diff --git a/ansible-role-requirements.yml b/ansible-role-requirements.yml new file mode 100644 index 0000000000..74c8a3eca6 --- /dev/null +++ b/ansible-role-requirements.yml @@ -0,0 +1,9 @@ +# Use this file to fill in your third party roles that you'd like to have added to the list of available roles. +# Example: +# - github_api: https://api.github.com/repos/os-cloud/opc_role-galera_client +# name: galera_client +# src: https://github.com/os-cloud/opc_role-galera_client +# version: master +- src: evrardjp.keepalived + name: keepalived + version: '1.3' diff --git a/doc/source/install-guide/configure-haproxy.rst b/doc/source/install-guide/configure-haproxy.rst index 190fe2eba4..f19d7782ac 100644 --- a/doc/source/install-guide/configure-haproxy.rst +++ b/doc/source/install-guide/configure-haproxy.rst @@ -23,8 +23,61 @@ balancer prior to deploying OSA. 123458-infra03: ip: 172.29.236.53 +Making HAProxy highly-available +############################### + +HAProxy will be deployed in a highly-available manner, by installing +keepalived if multiple hosts are found in the inventory. + +To skip the deployment of keepalived along HAProxy when installing +HAProxy on multiple hosts, edit the +``/etc/openstack_deploy/user_variables.yml`` by setting: + +.. code-block:: yaml + + haproxy_use_keepalived: False + +Otherwise, edit at least the following variables in +``user_variables.yml`` to make keepalived work: + +.. code-block:: yaml + + haproxy_keepalived_external_vip_cidr: 192.168.0.4/25 + haproxy_keepalived_internal_vip_cidr: 172.29.236.54/16 + haproxy_keepalived_external_interface: br-flat + haproxy_keepalived_internal_interface: br-mgmt + +``haproxy_keepalived_internal_interface`` represents the interface +on the deployed node where the keepalived master will bind the +internal vip. By default the ``br-mgmt`` will be used. + +``haproxy_keepalived_external_interface`` represents the interface +on the deployed node where the keepalived master will bind the +external vip. By default the ``br-mgmt`` will be used. + +``haproxy_keepalived_external_vip_cidr`` represents the external +vip (and its netmask) that will be used on keepalived master host. + +``haproxy_keepalived_internal_vip_cidr`` represents the internal +vip (and its netmask) that will be used on keepalived master host. + +Additional variables can be set to adapt keepalived in the deployed +environment. Please refer to the ``user_variables.yml`` +for more descriptions. + +All the variables mentionned here before are used in the variable +files ``vars/configs/keepalived_haproxy_master.yml`` and +``vars/configs/keepalived_haproxy_backup.yml`` to feed the +keepalived role. More information can be found on the keepalived +role documentation. You can use your own file by setting their path +in your ``/etc/openstack_deploy/user_variables.yml``: + +.. code-block:: yaml + + haproxy_keepalived_vars_file: + Securing HAProxy communication with SSL certificates -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +#################################################### The openstack-ansible project provides the ability to secure HAProxy communications with self-signed or user-provided SSL certificates. diff --git a/doc/source/install-guide/install-foundation-run.rst b/doc/source/install-guide/install-foundation-run.rst index 124310c310..e9c2fa1ef5 100644 --- a/doc/source/install-guide/install-foundation-run.rst +++ b/doc/source/install-guide/install-foundation-run.rst @@ -21,11 +21,30 @@ Running the foundation playbook ... deployment_host : ok=18 changed=11 unreachable=0 failed=0 -#. If using HAProxy, run the playbook to deploy it: +#. If using HAProxy: - .. code-block:: bash + .. note:: - $ openstack-ansible haproxy-install.yml + If you plan to run haproxy on multiple hosts, you'll need keepalived + to make haproxy highly-available. The keepalived role should have + been downloaded during the bootstrap-ansible stage. If not, you should + rerun the following command before running the haproxy playbook: + + .. code-block:: shell + + $ ../scripts/bootstrap-ansible.sh + + or + + .. code-block:: shell + + $ ansible-galaxy install -r ../ansible-role-requirements.yml + + Run the playbook to deploy haproxy: + + .. code-block:: bash + + $ openstack-ansible haproxy-install.yml -------------- diff --git a/etc/openstack_deploy/conf.d/haproxy.yml.example b/etc/openstack_deploy/conf.d/haproxy.yml.example new file mode 100644 index 0000000000..7224ed5ff8 --- /dev/null +++ b/etc/openstack_deploy/conf.d/haproxy.yml.example @@ -0,0 +1,6 @@ +# The nodes where haproxy will run +haproxy_hosts: + infra1: + ip: 172.20.236.110 + infra2: + ip: 172.20.236.111 diff --git a/etc/openstack_deploy/env.d/haproxy.yml b/etc/openstack_deploy/env.d/haproxy.yml new file mode 100644 index 0000000000..39ca70c428 --- /dev/null +++ b/etc/openstack_deploy/env.d/haproxy.yml @@ -0,0 +1,39 @@ +--- +# Copyright 2015, Jean-Philippe Evrard +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +component_skel: + haproxy: + belongs_to: + # This is a meta group of a given component type. + - haproxy_all + +container_skel: + haproxy_container: + belongs_to: + - haproxy_containers + contains: + - haproxy + properties: + service_name: haproxy + #container_release: trusty + is_metal: true + +physical_skel: + haproxy_containers: + belongs_to: + - all_containers + haproxy_hosts: + belongs_to: + - hosts diff --git a/etc/openstack_deploy/user_secrets.yml b/etc/openstack_deploy/user_secrets.yml index 2d9fe5e87e..d197e4e83f 100644 --- a/etc/openstack_deploy/user_secrets.yml +++ b/etc/openstack_deploy/user_secrets.yml @@ -95,3 +95,4 @@ swift_hash_path_prefix: ## haproxy stats password haproxy_stats_password: +haproxy_keepalived_authentication_password: diff --git a/etc/openstack_deploy/user_variables.yml b/etc/openstack_deploy/user_variables.yml index 10245f0934..782a661ed6 100644 --- a/etc/openstack_deploy/user_variables.yml +++ b/etc/openstack_deploy/user_variables.yml @@ -167,3 +167,25 @@ ssl_cipher_suite: "ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AE # If an increased delay for the ssh connection check is desired, # uncomment this variable and set it appropriately. #ssh_delay: 5 + +## HAProxy +# Uncomment this to disable keepalived installation (cf. documentation) +#haproxy_use_keepalived: False +# +# HAProxy Keepalived configuration (cf. documentation) +haproxy_keepalived_external_vip_cidr: "{{external_lb_vip_address}}/32" +haproxy_keepalived_internal_vip_cidr: "{{internal_lb_vip_address}}/32" +#haproxy_keepalived_external_interface: +#haproxy_keepalived_internal_interface: +# Defines the default VRRP id used for keepalived with haproxy. +# Overwrite it to your value to make sure you don't overlap +# with existing VRRPs id on your network. Default is 10 for the external and 11 for the +# internal VRRPs +#haproxy_keepalived_external_virtual_router_id: +#haproxy_keepalived_internal_virtual_router_id: +# Defines the VRRP master/backup priority. Defaults respectively to 100 and 20 +#haproxy_keepalived_priority_master: +#haproxy_keepalived_priority_backup: +# All the previous variables are used in a var file, fed to the keepalived role. +# To use another file to feed the role, override the following var: +#haproxy_keepalived_vars_file: 'vars/configs/keepalived_haproxy.yml' diff --git a/playbooks/haproxy-install.yml b/playbooks/haproxy-install.yml index 62e1e799f1..95f4b86b39 100644 --- a/playbooks/haproxy-install.yml +++ b/playbooks/haproxy-install.yml @@ -13,6 +13,25 @@ # See the License for the specific language governing permissions and # limitations under the License. +- hosts: haproxy_hosts + vars_files: + - "{{ haproxy_keepalived_vars_file | default('vars/configs/keepalived_haproxy.yml')}}" + roles: + - role: "keepalived" + keepalived_sync_groups: "{{ keepalived_master_sync_groups }}" + keepalived_scripts: "{{ keepalived_master_scripts }}" + keepalived_instances: "{{ keepalived_master_instances }}" + when: > + haproxy_use_keepalived|bool and + inventory_hostname in groups['haproxy_hosts'][0] + - role: "keepalived" + keepalived_sync_groups: "{{ keepalived_backup_sync_groups }}" + keepalived_scripts: "{{ keepalived_backup_scripts }}" + keepalived_instances: "{{ keepalived_backup_instances }}" + when: > + haproxy_use_keepalived|bool and + inventory_hostname in groups['haproxy_hosts'][1:] + - name: Install haproxy hosts: haproxy_hosts max_fail_percentage: 20 diff --git a/playbooks/inventory/group_vars/hosts.yml b/playbooks/inventory/group_vars/hosts.yml index 382349f1e7..f53705b46c 100644 --- a/playbooks/inventory/group_vars/hosts.yml +++ b/playbooks/inventory/group_vars/hosts.yml @@ -232,3 +232,4 @@ swift_service_region: "{{ service_region }}" ## HAProxy haproxy_bind_on_non_local: "{% if groups.haproxy_hosts[1] is defined and internal_lb_vip_address != external_lb_vip_address %}True{% else %}False{% endif %}" +haproxy_use_keepalived: "{% if groups.haproxy_hosts|length > 1 %}True{% else %}False{% endif %}" diff --git a/playbooks/vars/configs/keepalived_haproxy.yml b/playbooks/vars/configs/keepalived_haproxy.yml new file mode 100644 index 0000000000..bf9f42c9a1 --- /dev/null +++ b/playbooks/vars/configs/keepalived_haproxy.yml @@ -0,0 +1,90 @@ +--- +# Copyright 2015, Jean-Philippe Evrard +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +keepalived_global_sync_groups: + haproxy: + instances: + - external + - internal + notify_script: /etc/keepalived/haproxy_notify.sh + ##if a src_*_script is defined, it will be uploaded from src_*_script on the deploy host to the *_script location. Make sure *_script is a location in that case. + #src_notify_script: /opt/os-ansible-deployment/playbooks/vars/configs/keepalived_haproxy_notifications.sh + src_notify_script: vars/configs/keepalived_haproxy_notifications.sh + +# Master and backup sync groups should normally be the same. +keepalived_master_sync_groups: "{{ keepalived_global_sync_groups }}" +keepalived_backup_sync_groups: "{{ keepalived_global_sync_groups }}" + +keepalived_global_scripts: + haproxy_check_script: + check_script: "killall -0 haproxy" + pingable_check_script: + check_script: "ping -c 1 193.0.14.129 1>&2" + interval: 10 + fall: 2 + rise: 4 + +# Master and backup scripts should be the same. +# The two variables (master/backup) are kept if the deployer wants different checks for backup and master. +keepalived_master_scripts: "{{ keepalived_global_scripts }}" +keepalived_backup_scripts: "{{ keepalived_global_scripts }}" + +keepalived_master_instances: + external: + interface: "{{ haproxy_keepalived_external_interface | default(management_bridge) }}" + state: MASTER + virtual_router_id: "{{ haproxy_keepalived_external_virtual_router_id | default ('10') }}" + priority: "{{ haproxy_keepalived_priority_master | default('100') }}" + authentication_password: "{{ haproxy_keepalived_authentication_password }}" + vips: + - "{{ haproxy_keepalived_external_vip_cidr }} dev {{ haproxy_keepalived_external_interface | default(management_bridge) }}" + track_scripts: + - haproxy_check_script + - pingable_check_script + internal: + interface: "{{ haproxy_keepalived_internal_interface | default(management_bridge) }}" + state: MASTER + virtual_router_id: "{{ haproxy_keepalived_internal_virtual_router_id | default ('11') }}" + priority: "{{ haproxy_keepalived_priority_master | default('100') }}" + authentication_password: "{{ haproxy_keepalived_authentication_password }}" + track_scripts: + - haproxy_check_script + - pingable_check_script + vips: + - "{{ haproxy_keepalived_internal_vip_cidr }} dev {{ haproxy_keepalived_internal_interface | default(management_bridge) }}" + +keepalived_backup_instances: + external: + interface: "{{ haproxy_keepalived_external_interface | default(management_bridge) }}" + state: BACKUP + virtual_router_id: "{{ haproxy_keepalived_external_virtual_router_id | default ('10') }}" + priority: "{{ haproxy_keepalived_priority_backup | default('20') }}" + authentication_password: "{{ haproxy_keepalived_authentication_password }}" + vips: + - "{{ haproxy_keepalived_external_vip_cidr }} dev {{ haproxy_keepalived_external_interface | default(management_bridge) }}" + track_scripts: + - haproxy_check_script + - pingable_check_script + internal: + interface: "{{ haproxy_keepalived_internal_interface | default(management_bridge) }}" + state: BACKUP + virtual_router_id: "{{ haproxy_keepalived_internal_virtual_router_id | default ('11') }}" + priority: "{{ haproxy_keepalived_priority_backup | default('20') }}" + authentication_password: "{{ haproxy_keepalived_authentication_password }}" + track_scripts: + - haproxy_check_script + - pingable_check_script + vips: + - "{{ haproxy_keepalived_internal_vip_cidr }} dev {{ haproxy_keepalived_internal_interface | default(management_bridge) }}" diff --git a/playbooks/vars/configs/keepalived_haproxy_notifications.sh b/playbooks/vars/configs/keepalived_haproxy_notifications.sh new file mode 100644 index 0000000000..1d6bead9fb --- /dev/null +++ b/playbooks/vars/configs/keepalived_haproxy_notifications.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Copyright 2015, Jean-Philippe Evrard +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +TYPE=$1 +NAME=$2 +NOW=`date "+%Y-%m-%d %H:%M:%S"` +NEWSTATE=$3 +OLDSTATE=$(cat /var/run/keepalived.state) + +echo "$NEWSTATE" > /var/run/keepalived.state + +case $NEWSTATE in + "FAULT") echo "$NOW Trying to restart haproxy to get out"\ + "of faulty state" >> /var/log/keepalived-notifications.log + /etc/init.d/haproxy stop + /etc/init.d/haproxy start + exit 0 + ;; + *) echo "$NOW Unknown state" >> /var/log/keepalived-notifications.log + exit 1 + ;; +esac diff --git a/scripts/gate-check-lint.sh b/scripts/gate-check-lint.sh index 62568e958d..4b394e7a5c 100755 --- a/scripts/gate-check-lint.sh +++ b/scripts/gate-check-lint.sh @@ -24,6 +24,8 @@ info_block "Checking for required libraries." 2> /dev/null || source $(dirname $ ## Main ---------------------------------------------------------------------- info_block "Running Basic Ansible Lint Check" +# next, bootstrap Ansible +source $(dirname ${0})/bootstrap-ansible.sh # Install the development requirements. if [ -f "dev-requirements.txt" ]; then