From d632847311341f507f77cc2e9d9c5c0d0691c1b4 Mon Sep 17 00:00:00 2001 From: Michele Baldessari Date: Sun, 20 Dec 2020 10:37:22 +0100 Subject: [PATCH] Allow to specify a nic for the VIPs + Fix nic selection when no nic is specified MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This review is a merge of: Revert of Revert "Cleanup old workaround for ipv6 VIPs": Ia41046f148e0593ea773e8409494ce5dcca4b7a2 Fix nic selection when no nic is specified : I14bfaf0060093811cddf0cdd9a130b91a3cd0477 Allow to specify a nic for the VIPs : Iaa2676655d97a66237a3bd6066985f87d3565a4d↲ We do this merge only for ussuri and train because in master and victoria we landed "Allow to specify a nic for the VIPs" before doing the revert of the "Cleanup old workaround for ipv6 VIPs". So this change restores the exact state as we have in master and victoria with a single review. We do not split the reviews because if the Ia41046f148e0593ea773e8409494ce5dcca4b7a2 merges without the other two we will have ipv6 broken. This review allows the operator to specify a nic to bind to a VIP to. It does so in two different ways: 1) Via a global 'tripleo::pacemaker::force_nic' hiera key. For example setting 'tripleo::pacemaker::force_nic: lo' will make sure all VIPs are added to the 'lo' network interface. 2) The global setting can be overriden via a custom hiera hash called 'force_vip_nic_overrides' so that we can override the binding nic for specific VIPs. For example: force_vip_nic_overrides: redis_vip: vip-test-nic ovn_dbs_vip: vip-test-nic Will make sure that the redis_vip and the ovn_dbs_vip will be bound to the 'vip-test-nic'. Tested with: tripleo::pacemaker::force_nic: lo force_vip_nic_overrides: redis_vip: vip-test-nic ovn_dbs_vip: vip-test-nic And correctly got all VIPs running on lo: 1: lo inet 192.168.24.9/32 scope global lo\ valid_lft forever preferred_lft forever 1: lo inet 172.23.1.9/32 scope global lo\ valid_lft forever preferred_lft forever except for redis and ovn_dbs which are on the vip-test-nic: 5: vip-test-nic inet 172.25.1.7/32 scope global vip-test-nic\ valid_lft forever preferred_lft forever Note: The interface is intentionally kept in hiera as this change will only be needed whenever BGP runs on all nodes and advertises the VIPs across the ASN (i.e. we do not want to make it very exposed just yet) Change-Id: I0630ae124dac304614431e67975767206555e9aa --- manifests/pacemaker/haproxy_with_vip.pp | 18 +++++++--- manifests/profile/pacemaker/haproxy_bundle.pp | 33 +++++++++++++++++++ manifests/profile/pacemaker/ovn_dbs_bundle.pp | 29 ++++++++++++++-- .../notes/vip-bind-nic-11e80207fcb78a20.yaml | 10 ++++++ 4 files changed, 82 insertions(+), 8 deletions(-) create mode 100644 releasenotes/notes/vip-bind-nic-11e80207fcb78a20.yaml diff --git a/manifests/pacemaker/haproxy_with_vip.pp b/manifests/pacemaker/haproxy_with_vip.pp index c3d1c3af9..559701176 100644 --- a/manifests/pacemaker/haproxy_with_vip.pp +++ b/manifests/pacemaker/haproxy_with_vip.pp @@ -52,6 +52,10 @@ # (Optional) The number of times pcs commands should be retried. # Defaults to 1 # +# [*nic*] +# (Optional) Specifies the nic interface on which the VIP should be added +# Defaults to undef +# # [*ensure*] # (Boolean) Create the all the resources only if true. False won't # destroy the resource, it will just not create them. @@ -64,23 +68,27 @@ define tripleo::pacemaker::haproxy_with_vip( $meta_params = '', $op_params = '', $pcs_tries = 1, + $nic = undef, $ensure = true) { if($ensure) { if !is_ip_addresses($ip_address) { fail("Haproxy VIP: ${ip_address} is not a proper IP address.") } - # NB: Until the IPaddr2 RA has a fix for https://bugzilla.redhat.com/show_bug.cgi?id=1445628 - # we need to specify the nic when creating the ipv6 vip. if is_ipv6_address($ip_address) { $netmask = '128' - $nic = interface_for_ip($ip_address) + $vip_nic = interface_for_ip($ip_address) $ipv6_addrlabel = '99' } else { $netmask = '32' - $nic = '' + $vip_nic = '' $ipv6_addrlabel = '' } + if $nic != undef { + $nic_real = $nic + } else { + $nic_real = $vip_nic + } $haproxy_in_container = hiera('haproxy_docker', false) $constraint_target_name = $haproxy_in_container ? { @@ -91,7 +99,7 @@ define tripleo::pacemaker::haproxy_with_vip( pacemaker::resource::ip { "${vip_name}_vip": ip_address => $ip_address, cidr_netmask => $netmask, - nic => $nic, + nic => $nic_real, ipv6_addrlabel => $ipv6_addrlabel, meta_params => "resource-stickiness=INFINITY ${meta_params}", location_rule => $location_rule, diff --git a/manifests/profile/pacemaker/haproxy_bundle.pp b/manifests/profile/pacemaker/haproxy_bundle.pp index 0eb1b4ef2..79030ce67 100644 --- a/manifests/profile/pacemaker/haproxy_bundle.pp +++ b/manifests/profile/pacemaker/haproxy_bundle.pp @@ -98,6 +98,12 @@ # (optional) Set the --user= switch to be passed to pcmk # Defaults to 'root' # +# [*force_nic*] +# (optional) Force a specific nic interface name when creating all the VIPs +# The listening nic can be customized on a per-VIP basis by creating a hiera +# dict called: force_vip_nic_overrides[] = 'dummy' +# Defaults to hiera('tripleo::pacemaker::force_nic', undef) +# class tripleo::profile::pacemaker::haproxy_bundle ( $haproxy_docker_image = hiera('tripleo::profile::pacemaker::haproxy::haproxy_docker_image', undef), $bootstrap_node = hiera('haproxy_short_bootstrap_node_name'), @@ -113,6 +119,7 @@ class tripleo::profile::pacemaker::haproxy_bundle ( $container_backend = 'docker', $tls_priorities = hiera('tripleo::pacemaker::tls_priorities', undef), $bundle_user = 'root', + $force_nic = hiera('tripleo::pacemaker::force_nic', undef), $log_driver = undef, $log_file = '/var/log/containers/stdouts/haproxy-bundle.log', $step = Integer(hiera('step')), @@ -140,6 +147,8 @@ class tripleo::profile::pacemaker::haproxy_bundle ( } else { $log_file_real = '' } + $force_vip_nic_overrides = hiera('force_vip_nic_overrides', {}) + validate_legacy(Hash, 'validate_hash', $force_vip_nic_overrides) if $step >= 2 and $enable_load_balancer { if $pacemaker_master { @@ -296,16 +305,27 @@ class tripleo::profile::pacemaker::haproxy_bundle ( tries => $pcs_tries, } $control_vip = hiera('controller_virtual_ip') + if has_key($force_vip_nic_overrides, 'controller_virtual_ip') { + $control_vip_nic = $force_vip_nic_overrides['controller_virtual_ip'] + } else { + $control_vip_nic = $force_nic + } tripleo::pacemaker::haproxy_with_vip { 'haproxy_and_control_vip': vip_name => 'control', ip_address => $control_vip, location_rule => $haproxy_location_rule, meta_params => $meta_params, op_params => $op_params, + nic => $control_vip_nic, pcs_tries => $pcs_tries, } $public_vip = hiera('public_virtual_ip') + if has_key($force_vip_nic_overrides, 'public_virtual_ip') { + $public_vip_nic = $force_vip_nic_overrides['public_virtual_ip'] + } else { + $public_vip_nic = $force_nic + } tripleo::pacemaker::haproxy_with_vip { 'haproxy_and_public_vip': ensure => $public_vip and $public_vip != $control_vip, vip_name => 'public', @@ -313,12 +333,18 @@ class tripleo::profile::pacemaker::haproxy_bundle ( location_rule => $haproxy_location_rule, meta_params => $meta_params, op_params => $op_params, + nic => $public_vip_nic, pcs_tries => $pcs_tries, } $redis = hiera('redis_enabled', false) if $redis { $redis_vip = hiera('redis_vip') + if has_key($force_vip_nic_overrides, 'redis_vip') { + $redis_vip_nic = $force_vip_nic_overrides['redis_vip'] + } else { + $redis_vip_nic = $force_nic + } tripleo::pacemaker::haproxy_with_vip { 'haproxy_and_redis_vip': ensure => $redis_vip and $redis_vip != $control_vip, vip_name => 'redis', @@ -326,6 +352,7 @@ class tripleo::profile::pacemaker::haproxy_bundle ( location_rule => $haproxy_location_rule, meta_params => $meta_params, op_params => $op_params, + nic => $redis_vip_nic, pcs_tries => $pcs_tries, } } @@ -334,6 +361,11 @@ class tripleo::profile::pacemaker::haproxy_bundle ( $network_vips = hiera('network_virtual_ips', {}) $network_vips.each |String $net_name, $vip_info| { $virtual_ip = $vip_info[ip_address] + if has_key($force_vip_nic_overrides, $net_name) { + $vip_nic = $force_vip_nic_overrides[$net_name] + } else { + $vip_nic = $force_nic + } tripleo::pacemaker::haproxy_with_vip {"haproxy_and_${net_name}_vip": ensure => $virtual_ip and $virtual_ip != $control_vip, vip_name => $net_name, @@ -341,6 +373,7 @@ class tripleo::profile::pacemaker::haproxy_bundle ( location_rule => $haproxy_location_rule, meta_params => $meta_params, op_params => $op_params, + nic => $vip_nic, pcs_tries => $pcs_tries, } } diff --git a/manifests/profile/pacemaker/ovn_dbs_bundle.pp b/manifests/profile/pacemaker/ovn_dbs_bundle.pp index 47561507f..028c8c7cf 100644 --- a/manifests/profile/pacemaker/ovn_dbs_bundle.pp +++ b/manifests/profile/pacemaker/ovn_dbs_bundle.pp @@ -104,6 +104,13 @@ # [*force_ocf*] # (optional) Use --force when creating the ocf resource via pcs # Defaults to false +# +# [*force_nic*] +# (optional) Force a specific nic interface name when creating all the VIPs +# The listening nic can be customized on a per-VIP basis by creating a hiera +# dict called: force_vip_nic_overrides[] = 'dummy' +# Defaults to hiera('tripleo::pacemaker::force_nic', undef) +# class tripleo::profile::pacemaker::ovn_dbs_bundle ( $ovn_dbs_docker_image = undef, @@ -126,6 +133,7 @@ class tripleo::profile::pacemaker::ovn_dbs_bundle ( $dbs_timeout = 60, $listen_on_master_ip_only = 'yes', $force_ocf = false, + $force_nic = hiera('tripleo::pacemaker::force_nic', undef), ) { if $bootstrap_node and $::hostname == downcase($bootstrap_node) { @@ -148,6 +156,8 @@ class tripleo::profile::pacemaker::ovn_dbs_bundle ( } else { $log_file_real = '' } + $force_vip_nic_overrides = hiera('force_vip_nic_overrides', {}) + validate_legacy(Hash, 'validate_hash', $force_vip_nic_overrides) if $step >= 3 { if $pacemaker_master { @@ -295,11 +305,18 @@ monitor interval=30s role=Slave timeout=${dbs_timeout}s", $old_vip = $net_vip_map[$ovn_dbs_network]['ip_address'] if $old_vip != $ovn_dbs_vip { $ovn_separate_vip = true + if has_key($force_vip_nic_overrides, 'ovn_dbs_vip') { + $ovn_dbs_vip_nic = $force_vip_nic_overrides['ovn_dbs_vip'] + } else { + $ovn_dbs_vip_nic = $force_nic + } } else { $ovn_separate_vip = false + $ovn_dbs_vip_nic = $force_nic } } else { $ovn_separate_vip = false + $ovn_dbs_vip_nic = $force_nic } # We create a separate VIP only in case OVN has been configured so via THT @@ -307,18 +324,24 @@ monitor interval=30s role=Slave timeout=${dbs_timeout}s", if $ovn_separate_vip { if is_ipv6_address($ovn_dbs_vip) { $netmask = '128' - $nic = interface_for_ip($ovn_dbs_vip) + $vip_nic = interface_for_ip($ovn_dbs_vip) $ipv6_addrlabel = '99' } else { $netmask = '32' - $nic = '' + $vip_nic = '' $ipv6_addrlabel = '' } + if $ovn_dbs_vip_nic != undef { + $nic_real = $ovn_dbs_vip_nic + } else { + $nic_real = $vip_nic + } + pacemaker::resource::ip { "${ovndb_vip_resource_name}": ip_address => $ovn_dbs_vip, cidr_netmask => $netmask, - nic => $nic, + nic => $nic_real, ipv6_addrlabel => $ipv6_addrlabel, location_rule => $ovn_dbs_location_rule, meta_params => "resource-stickiness=INFINITY ${meta_params}", diff --git a/releasenotes/notes/vip-bind-nic-11e80207fcb78a20.yaml b/releasenotes/notes/vip-bind-nic-11e80207fcb78a20.yaml new file mode 100644 index 000000000..431ff55fd --- /dev/null +++ b/releasenotes/notes/vip-bind-nic-11e80207fcb78a20.yaml @@ -0,0 +1,10 @@ +--- +features: + - | + This change introduces two hiera keys that allow an operator to specify + which NIC (or NICs) the VIPs will be bound to. One hiera key has global effect + (tripleo::pacemaker::force_nic) and forces all VIPs to listen to that NIC. + There is also the possibility to override that for specific VIPs with the + force_vip_nic_overrides hiera hash. + This change is only useful for deployments where BGP is used to advertise + IP addresses from the host across multiple L3 networks.