Tune haproxy for long running sessions to use leastconn

For long running sessions, the leastconn LB is more prefered than the
default roundrobin. For long connections, it picks the least recently
used of the servers with the lowest connection count.

LRU LB strategy that leastconn uses also indirectly reduces the
possibility of cascaded failures by smoothing out the high traffic
spikes, which otherwise may be caused by unequal round-robin
distribution of client connections upon executing it a fail-over to
another backend.

Heat API/cfn provides Orchestration API with a notion of long running
tasks. Neutron server may maintain long running RPC calls to its
agents.  Cinder BlockStorage API provides long running volume and
backup actions.  Swift Proxy's PutObject etc. and Ceph RGW APIs, like
https://docs.ceph.com/docs/master/radosgw/s3/bucketops/, have a notion
of "long running" as well. Ironic Inspector API provides long running
operations, like introspection of BM nodes. All those should benefit
from the leastconn LB switch controlled by the new parameter.

Closes-Bug: #1882927
Change-Id: I9515af738113a3f7aa2ea07315889d4a6595d4eb
Signed-off-by: Bogdan Dobrelya <bdobreli@redhat.com>

manifests/haproxy.pp

@@ -885,6 +885,7 @@ class tripleo::haproxy (
       ip_addresses      => hiera('neutron_api_node_ips', $controller_hosts_real),
       server_names      => hiera('neutron_api_node_names', $controller_hosts_names_real),
       mode              => 'http',
+      listen_options    => merge($default_listen_options, { 'balance' => $haproxy_lb_mode_longrunning }),
       public_ssl_port   => $ports[neutron_api_ssl_port],
       service_network   => $neutron_network,
       member_options    => union($haproxy_member_options, $internal_tls_member_options),
@@ -899,6 +900,7 @@ class tripleo::haproxy (
       ip_addresses      => hiera('cinder_api_node_ips', $controller_hosts_real),
       server_names      => hiera('cinder_api_node_names', $controller_hosts_names_real),
       mode              => 'http',
+      listen_options    => merge($default_listen_options, { 'balance' => $haproxy_lb_mode_longrunning }),
       public_ssl_port   => $ports[cinder_api_ssl_port],
       service_network   => $cinder_network,
       member_options    => union($haproxy_member_options, $internal_tls_member_options),
@@ -1120,6 +1122,7 @@ class tripleo::haproxy (
   if $swift_proxy_server {
     $swift_proxy_server_listen_options = {
       'option'         => [ 'httpchk GET /healthcheck', ],
+      'balance'        => $haproxy_lb_mode_longrunning,
       'timeout client' => '2m',
       'timeout server' => '2m',
     }
@@ -1227,6 +1230,7 @@ class tripleo::haproxy (
       public_ssl_port   => $ports[ironic_inspector_ssl_port],
       service_network   => $ironic_inspector_network,
       mode              => 'http',
+      listen_options    => merge($default_listen_options, { 'balance' => $haproxy_lb_mode_longrunning }),
     }
   }
 
@@ -1449,7 +1453,13 @@ class tripleo::haproxy (
       mode              => 'http',
       public_ssl_port   => $ports[ceph_rgw_ssl_port],
       service_network   => $ceph_rgw_network,
-      listen_options    => merge($default_listen_options, { 'option' => [ 'httpchk GET /swift/healthcheck' ] }),
+      listen_options    => merge(
+        $default_listen_options,
+        {
+          'option'  => [ 'httpchk GET /swift/healthcheck' ],
+          'balance' => $haproxy_lb_mode_longrunning
+        }
+      ),
       member_options    => union($haproxy_member_options, $internal_tls_member_options),
     }
   }

releasenotes/notes/haproxy-leastconn-overrides-bdb2068ef794ff1d.yaml

@@ -1,11 +1,9 @@
 ---
-features:
-  - |
-    Add `haproxy_lb_mode_longrunning` parameter for the use with the services
-    that may have longrunning API requests. Defaults to 'leastconn' (replaces
-    the HAProxy default 'roundrobin').
 upgrade:
   - |
     Since Heat API can be given longrunning API requests its backends will
     become load-balanced based on LRU 'leastconn' algorithm and its sessions
     will also benefit from the TCP-keepalive feature of HAProxy.
+
+    Some of the other services with the longrunning requests notion will start
+    using the 'leastconn' LRU LB as well.

spec/classes/tripleo_haproxy_spec.rb

@@ -187,6 +187,48 @@ describe 'tripleo::haproxy' do
         )
       end
     end
+
+    describe "APIs with long running actions to use leastconn" do
+      before :each do
+        params.merge!({
+          :neutron            => true,
+          :cinder             => true,
+          :swift_proxy_server => true,
+          :heat_api           => true,
+          :heat_cfn           => true,
+          :ironic_inspector   => true,
+          :ceph_rgw           => true,
+        })
+      end
+
+      %w(neutron cinder swift_proxy_server heat_cfn ironic-inspector ceph_rgw).each do |api|
+        it 'should configure haproxy ' + api + ' endpoint' do
+          is_expected.to contain_haproxy__listen(api)
+          p = catalogue.resource('tripleo::haproxy::endpoint', api).send(:parameters)
+          expect(p).to include(listen_options: a_hash_including('balance' => 'leastconn'))
+        end
+      end
+    end
+
+    describe "source-based sticky sessions w/o use of consistent hashing" do
+      before :each do
+        params.merge!({
+          :etcd            => true,
+          :ceph_grafana    => true,
+          :ceph_dashboard  => true,
+          :nova_novncproxy => true,
+        })
+      end
+
+      %w(etcd ceph_grafana ceph_dashboard nova_novncproxy).each do |svc|
+        it 'should configure haproxy ' + svc + ' endpoint' do
+          is_expected.to contain_haproxy__listen(svc)
+          p = catalogue.resource('tripleo::haproxy::endpoint', svc).send(:parameters)
+          expect(p).to include(listen_options: a_hash_including(
+            'balance' => 'source'))
+        end
+      end
+    end
   end
 
   on_supported_os.each do |os, facts|