Reduce tune.ssl.cachesize for HTTPS terminating listeners

454cff5 introduced haproxy's `tune.ssl.cachesize` for TERMINATED_HTTPS
listeners. During a reload of haproxy the old worker process stays
running until the new worker process is ready. This means that two TLS
session caches are allocated/held simultaneously during a reload of the
service.

For small Amphorae, this works fine. The default connection limit is
50000, which takes enough of a chunk out of the 50% allocation that
there is enough wiggle room for the new haproxy worker to allocate its
cache and coexist with the old worker for some time.

However, for larger amphorae, the memory calculated for the session
cache approaches 50%.

haproxy allocates an additional 48 bytes for each 200 byte chunk, so
the total memory allocated exceeds 50% of the available memory,
triggering the OOM-killer on haproxy reload.

Out of an abundance of caution this also reduces the proportion of
memory Octavia considers "available" for the TLS session cache from 1/2
to 2/5.

Closes-Bug: #2119987
Change-Id: I91b6907c3e3e456860f7274153e0ecf030e0519e
Signed-off-by: Wesley Hershberger <wesley.hershberger@canonical.com>

octavia/common/jinja/haproxy/combined_listeners/jinja_cfg.py

@@ -221,14 +221,14 @@ class JinjaTemplater:
                 # because that is what ulimit -n typically returns.
                 max_conn_mem_kb = 32 * loadbalancer.get(
                     "global_connection_limit", 1024)
-                # Use half of the remaining memory for SSL caches
+                # LP #2119987: Use 2/5 of the remaining memory for SSL caches
                 ssl_cache_mem_kb = (mem["free"] + mem["buffers"] +
-                                    mem["cached"] - max_conn_mem_kb) // 2
-                # A cache block uses about 200 bytes of data.
+                                    mem["cached"] - max_conn_mem_kb) * 2 // 5
+                # A cache block uses about 250 bytes of data.
                 # The HAProxy default of ssl_cache (20000) would take up
-                # 4000 KB. We don't want to go below that.
-                if ssl_cache_mem_kb > 4000:
-                    jinja_dict["ssl_cache"] = ssl_cache_mem_kb * 5
+                # 5000 KB. We don't want to go below that.
+                if ssl_cache_mem_kb > 5000:
+                    jinja_dict["ssl_cache"] = ssl_cache_mem_kb * 4
             except (KeyError, TypeError):
                 pass
 

octavia/tests/unit/common/jinja/haproxy/combined_listeners/test_jinja_cfg.py

@@ -1140,6 +1140,42 @@ class TestHaproxyCfg(base.TestCase):
             frontend=fe, logging=lg, backend=be, global_opts=g_opts),
             rendered_obj)
 
+    def test_render_template_tls_cachesize(self):
+        g_opts = (f"    maxconn {constants.HAPROXY_DEFAULT_MAXCONN}\n"
+                  f"    tune.ssl.cachesize 101722232\n\n")
+        fe = ("frontend sample_listener_id_1\n"
+              f"    maxconn {constants.HAPROXY_DEFAULT_MAXCONN}\n"
+              "    redirect scheme https if !{ ssl_fc }\n"
+              "    http-response set-header Strict-Transport-Security "
+              "\"max-age=10000000; includeSubDomains; preload;\"\n"
+              "    bind 10.0.0.2:443 "
+              f"ciphers {constants.CIPHERS_OWASP_SUITE_B} "
+              "no-sslv3 no-tlsv10 no-tlsv11 alpn "
+              f"{','.join(constants.AMPHORA_SUPPORTED_ALPN_PROTOCOLS)}\n"
+              "    mode http\n"
+              "    default_backend sample_pool_id_1:sample_listener_id_1\n"
+              "    timeout client 50000\n")
+        tls_tupe = {'cont_id_1':
+                    sample_configs_combined.sample_tls_container_tuple(
+                        id='tls_container_id',
+                        certificate='imaCert1', private_key='imaPrivateKey1',
+                        primary_cn='FakeCN'),
+                    'cont_id_ca': 'client_ca.pem',
+                    'cont_id_crl': 'SHA_ID.pem'}
+        rendered_obj = self.jinja_cfg.render_loadbalancer_obj(
+            sample_configs_combined.sample_amphora_tuple(),
+            [sample_configs_combined.sample_listener_tuple(
+                proto='TERMINATED_HTTPS')],
+            tls_tupe,
+            # 32GiB total
+            amp_details={"memory": {
+                "free": 32864004,
+                "buffers": 32312392 // 2,
+                "cached": 32312392 // 2,
+            }})
+        self.assertEqual(sample_configs_combined.sample_base_expected_config(
+            frontend=fe, global_opts=g_opts), rendered_obj)
+
     def test_render_template_l7policies(self):
         fe = ("frontend sample_listener_id_1\n"
               "    maxconn {maxconn}\n"

releasenotes/notes/fix-haproxy-ssl-cache-size-5d5842100a87de54.yaml

@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    Reduce the value of tune.ssl.cachesize for HTTPS termination listeners to
+    prevent OOM during haproxy reload (LP: #2119987).