Merge "Make /healthcheck cache results" into stable/victoria

doc/source/admin/healthcheck.rst

@@ -527,6 +527,25 @@ You will then need to select the desired monitoring backend plugins:
   changes. Not only does it not run any tests, it will return 204 results
   instead of 200.
 
+The Octavia API health monitoring endpoint does not require a keystone token
+for access to allow external load balancers to query the endpoint. For this
+reason we recommend you restrict access to it on your external load balancer
+to prevent abuse.
+
+As an additional protection, the API will cache results for a configurable
+period of time. This means that queries to the health monitoring endpoint
+will return cached results until the refresh interval has expired, at which
+point the health check plugin will rerun the check.
+
+By default, the refresh interval is five seconds. This can be configured by
+adjusting the healthcheck_refresh_interval setting in the Octavia configuration
+file:
+
+.. code-block:: ini
+
+    [api_settings]
+    healthcheck_refresh_interval = 5
+
 Optionally you can enable the "detailed" mode in Oslo middleware healthcheck.
 This will cause Oslo middleware healthcheck to return additional information
 about the API instance. It will also provide exception details if one was

etc/octavia.conf

@@ -56,6 +56,9 @@
 # Boolean to enable/disable oslo middleware /healthcheck in the Octavia API
 # healthcheck_enabled = False
 
+# The interval healthcheck plugins should cache results, in seconds.
+# healthcheck_refresh_interval = 5
+
 # Default cipher string for new TLS-terminated listeners
 # Cipher strings are in OpenSSL format, see https://www.openssl.org/docs/man1.1.1/man1/ciphers.html
 # This example is the "Broad Compatibility" cipher string from OWASP,

octavia/api/healthcheck/healthcheck_plugins.py

@@ -11,23 +11,42 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
+import datetime
+
+from oslo_config import cfg
 from oslo_middleware.healthcheck import pluginbase
 
 from octavia.db import api as db_apis
 from octavia.db import healthcheck
 
+CONF = cfg.CONF
+
 
 class OctaviaDBHealthcheck(pluginbase.HealthcheckBaseExtension):
 
     UNAVAILABLE_REASON = 'The Octavia database is unavailable.'
 
+    last_check = None
+    last_result = None
+    last_message = None
+
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
     def healthcheck(self, server_port):
         try:
-            result, message = healthcheck.check_database_connection(
+            if (self.last_check is not None and
-                db_apis.get_session())
+                    ((datetime.datetime.now() -
+                      self.last_check).total_seconds()) <
+                    CONF.api_settings.healthcheck_refresh_interval):
+                result = self.last_result
+                message = self.last_message
+            else:
+                result, message = healthcheck.check_database_connection(
+                    db_apis.get_session())
+                self.last_check = datetime.datetime.now()
+                self.last_result = result
+                self.last_message = message
             if result:
                 return OctaviaDBCheckResult(available=True, reason="OK")
             else:

octavia/common/config.py

@@ -105,6 +105,9 @@ api_opts = [
     cfg.BoolOpt('healthcheck_enabled', default=False,
                 help=_("When True, the oslo middleware healthcheck endpoint "
                        "is enabled in the Octavia API.")),
+    cfg.IntOpt('healthcheck_refresh_interval', default=5,
+               help=_("The interval healthcheck plugins should cache results, "
+                      "in seconds.")),
     cfg.StrOpt('default_listener_ciphers',
                default=constants.CIPHERS_OWASP_SUITE_B,
                help=_("Default OpenSSL cipher string (colon-separated) for "

octavia/tests/functional/api/test_healthcheck.py

@@ -49,6 +49,7 @@ class TestHealthCheck(base_db_test.OctaviaDBTestBase):
 
         self.conf = self.useFixture(oslo_fixture.Config(cfg.CONF))
         self.conf.config(group='healthcheck', backends=['octavia_db_check'])
+        self.conf.config(group='api_settings', healthcheck_refresh_interval=5)
         self.UNAVAILABLE = (healthcheck_plugins.OctaviaDBHealthcheck.
                             UNAVAILABLE_REASON)
 
@@ -145,6 +146,14 @@ class TestHealthCheck(base_db_test.OctaviaDBTestBase):
         self.assertIn('OK', response.text)
         self.assertIn('Garbage collector', response.text)
 
+    def test_healthcheck_get_text_cached(self):
+        self.conf.config(group='healthcheck', detailed=False)
+        app = self._get_enabled_app()
+        for i in range(10):
+            response = self._get(app, '/healthcheck')
+            self.assertEqual(200, response.status_code)
+            self.assertEqual('OK', response.text)
+
     def test_healthcheck_disabled_get(self):
         self._get(self._get_disabled_app(), '/healthcheck', status=404)
 

releasenotes/notes/healthcheck-cache-641f0a64e6f5856c.yaml

@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    Fixed the healthcheck endpoint always querying the backends by caching
+    results for a configurable time. The default is five seconds.