Make /healthcheck cache results
The healthcheck endpoint should cache results to reduce the potential load on the backend systems being tested.
This patch adds the caching and a configuration setting for the interval
between cache refreshes.
Change-Id: Ic97a991437144f3a220d9b96839cec5b63565f8c
Story: 2008203
Task: 40987
(cherry picked from commit 6c54eab5b5
)
This commit is contained in:
parent
035d6d6daa
commit
582b4bab0e
|
@ -527,6 +527,25 @@ You will then need to select the desired monitoring backend plugins:
|
||||||
changes. Not only does it not run any tests, it will return 204 results
|
changes. Not only does it not run any tests, it will return 204 results
|
||||||
instead of 200.
|
instead of 200.
|
||||||
|
|
||||||
|
The Octavia API health monitoring endpoint does not require a keystone token
|
||||||
|
for access to allow external load balancers to query the endpoint. For this
|
||||||
|
reason we recommend you restrict access to it on your external load balancer
|
||||||
|
to prevent abuse.
|
||||||
|
|
||||||
|
As an additional protection, the API will cache results for a configurable
|
||||||
|
period of time. This means that queries to the health monitoring endpoint
|
||||||
|
will return cached results until the refresh interval has expired, at which
|
||||||
|
point the health check plugin will rerun the check.
|
||||||
|
|
||||||
|
By default, the refresh interval is five seconds. This can be configured by
|
||||||
|
adjusting the healthcheck_refresh_interval setting in the Octavia configuration
|
||||||
|
file:
|
||||||
|
|
||||||
|
.. code-block:: ini
|
||||||
|
|
||||||
|
[api_settings]
|
||||||
|
healthcheck_refresh_interval = 5
|
||||||
|
|
||||||
Optionally you can enable the "detailed" mode in Oslo middleware healthcheck.
|
Optionally you can enable the "detailed" mode in Oslo middleware healthcheck.
|
||||||
This will cause Oslo middleware healthcheck to return additional information
|
This will cause Oslo middleware healthcheck to return additional information
|
||||||
about the API instance. It will also provide exception details if one was
|
about the API instance. It will also provide exception details if one was
|
||||||
|
|
|
@ -56,6 +56,9 @@
|
||||||
# Boolean to enable/disable oslo middleware /healthcheck in the Octavia API
|
# Boolean to enable/disable oslo middleware /healthcheck in the Octavia API
|
||||||
# healthcheck_enabled = False
|
# healthcheck_enabled = False
|
||||||
|
|
||||||
|
# The interval healthcheck plugins should cache results, in seconds.
|
||||||
|
# healthcheck_refresh_interval = 5
|
||||||
|
|
||||||
# Default cipher string for new TLS-terminated listeners
|
# Default cipher string for new TLS-terminated listeners
|
||||||
# Cipher strings are in OpenSSL format, see https://www.openssl.org/docs/man1.1.1/man1/ciphers.html
|
# Cipher strings are in OpenSSL format, see https://www.openssl.org/docs/man1.1.1/man1/ciphers.html
|
||||||
# This example is the "Broad Compatibility" cipher string from OWASP,
|
# This example is the "Broad Compatibility" cipher string from OWASP,
|
||||||
|
|
|
@ -11,23 +11,42 @@
|
||||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
# License for the specific language governing permissions and limitations
|
# License for the specific language governing permissions and limitations
|
||||||
# under the License.
|
# under the License.
|
||||||
|
import datetime
|
||||||
|
|
||||||
|
from oslo_config import cfg
|
||||||
from oslo_middleware.healthcheck import pluginbase
|
from oslo_middleware.healthcheck import pluginbase
|
||||||
|
|
||||||
from octavia.db import api as db_apis
|
from octavia.db import api as db_apis
|
||||||
from octavia.db import healthcheck
|
from octavia.db import healthcheck
|
||||||
|
|
||||||
|
CONF = cfg.CONF
|
||||||
|
|
||||||
|
|
||||||
class OctaviaDBHealthcheck(pluginbase.HealthcheckBaseExtension):
|
class OctaviaDBHealthcheck(pluginbase.HealthcheckBaseExtension):
|
||||||
|
|
||||||
UNAVAILABLE_REASON = 'The Octavia database is unavailable.'
|
UNAVAILABLE_REASON = 'The Octavia database is unavailable.'
|
||||||
|
|
||||||
|
last_check = None
|
||||||
|
last_result = None
|
||||||
|
last_message = None
|
||||||
|
|
||||||
def __init__(self, *args, **kwargs):
|
def __init__(self, *args, **kwargs):
|
||||||
super().__init__(*args, **kwargs)
|
super().__init__(*args, **kwargs)
|
||||||
|
|
||||||
def healthcheck(self, server_port):
|
def healthcheck(self, server_port):
|
||||||
try:
|
try:
|
||||||
result, message = healthcheck.check_database_connection(
|
if (self.last_check is not None and
|
||||||
db_apis.get_session())
|
((datetime.datetime.now() -
|
||||||
|
self.last_check).total_seconds()) <
|
||||||
|
CONF.api_settings.healthcheck_refresh_interval):
|
||||||
|
result = self.last_result
|
||||||
|
message = self.last_message
|
||||||
|
else:
|
||||||
|
result, message = healthcheck.check_database_connection(
|
||||||
|
db_apis.get_session())
|
||||||
|
self.last_check = datetime.datetime.now()
|
||||||
|
self.last_result = result
|
||||||
|
self.last_message = message
|
||||||
if result:
|
if result:
|
||||||
return OctaviaDBCheckResult(available=True, reason="OK")
|
return OctaviaDBCheckResult(available=True, reason="OK")
|
||||||
else:
|
else:
|
||||||
|
|
|
@ -105,6 +105,9 @@ api_opts = [
|
||||||
cfg.BoolOpt('healthcheck_enabled', default=False,
|
cfg.BoolOpt('healthcheck_enabled', default=False,
|
||||||
help=_("When True, the oslo middleware healthcheck endpoint "
|
help=_("When True, the oslo middleware healthcheck endpoint "
|
||||||
"is enabled in the Octavia API.")),
|
"is enabled in the Octavia API.")),
|
||||||
|
cfg.IntOpt('healthcheck_refresh_interval', default=5,
|
||||||
|
help=_("The interval healthcheck plugins should cache results, "
|
||||||
|
"in seconds.")),
|
||||||
cfg.StrOpt('default_listener_ciphers',
|
cfg.StrOpt('default_listener_ciphers',
|
||||||
default=constants.CIPHERS_OWASP_SUITE_B,
|
default=constants.CIPHERS_OWASP_SUITE_B,
|
||||||
help=_("Default OpenSSL cipher string (colon-separated) for "
|
help=_("Default OpenSSL cipher string (colon-separated) for "
|
||||||
|
|
|
@ -49,6 +49,7 @@ class TestHealthCheck(base_db_test.OctaviaDBTestBase):
|
||||||
|
|
||||||
self.conf = self.useFixture(oslo_fixture.Config(cfg.CONF))
|
self.conf = self.useFixture(oslo_fixture.Config(cfg.CONF))
|
||||||
self.conf.config(group='healthcheck', backends=['octavia_db_check'])
|
self.conf.config(group='healthcheck', backends=['octavia_db_check'])
|
||||||
|
self.conf.config(group='api_settings', healthcheck_refresh_interval=5)
|
||||||
self.UNAVAILABLE = (healthcheck_plugins.OctaviaDBHealthcheck.
|
self.UNAVAILABLE = (healthcheck_plugins.OctaviaDBHealthcheck.
|
||||||
UNAVAILABLE_REASON)
|
UNAVAILABLE_REASON)
|
||||||
|
|
||||||
|
@ -145,6 +146,14 @@ class TestHealthCheck(base_db_test.OctaviaDBTestBase):
|
||||||
self.assertIn('OK', response.text)
|
self.assertIn('OK', response.text)
|
||||||
self.assertIn('Garbage collector', response.text)
|
self.assertIn('Garbage collector', response.text)
|
||||||
|
|
||||||
|
def test_healthcheck_get_text_cached(self):
|
||||||
|
self.conf.config(group='healthcheck', detailed=False)
|
||||||
|
app = self._get_enabled_app()
|
||||||
|
for i in range(10):
|
||||||
|
response = self._get(app, '/healthcheck')
|
||||||
|
self.assertEqual(200, response.status_code)
|
||||||
|
self.assertEqual('OK', response.text)
|
||||||
|
|
||||||
def test_healthcheck_disabled_get(self):
|
def test_healthcheck_disabled_get(self):
|
||||||
self._get(self._get_disabled_app(), '/healthcheck', status=404)
|
self._get(self._get_disabled_app(), '/healthcheck', status=404)
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,5 @@
|
||||||
|
---
|
||||||
|
fixes:
|
||||||
|
- |
|
||||||
|
Fixed the healthcheck endpoint always querying the backends by caching
|
||||||
|
results for a configurable time. The default is five seconds.
|
Loading…
Reference in New Issue