Healthcheck Middleware

Provide healthcheck middleware from oslo_middleware to be able to
disable given nodes from loadbalancer. It's achieved by adding a new
pipeline which depending on existing of the
/etc/glance/healthcheck_disable file can return one of the following
results:

- 200 OK (if file does not exist)
- 503 DISABLED BY FILE (if file exist)

The healthcheck is available under /healthcheck URL, and the whole
mechanism behave similar to the Swift healthcheck system.

implements bp: healthcheck-middleware

Co-Authored-By: Erno Kuvaja <jokke@usr.fi>
Co-Authored-By: Kamil Rykowski <kamil.rykowski@intel.com>

DocImpact

Change-Id: I45f6a8c59ec3040aaf06f8bab46d8001c44dac7a

doc/source/configuring.rst

@@ -1502,7 +1502,7 @@ return a ValueError exception with "No such digest method" error.
 Optional. Default: ``sha1``
 
 Configuring http_keepalive option
-----------------------------------
+---------------------------------
 
 * ``http_keepalive=<True|False>``
 
@@ -1511,3 +1511,32 @@ will return "Connection: Keep-Alive" in its responses. In order to close the
 client socket connection explicitly after the response is sent and read
 successfully by the client, you simply have to set this option to False when
 you create a wsgi server.
+
+Configuring the Health Check
+----------------------------
+
+This setting allows an operator to configure the endpoint URL that will
+provide information to load balancer if given API endpoint at the node should
+be available or not. Both Glance API and Glance Registry servers can be
+configured to expose a health check URL.
+
+To enable the health check middleware, it must occur in the beginning of the
+application pipeline.
+
+The health check middleware should be placed in your
+``glance-api-paste.ini`` / ``glance-registry-paste.ini`` in a section
+titled ``[filter:healthcheck]``. It should look like this::
+
+  [filter:healthcheck]
+  paste.filter_factory = oslo_middleware:Healthcheck.factory
+  backends = disable_by_file
+  disable_by_file_path = /etc/glance/healthcheck_disable
+
+A ready-made application pipeline including this filter is defined e.g. in
+the ``glance-api-paste.ini`` file, looking like so::
+
+  [pipeline:glance-api]
+  pipeline = healthcheck versionnegotiation osprofiler unauthenticated-context rootapp
+
+For more information see
+`oslo.middleware <http://docs.openstack.org/developer/oslo.middleware/api.html#oslo_middleware.Healthcheck>`_.

etc/glance-api-paste.ini

@@ -1,38 +1,38 @@
 # Use this pipeline for no auth or image caching - DEFAULT
 [pipeline:glance-api]
-pipeline = versionnegotiation osprofiler unauthenticated-context rootapp
+pipeline = healthcheck versionnegotiation osprofiler unauthenticated-context rootapp
 
 # Use this pipeline for image caching and no auth
 [pipeline:glance-api-caching]
-pipeline = versionnegotiation osprofiler unauthenticated-context cache rootapp
+pipeline = healthcheck versionnegotiation osprofiler unauthenticated-context cache rootapp
 
 # Use this pipeline for caching w/ management interface but no auth
 [pipeline:glance-api-cachemanagement]
-pipeline = versionnegotiation osprofiler unauthenticated-context cache cachemanage rootapp
+pipeline = healthcheck versionnegotiation osprofiler unauthenticated-context cache cachemanage rootapp
 
 # Use this pipeline for keystone auth
 [pipeline:glance-api-keystone]
-pipeline = versionnegotiation osprofiler authtoken context  rootapp
+pipeline = healthcheck versionnegotiation osprofiler authtoken context  rootapp
 
 # Use this pipeline for keystone auth with image caching
 [pipeline:glance-api-keystone+caching]
-pipeline = versionnegotiation osprofiler authtoken context cache rootapp
+pipeline = healthcheck versionnegotiation osprofiler authtoken context cache rootapp
 
 # Use this pipeline for keystone auth with caching and cache management
 [pipeline:glance-api-keystone+cachemanagement]
-pipeline = versionnegotiation osprofiler authtoken context cache cachemanage rootapp
+pipeline = healthcheck versionnegotiation osprofiler authtoken context cache cachemanage rootapp
 
 # Use this pipeline for authZ only. This means that the registry will treat a
 # user as authenticated without making requests to keystone to reauthenticate
 # the user.
 [pipeline:glance-api-trusted-auth]
-pipeline = versionnegotiation osprofiler context rootapp
+pipeline = healthcheck versionnegotiation osprofiler context rootapp
 
 # Use this pipeline for authZ only. This means that the registry will treat a
 # user as authenticated without making requests to keystone to reauthenticate
 # the user and uses cache management
 [pipeline:glance-api-trusted-auth+cachemanagement]
-pipeline = versionnegotiation osprofiler context cache cachemanage rootapp
+pipeline = healthcheck versionnegotiation osprofiler context cache cachemanage rootapp
 
 [composite:rootapp]
 paste.composite_factory = glance.api:root_app_factory
@@ -53,6 +53,11 @@ paste.app_factory = glance.api.v2.router:API.factory
 [app:apiv3app]
 paste.app_factory = glance.api.v3.router:API.factory
 
+[filter:healthcheck]
+paste.filter_factory = oslo_middleware:Healthcheck.factory
+backends = disable_by_file
+disable_by_file_path = /etc/glance/healthcheck_disable
+
 [filter:versionnegotiation]
 paste.filter_factory = glance.api.middleware.version_negotiation:VersionNegotiationFilter.factory
 

etc/glance-registry-paste.ini

@@ -1,20 +1,25 @@
 # Use this pipeline for no auth - DEFAULT
 [pipeline:glance-registry]
-pipeline = osprofiler unauthenticated-context registryapp
+pipeline = healthcheck osprofiler unauthenticated-context registryapp
 
 # Use this pipeline for keystone auth
 [pipeline:glance-registry-keystone]
-pipeline = osprofiler authtoken context registryapp
+pipeline = healthcheck osprofiler authtoken context registryapp
 
 # Use this pipeline for authZ only. This means that the registry will treat a
 # user as authenticated without making requests to keystone to reauthenticate
 # the user.
 [pipeline:glance-registry-trusted-auth]
-pipeline = osprofiler context registryapp
+pipeline = healthcheck osprofiler context registryapp
 
 [app:registryapp]
 paste.app_factory = glance.registry.api:API.factory
 
+[filter:healthcheck]
+paste.filter_factory = oslo_middleware:Healthcheck.factory
+backends = disable_by_file
+disable_by_file_path = /etc/glance/healthcheck_disable
+
 [filter:context]
 paste.filter_factory = glance.api.middleware.context:ContextMiddleware.factory
 

glance/tests/functional/__init__.py

@@ -304,6 +304,7 @@ class ApiServer(Server):
         self.image_property_quota = 10
         self.image_tag_quota = 10
         self.image_location_quota = 2
+        self.disable_path = None
 
         self.needs_database = True
         default_sql_connection = 'sqlite:////%s/tests.sqlite' % self.test_dir
@@ -368,13 +369,15 @@ filesystem_store_datadir=%(image_dir)s
 default_store = %(default_store)s
 """
         self.paste_conf_base = """[pipeline:glance-api]
-pipeline = versionnegotiation gzip unauthenticated-context rootapp
+pipeline = healthcheck versionnegotiation gzip unauthenticated-context rootapp
 
 [pipeline:glance-api-caching]
-pipeline = versionnegotiation gzip unauthenticated-context cache rootapp
+pipeline = healthcheck versionnegotiation gzip unauthenticated-context
+ cache rootapp
 
 [pipeline:glance-api-cachemanagement]
 pipeline =
+    healthcheck
     versionnegotiation
     gzip
     unauthenticated-context
@@ -383,10 +386,10 @@ pipeline =
     rootapp
 
 [pipeline:glance-api-fakeauth]
-pipeline = versionnegotiation gzip fakeauth context rootapp
+pipeline = healthcheck versionnegotiation gzip fakeauth context rootapp
 
 [pipeline:glance-api-noauth]
-pipeline = versionnegotiation gzip context rootapp
+pipeline = healthcheck versionnegotiation gzip context rootapp
 
 [composite:rootapp]
 paste.composite_factory = glance.api:root_app_factory
@@ -407,6 +410,11 @@ paste.app_factory = glance.api.v2.router:API.factory
 [app:apiv3app]
 paste.app_factory = glance.api.v3.router:API.factory
 
+[filter:healthcheck]
+paste.filter_factory = oslo_middleware:Healthcheck.factory
+backends = disable_by_file
+disable_by_file_path = %(disable_path)s
+
 [filter:versionnegotiation]
 paste.filter_factory =
  glance.api.middleware.version_negotiation:VersionNegotiationFilter.factory
@@ -458,6 +466,7 @@ class RegistryServer(Server):
         self.metadata_encryption_key = "012345678901234567890123456789ab"
         self.policy_file = policy_file
         self.policy_default_rule = 'default'
+        self.disable_path = None
 
         self.conf_base = """[DEFAULT]
 verbose = %(verbose)s
@@ -481,17 +490,22 @@ policy_default_rule = %(policy_default_rule)s
 flavor = %(deployment_flavor)s
 """
         self.paste_conf_base = """[pipeline:glance-registry]
-pipeline = unauthenticated-context registryapp
+pipeline = healthcheck unauthenticated-context registryapp
 
 [pipeline:glance-registry-fakeauth]
-pipeline = fakeauth context registryapp
+pipeline = healthcheck fakeauth context registryapp
 
 [pipeline:glance-registry-trusted-auth]
-pipeline = context registryapp
+pipeline = healthcheck context registryapp
 
 [app:registryapp]
 paste.app_factory = glance.registry.api:API.factory
 
+[filter:healthcheck]
+paste.filter_factory = oslo_middleware:Healthcheck.factory
+backends = disable_by_file
+disable_by_file_path = %(disable_path)s
+
 [filter:context]
 paste.filter_factory = glance.api.middleware.context:ContextMiddleware.factory
 

glance/tests/functional/test_healthcheck_middleware.py

@@ -0,0 +1,54 @@
+# Copyright 2015 Hewlett Packard
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Tests healthcheck middleware."""
+
+import tempfile
+
+import httplib2
+
+from glance.tests import functional
+from glance.tests import utils
+
+
+class HealthcheckMiddlewareTest(functional.FunctionalTest):
+
+    def request(self):
+        url = 'http://127.0.0.1:%s/healthcheck' % self.api_port
+        http = httplib2.Http()
+        return http.request(url, 'GET')
+
+    @utils.skip_if_disabled
+    def test_healthcheck_enabled(self):
+        self.cleanup()
+        self.start_servers(**self.__dict__.copy())
+
+        response, content = self.request()
+        self.assertEqual('OK', content)
+        self.assertEqual(200, response.status)
+
+        self.stop_servers()
+
+    def test_healthcheck_disabled(self):
+        with tempfile.NamedTemporaryFile() as test_disable_file:
+            self.cleanup()
+            self.api_server.disable_path = test_disable_file.name
+            self.start_servers(**self.__dict__.copy())
+
+            response, content = self.request()
+            self.assertEqual('DISABLED BY FILE', content)
+            self.assertEqual(503, response.status)
+
+            self.stop_servers()

glance/tests/unit/common/test_config.py

@@ -17,8 +17,8 @@ import os.path
 import shutil
 
 import fixtures
+import oslo_middleware
 from oslotest import moxstubout
-import osprofiler.web
 
 from glance.api.middleware import context
 from glance.common import config
@@ -72,7 +72,7 @@ class TestPasteApp(test_utils.BaseTestCase):
         self.assertIsInstance(app, expected_app_type)
 
     def test_load_paste_app(self):
-        expected_middleware = osprofiler.web.WsgiMiddleware
+        expected_middleware = oslo_middleware.Healthcheck
         self._do_test_load_paste_app(expected_middleware)
 
     def test_load_paste_app_paste_config_not_found(self):
@@ -91,7 +91,7 @@ class TestPasteApp(test_utils.BaseTestCase):
     def test_load_paste_app_with_paste_config_file(self):
         paste_config_file = os.path.join(os.getcwd(),
                                          'etc/glance-registry-paste.ini')
-        expected_middleware = osprofiler.web.WsgiMiddleware
+        expected_middleware = oslo_middleware.Healthcheck
         self._do_test_load_paste_app(expected_middleware,
                                      paste_config_file=paste_config_file)
 

requirements.txt

@@ -52,6 +52,7 @@ oslo.db>=1.12.0 # Apache-2.0
 oslo.i18n>=1.5.0 # Apache-2.0
 oslo.log>=1.2.0 # Apache-2.0
 oslo.messaging!=1.12.0,>=1.8.0 # Apache-2.0
+oslo.middleware>=1.2.0,!=2.0.0  # Apache-2.0
 oslo.policy>=0.5.0 # Apache-2.0
 oslo.serialization>=1.4.0 # Apache-2.0