openstack/aodh
Code Issues Proposed changes

Handle database failures on api startup

Browse Source 
Storage layer connections are initialized
for metering, alarm and event in Pecan hook
middleware.
If any of the backends are not responding,
the connections are retried till maximum
retries and the exception propagated to the
Ceilometer-api which fails to startup.

This fix logs and handles the exception in
the Pecan hook and allows all connection
types to be initialized.

Closes-Bug: 1456944

Change-Id: Id70693dafc823927b3d72e0371e3e55d4e5588ce
This commit is contained in:
Rohit Jaiswal
2015-05-21 00:05:38 +00:00
parent 89089c2a70
commit 1f30839494
3 changed files with 127 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
ceilometer/api/app.py
View File

@@ -28,7 +28,6 @@ from ceilometer.api import middleware
from ceilometer.i18n import _
from ceilometer.i18n import _LW
from ceilometer import service
from ceilometer import storage
LOG = log.getLogger(__name__)
@@ -62,10 +61,7 @@ def get_pecan_config():
def setup_app(pecan_config=None, extra_hooks=None):
# FIXME: Replace DBHook with a hooks.TransactionHook
app_hooks = [hooks.ConfigHook(),
hooks.DBHook(
storage.get_connection_from_config(cfg.CONF, 'metering'),
storage.get_connection_from_config(cfg.CONF, 'event'),
storage.get_connection_from_config(cfg.CONF, 'alarm'),),
hooks.DBHook(),
hooks.PipelineHook(),
hooks.TranslationHook()]
if extra_hooks:

30
ceilometer/api/hooks.py
View File

@@ -16,9 +16,15 @@
import threading
from oslo_config import cfg
from oslo_log import log
from pecan import hooks
from ceilometer.i18n import _LE
from ceilometer import pipeline
from ceilometer import storage
LOG = log.getLogger(__name__)
class ConfigHook(hooks.PecanHook):
@@ -34,16 +40,32 @@ class ConfigHook(hooks.PecanHook):
class DBHook(hooks.PecanHook):
def __init__(self, conn, event_conn, alarm_conn):
self.storage_connection = conn
self.event_storage_connection = event_conn
self.alarm_storage_connection = alarm_conn
def __init__(self):
self.storage_connection = DBHook.get_connection('metering')
self.event_storage_connection = DBHook.get_connection('event')
self.alarm_storage_connection = DBHook.get_connection('alarm')
if (not self.storage_connection and
not self.event_storage_connection and
not self.alarm_storage_connection):
raise Exception("Api failed to start. Failed to connect to "
"databases, purpose: %s" %
', '.join(['metering', 'event', 'alarm']))
def before(self, state):
state.request.storage_conn = self.storage_connection
state.request.event_storage_conn = self.event_storage_connection
state.request.alarm_storage_conn = self.alarm_storage_connection
@staticmethod
def get_connection(purpose):
try:
return storage.get_connection_from_config(cfg.CONF, purpose)
except Exception as err:
params = {"purpose": purpose, "err": err}
LOG.exception(_LE("Failed to connect to db, purpose %(purpose)s "
"retry later: %(err)s") % params)
class PipelineHook(hooks.PecanHook):
"""Create and attach a pipeline to the request.

126
ceilometer/tests/test_bin.py
View File

@@ -139,35 +139,16 @@ class BinApiTestCase(base.BaseTestCase):
# create ceilometer.conf file
self.api_port = random.randint(10000, 11000)
self.http = httplib2.Http(proxy_info=None)
pipeline_cfg_file = self.path_get('etc/ceilometer/pipeline.yaml')
policy_file = self.path_get('etc/ceilometer/policy.json')
content = ("[DEFAULT]\n"
"rpc_backend=fake\n"
"auth_strategy=noauth\n"
"debug=true\n"
"pipeline_cfg_file={0}\n"
"policy_file={1}\n"
"api_paste_config={2}\n"
"[api]\n"
"port={3}\n"
"[database]\n"
"connection=log://localhost\n".format(pipeline_cfg_file,
policy_file,
self.paste,
self.api_port))
if six.PY3:
content = content.encode('utf-8')
self.tempfile = fileutils.write_to_tempfile(content=content,
prefix='ceilometer',
suffix='.conf')
self.subp = subprocess.Popen(['ceilometer-api',
"--config-file=%s" % self.tempfile])
self.pipeline_cfg_file = self.path_get('etc/ceilometer/pipeline.yaml')
self.policy_file = self.path_get('etc/ceilometer/policy.json')
def tearDown(self):
super(BinApiTestCase, self).tearDown()
self.subp.kill()
self.subp.wait()
try:
self.subp.kill()
self.subp.wait()
except OSError:
pass
os.remove(self.tempfile)
def get_response(self, path):
@@ -183,13 +164,106 @@ class BinApiTestCase(base.BaseTestCase):
return r, c
return None, None
def run_api(self, content, err_pipe=None):
if six.PY3:
content = content.encode('utf-8')
self.tempfile = fileutils.write_to_tempfile(content=content,
prefix='ceilometer',
suffix='.conf')
if err_pipe:
return subprocess.Popen(['ceilometer-api',
"--config-file=%s" % self.tempfile],
stderr=subprocess.PIPE)
else:
return subprocess.Popen(['ceilometer-api',
"--config-file=%s" % self.tempfile])
def test_v2(self):
content = ("[DEFAULT]\n"
"rpc_backend=fake\n"
"auth_strategy=noauth\n"
"debug=true\n"
"pipeline_cfg_file={0}\n"
"policy_file={1}\n"
"api_paste_config={2}\n"
"[api]\n"
"port={3}\n"
"[database]\n"
"connection=log://localhost\n".
format(self.pipeline_cfg_file,
self.policy_file,
self.paste,
self.api_port))
self.subp = self.run_api(content)
response, content = self.get_response('v2/meters')
self.assertEqual(200, response.status)
if six.PY3:
content = content.decode('utf-8')
self.assertEqual([], json.loads(content))
def test_v2_with_bad_storage_conn(self):
content = ("[DEFAULT]\n"
"rpc_backend=fake\n"
"auth_strategy=noauth\n"
"debug=true\n"
"pipeline_cfg_file={0}\n"
"policy_file={1}\n"
"api_paste_config={2}\n"
"[api]\n"
"port={3}\n"
"[database]\n"
"max_retries=1\n"
"alarm_connection=log://localhost\n"
"connection=dummy://localhost\n".
format(self.pipeline_cfg_file,
self.policy_file,
self.paste,
self.api_port))
self.subp = self.run_api(content)
response, content = self.get_response('v2/alarms')
self.assertEqual(200, response.status)
if six.PY3:
content = content.decode('utf-8')
self.assertEqual([], json.loads(content))
response, content = self.get_response('v2/meters')
self.assertEqual(500, response.status)
def test_v2_with_all_bad_conns(self):
content = ("[DEFAULT]\n"
"rpc_backend=fake\n"
"auth_strategy=noauth\n"
"debug=true\n"
"pipeline_cfg_file={0}\n"
"policy_file={1}\n"
"api_paste_config={2}\n"
"[api]\n"
"port={3}\n"
"[database]\n"
"max_retries=1\n"
"alarm_connection=dummy://localhost\n"
"connection=dummy://localhost\n"
"event_connection=dummy://localhost\n".
format(self.pipeline_cfg_file,
self.policy_file,
self.paste,
self.api_port))
self.subp = self.run_api(content, err_pipe=True)
__, err = self.subp.communicate()
self.assertIn(b"Api failed to start. Failed to connect to"
b" databases, purpose: metering, event, alarm", err)
class BinCeilometerPollingServiceTestCase(base.BaseTestCase):
def setUp(self):