Add retries to Barbican secret create

Occasionally when Deckhand is creating secrets in Barbican, Barbican
encounters an error in which a subsequent attempt at creating the
secret would succeed. This patch set adds logic to the Deckhand
Barbican driver to retry secret creates a configurable number of
times to work around this Barbican issue.

Change-Id: I52293195dd708255508949723d89117ce2e32b71
This commit is contained in:
Carter, Matt (mc981n) 2019-10-02 11:22:43 -05:00
parent 84661f9159
commit 460eb7fb6c
2 changed files with 38 additions and 6 deletions

View File

@ -13,8 +13,10 @@
# limitations under the License.
import ast
import time
import barbicanclient
from oslo_config import cfg
from oslo_log import log as logging
from oslo_serialization import base64
from oslo_utils import excutils
@ -25,6 +27,7 @@ from deckhand.barbican import client_wrapper
from deckhand import errors
from deckhand import types
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
@ -138,15 +141,43 @@ class BarbicanDriver(object):
return payload
# Store secret_ref in database for `secret_doc`.
kwargs = {
secret_args = {
'name': secret_doc['metadata']['name'],
'secret_type': secret_type,
'payload': payload
}
LOG.info('Storing encrypted document data in Barbican.')
LOG.info('Storing encrypted data in Barbican for document [{}, {}]'
.format(secret_doc.schema, secret_doc.name))
for i in range(CONF.secret_create_attempts):
LOG.debug('Creating secret in Barbican, attempt {} of {}'
.format((i + 1), CONF.secret_create_attempts))
try:
return self._do_create_secret(secret_args)
except Exception as e:
if i == (CONF.secret_create_attempts - 1):
# This was the last attempt, re-raise any error
raise
else:
# This was not the last attempt, suppress the error and
# try again after a brief sleep
sleep_amount = (i + 1)
LOG.error('Caught an error while trying to create a '
'secret in Barbican, will try again in {} second'
.format(sleep_amount))
time.sleep(sleep_amount)
def _do_create_secret(self, secret_args):
"""Using the cache construct, and the barbican client, create a secret
:param secret_args: Dict containing the data for the secret to create
:type secret_args: dict
:returns: Secret reference returned by Barbican
:rtype: str
"""
try:
secret_ref = cache.lookup_by_payload(self.barbicanclient, **kwargs)
return cache.lookup_by_payload(self.barbicanclient,
**secret_args)
except (barbicanclient.exceptions.HTTPAuthError,
barbicanclient.exceptions.HTTPClientError) as e:
LOG.exception(str(e))
@ -162,8 +193,6 @@ class BarbicanDriver(object):
'payload type is unsupported.', e.__class__.__name__)
raise errors.BarbicanServerException(details=str(e))
return secret_ref
def _base64_decode_payload(self, payload):
# If the secret_type is 'opaque' then this implies the
# payload was encoded to base64 previously. Reverse the

View File

@ -88,7 +88,10 @@ default_opts = [
"production."),
cfg.BoolOpt('development_mode', default=False,
help="Enables development mode, which disables Keystone "
"authentication. Do NOT use in production.")
"authentication. Do NOT use in production."),
cfg.IntOpt('secret_create_attempts', default=2,
help="How many times Deckhand should attempt to create a "
"secret in Barbican before raising an exception.")
]