airship/shipyard
Code Issues Proposed changes

Logging to use request scope provided by uwsgi and standard logging filters

Browse Source 
This change introduces a logging filter that sniffs for the presence of uwsgi
being provided to the application, and if so, uses it to provide request scoped
logging variables. This change allows for more standard logging pattersn to be
followed throughout the rest of the API code while still providing information
related to the request, such as the request ID and the external context marker
that may be used by a client of the software.

Change-Id: I82f9070e25a97043ddf4660635595c473a38cda2
This commit is contained in:
Bryan Strassner 2017-10-28 23:52:00 -05:00
parent 7f1f5319d2
commit 5d0b55b272
21 changed files with 337 additions and 294 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
shipyard_airflow/control/action/actions_api.py
View File

@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from datetime import datetime
import logging
import falcon
import requests
@ -29,6 +30,7 @@ from shipyard_airflow.db.db import AIRFLOW_DB, SHIPYARD_DB
from shipyard_airflow.errors import ApiError
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
# Mappings of actions to dags
SUPPORTED_ACTION_MAPPINGS = {
@ -64,7 +66,6 @@ class ActionsResource(BaseResource):
"""
resp.body = self.to_json(self.get_all_actions())
resp.status = falcon.HTTP_200
self.info(req.context, 'response data is %s' % resp.body)
@policy.ApiEnforcer('workflow_orchestrator:create_action')
def on_post(self, req, resp, **kwargs):
@ -73,8 +74,7 @@ class ActionsResource(BaseResource):
"""
input_action = self.req_json(req, validate_json_schema=ACTION)
action = self.create_action(action=input_action, context=req.context)
self.info(req.context, "Id %s generated for action %s " %
(action['id'], action['name']))
LOG.info("Id %s generated for action %s", action['id'], action['name'])
# respond with the action and location for checking status
resp.status = falcon.HTTP_201
resp.body = self.to_json(action)
@ -88,7 +88,7 @@ class ActionsResource(BaseResource):
# add current timestamp (UTC) to the action.
action['timestamp'] = str(datetime.utcnow())
# validate that action is supported.
self.info(context, "Attempting action: %s" % action['name'])
LOG.info("Attempting action: %s", action['name'])
if action['name'] not in SUPPORTED_ACTION_MAPPINGS:
raise ApiError(
title='Unable to start action',
@ -244,17 +244,14 @@ class ActionsResource(BaseResource):
try:
resp = requests.get(req_url, timeout=(5, 15))
self.info(context,
'Response code from Airflow trigger_dag: %s' %
resp.status_code)
LOG.info('Response code from Airflow trigger_dag: %s',
resp.status_code)
# any 4xx/5xx will be HTTPError, which are RequestException
resp.raise_for_status()
response = resp.json()
self.info(context,
'Response from Airflow trigger_dag: %s' %
response)
LOG.info('Response from Airflow trigger_dag: %s', response)
except RequestException as rex:
self.error(context, "Request to airflow failed: %s" % rex.args)
LOG.error("Request to airflow failed: %s", rex.args)
raise ApiError(
title='Unable to complete request to Airflow',
description=(

2
shipyard_airflow/control/af_monitoring/workflows_api.py
View File

@ -42,7 +42,6 @@ class WorkflowResource(BaseResource):
self.get_all_workflows(helper=helper, since_date=since_date)
)
resp.status = falcon.HTTP_200
self.info(req.context, 'response data is %s' % resp.body)
def get_all_workflows(self, helper, since_date=None):
"""
@ -72,7 +71,6 @@ class WorkflowIdResource(BaseResource):
self.get_workflow_detail(helper=helper, workflow_id=workflow_id)
)
resp.status = falcon.HTTP_200
self.info(req.context, 'response data is %s' % resp.body)
def get_workflow_detail(self, helper, workflow_id):
"""

6
shipyard_airflow/control/api.py
View File

@ -34,9 +34,9 @@ from shipyard_airflow.control.configdocs.configdocs_api import (
from shipyard_airflow.control.configdocs.rendered_configdocs_api import \
RenderedConfigDocsResource
from shipyard_airflow.control.health import HealthResource
from shipyard_airflow.control.middleware import (AuthMiddleware,
ContextMiddleware,
LoggingMiddleware)
from shipyard_airflow.control.middleware.auth import AuthMiddleware
from shipyard_airflow.control.middleware.context import ContextMiddleware
from shipyard_airflow.control.middleware.logging_mw import LoggingMiddleware
from shipyard_airflow.errors import (AppError, default_error_serializer,
default_exception_handler)

55
shipyard_airflow/control/base.py
View File

@ -22,15 +22,14 @@ import falcon.routing as routing
from shipyard_airflow.control.json_schemas import validate_json
from shipyard_airflow.errors import InvalidFormatError
LOG = logging.getLogger(__name__)
class BaseResource(object):
"""
The base resource for Shipyard entities/api handlers. This class
provides some reusable functionality.
"""
def __init__(self):
self.logger = logging.getLogger('shipyard.control')
def on_options(self, req, resp, **kwargs):
"""Handle options requests"""
method_map = routing.create_http_method_map(self)
@ -53,9 +52,9 @@ class BaseResource(object):
raw_body = req.stream.read(req.content_length or 0)
if raw_body is not None:
has_input = True
self.info(req.context, 'Input message body: %s' % raw_body)
LOG.info('Input message body: %s', raw_body)
else:
self.info(req.context, 'No message body specified')
LOG.info('No message body specified')
if has_input:
# read the json and validate if necessary
try:
@ -66,15 +65,14 @@ class BaseResource(object):
validate_json(json_body, validate_json_schema)
return json_body
except json.JSONDecodeError as jex:
self.error(req.context, "Invalid JSON in request: \n%s" %
raw_body)
LOG.error("Invalid JSON in request: %s", raw_body)
raise InvalidFormatError(
title='JSON could not be decoded',
description='%s: Invalid JSON in body: %s' %
(req.path, jex)
)
else:
# No body passed as input. Fail validation if it was asekd for
# No body passed as input. Fail validation if it was asked for
if validate_json_schema is not None:
raise InvalidFormatError(
title='Json body is required',
@ -88,42 +86,6 @@ class BaseResource(object):
"""Thin wrapper around json.dumps, providing the default=str config"""
return json.dumps(body_dict, default=str)
def log_message(self, ctx, level, msg):
"""Logs a message with context, and extra populated."""
extra = {'user': 'N/A', 'req_id': 'N/A', 'external_ctx': 'N/A'}
if ctx is not None:
extra = {
'user': ctx.user,
'req_id': ctx.request_id,
'external_ctx': ctx.external_marker,
}
self.logger.log(level, msg, extra=extra)
def debug(self, ctx, msg):
"""
Debug logger for resources, incorporating context.
"""
self.log_message(ctx, logging.DEBUG, msg)
def info(self, ctx, msg):
"""
Info logger for resources, incorporating context.
"""
self.log_message(ctx, logging.INFO, msg)
def warn(self, ctx, msg):
"""
Warn logger for resources, incorporating context.
"""
self.log_message(ctx, logging.WARN, msg)
def error(self, ctx, msg):
"""
Error logger for resources, incorporating context.
"""
self.log_message(ctx, logging.ERROR, msg)
class ShipyardRequestContext(object):
"""
@ -131,7 +93,6 @@ class ShipyardRequestContext(object):
"""
def __init__(self):
self.log_level = 'error'
self.user = None
self.roles = ['anyone']
self.request_id = str(uuid.uuid4())
@ -144,10 +105,6 @@ class ShipyardRequestContext(object):
self.is_admin_project = False
self.authenticated = False
def set_log_level(self, level):
if level in ['error', 'info', 'debug']:
self.log_level = level
def set_user(self, user):
self.user = user

4
shipyard_airflow/control/configdocs/configdocs_api.py
View File

@ -44,7 +44,7 @@ class ConfigDocsResource(BaseResource):
Ingests a collection of documents
"""
document_data = req.stream.read(req.content_length or 0)
helper = ConfigdocsHelper(req.context.external_marker)
helper = ConfigdocsHelper(req.context)
validations = self.post_collection(
helper=helper,
collection_id=collection_id,
@ -62,7 +62,7 @@ class ConfigDocsResource(BaseResource):
"""
version = (req.params.get('version') or 'buffer')
self._validate_version_parameter(version)
helper = ConfigdocsHelper(req.context.external_marker)
helper = ConfigdocsHelper(req.context)
# Not reformatting to JSON or YAML since just passing through
resp.body = self.get_collection(
helper=helper,

26
shipyard_airflow/control/configdocs/configdocs_helper.py
View File

@ -71,13 +71,13 @@ class ConfigdocsHelper(object):
service.
"""
def __init__(self, context_marker):
def __init__(self, context):
"""
Sets up this Configdocs helper with the supplied
context marker
request context
"""
self.deckhand = DeckhandClient(context_marker)
self.context_marker = context_marker
self.deckhand = DeckhandClient(context.external_marker)
self.ctx = context
# The revision_dict indicates the revisions that are
# associated with the buffered and committed doc sets. There
# is a risk of this being out of sync if there is high volume
@ -367,7 +367,7 @@ class ConfigdocsHelper(object):
@staticmethod
def _get_validation_threads(validation_endpoints,
revision_id,
context_marker):
ctx):
# create a list of validation threads from the endpoints
validation_threads = []
for endpoint in validation_endpoints:
@ -385,8 +385,15 @@ class ConfigdocsHelper(object):
),
response,
exception,
context_marker
)
ctx.external_marker
),
kwargs={
'log_extra': {
'req_id': ctx.request_id,
'external_ctx': ctx.external_marker,
'user': ctx.user
}
}
),
'name': endpoint['name'],
'url': endpoint['url'],
@ -401,7 +408,8 @@ class ConfigdocsHelper(object):
design_reference,
response,
exception,
context_marker):
context_marker,
**kwargs):
# Invoke the POST for validation
try:
headers = {
@ -435,7 +443,7 @@ class ConfigdocsHelper(object):
validation_threads = ConfigdocsHelper._get_validation_threads(
ConfigdocsHelper._get_validation_endpoints(),
revision_id,
self.context_marker
self.ctx
)
# trigger each validation in parallel
for validation_thread in validation_threads:

2
shipyard_airflow/control/configdocs/rendered_configdocs_api.py
View File

@ -41,7 +41,7 @@ class RenderedConfigDocsResource(BaseResource):
"""
version = (req.params.get('version') or 'buffer')
self._validate_version_parameter(version)
helper = ConfigdocsHelper(req.context.external_marker)
helper = ConfigdocsHelper(req.context)
resp.body = self.get_rendered_configdocs(
helper=helper,
version=version

14
shipyard_airflow/control/json_schemas.py
View File

@ -25,6 +25,8 @@ from jsonschema.exceptions import FormatError, SchemaError, ValidationError
from shipyard_airflow.errors import AppError, InvalidFormatError
LOG = logging.getLogger(__name__)
def validate_json(json_string, schema):
"""
@ -40,8 +42,8 @@ def validate_json(json_string, schema):
err.validator,
err.validator_value
)
logging.error(title)
logging.error(description)
LOG.error(title)
LOG.error(description)
raise InvalidFormatError(
title=title,
description=description,
@ -49,8 +51,8 @@ def validate_json(json_string, schema):
except SchemaError as err:
title = 'SchemaError: Unable to validate JSON: {}'.format(err)
description = 'Invalid Schema: {}'.format(schema_title)
logging.error(title)
logging.error(description)
LOG.error(title)
LOG.error(description)
raise AppError(
title=title,
description=description
@ -58,8 +60,8 @@ def validate_json(json_string, schema):
except FormatError as err:
title = 'FormatError: Unable to validate JSON: {}'.format(err)
description = 'Invalid Format: {}'.format(schema_title)
logging.error(title)
logging.error(description)
LOG.error(title)
LOG.error(description)
raise AppError(
title=title,
description=description

63
shipyard_airflow/control/middleware.py → shipyard_airflow/control/middleware/auth.py
View File

@ -11,25 +11,24 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" AuthMiddleware provides header processing that will decorate the
request context with auth values provided by the identity service.
"""
import logging
from oslo_utils import uuidutils
from shipyard_airflow import policy
LOG = logging.getLogger(__name__)
class AuthMiddleware(object):
def __init__(self):
self.logger = logging.getLogger('shipyard')
# Authentication
""" Authentication middleware class that handles auth headers
and adds them to the request context
"""
def process_request(self, req, resp):
ctx = req.context
ctx.set_policy_engine(policy.policy_engine)
for k, v in req.headers.items():
self.logger.debug("Request with header %s: %s" % (k, v))
auth_status = req.get_header(
'X-SERVICE-IDENTITY-STATUS') # will be set to Confirmed or Invalid
service = True
@ -73,49 +72,7 @@ class AuthMiddleware(object):
else:
ctx.is_admin_project = False
self.logger.debug(
'Request from authenticated user %s with roles %s',
ctx.user, ','.join(ctx.roles)
)
LOG.debug('Request from authenticated user %s with roles %s',
ctx.user, ','.join(ctx.roles))
else:
ctx.authenticated = False
class ContextMiddleware(object):
"""
Handle looking at the X-Context_Marker to see if it has value and that
value is a UUID (or close enough). If not, generate one.
"""
def process_request(self, req, resp):
ctx = req.context
ext_marker = req.get_header('X-Context-Marker')
if ext_marker is not None and uuidutils.is_uuid_like(ext_marker):
# external passed in an ok context marker
ctx.set_external_marker(ext_marker)
else:
# use the request id
ctx.set_external_marker(ctx.request_id)
class LoggingMiddleware(object):
def __init__(self):
self.logger = logging.getLogger('shipyard.control')
def process_response(self, req, resp, resource, req_succeeded):
ctx = req.context
extra = {
'user': ctx.user,
'req_id': ctx.request_id,
'external_ctx': ctx.external_marker,
}
resp.append_header('X-Shipyard-Req', ctx.request_id)
self.logger.info('%s %s - %s',
req.method,
req.uri,
resp.status,
extra=extra)
self.logger.debug('Response body:\n%s',
resp.body,
extra=extra)

33
shipyard_airflow/control/middleware/context.py Normal file
View File

@ -0,0 +1,33 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" ContextMiddleware handles setting the external marker provided
by the invoker of a service and adds it to the request contest.
"""
from oslo_utils import uuidutils
class ContextMiddleware(object):
"""
Handle looking at the X-Context_Marker to see if it has value and that
value is a UUID (or close enough). If not, generate one.
"""
def process_request(self, req, resp):
ctx = req.context
ext_marker = req.get_header('X-Context-Marker')
if ext_marker is not None and uuidutils.is_uuid_like(ext_marker):
# external passed in an ok context marker
ctx.set_external_marker(ext_marker)
else:
# use the request id
ctx.set_external_marker(ctx.request_id)

44
shipyard_airflow/control/middleware/logging_mw.py Normal file
View File

@ -0,0 +1,44 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" Module for logging related middleware
"""
import logging
from shipyard_airflow.control import ucp_logging
LOG = logging.getLogger(__name__)
class LoggingMiddleware(object):
""" Sets values to the request scope, and logs request and
response information
"""
def process_request(self, req, resp):
""" Set up values to be logged across the request
"""
ucp_logging.set_logvar('req_id', req.context.request_id)
ucp_logging.set_logvar('external_ctx', req.context.external_marker)
ucp_logging.set_logvar('user', req.context.user)
LOG.info("Request %s %s", req.method, req.url)
for header, header_value in req.headers.items():
LOG.info("Request header %s: %s", header, header_value)
def process_response(self, req, resp, resource, req_succeeded):
""" Log the response information
"""
ctx = req.context
resp.append_header('X-Shipyard-Req', ctx.request_id)
LOG.info('%s %s - %s', req.method, req.uri, resp.status)
LOG.debug('Response body:%s', resp.body)

38
shipyard_airflow/control/start_shipyard.py
View File

@ -11,49 +11,27 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
"""Shipyard startup
Sets up the global configurations for the Shipyard service. Hands off
to the api startup to handle the Falcon specific setup.
"""
from oslo_config import cfg
import shipyard_airflow.control.api as api
from shipyard_airflow import policy
from shipyard_airflow.conf import config
import shipyard_airflow.control.api as api
from shipyard_airflow.control import ucp_logging
from shipyard_airflow.db import db
from shipyard_airflow import policy
CONF = cfg.CONF
def start_shipyard(default_config_files=None):
"""Initializer for shipyard service.
Sets up global options before setting up API endpoints.
"""
# Trigger configuration resolution.
config.parse_args(args=[], default_config_files=default_config_files)
# Setup root logger
base_console_handler = logging.StreamHandler()
logging.basicConfig(level=logging.INFO,
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[base_console_handler])
logging.getLogger().info("Setting logging level to: %s",
logging.getLevelName(CONF.logging.log_level))
logging.basicConfig(level=CONF.logging.log_level,
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[base_console_handler])
# Specalized format for API logging
logger = logging.getLogger('shipyard.control')
logger.propagate = False
formatter = logging.Formatter(
('%(asctime)s - %(levelname)s - %(user)s - %(req_id)s - '
'%(external_ctx)s - %(message)s'))
console_handler = logging.StreamHandler()
console_handler.setFormatter(formatter)
logger.addHandler(console_handler)
ucp_logging.setup_logging(CONF.logging.log_level)
# Setup the RBAC policy enforcer
policy.policy_engine = policy.ShipyardPolicy()

147
shipyard_airflow/control/ucp_logging.py Normal file
View File

@ -0,0 +1,147 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
""" A logging filter to prepend UWSGI-handled formatting to all logging
records that use this filter. Request-based values will cause the log
records to have correlation values that can be used to better trace
logs. If uwsgi is not present, does not attempt to change the logs in
any way
Threads initiated using threading.Thread can be correlated to the request
they came from by setting a kwarg of log_extra, containing a dictionary
of valeus matching the VALID_ADDL_FIELDS below and any fields that are set
as additional_fields by the setup_logging function. This mechanism assumes
that the thread will maintain the correlation values for the life
of the thread.
"""
import logging
import threading
# Import uwsgi to determine if it has been provided to the application.
try:
import uwsgi
except ImportError:
uwsgi = None
VALID_ADDL_FIELDS = ['req_id', 'external_ctx', 'user']
_DEFAULT_LOG_FORMAT = (
'%(asctime)s %(levelname)-8s %(req_id)s %(external_ctx)s %(user)s '
'%(module)s(%(lineno)d) %(funcName)s - %(message)s'
)
_LOG_FORMAT_IN_USE = None
def setup_logging(level, format_string=None, additional_fields=None):
""" Establishes the base logging using the appropriate filter
attached to the console/stream handler.
:param level: The level value to set as the threshold for
logging. Ideally a client would use logging.INFO or
the desired logging constant to set this level value
:param format_string: Optional value allowing for override of the
logging format string. If new values beyond
the default value are introduced, the
additional_fields must contain those fields
to ensure they are set upon using the logging
filter.
:param additional_fields: Optionally allows for specifying more
fields that will be set on each logging
record. If specified, the format_string
parameter should be set with matching
fields, otherwise they will not be
displayed.
"""
global _LOG_FORMAT_IN_USE
_LOG_FORMAT_IN_USE = format_string or _DEFAULT_LOG_FORMAT
console_handler = logging.StreamHandler()
if uwsgi:
console_handler.addFilter(UwsgiLogFilter(additional_fields))
logging.basicConfig(level=level,
format=_LOG_FORMAT_IN_USE,
handlers=[console_handler])
logger = logging.getLogger(__name__)
logger.info('Established logging defaults')
def get_log_format():
""" Returns the common log format being used by this application
"""
return _LOG_FORMAT_IN_USE
def set_logvar(key, value):
""" Attempts to set the logvar in the request scope , or ignores it
if not running in uwsgi
"""
if uwsgi and value:
uwsgi.set_logvar(key, value)
class UwsgiLogFilter(logging.Filter):
""" A filter that preepends log records with additional request
based information, or information provided by log_extra in the
kwargs provided to a thread
"""
def __init__(self, additional_fields=None):
super().__init__()
if additional_fields is None:
additional_fields = []
self.log_fields = [*VALID_ADDL_FIELDS, *additional_fields]
def filter(self, record):
""" Checks for thread provided values, or attempts to get values
from uwsgi
"""
if self._thread_has_log_extra():
value_setter = self._set_values_from_log_extra
else:
value_setter = self._set_value
for field_nm in self.log_fields:
value_setter(record, field_nm)
return True
def _set_value(self, record, logvar):
# handles setting the logvars from uwsgi or '' in case of none/empty
try:
logvar_value = None
if uwsgi:
logvar_value = uwsgi.get_logvar(logvar)
if logvar_value:
setattr(record, logvar, logvar_value.decode('UTF-8'))
else:
setattr(record, logvar, '')
except SystemError:
# This happens if log_extra is not on a thread that is spawned
# by a process running under uwsgi
setattr(record, logvar, '')
def _set_values_from_log_extra(self, record, logvar):
# sets the values from the log_extra on the thread
setattr(record, logvar, self._get_value_from_thread(logvar) or '')
def _thread_has_log_extra(self):
# Checks to see if log_extra is present on the current thread
if self._get_log_extra_from_thread():
return True
return False
def _get_value_from_thread(self, logvar):
# retrieve the logvar from the log_extra from kwargs for the thread
return self._get_log_extra_from_thread().get(logvar, '')
def _get_log_extra_from_thread(self):
# retrieves the log_extra value from kwargs or {} if it doesn't
# exist
return threading.current_thread()._kwargs.get('log_extra', {})

2
shipyard_airflow/errors.py
View File

@ -136,7 +136,7 @@ def default_exception_handler(ex, req, resp, params):
else:
# take care of the uncaught stuff
exc_string = traceback.format_exc()
logging.error('Unhanded Exception being handled: \n%s', exc_string)
logging.error('Unhandled Exception being handled: \n%s', exc_string)
format_error_resp(
req,
resp,

26
shipyard_airflow/policy.py
View File

@ -186,15 +186,14 @@ class ApiEnforcer(object):
def secure_handler(slf, req, resp, *args, **kwargs):
ctx = req.context
policy_eng = ctx.policy_engine
slf.info(ctx, "Policy Engine: %s" % policy_eng.__class__.__name__)
LOG.info("Policy Engine: %s", policy_eng.__class__.__name__)
# perform auth
slf.info(ctx, "Enforcing policy %s on request %s" %
(self.action, ctx.request_id))
LOG.info("Enforcing policy %s on request %s",
self.action, ctx.request_id)
# policy engine must be configured
if policy_eng is None:
slf.error(
ctx,
"Error-Policy engine required-action: %s" % self.action)
LOG.error(
"Error-Policy engine required-action: %s", self.action)
raise AppError(
title="Auth is not being handled by any policy engine",
status=falcon.HTTP_500,
@ -204,13 +203,12 @@ class ApiEnforcer(object):
try:
if policy_eng.authorize(self.action, ctx):
# authorized
slf.info(ctx, "Request is authorized")
LOG.info("Request is authorized")
authorized = True
except:
# couldn't service the auth request
slf.error(
ctx,
"Error - Expectation Failed - action: %s" % self.action)
LOG.error(
"Error - Expectation Failed - action: %s", self.action)
raise ApiError(
title="Expectation Failed",
status=falcon.HTTP_417,
@ -219,13 +217,11 @@ class ApiEnforcer(object):
if authorized:
return f(slf, req, resp, *args, **kwargs)
else:
slf.error(ctx,
"Auth check failed. Authenticated:%s" %
LOG.error("Auth check failed. Authenticated:%s",
ctx.authenticated)
# raise the appropriate response exeception
if ctx.authenticated:
slf.error(ctx,
"Error: Forbidden access - action: %s" %
LOG.error("Error: Forbidden access - action: %s",
self.action)
raise ApiError(
title="Forbidden",
@ -234,7 +230,7 @@ class ApiEnforcer(object):
retry=False
)
else:
slf.error(ctx, "Error - Unauthenticated access")
LOG.error("Error - Unauthenticated access")
raise ApiError(
title="Unauthenticated",
status=falcon.HTTP_401,

1
shipyard_airflow/shipyard.py
View File

@ -18,6 +18,7 @@ Bootstraps to the start_shipyard module.
from shipyard_airflow.control.start_shipyard import start_shipyard
# Initialization compatible with PasteDeploy
def paste_start_shipyard(global_conf, **kwargs):
"""Paste deploy compatible initializer"""
return shipyard

72
tests/unit/control/test_base_resource.py
View File

@ -13,8 +13,6 @@
# limitations under the License.
import json
import logging
from mock import patch
import pytest
import falcon
@ -80,8 +78,7 @@ def test_req_json_no_body():
assert result is None
@patch('shipyard_airflow.control.base.BaseResource.log_message')
def test_req_json_with_body(mock_logger):
def test_req_json_with_body():
'''test req_json when there is a body'''
baseResource = BaseResource()
ctx = ShipyardRequestContext()
@ -96,9 +93,6 @@ def test_req_json_with_body(mock_logger):
req = create_req(ctx, body=json_body)
result = baseResource.req_json(req, validate_json_schema=ACTION)
mock_logger.assert_called_with(
ctx, logging.INFO,
'Input message body: b\'' + json_body.decode('utf-8') + '\'')
assert result == json.loads(json_body.decode('utf-8'))
req = create_req(ctx, body=json_body)
@ -108,9 +102,6 @@ def test_req_json_with_body(mock_logger):
with pytest.raises(InvalidFormatError) as expected_exc:
baseResource.req_json(req)
mock_logger.assert_called_with(
ctx, logging.ERROR,
'Invalid JSON in request: \n' + json_body.decode('utf-8'))
assert 'JSON could not be decoded' in str(expected_exc)
assert str(req.path) in str(expected_exc)
@ -126,64 +117,3 @@ def test_to_json():
}
results = baseResource.to_json(body_dict)
assert results == json.dumps(body_dict)
@patch('logging.Logger.log')
def test_log_message(mock_log):
'''test log_message'''
baseResource = BaseResource()
ctx = None
level = logging.ERROR
msg = 'test_message'
extra = {'user': 'N/A', 'req_id': 'N/A', 'external_ctx': 'N/A'}
baseResource.log_message(ctx, level, msg)
mock_log.assert_called_with(level, msg, extra=extra)
ctx = ShipyardRequestContext()
extra = {
'user': ctx.user,
'req_id': ctx.request_id,
'external_ctx': ctx.external_marker,
}
baseResource.log_message(ctx, level, msg)
mock_log.assert_called_with(level, msg, extra=extra)
def test_debug():
'''test debug'''
baseResource = BaseResource()
ctx = ShipyardRequestContext()
msg = 'test_msg'
with patch.object(BaseResource, 'log_message') as mock_method:
baseResource.debug(ctx, msg)
mock_method.assert_called_once_with(ctx, logging.DEBUG, msg)
def test_info():
'''test info'''
baseResource = BaseResource()
ctx = ShipyardRequestContext()
msg = 'test_msg'
with patch.object(BaseResource, 'log_message') as mock_method:
baseResource.info(ctx, msg)
mock_method.assert_called_once_with(ctx, logging.INFO, msg)
def test_warn():
'''test warn '''
baseResource = BaseResource()
ctx = ShipyardRequestContext()
msg = 'test_msg'
with patch.object(BaseResource, 'log_message') as mock_method:
baseResource.warn(ctx, msg)
mock_method.assert_called_once_with(ctx, logging.WARN, msg)
def test_error():
'''test error'''
baseResource = BaseResource()
ctx = ShipyardRequestContext()
msg = 'test_msg'
with patch.object(BaseResource, 'log_message') as mock_method:
baseResource.error(ctx, msg)
mock_method.assert_called_once_with(ctx, logging.ERROR, msg)

19
tests/unit/control/test_configdocs_api.py
View File

@ -16,7 +16,7 @@ from mock import patch
import pytest
from tests.unit.control import common
from shipyard_airflow.control.base import ShipyardRequestContext
from shipyard_airflow.control.configdocs.configdocs_api import (
CommitConfigDocsResource,
ConfigDocsResource
@ -24,6 +24,9 @@ from shipyard_airflow.control.configdocs.configdocs_api import (
from shipyard_airflow.control.configdocs.configdocs_helper import \
ConfigdocsHelper
from shipyard_airflow.errors import ApiError
from tests.unit.control import common
CTX = ShipyardRequestContext()
def test__validate_version_parameter():
@ -46,7 +49,7 @@ def test_get_collection():
helper = None
with patch.object(ConfigdocsHelper, 'get_collection_docs') as mock_method:
cdr = ConfigDocsResource()
helper = ConfigdocsHelper('')
helper = ConfigdocsHelper(CTX)
cdr.get_collection(helper, 'apples')
mock_method.assert_called_once_with('buffer', 'apples')
@ -61,7 +64,7 @@ def test_post_collection():
document_data = 'lots of info'
with patch.object(ConfigdocsHelper, 'add_collection') as mock_method:
cdr = ConfigDocsResource()
helper = ConfigdocsHelper('')
helper = ConfigdocsHelper(CTX)
helper.is_buffer_valid_for_bucket = lambda a, b: True
helper.get_deckhand_validation_status = (
lambda a: ConfigdocsHelper._format_validations_to_status([], 0)
@ -74,7 +77,7 @@ def test_post_collection():
with pytest.raises(ApiError):
cdr = ConfigDocsResource()
helper = ConfigdocsHelper('')
helper = ConfigdocsHelper(CTX)
# not valid for bucket
helper.is_buffer_valid_for_bucket = lambda a, b: False
helper.get_deckhand_validation_status = (
@ -92,7 +95,7 @@ def test_commit_configdocs():
ccdr = CommitConfigDocsResource()
commit_resp = None
with patch.object(ConfigdocsHelper, 'tag_buffer') as mock_method:
helper = ConfigdocsHelper('')
helper = ConfigdocsHelper(CTX)
helper.is_buffer_empty = lambda: False
helper.get_validations_for_buffer = lambda: {'status': 'Valid'}
commit_resp = ccdr.commit_configdocs(helper, False)
@ -102,7 +105,7 @@ def test_commit_configdocs():
commit_resp = None
with patch.object(ConfigdocsHelper, 'tag_buffer') as mock_method:
helper = ConfigdocsHelper('')
helper = ConfigdocsHelper(CTX)
helper.is_buffer_empty = lambda: False
helper.get_validations_for_buffer = (
lambda: {
@ -124,7 +127,7 @@ def test_commit_configdocs_force():
ccdr = CommitConfigDocsResource()
commit_resp = None
with patch.object(ConfigdocsHelper, 'tag_buffer') as mock_method:
helper = ConfigdocsHelper('')
helper = ConfigdocsHelper(CTX)
helper.is_buffer_empty = lambda: False
helper.get_validations_for_buffer = lambda: {'status': 'Invalid'}
commit_resp = ccdr.commit_configdocs(helper, True)
@ -143,7 +146,7 @@ def test_commit_configdocs_buffer_err():
ccdr = CommitConfigDocsResource()
with pytest.raises(ApiError):
helper = ConfigdocsHelper('')
helper = ConfigdocsHelper(CTX)
helper.is_buffer_empty = lambda: True
helper.get_validations_for_buffer = lambda: {'status': 'Valid'}
ccdr.commit_configdocs(helper, False)

40
tests/unit/control/test_configdocs_helper.py
View File

@ -18,6 +18,7 @@ import yaml
import pytest
from .fake_response import FakeResponse
from shipyard_airflow.control.base import ShipyardRequestContext
from shipyard_airflow.control.configdocs import configdocs_helper
from shipyard_airflow.control.configdocs.configdocs_helper import (
BufferMode,
@ -30,6 +31,8 @@ from shipyard_airflow.control.configdocs.deckhand_client import (
)
from shipyard_airflow.errors import ApiError, AppError
CTX = ShipyardRequestContext()
REV_BUFFER_DICT = {
'committed': {'id': 3,
'url': 'url3',
@ -138,13 +141,11 @@ DIFF_COMMIT_AND_BUFFER_DICT = {
def test_construct_configdocs_helper():
"""
Creates a configdoc helper, tests that the context_marker
Creates a configdoc helper, tests that the context
is passed to the sub-helper
"""
marker = 'marker'
helper = ConfigdocsHelper(marker)
assert helper.deckhand.context_marker == marker
assert helper.context_marker == marker
helper = ConfigdocsHelper(CTX)
assert helper.ctx == CTX
def test_get_buffer_mode():
@ -185,7 +186,7 @@ def test_is_buffer_emtpy():
"""
Test the method to check if the configdocs buffer is empty
"""
helper = ConfigdocsHelper('')
helper = ConfigdocsHelper(CTX)
helper._get_revision_dict = lambda: REV_BUFFER_DICT
assert not helper.is_buffer_empty()
@ -203,7 +204,7 @@ def test_is_collection_in_buffer():
"""
Test that collections are found in the buffer
"""
helper = ConfigdocsHelper('')
helper = ConfigdocsHelper(CTX)
helper._get_revision_dict = lambda: REV_BUFFER_DICT
helper.deckhand.get_diff = (
lambda old_revision_id, new_revision_id: DIFF_BUFFER_DICT
@ -237,7 +238,7 @@ def test_is_buffer_valid_for_bucket():
helper._get_revision_dict = lambda: revision_dict
helper.deckhand.get_diff = lambda: diff_dict
helper = ConfigdocsHelper('')
helper = ConfigdocsHelper(CTX)
helper._get_revision_dict = lambda: REV_BUFFER_DICT
helper.deckhand.get_diff = (
lambda old_revision_id, new_revision_id: DIFF_BUFFER_DICT
@ -308,7 +309,7 @@ def test__get_revision_dict_no_commit():
Tests the processing of revision dict response from dechand
with a buffer version, but no committed revision
"""
helper = ConfigdocsHelper('')
helper = ConfigdocsHelper(CTX)
helper.deckhand.get_revision_list = lambda: yaml.load("""
---
- id: 1
@ -345,7 +346,7 @@ def test__get_revision_dict_empty():
Tests the processing of revision dict response from dechand
where the response is an empty list
"""
helper = ConfigdocsHelper('')
helper = ConfigdocsHelper(CTX)
helper.deckhand.get_revision_list = lambda: []
rev_dict = helper._get_revision_dict()
committed = rev_dict.get(configdocs_helper.COMMITTED)
@ -363,7 +364,7 @@ def test__get_revision_dict_commit_no_buff():
Tests the processing of revision dict response from dechand
with a committed and no buffer revision
"""
helper = ConfigdocsHelper('')
helper = ConfigdocsHelper(CTX)
helper.deckhand.get_revision_list = lambda: yaml.load("""
---
- id: 1
@ -400,7 +401,7 @@ def test__get_revision_dict_commit_and_buff():
Tests the processing of revision dict response from dechand
with a committed and a buffer revision
"""
helper = ConfigdocsHelper('')
helper = ConfigdocsHelper(CTX)
helper.deckhand.get_revision_list = lambda: yaml.load("""
---
- id: 1
@ -446,7 +447,7 @@ def test__get_revision_dict_errs():
def _raise_nree():
raise NoRevisionsExistError()
helper = ConfigdocsHelper('')
helper = ConfigdocsHelper(CTX)
helper.deckhand.get_revision_list = _raise_dre
with pytest.raises(AppError):
@ -468,7 +469,7 @@ def test_get_collection_docs():
"""
Returns the representation of the yaml docs from deckhand
"""
helper = ConfigdocsHelper('')
helper = ConfigdocsHelper(CTX)
helper.deckhand.get_docs_from_revision = (
lambda revision_id, bucket_id: "{'yaml': 'yaml'}"
)
@ -508,7 +509,8 @@ def _fake_get_validations_for_component(url,
design_reference,
response,
exception,
context_marker):
context_marker,
**kwargs):
"""
Responds with a status response
"""
@ -537,7 +539,7 @@ def test_get_validations_for_revision():
"""
Tets the functionality of the get_validations_for_revision method
"""
helper = ConfigdocsHelper('')
helper = ConfigdocsHelper(CTX)
hold_ve = helper.__class__._get_validation_endpoints
hold_vfc = helper.__class__._get_validations_for_component
helper.__class__._get_validation_endpoints = (
@ -609,7 +611,7 @@ def test__get_deckhand_validations():
"""
Tets the functionality of processing a response from deckhand
"""
helper = ConfigdocsHelper('')
helper = ConfigdocsHelper(CTX)
helper.deckhand._get_base_validation_resp = (
lambda revision_id: FK_VAL_BASE_RESP
)
@ -627,7 +629,7 @@ def test_tag_buffer():
Tests that the tag buffer method attempts to tag the right version
"""
with patch.object(ConfigdocsHelper, 'tag_revision') as mock_method:
helper = ConfigdocsHelper('')
helper = ConfigdocsHelper(CTX)
helper._get_revision_dict = lambda: REV_BUFFER_DICT
helper.tag_buffer('artful')
@ -640,7 +642,7 @@ def test_add_collection():
error handling
"""
with patch.object(DeckhandClient, 'put_bucket') as mock_method:
helper = ConfigdocsHelper('')
helper = ConfigdocsHelper(CTX)
helper._get_revision_dict = lambda: REV_BUFFER_DICT
assert helper.add_collection('mop', 'yaml:yaml') == 5

5
tests/unit/control/test_rendered_configdocs_api.py
View File

@ -15,12 +15,15 @@ from mock import patch
import pytest
from shipyard_airflow.control.base import ShipyardRequestContext
from shipyard_airflow.control.configdocs.rendered_configdocs_api import \
RenderedConfigDocsResource
from shipyard_airflow.control.configdocs.configdocs_helper import \
ConfigdocsHelper
from shipyard_airflow.errors import ApiError
CTX = ShipyardRequestContext()
def test__validate_version_parameter():
"""
@ -46,7 +49,7 @@ def test_get_rendered_configdocs():
with patch.object(
ConfigdocsHelper, 'get_rendered_configdocs'
) as mock_method:
helper = ConfigdocsHelper('')
helper = ConfigdocsHelper(CTX)
rcdr.get_rendered_configdocs(helper, version='buffer')
mock_method.assert_called_once_with('buffer')

13
tests/unit/control/test_shipyard_request_context.py
View File

@ -15,19 +15,6 @@
from shipyard_airflow.control.base import ShipyardRequestContext
def test_set_log_level():
'''test set_log_level'''
ctx = ShipyardRequestContext()
ctx.set_log_level('error')
assert ctx.log_level == 'error'
ctx.set_log_level('info')
assert ctx.log_level == 'info'
ctx.set_log_level('debug')
assert ctx.log_level == 'debug'
def test_set_user():
'''test set_user '''
ctx = ShipyardRequestContext()