Rajaram/Vinkesh|used common framework code that exists in the openstack-common project
This commit is contained in:
Rajaram Mallya
@@ -19,311 +19,11 @@
|
||||
"""
|
||||
Routines for configuring Melange
|
||||
"""
|
||||
|
||||
from gettext import gettext as _
|
||||
import logging
|
||||
import logging.config
|
||||
import logging.handlers
|
||||
import optparse
|
||||
import os
|
||||
from paste import deploy
|
||||
import sys
|
||||
|
||||
|
||||
DEFAULT_LOG_FORMAT = "%(asctime)s %(levelname)8s [%(name)s] %(message)s"
|
||||
DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
|
||||
|
||||
|
||||
def parse_options(parser, cli_args=None):
|
||||
"""
|
||||
Returns the parsed CLI options, command to run and its arguments, merged
|
||||
with any same-named options found in a configuration file.
|
||||
|
||||
The function returns a tuple of (options, args), where options is a
|
||||
mapping of option key/str(value) pairs, and args is the set of arguments
|
||||
(not options) supplied on the command-line.
|
||||
|
||||
The reason that the option values are returned as strings only is that
|
||||
ConfigParser and paste.deploy only accept string values...
|
||||
|
||||
:param parser: The option parser
|
||||
:param cli_args: (Optional) Set of arguments to process. If not present,
|
||||
sys.argv[1:] is used.
|
||||
:retval tuple of (options, args)
|
||||
"""
|
||||
|
||||
(options, args) = parser.parse_args(cli_args)
|
||||
|
||||
return (vars(options), args)
|
||||
|
||||
|
||||
def add_common_options(parser):
|
||||
"""
|
||||
Given a supplied optparse.OptionParser, adds an OptionGroup that
|
||||
represents all common configuration options.
|
||||
|
||||
:param parser: optparse.OptionParser
|
||||
"""
|
||||
help_text = _("The following configuration options are common to "
|
||||
"all melange programs.")
|
||||
|
||||
group = optparse.OptionGroup(parser, "Common Options", help_text)
|
||||
group.add_option('-v', '--verbose', default=False, dest="verbose",
|
||||
action="store_true",
|
||||
help=_("Print more verbose output"))
|
||||
group.add_option('-d', '--debug', default=False, dest="debug",
|
||||
action="store_true",
|
||||
help=_("Print debugging output"))
|
||||
group.add_option('--config-file', default=None, metavar="PATH",
|
||||
help=_("Path to the config file to use. When not "
|
||||
"specified (the default), we generally look at the"
|
||||
" first argument specified to be a config file,"
|
||||
" and if that is also missing, we search standard "
|
||||
"directories for a config file."))
|
||||
parser.add_option_group(group)
|
||||
|
||||
|
||||
def add_log_options(parser):
|
||||
"""
|
||||
Given a supplied optparse.OptionParser, adds an OptionGroup that
|
||||
represents all the configuration options around logging.
|
||||
|
||||
:param parser: optparse.OptionParser
|
||||
"""
|
||||
help_text = _("The following configuration options are specific to "
|
||||
"logging functionality for this program.")
|
||||
|
||||
group = optparse.OptionGroup(parser, "Logging Options", help_text)
|
||||
group.add_option('--log-config', default=None, metavar="PATH",
|
||||
help=_("If this option is specified, the logging "
|
||||
"configuration file specified is used and overrides "
|
||||
"any other logging options specified. Please see "
|
||||
"the Python logging module documentation for "
|
||||
"details on logging configuration files."))
|
||||
group.add_option('--log-date-format', metavar="FORMAT",
|
||||
default=DEFAULT_LOG_DATE_FORMAT,
|
||||
help=_("Format string for %(asctime)s in log records. "
|
||||
"Default: %default"))
|
||||
group.add_option('--log-file', default=None, metavar="PATH",
|
||||
help=_("(Optional) Name of log file to output to. "
|
||||
"If not set, logging will go to stdout."))
|
||||
group.add_option("--log-dir", default=None,
|
||||
help=_("(Optional) The directory to keep log files in "
|
||||
"(will be prepended to --logfile)"))
|
||||
parser.add_option_group(group)
|
||||
|
||||
|
||||
def setup_logging(options, conf):
|
||||
"""
|
||||
Sets up the logging options for a log with supplied name
|
||||
|
||||
:param options: Mapping of typed option key/values
|
||||
:param conf: Mapping of untyped key/values from config file
|
||||
"""
|
||||
|
||||
if options.get('log_config', None):
|
||||
# Use a logging configuration file for all settings...
|
||||
if os.path.exists(options['log_config']):
|
||||
logging.config.fileConfig(options['log_config'])
|
||||
return
|
||||
else:
|
||||
raise RuntimeError(_("Unable to locate specified logging "
|
||||
"config file: %s") % options['log_config'])
|
||||
|
||||
# If either the CLI option or the conf value
|
||||
# is True, we set to True
|
||||
debug = options.get('debug') or \
|
||||
get_option(conf, 'debug', type='bool', default=False)
|
||||
verbose = options.get('verbose') or \
|
||||
get_option(conf, 'verbose', type='bool', default=False)
|
||||
root_logger = logging.root
|
||||
if debug:
|
||||
root_logger.setLevel(logging.DEBUG)
|
||||
elif verbose:
|
||||
root_logger.setLevel(logging.INFO)
|
||||
else:
|
||||
root_logger.setLevel(logging.WARNING)
|
||||
|
||||
# Set log configuration from options...
|
||||
# Note that we use a hard-coded log format in the options
|
||||
# because of Paste.Deploy bug #379
|
||||
# http://trac.pythonpaste.org/pythonpaste/ticket/379
|
||||
log_format = options.get('log_format', DEFAULT_LOG_FORMAT)
|
||||
log_date_format = options.get('log_date_format', DEFAULT_LOG_DATE_FORMAT)
|
||||
formatter = logging.Formatter(log_format, log_date_format)
|
||||
|
||||
logfile = options.get('log_file')
|
||||
if not logfile:
|
||||
logfile = conf.get('log_file')
|
||||
|
||||
if logfile:
|
||||
logdir = options.get('log_dir')
|
||||
if not logdir:
|
||||
logdir = conf.get('log_dir')
|
||||
if logdir:
|
||||
logfile = os.path.join(logdir, logfile)
|
||||
logfile = logging.FileHandler(logfile)
|
||||
logfile.setFormatter(formatter)
|
||||
logfile.setFormatter(formatter)
|
||||
root_logger.addHandler(logfile)
|
||||
else:
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
handler.setFormatter(formatter)
|
||||
root_logger.addHandler(handler)
|
||||
|
||||
|
||||
def find_config_file(options, args):
|
||||
"""
|
||||
Return the first config file found.
|
||||
|
||||
We search for the paste config file in the following order:
|
||||
* If --config-file option is used, use that
|
||||
* If args[0] is a file, use that
|
||||
* Search for melange.conf in standard directories:
|
||||
* .
|
||||
* ~.melange/
|
||||
* ~
|
||||
* /etc/melange
|
||||
* /etc
|
||||
|
||||
:retval Full path to config file, or None if no config file found
|
||||
"""
|
||||
|
||||
fix_path = lambda p: os.path.abspath(os.path.expanduser(p))
|
||||
if options.get('config_file'):
|
||||
if os.path.exists(options['config_file']):
|
||||
return fix_path(options['config_file'])
|
||||
elif args:
|
||||
if os.path.exists(args[0]):
|
||||
return fix_path(args[0])
|
||||
|
||||
# Handle standard directory search for melange.conf
|
||||
config_file_dirs = [fix_path(os.getcwd()),
|
||||
fix_path(os.path.join(os.getcwd(), "etc")),
|
||||
fix_path(os.path.join('~', '.melange')),
|
||||
fix_path('~'),
|
||||
'/etc/melange/',
|
||||
'/etc']
|
||||
|
||||
for cfg_dir in config_file_dirs:
|
||||
cfg_file = os.path.join(cfg_dir, 'melange.conf')
|
||||
if os.path.exists(cfg_file):
|
||||
return cfg_file
|
||||
|
||||
|
||||
def load_paste_config(app_name, options, args):
|
||||
"""
|
||||
Looks for a config file to use for an app and returns the
|
||||
config file path and a configuration mapping from a paste config file.
|
||||
|
||||
We search for the paste config file in the following order:
|
||||
* If --config-file option is used, use that
|
||||
* If args[0] is a file, use that
|
||||
* Search for melange.conf in standard directories:
|
||||
* .
|
||||
* ~.melange/
|
||||
* ~
|
||||
* /etc/melange
|
||||
* /etc
|
||||
|
||||
:param app_name: Name of the application to load config for, or None.
|
||||
None signifies to only load the [DEFAULT] section of
|
||||
the config file.
|
||||
:param options: Set of typed options returned from parse_options()
|
||||
:param args: Command line arguments from argv[1:]
|
||||
:retval Tuple of (conf_file, conf)
|
||||
|
||||
:raises RuntimeError when config file cannot be located or there was a
|
||||
problem loading the configuration file.
|
||||
"""
|
||||
conf_file = find_config_file(options, args)
|
||||
if not conf_file:
|
||||
raise RuntimeError(_("Unable to locate any configuration file. "
|
||||
"Cannot load application %s") % app_name)
|
||||
try:
|
||||
conf = deploy.appconfig("config:%s" % conf_file, name=app_name)
|
||||
return conf_file, conf
|
||||
except Exception, error:
|
||||
raise RuntimeError(_("Error trying to load config %(conf_file)s:"
|
||||
" %(error)s") % locals())
|
||||
|
||||
|
||||
def load_paste_app(app_name, options, args):
|
||||
"""
|
||||
Builds and returns a WSGI app from a paste config file.
|
||||
|
||||
We search for the paste config file in the following order:
|
||||
* If --config-file option is used, use that
|
||||
* If args[0] is a file, use that
|
||||
* Search for melange.conf in standard directories:
|
||||
* .
|
||||
* ~.melange/
|
||||
* ~
|
||||
* /etc/melange
|
||||
* /etc
|
||||
|
||||
:param app_name: Name of the application to load
|
||||
:param options: Set of typed options returned from parse_options()
|
||||
:param args: Command line arguments from argv[1:]
|
||||
|
||||
:raises RuntimeError when config file cannot be located or application
|
||||
cannot be loaded from config file
|
||||
"""
|
||||
conf_file, conf = load_paste_config(app_name, options, args)
|
||||
|
||||
try:
|
||||
# Setup logging early, supplying both the CLI options and the
|
||||
# configuration mapping from the config file
|
||||
setup_logging(options, conf)
|
||||
|
||||
# We only update the conf dict for the verbose and debug
|
||||
# flags. Everything else must be set up in the conf file...
|
||||
debug = options.get('debug') or \
|
||||
get_option(conf, 'debug', type='bool', default=False)
|
||||
verbose = options.get('verbose') or \
|
||||
get_option(conf, 'verbose', type='bool', default=False)
|
||||
conf['debug'] = debug
|
||||
conf['verbose'] = verbose
|
||||
|
||||
# Log the options used when starting if we're in debug mode...
|
||||
if debug:
|
||||
logger = logging.getLogger(app_name)
|
||||
logger.debug("*" * 80)
|
||||
logger.debug("Configuration options gathered from config file:")
|
||||
logger.debug(conf_file)
|
||||
logger.debug("================================================")
|
||||
items = dict([(k, v) for k, v in conf.items()
|
||||
if k not in ('__file__', 'here')])
|
||||
for key, value in sorted(items.items()):
|
||||
logger.debug("%(key)-30s %(value)s" % locals())
|
||||
logger.debug("*" * 80)
|
||||
app = deploy.loadapp("config:%s" % conf_file, name=app_name)
|
||||
except (LookupError, ImportError), e:
|
||||
raise RuntimeError(_("Unable to load %(app_name)s from "
|
||||
"configuration file %(conf_file)s."
|
||||
"\nGot: %(e)r") % locals())
|
||||
return conf, app
|
||||
|
||||
|
||||
def get_option(options, option, **kwargs):
|
||||
if option in options:
|
||||
value = options[option]
|
||||
type_ = kwargs.get('type', 'str')
|
||||
if type_ == 'bool':
|
||||
if hasattr(value, 'lower'):
|
||||
return value.lower() == 'true'
|
||||
else:
|
||||
return value
|
||||
elif type_ == 'int':
|
||||
return int(value)
|
||||
elif type_ == 'float':
|
||||
return float(value)
|
||||
else:
|
||||
return value
|
||||
elif 'default' in kwargs:
|
||||
return kwargs['default']
|
||||
else:
|
||||
raise KeyError(_("option '%s' not found") % option)
|
||||
from openstack.common.config import (parse_options,
|
||||
add_log_options,
|
||||
add_common_options,
|
||||
load_paste_config,
|
||||
load_paste_app, get_option)
|
||||
|
||||
|
||||
class Config(object):
|
||||
|
||||
@@ -20,85 +20,15 @@ Nova base exception handling, including decorator for re-raising
|
||||
Nova-type exceptions. SHOULD include dedicated exception logging.
|
||||
"""
|
||||
|
||||
from gettext import gettext as _
|
||||
import logging
|
||||
from openstack.common.exception import (Error, ProcessExecutionError,
|
||||
DatabaseMigrationError,
|
||||
InvalidContentType)
|
||||
|
||||
|
||||
class ProcessExecutionError(IOError):
|
||||
def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None,
|
||||
description=None):
|
||||
if description is None:
|
||||
description = _("Unexpected error while running command.")
|
||||
if exit_code is None:
|
||||
exit_code = '-'
|
||||
message = "%s\nCommand: %s\nExit code: %s\nStdout: %r\nStderr: %r" % (
|
||||
description, cmd, exit_code, stdout, stderr)
|
||||
IOError.__init__(self, message)
|
||||
|
||||
|
||||
class MelangeError(Exception):
|
||||
class MelangeError(Error):
|
||||
|
||||
def __init__(self, message=None):
|
||||
super(MelangeError, self).__init__(message or self._error_message())
|
||||
|
||||
def _error_message(self):
|
||||
pass
|
||||
|
||||
|
||||
class ApiError(MelangeError):
|
||||
def __init__(self, message='Unknown', code='Unknown'):
|
||||
self.message = message
|
||||
self.code = code
|
||||
super(ApiError, self).__init__('%s: %s' % (code, message))
|
||||
|
||||
|
||||
class NotFound(MelangeError):
|
||||
pass
|
||||
|
||||
|
||||
class Duplicate(MelangeError):
|
||||
pass
|
||||
|
||||
|
||||
class NotAuthorized(MelangeError):
|
||||
pass
|
||||
|
||||
|
||||
class NotEmpty(MelangeError):
|
||||
pass
|
||||
|
||||
|
||||
class Invalid(MelangeError):
|
||||
pass
|
||||
|
||||
|
||||
class InvalidContentType(Invalid):
|
||||
message = _("Invalid content type %(content_type)s.")
|
||||
|
||||
|
||||
class BadInputError(Exception):
|
||||
"""Error resulting from a client sending bad input to a server"""
|
||||
pass
|
||||
|
||||
|
||||
class MissingArgumentError(MelangeError):
|
||||
pass
|
||||
|
||||
|
||||
class DatabaseMigrationError(MelangeError):
|
||||
pass
|
||||
|
||||
|
||||
def wrap_exception(f):
|
||||
def _wrap(*args, **kw):
|
||||
try:
|
||||
return f(*args, **kw)
|
||||
except Exception, e:
|
||||
if not isinstance(e, Error):
|
||||
#exc_type, exc_value, exc_traceback = sys.exc_info()
|
||||
logging.exception('Uncaught exception')
|
||||
#logging.error(traceback.extract_stack(exc_traceback))
|
||||
raise Error(str(e))
|
||||
raise
|
||||
_wrap.func_name = f.func_name
|
||||
return _wrap
|
||||
|
||||
@@ -28,6 +28,7 @@ import subprocess
|
||||
import sys
|
||||
import uuid
|
||||
|
||||
from openstack.common.utils import import_class, import_object
|
||||
from melange.common import exception
|
||||
from melange.common.exception import ProcessExecutionError
|
||||
|
||||
@@ -42,26 +43,6 @@ def parse_int(subject):
|
||||
return None
|
||||
|
||||
|
||||
def import_class(import_str):
|
||||
"""Returns a class from a string including module and class"""
|
||||
mod_str, _sep, class_str = import_str.rpartition('.')
|
||||
try:
|
||||
__import__(mod_str)
|
||||
return getattr(sys.modules[mod_str], class_str)
|
||||
except (ImportError, ValueError, AttributeError):
|
||||
raise exception.NotFound('Class %s cannot be found' % class_str)
|
||||
|
||||
|
||||
def import_object(import_str):
|
||||
"""Returns an object including a module or module and class"""
|
||||
try:
|
||||
__import__(import_str)
|
||||
return sys.modules[import_str]
|
||||
except ImportError:
|
||||
cls = import_class(import_str)
|
||||
return cls()
|
||||
|
||||
|
||||
def execute(cmd, process_input=None, addl_env=None, check_exit_code=True):
|
||||
logging.debug("Running cmd: %s", cmd)
|
||||
env = os.environ.copy()
|
||||
@@ -101,20 +82,10 @@ def generate_uid(topic, size=8):
|
||||
for x in xrange(size)]))
|
||||
|
||||
|
||||
def isotime(at=None):
|
||||
if not at:
|
||||
at = datetime.datetime.utcnow()
|
||||
return at.strftime(TIME_FORMAT)
|
||||
|
||||
|
||||
def utcnow():
|
||||
return datetime.datetime.utcnow()
|
||||
|
||||
|
||||
def parse_isotime(timestr):
|
||||
return datetime.datetime.strptime(timestr, TIME_FORMAT)
|
||||
|
||||
|
||||
class LazyPluggable(object):
|
||||
"""A pluggable backend loaded lazily based on some value."""
|
||||
|
||||
|
||||
@@ -27,8 +27,6 @@ import json
|
||||
import logging
|
||||
import paste.urlmap
|
||||
import re
|
||||
import routes.middleware
|
||||
import sys
|
||||
import traceback
|
||||
from webob import Response
|
||||
import webob.dec
|
||||
@@ -40,6 +38,7 @@ from webob.exc import HTTPNotAcceptable
|
||||
from webob.exc import HTTPNotFound
|
||||
from xml.dom import minidom
|
||||
|
||||
from openstack.common.wsgi import Router, Server, Middleware
|
||||
from melange.common.exception import InvalidContentType
|
||||
from melange.common.exception import MelangeError
|
||||
from melange.common.utils import cached_property
|
||||
@@ -73,23 +72,6 @@ class VersionedURLMap(object):
|
||||
return app(environ, start_response)
|
||||
|
||||
|
||||
class WritableLogger(object):
|
||||
"""A thin wrapper that responds to `write` and logs."""
|
||||
|
||||
def __init__(self, logger, level=logging.DEBUG):
|
||||
self.logger = logger
|
||||
self.level = level
|
||||
|
||||
def write(self, msg):
|
||||
self.logger.log(self.level, msg.strip("\n"))
|
||||
|
||||
|
||||
def run_server(application, port):
|
||||
"""Run a WSGI server with the given application."""
|
||||
sock = eventlet.listen(('0.0.0.0', port))
|
||||
eventlet.wsgi.server(sock, application)
|
||||
|
||||
|
||||
class Request(webob.Request):
|
||||
|
||||
@property
|
||||
@@ -144,31 +126,6 @@ class Request(webob.Request):
|
||||
return match.group("version_no") if match else None
|
||||
|
||||
|
||||
class Server(object):
|
||||
"""Server class to manage multiple WSGI sockets and applications."""
|
||||
|
||||
def __init__(self, threads=1000):
|
||||
self.pool = eventlet.GreenPool(threads)
|
||||
|
||||
def start(self, application, port, host='0.0.0.0', backlog=128):
|
||||
"""Run a WSGI server with the given application."""
|
||||
socket = eventlet.listen((host, port), backlog=backlog)
|
||||
self.pool.spawn_n(self._run, application, socket)
|
||||
|
||||
def wait(self):
|
||||
"""Wait until all servers have completed running."""
|
||||
try:
|
||||
self.pool.waitall()
|
||||
except KeyboardInterrupt:
|
||||
eventlet.convenience.StopServe()
|
||||
|
||||
def _run(self, application, socket):
|
||||
"""Start a WSGI server in a new green thread."""
|
||||
logger = logging.getLogger('eventlet.wsgi.server')
|
||||
eventlet.wsgi.server(socket, application, custom_pool=self.pool,
|
||||
log=WritableLogger(logger))
|
||||
|
||||
|
||||
class Middleware(object):
|
||||
"""
|
||||
Base WSGI middleware wrapper. These classes require an application to be
|
||||
@@ -232,100 +189,6 @@ class Middleware(object):
|
||||
return self.process_response(response)
|
||||
|
||||
|
||||
class Debug(Middleware):
|
||||
"""
|
||||
Helper class that can be inserted into any WSGI application chain
|
||||
to get information about the request and response.
|
||||
"""
|
||||
|
||||
@webob.dec.wsgify
|
||||
def __call__(self, req):
|
||||
print ("*" * 40) + " REQUEST ENVIRON"
|
||||
for key, value in req.environ.items():
|
||||
print key, "=", value
|
||||
print
|
||||
resp = req.get_response(self.application)
|
||||
|
||||
print ("*" * 40) + " RESPONSE HEADERS"
|
||||
for (key, value) in resp.headers.iteritems():
|
||||
print key, "=", value
|
||||
print
|
||||
|
||||
resp.app_iter = self.print_generator(resp.app_iter)
|
||||
|
||||
return resp
|
||||
|
||||
@staticmethod
|
||||
def print_generator(app_iter):
|
||||
"""
|
||||
Iterator that prints the contents of a wrapper string iterator
|
||||
when iterated.
|
||||
"""
|
||||
print ("*" * 40) + " BODY"
|
||||
for part in app_iter:
|
||||
sys.stdout.write(part)
|
||||
sys.stdout.flush()
|
||||
yield part
|
||||
print
|
||||
|
||||
|
||||
class Router(object):
|
||||
"""
|
||||
WSGI middleware that maps incoming requests to WSGI apps.
|
||||
"""
|
||||
|
||||
def __init__(self, mapper):
|
||||
"""
|
||||
Create a router for the given routes.Mapper.
|
||||
|
||||
Each route in `mapper` must specify a 'controller', which is a
|
||||
WSGI app to call. You'll probably want to specify an 'action' as
|
||||
well and have your controller be a service.Controller, who will route
|
||||
the request to the action method.
|
||||
|
||||
Examples:
|
||||
mapper = routes.Mapper()
|
||||
sc = ServerController()
|
||||
|
||||
# Explicit mapping of one route to a controller+action
|
||||
mapper.connect(None, "/svrlist", controller=sc, action="list")
|
||||
|
||||
# Actions are all implicitly defined
|
||||
mapper.resource("server", "servers", controller=sc)
|
||||
|
||||
# Pointing to an arbitrary WSGI app. You can specify the
|
||||
# {path_info:.*} parameter so the target app can be handed just that
|
||||
# section of the URL.
|
||||
mapper.connect(None, "/v1.0/{path_info:.*}", controller=BlogApp())
|
||||
"""
|
||||
self.map = mapper
|
||||
self._router = routes.middleware.RoutesMiddleware(self._dispatch,
|
||||
self.map)
|
||||
|
||||
@webob.dec.wsgify
|
||||
def __call__(self, req):
|
||||
|
||||
"""
|
||||
Route the incoming request to a controller based on self.map.
|
||||
If no match, return a 404.
|
||||
"""
|
||||
return self._router
|
||||
|
||||
@staticmethod
|
||||
@webob.dec.wsgify
|
||||
def _dispatch(req):
|
||||
"""
|
||||
Called by self._router after matching the incoming request to a route
|
||||
and putting the information into req.environ. Either returns 404
|
||||
or the routed WSGI app's response.
|
||||
"""
|
||||
match = req.environ['wsgiorg.routing_args'][1]
|
||||
if not match:
|
||||
return webob.exc.HTTPNotFound()
|
||||
app = match['controller']
|
||||
return app
|
||||
|
||||
|
||||
class Result(object):
|
||||
|
||||
def __init__(self, data, status=200):
|
||||
|
||||
Reference in New Issue
Block a user