x
/
radar
Code Issues Proposed changes

add new dashboard code 

Adding new dashboard code for the Radar CI Dashboard

old radar code was moved to the directory scripts

Change-Id: I8a588a56aa9fd044826c889813986041fcd142d6
This commit is contained in:
Steven Weston 2014-12-30 17:10:03 -05:00
parent f12156a422
commit 59b14fe68c
199 changed files with 23162 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
README.md Normal file
View File

@ -0,0 +1,22 @@
Radar Third Party CI Dashboard for OpenStack
=====================
* Application Installation
* apt-get install libpq-dev libmysqlclient-dev
* apt-get install mysql-server
* apt-get install rabbitmq-server
* pip install --upgrade -r requirements.txt
* python setup.py build
* python setup.py install
* Database
* CREATE user 'radar'@'localhost' IDENTIFIED BY 'radar';
* GRANT ALL PRIVILEGES ON radar.* to 'radar'@'localhost';
* FLUSH PRIVILEGES;
* radar-db-manage upgrade head
* API
* radar-api
* Update Daemon
* radar-update-daemon

133
etc/radar.conf.sample Normal file
View File

@ -0,0 +1,133 @@
[DEFAULT]
# Default log level is INFO
# verbose and debug has the same result.
# One of them will set DEBUG log level output
# debug = True
# verbose = True
# Where to store lock files
lock_path = $state_path/lock
# Radar's working directory. Please ensure that the radar user has
# read/write access to this directory.
# working_directory = ~/.radar
# log_format = %(asctime)s %(levelname)8s [%(name)s] %(message)s
# log_date_format = %Y-%m-%d %H:%M:%S
# use_syslog -> syslog
# log_file and log_dir -> log_dir/log_file
# (not log_file) and log_dir -> log_dir/{binary_name}.log
# use_stderr -> stderr
# (not user_stderr) and (not log_file) -> stdout
# publish_errors -> notification system
# use_syslog = False
# syslog_log_facility = LOG_USER
# use_stderr = True
# log_file =
# log_dir =
# publish_errors = False
# Address to bind the API server
# bind_host = 0.0.0.0
# Port the bind the API server to
# bind_port = 8080
# OpenId Authentication endpoint
# openid_url = https://login.launchpad.net/+openid
# Time in seconds before an access_token expires
# access_token_ttl = 3600
# Time in seconds before an refresh_token expires
# refresh_token_ttl = 604800
# List paging configuration options.
# page_size_maximum = 500
# page_size_default = 100
# Enable notifications. This feature drives deferred processing, reporting,
# and subscriptions.
# enable_notifications = True
[cors]
# W3C CORS configuration. For more information, see http://www.w3.org/TR/cors/
# List of permitted CORS domains.
# allowed_origins = https://radar.openstack.org, http://localhost:9000
# CORS browser options cache max age (in seconds)
# max_age=3600
[notifications]
# Host of the rabbitmq server.
# rabbit_host=localhost
# The RabbitMQ login method
# rabbit_login_method = AMQPLAIN
# The RabbitMQ userid.
# rabbit_userid = guest
# The RabbitMQ password.
# rabbit_password = guest
# The RabbitMQ broker port where a single node is used.
# rabbit_port = 5672
# The virtual host within which our queues and exchanges live.
# rabbit_virtual_host = /
# Application name that binds to rabbit.
# rabbit_application_name=radar
# The name of the topic exchange to which radar will broadcast its events.
# rabbit_exchange_name=radar
# The name of the queue that will be created for API events.
# rabbit_event_queue_name=radar_events
[database]
# This line MUST be changed to actually run radar
# Example:
# connection = mysql://radar:radar@127.0.0.1:3306/radar
# Replace 127.0.0.1 above with the IP address of the database used by the
# main radar server. (Leave it as is if the database runs on this host.)
# connection=sqlite://
# The SQLAlchemy connection string used to connect to the slave database
# slave_connection =
# Database reconnection retry times - in event connectivity is lost
# set to -1 implies an infinite retry count
# max_retries = 10
# Database reconnection interval in seconds - if the initial connection to the
# database fails
# retry_interval = 10
# Minimum number of SQL connections to keep open in a pool
# min_pool_size = 1
# Maximum number of SQL connections to keep open in a pool
# max_pool_size = 10
# Timeout in seconds before idle sql connections are reaped
# idle_timeout = 3600
# If set, use this value for max_overflow with sqlalchemy
# max_overflow = 20
# Verbosity of SQL debugging information. 0=None, 100=Everything
# connection_debug = 100
# Add python stack traces to SQL as comment strings
# connection_trace = True
# If set, use this value for pool_timeout with sqlalchemy
# pool_timeout = 10

0
radar/__init__.py Normal file
View File

0
radar/api/__init__.py Normal file
View File

151
radar/api/app.py Normal file
View File

@ -0,0 +1,151 @@
# Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from oslo.config import cfg
import pecan
from wsgiref import simple_server
from radar.api.auth.token_storage import impls as storage_impls
from radar.api.auth.token_storage import storage
from radar.api import config as api_config
from radar.api.middleware.cors_middleware import CORSMiddleware
from radar.api.middleware import token_middleware
from radar.api.middleware import user_id_hook
from radar.api.v1.search import impls as search_engine_impls
from radar.api.v1.search import search_engine
from radar.notifications.notification_hook import NotificationHook
from radar.openstack.common.gettextutils import _LI # noqa
from radar.openstack.common import log
from radar.plugin.user_preferences import initialize_user_preferences
CONF = cfg.CONF
LOG = log.getLogger(__name__)
API_OPTS = [
cfg.StrOpt('bind_host',
default='0.0.0.0',
help='API host'),
cfg.IntOpt('bind_port',
default=8080,
help='API port'),
cfg.BoolOpt('enable_notifications',
default=False,
help='Enable Notifications')
]
CORS_OPTS = [
cfg.ListOpt('allowed_origins',
default=None,
help='List of permitted CORS origins.'),
cfg.IntOpt('max_age',
default=3600,
help='Maximum cache age of CORS preflight requests.')
]
CONF.register_opts(API_OPTS)
CONF.register_opts(CORS_OPTS, 'cors')
def get_pecan_config():
# Set up the pecan configuration
filename = api_config.__file__.replace('.pyc', '.py')
return pecan.configuration.conf_from_file(filename)
def setup_app(pecan_config=None):
if not pecan_config:
pecan_config = get_pecan_config()
# Setup logging
cfg.set_defaults(log.log_opts,
default_log_levels=[
'radar=INFO',
'radar.openstack.common.db=WARN',
'sqlalchemy=WARN'
])
log.setup('radar')
hooks = [
user_id_hook.UserIdHook()
]
# Setup token storage
token_storage_type = CONF.token_storage_type
storage_cls = storage_impls.STORAGE_IMPLS[token_storage_type]
storage.set_storage(storage_cls())
# Setup search engine
search_engine_name = CONF.search_engine
search_engine_cls = search_engine_impls.ENGINE_IMPLS[search_engine_name]
search_engine.set_engine(search_engine_cls())
# Load user preference plugins
initialize_user_preferences()
# Setup notifier
if CONF.enable_notifications:
hooks.append(NotificationHook())
app = pecan.make_app(
pecan_config.app.root,
debug=CONF.debug,
hooks=hooks,
force_canonical=getattr(pecan_config.app, 'force_canonical', True),
guess_content_type_from_ext=False
)
app = token_middleware.AuthTokenMiddleware(app)
# Setup CORS
if CONF.cors.allowed_origins:
app = CORSMiddleware(app,
allowed_origins=CONF.cors.allowed_origins,
allowed_methods=['GET', 'POST', 'PUT', 'DELETE',
'OPTIONS'],
allowed_headers=['origin', 'authorization',
'accept', 'x-total', 'x-limit',
'x-marker', 'x-client',
'content-type'],
max_age=CONF.cors.max_age)
return app
def start():
CONF(project='radar')
api_root = setup_app()
# Create the WSGI server and start it
host = cfg.CONF.bind_host
port = cfg.CONF.bind_port
srv = simple_server.make_server(host, port, api_root)
LOG.info(_LI('Starting server in PID %s') % os.getpid())
LOG.info(_LI("Configuration:"))
if host == '0.0.0.0':
LOG.info(_LI(
'serving on 0.0.0.0:%(port)s, view at http://127.0.0.1:%(port)s')
% ({'port': port}))
else:
LOG.info(_LI("serving on http://%(host)s:%(port)s") % (
{'host': host, 'port': port}))
srv.serve_forever()
if __name__ == '__main__':
start()

0
radar/api/auth/__init__.py Normal file
View File

63
radar/api/auth/authorization_checks.py Normal file
View File

@ -0,0 +1,63 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pecan import abort
from pecan import request
from radar.api.auth.token_storage import storage
from radar.db.api import users as user_api
from radar.openstack.common.gettextutils import _ # noqa
def _get_token():
if request.authorization and len(request.authorization) == 2:
return request.authorization[1]
else:
return None
def guest():
token_storage = storage.get_storage()
token = _get_token()
# Public resources do not require a token.
if not token:
return True
# But if there is a token, it should be valid.
return token_storage.check_access_token(token)
def authenticated():
token_storage = storage.get_storage()
token = _get_token()
return token and token_storage.check_access_token(token)
def superuser():
token_storage = storage.get_storage()
token = _get_token()
if not token:
return False
token_info = token_storage.get_access_token_info(token)
user = user_api.user_get(token_info.user_id)
if not user.is_superuser:
abort(403, _("This action is limited to superusers only."))
return user.is_superuser

268
radar/api/auth/oauth_validator.py Normal file
View File

@ -0,0 +1,268 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from datetime import datetime
from oauthlib.oauth2 import RequestValidator
from oauthlib.oauth2 import WebApplicationServer
from oslo.config import cfg
from radar.api.auth.token_storage import storage
from radar.db.api import users as user_api
from radar.openstack.common import log
CONF = cfg.CONF
LOG = log.getLogger(__name__)
TOKEN_OPTS = [
cfg.IntOpt("access_token_ttl",
default=60 * 60, # One hour
help="Time in seconds before an access_token expires"),
cfg.IntOpt("refresh_token_ttl",
default=60 * 60 * 24 * 7, # One week
help="Time in seconds before an refresh_token expires")
]
CONF.register_opts(TOKEN_OPTS)
class SkeletonValidator(RequestValidator):
"""This is oauth skeleton for handling all kind of validations and storage
manipulations.
As it is and OAuth2, not OpenId-connect, some methods are not required to
be implemented.
Scope parameter validation is skipped as it is not a part of OpenId-connect
protocol.
"""
def __init__(self):
super(SkeletonValidator, self).__init__()
self.token_storage = storage.get_storage()
def validate_client_id(self, client_id, request, *args, **kwargs):
"""Check that a valid client is connecting
"""
# Let's think about valid clients later
return True
def validate_redirect_uri(self, client_id, redirect_uri, request, *args,
**kwargs):
"""Check that the client is allowed to redirect using the given
redirect_uri.
"""
#todo(nkonovalov): check an uri based on CONF.domain
return True
def get_default_redirect_uri(self, client_id, request, *args, **kwargs):
return request.sb_redirect_uri
def validate_scopes(self, client_id, scopes, client, request, *args,
**kwargs):
"""Scopes are not supported in OpenId-connect
The "user" value is hardcoded here to fill the difference between
the protocols.
"""
# Verify that the claimed user is allowed to log in.
openid = request._params["openid.claimed_id"]
user = user_api.user_get_by_openid(openid)
if user and not user.enable_login:
return False
return scopes == "user"
def get_default_scopes(self, client_id, request, *args, **kwargs):
"""Scopes a client will authorize for if none are supplied in the
authorization request.
"""
return ["user"]
def validate_response_type(self, client_id, response_type, client, request,
*args, **kwargs):
"""Clients should only be allowed to use one type of response type, the
one associated with their one allowed grant type.
In this case it must be "code".
"""
return response_type == "code"
# Post-authorization
def save_authorization_code(self, client_id, code, request, *args,
**kwargs):
"""Save the code to the storage and remove the state as it is persisted
in the "code" argument
"""
openid = request._params["openid.claimed_id"]
email = request._params["openid.sreg.email"]
full_name = request._params["openid.sreg.fullname"]
username = request._params["openid.sreg.nickname"]
last_login = datetime.utcnow()
user = user_api.user_get_by_openid(openid)
user_dict = {"full_name": full_name,
"username": username,
"email": email,
"last_login": last_login}
if not user:
user_dict.update({"openid": openid})
user = user_api.user_create(user_dict)
else:
user = user_api.user_update(user.id, user_dict)
self.token_storage.save_authorization_code(code, user_id=user.id)
# Token request
def authenticate_client(self, request, *args, **kwargs):
"""Skip the authentication here. It is handled through an OpenId client
The parameters are set to match th OAuth protocol.
"""
setattr(request, "client", type("Object", (object,), {})())
setattr(request.client, "client_id", "1")
return True
def validate_code(self, client_id, code, client, request, *args, **kwargs):
"""Validate the code belongs to the client."""
return self.token_storage.check_authorization_code(code)
def confirm_redirect_uri(self, client_id, code, redirect_uri, client,
*args, **kwargs):
"""Check that the client is allowed to redirect using the given
redirect_uri.
"""
#todo(nkonovalov): check an uri based on CONF.domain
return True
def validate_grant_type(self, client_id, grant_type, client, request,
*args, **kwargs):
"""Clients should only be allowed to use one type of grant.
In this case, it must be "authorization_code" or "refresh_token"
"""
return (grant_type == "authorization_code"
or grant_type == "refresh_token")
def _resolve_user_id(self, request):
# Try authorization code
code = request._params.get("code")
if code:
code_info = self.token_storage.get_authorization_code_info(code)
return code_info.user_id
# Try refresh token
refresh_token = request._params.get("refresh_token")
refresh_token_entry = self.token_storage.get_refresh_token_info(
refresh_token)
if refresh_token_entry:
return refresh_token_entry.user_id
return None
def save_bearer_token(self, token, request, *args, **kwargs):
"""Save all token information to the storage."""
user_id = self._resolve_user_id(request)
# If a refresh_token was used to obtain a new access_token, it should
# be removed.
self.invalidate_refresh_token(request)
self.token_storage.save_token(access_token=token["access_token"],
expires_in=token["expires_in"],
refresh_token=token["refresh_token"],
user_id=user_id)
def invalidate_authorization_code(self, client_id, code, request, *args,
**kwargs):
"""Authorization codes are use once, invalidate it when a token has
been acquired.
"""
self.token_storage.invalidate_authorization_code(code)
# Protected resource request
def validate_bearer_token(self, token, scopes, request):
"""The check will be performed in a separate middleware."""
pass
# Token refresh request
def get_original_scopes(self, refresh_token, request, *args, **kwargs):
"""Scopes a client will authorize for if none are supplied in the
authorization request.
"""
return ["user"]
def rotate_refresh_token(self, request):
"""The refresh token should be single use."""
return True
def validate_refresh_token(self, refresh_token, client, request, *args,
**kwargs):
"""Check that the refresh token exists in the db."""
return self.token_storage.check_refresh_token(refresh_token)
def invalidate_refresh_token(self, request):
"""Remove a used token from the storage."""
refresh_token = request._params.get("refresh_token")
# The request may have no token in parameters which means that the
# authorization code was used.
if not refresh_token:
return
self.token_storage.invalidate_refresh_token(refresh_token)
class OpenIdConnectServer(WebApplicationServer):
def __init__(self, request_validator):
access_token_ttl = CONF.access_token_ttl
super(OpenIdConnectServer, self).__init__(
request_validator,
token_expires_in=access_token_ttl)
validator = SkeletonValidator()
SERVER = OpenIdConnectServer(validator)

119
radar/api/auth/openid_client.py Normal file
View File

@ -0,0 +1,119 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo.config import cfg
import requests
from radar.api.auth import utils
from radar.openstack.common import log
LOG = log.getLogger(__name__)
CONF = cfg.CONF
OPENID_OPTS = [
cfg.StrOpt('openid_url',
default='https://login.launchpad.net/+openid',
help='OpenId Authentication endpoint')
]
CONF.register_opts(OPENID_OPTS)
class OpenIdClient(object):
def send_openid_redirect(self, request, response):
redirect_location = CONF.openid_url
response.status_code = 303
return_params = {
"scope": str(request.params.get("scope")),
"state": str(request.params.get("state")),
"client_id": str(request.params.get("client_id")),
"response_type": str(request.params.get("response_type")),
"sb_redirect_uri": str(request.params.get("redirect_uri"))
}
#TODO(krotscheck): URI base should be fully inferred from the request.
# assuming that the API is hosted at /api isn't good.
return_to_url = request.host_url + "/api/v1/openid/authorize_return?" \
+ utils.join_params(return_params, encode=True)
response.status_code = 303
openid_params = {
"openid.ns": "http://specs.openid.net/auth/2.0",
"openid.mode": "checkid_setup",
"openid.claimed_id": "http://specs.openid.net/auth/2.0/"
"identifier_select",
"openid.identity": "http://specs.openid.net/auth/2.0/"
"identifier_select",
"openid.realm": request.host_url,
"openid.return_to": return_to_url,
"openid.ns.sreg": "http://openid.net/sreg/1.0",
"openid.sreg.required": "fullname,email,nickname",
"openid.ns.ext2": "http://openid.net/srv/ax/1.0",
"openid.ext2.mode": "fetch_request",
"openid.ext2.type.FirstName": "http://schema.openid.net/"
"namePerson/first",
"openid.ext2.type.LastName": "http://schema.openid.net/"
"namePerson/last",
"openid.ext2.type.Email": "http://schema.openid.net/contact/email",
"openid.ext2.required": "FirstName,LastName,Email"
}
joined_params = utils.join_params(openid_params)
redirect_location = redirect_location + '?' + joined_params
response.headers["Location"] = redirect_location
return response
def verify_openid(self, request, response):
verify_params = dict(request.params.copy())
verify_params["openid.mode"] = "check_authentication"
verify_response = requests.post(CONF.openid_url, data=verify_params)
verify_data_tokens = verify_response.content.split()
verify_dict = dict((token.split(":")[0], token.split(":")[1])
for token in verify_data_tokens)
if (verify_response.status_code / 100 != 2
or verify_dict['is_valid'] != 'true'):
response.status_code = 401 # Unauthorized
return False
return True
def create_association(self, op_location):
# Let's skip it for MVP at least
query_dict = {
"openid.ns": "http://specs.openid.net/auth/2.0",
"openid.mode": "associate",
"openid.assoc_type": "HMAC-SHA256",
"openid.session_type": "no-encryption"
}
assoc_data = requests.post(op_location, data=query_dict).content
data_tokens = assoc_data.split()
data_dict = dict((token.split(":")[0], token.split(":")[1])
for token in data_tokens)
return data_dict["assoc_handle"]
client = OpenIdClient()

0
radar/api/auth/token_storage/__init__.py Normal file
View File

104
radar/api/auth/token_storage/db_storage.py Normal file
View File

@ -0,0 +1,104 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import datetime
from oslo.config import cfg
from radar.api.auth.token_storage import storage
from radar.db.api import auth as auth_api
CONF = cfg.CONF
class DBTokenStorage(storage.StorageBase):
def save_authorization_code(self, authorization_code, user_id):
values = {
"code": authorization_code["code"],
"state": authorization_code["state"],
"user_id": user_id
}
auth_api.authorization_code_save(values)
def get_authorization_code_info(self, code):
return auth_api.authorization_code_get(code)
def check_authorization_code(self, code):
db_code = auth_api.authorization_code_get(code)
return not db_code is None
def invalidate_authorization_code(self, code):
auth_api.authorization_code_delete(code)
def save_token(self, access_token, expires_in, refresh_token, user_id):
access_token_values = {
"access_token": access_token,
"expires_in": expires_in,
"expires_at": datetime.datetime.now() + datetime.timedelta(
seconds=expires_in),
"user_id": user_id
}
# Oauthlib does not provide a separate expiration time for a
# refresh_token so taking it from config directly.
refresh_expires_in = CONF.refresh_token_ttl
refresh_token_values = {
"refresh_token": refresh_token,
"user_id": user_id,
"expires_in": refresh_expires_in,
"expires_at": datetime.datetime.now() + datetime.timedelta(
seconds=refresh_expires_in),
}
auth_api.access_token_save(access_token_values)
auth_api.refresh_token_save(refresh_token_values)
def get_access_token_info(self, access_token):
return auth_api.access_token_get(access_token)
def check_access_token(self, access_token):
token_info = auth_api.access_token_get(access_token)
if not token_info:
return False
if datetime.datetime.now() > token_info.expires_at:
auth_api.access_token_delete(access_token)
return False
return True
def remove_token(self, access_token):
auth_api.access_token_delete(access_token)
def check_refresh_token(self, refresh_token):
refresh_token_entry = auth_api.refresh_token_get(refresh_token)
if not refresh_token_entry:
return False
if datetime.datetime.now() > refresh_token_entry.expires_at:
auth_api.refresh_token_delete(refresh_token)
return False
return True
def get_refresh_token_info(self, refresh_token):
return auth_api.refresh_token_get(refresh_token)
def invalidate_refresh_token(self, refresh_token):
auth_api.refresh_token_delete(refresh_token)

22
radar/api/auth/token_storage/impls.py Normal file
View File

@ -0,0 +1,22 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from radar.api.auth.token_storage import db_storage
from radar.api.auth.token_storage import memory_storage
STORAGE_IMPLS = {
"mem": memory_storage.MemoryTokenStorage,
"db": db_storage.DBTokenStorage
}

130
radar/api/auth/token_storage/memory_storage.py Normal file
View File

@ -0,0 +1,130 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import datetime
from radar.api.auth.token_storage import storage
class Token(object):
def __init__(self, access_token, refresh_token, expires_in, user_id,
is_valid=True):
self.access_token = access_token
self.refresh_token = refresh_token
self.expires_in = expires_in
self.expires_at = datetime.datetime.utcnow() + \
datetime.timedelta(seconds=expires_in)
self.user_id = user_id
self.is_valid = is_valid
class AuthorizationCode(object):
def __init__(self, code, user_id):
self.code = code
self.user_id = user_id
class MemoryTokenStorage(storage.StorageBase):
def __init__(self):
self.token_set = set([])
self.auth_code_set = set([])
def save_token(self, access_token, expires_in, refresh_token, user_id):
token_info = Token(access_token=access_token,
expires_in=expires_in,
refresh_token=refresh_token,
user_id=user_id)
self.token_set.add(token_info)
def check_access_token(self, access_token):
token_entry = None
for token_info in self.token_set:
if token_info.access_token == access_token:
token_entry = token_info
if not token_entry:
return False
now = datetime.datetime.utcnow()
if now > token_entry.expires_at:
token_entry.is_valid = False
return False
return True
def get_access_token_info(self, access_token):
for token_info in self.token_set:
if token_info.access_token == access_token:
return token_info
return None
def remove_token(self, token):
pass
def check_refresh_token(self, refresh_token):
for token_info in self.token_set:
if token_info.refresh_token == refresh_token:
return True
return False
def get_refresh_token_info(self, refresh_token):
for token_info in self.token_set:
if token_info.refresh_token == refresh_token:
return token_info
return None
def invalidate_refresh_token(self, refresh_token):
token_entry = None
for entry in self.token_set:
if entry.refresh_token == refresh_token:
token_entry = entry
break
self.token_set.remove(token_entry)
def save_authorization_code(self, authorization_code, user_id):
self.auth_code_set.add(AuthorizationCode(authorization_code, user_id))
def check_authorization_code(self, code):
code_entry = None
for entry in self.auth_code_set:
if entry.code["code"] == code:
code_entry = entry
break
if not code_entry:
return False
return True
def get_authorization_code_info(self, code):
for entry in self.auth_code_set:
if entry.code["code"] == code:
return entry
return None
def invalidate_authorization_code(self, code):
code_entry = None
for entry in self.auth_code_set:
if entry.code["code"] == code:
code_entry = entry
break
self.auth_code_set.remove(code_entry)

153
radar/api/auth/token_storage/storage.py Normal file
View File

@ -0,0 +1,153 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import abc
from oslo.config import cfg
CONF = cfg.CONF
STORAGE_OPTS = [
cfg.StrOpt('token_storage_type',
default='db',
help='Authorization token storage type.'
' Supported types are "mem" and "db".'
' Memory storage is not persistent between api launches')
]
CONF.register_opts(STORAGE_OPTS)
class StorageBase(object):
@abc.abstractmethod
def save_authorization_code(self, authorization_code, user_id):
"""This method should save an Authorization Code to the storage and
associate it with a user_id.
@param authorization_code: An object, containing state and a the code
itself.
@param user_id: The id of a User to associate the code with.
"""
pass
@abc.abstractmethod
def check_authorization_code(self, code):
"""Check that the given token exists in the storage.
@param code: The code to be checked.
@return bool
"""
pass
@abc.abstractmethod
def get_authorization_code_info(self, code):
"""Get the code info from the storage.
@param code: An authorization Code
@return object: The returned object should contain the state and the
user_id, which the given code is associated with.
"""
pass
@abc.abstractmethod
def invalidate_authorization_code(self, code):
"""Remove a code from the storage.
@param code: An authorization Code
"""
pass
@abc.abstractmethod
def save_token(self, access_token, expires_in, refresh_token, user_id):
"""Save a Bearer token to the storage with all associated fields
@param access_token: A token that will be used in authorized requests.
@param expires_in: The time in seconds while the access_token is valid.
@param refresh_token: A token that will be used in a refresh request
after an access_token gets expired.
@param user_id: The id of a User which owns a token.
"""
pass
@abc.abstractmethod
def check_access_token(self, access_token):
"""This method should say if a given token exists in the storage and
that it has not expired yet.
@param access_token: The token to be checked.
@return bool
"""
pass
@abc.abstractmethod
def get_access_token_info(self, access_token):
"""Get the Bearer token from the storage.
@param access_token: The token to get the information about.
@return object: The object should contain all fields associated with
the token (refresh_token, expires_in, user_id).
"""
pass
@abc.abstractmethod
def remove_token(self, token):
"""Invalidate a given token and remove it from the storage.
@param token: The token to be removed.
"""
pass
@abc.abstractmethod
def check_refresh_token(self, refresh_token):
"""This method should say if a given token exists in the storage and
that it has not expired yet.
@param refresh_token: The token to be checked.
@return bool
"""
pass
@abc.abstractmethod
def get_refresh_token_info(self, refresh_token):
"""Get the Bearer token from the storage.
@param refresh_token: The token to get the information about.
@return object: The object should contain all fields associated with
the token (refresh_token, expires_in, user_id).
"""
pass
@abc.abstractmethod
def invalidate_refresh_token(self, refresh_token):
"""Remove a token from the storage.
@param refresh_token: A refresh token
"""
pass
STORAGE = None
def get_storage():
global STORAGE
return STORAGE
def set_storage(impl):
global STORAGE
STORAGE = impl

24
radar/api/auth/utils.py Normal file
View File

@ -0,0 +1,24 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import six
import urllib
def join_params(params, encode=True):
return '&'.join(
["%s=%s" % (urllib.quote(key, safe='') if encode else key,
urllib.quote(val, safe='') if encode else val)
for key, val in six.iteritems(params)])

18
radar/api/config.py Normal file
View File

@ -0,0 +1,18 @@
from oslo.config import cfg
app = {
'root': 'radar.api.root_controller.RootController',
'modules': ['radar.api'],
'debug': False
}
cfg.CONF.register_opts([
cfg.IntOpt('page_size_maximum',
default=500,
help='The maximum number of results to allow a user to request '
'from the API'),
cfg.IntOpt('page_size_default',
default=20,
help='The maximum number of results to allow a user to request '
'from the API')
])

0
radar/api/middleware/__init__.py Normal file
View File

113
radar/api/middleware/cors_middleware.py Normal file
View File

@ -0,0 +1,113 @@
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing permissions and
# limitations under the License.
# Default allowed headers
ALLOWED_HEADERS = [
'origin',
'authorization',
'accept'
]
# Default allowed methods
ALLOWED_METHODS = [
'GET',
'POST',
'PUT',
'DELETE',
'OPTIONS'
]
class CORSMiddleware(object):
"""CORS Middleware.
By providing a list of allowed origins, methods, headers, and a max-age,
this middleware will detect and apply the appropriate CORS headers so
that your web application may elegantly overcome the browser's
same-origin sandbox.
For more information, see http://www.w3.org/TR/cors/
"""
def __init__(self, app, allowed_origins=None, allowed_methods=None,
allowed_headers=None, max_age=3600):
"""Create a new instance of the CORS middleware.
:param app: The application that is being wrapped.
:param allowed_origins: A list of allowed origins, as provided by the
'Origin:' Http header. Must include protocol, host, and port.
:param allowed_methods: A list of allowed HTTP methods.
:param allowed_headers: A list of allowed HTTP headers.
:param max_age: A maximum CORS cache age in seconds.
:return: A new middleware instance.
"""
# Wrapped app (or other middleware)
self.app = app
# Allowed origins
self.allowed_origins = allowed_origins or []
# List of allowed headers.
self.allowed_headers = ','.join(allowed_headers or ALLOWED_HEADERS)
# List of allowed methods.
self.allowed_methods = ','.join(allowed_methods or ALLOWED_METHODS)
# Cache age.
self.max_age = str(max_age)
def __call__(self, env, start_response):
"""Serve an application request.
:param env: Application environment parameters.
:param start_response: Wrapper method that starts the response.
:return:
"""
origin = env['HTTP_ORIGIN'] if 'HTTP_ORIGIN' in env else ''
method = env['REQUEST_METHOD'] if 'REQUEST_METHOD' in env else ''
def replacement_start_response(status, headers, exc_info=None):
"""Overrides the default response to attach CORS headers.
"""
# Decorate the headers
headers.append(('Access-Control-Allow-Origin',
origin))
headers.append(('Access-Control-Allow-Methods',
self.allowed_methods))
headers.append(('Access-Control-Expose-Headers',
self.allowed_headers))
headers.append(('Access-Control-Allow-Headers',
self.allowed_headers))
headers.append(('Access-Control-Max-Age',
self.max_age))
return start_response(status, headers, exc_info)
# Does this request match one of our origin domains?
if origin in self.allowed_origins:
# Is this an OPTIONS request?
if method == 'OPTIONS':
options_headers = [('Content-Length', '0')]
replacement_start_response('204 No Content', options_headers)
return ''
else:
# Handle the request.
return self.app(env, replacement_start_response)
else:
# This is not a request for a permitted CORS domain. Return
# the response without the appropriate headers and let the browser
# figure out the details.
return self.app(env, start_response)

51
radar/api/middleware/token_middleware.py Normal file
View File

@ -0,0 +1,51 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
AUTH_PREFIX = "/v1/openid"
class AuthTokenMiddleware(object):
def __init__(self, app, allow_unauthorized=None):
self.app = app
self.allow_unauthorized = allow_unauthorized or []
def _header_to_env_var(self, key):
"""Convert header to wsgi env variable.
"""
return 'HTTP_%s' % key.replace('-', '_').upper()
def _get_header(self, env, key, default=None):
"""Get http header from environment."""
env_key = self._header_to_env_var(key)
return env.get(env_key, default)
def _get_url(self, env):
return env.get("PATH_INFO")
def _get_method(self, env):
return env.get("REQUEST_METHOD")
def _clear_params(self, url):
return url.split("?")[0]
def __call__(self, env, start_response):
url = self._get_url(env)
if url and url.startswith(AUTH_PREFIX):
return self.app(env, start_response)
return self.app(env, start_response)

34
radar/api/middleware/user_id_hook.py Normal file
View File

@ -0,0 +1,34 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pecan import hooks
from radar.api.auth.token_storage import storage
class UserIdHook(hooks.PecanHook):
def before(self, state):
request = state.request
if request.authorization and len(request.authorization) == 2:
token = request.authorization[1]
token_info = storage.get_storage().get_access_token_info(token)
if token_info:
request.current_user_id = token_info.user_id
return
request.current_user_id = None

6
radar/api/root_controller.py Normal file
View File

@ -0,0 +1,6 @@
from radar.api.v1.v1_controller import V1Controller
from pecan import expose
from pecan.core import redirect
class RootController(object):
v1 = V1Controller()

0
radar/api/v1/__init__.py Normal file
View File

132
radar/api/v1/auth.py Normal file
View File

@ -0,0 +1,132 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import pecan
from pecan import request
from pecan import response
from pecan import rest
from radar.api.auth.oauth_validator import SERVER
from radar.api.auth.openid_client import client as openid_client
from radar.api.auth.token_storage import storage
from radar.openstack.common import log
LOG = log.getLogger(__name__)
class AuthController(rest.RestController):
_custom_actions = {
"authorize": ["GET"],
"authorize_return": ["GET"],
"token": ["POST"],
}
@pecan.expose()
def authorize(self):
"""Authorization code request."""
return openid_client.send_openid_redirect(request, response)
@pecan.expose()
def authorize_return(self):
"""Authorization code redirect endpoint.
At this point the server verifies an OpenId and retrieves user's
e-mail and full name from request
The client may already use both the e-mail and the fullname in the
templates, even though there was no token request so far.
"""
if not openid_client.verify_openid(request, response):
# The verify call will set unauthorized code
return response
headers, body, code = SERVER.create_authorization_response(
uri=request.url,
http_method=request.method,
body=request.body,
scopes=request.params.get("scope"),
headers=request.headers)
response.headers = dict((str(k), str(v))
for k, v in headers.iteritems())
response.status_code = code
response.body = body or ''
return response
def _access_token_by_code(self):
auth_code = request.params.get("code")
code_info = storage.get_storage() \
.get_authorization_code_info(auth_code)
headers, body, code = SERVER.create_token_response(
uri=request.url,
http_method=request.method,
body=request.body,
headers=request.headers)
response.headers = dict((str(k), str(v))
for k, v in headers.iteritems())
response.status_code = code
json_body = json.loads(body)
# Update a body with user_id only if a response is 2xx
if code / 100 == 2:
json_body.update({
'id_token': code_info.user_id
})
response.body = json.dumps(json_body)
return response
def _access_token_by_refresh_token(self):
refresh_token = request.params.get("refresh_token")
refresh_token_info = storage.get_storage().get_refresh_token_info(
refresh_token)
headers, body, code = SERVER.create_token_response(
uri=request.url,
http_method=request.method,
body=request.body,
headers=request.headers)
response.headers = dict((str(k), str(v))
for k, v in headers.iteritems())
response.status_code = code
json_body = json.loads(body)
# Update a body with user_id only if a response is 2xx
if code / 100 == 2:
json_body.update({
'id_token': refresh_token_info.user_id
})
response.body = json.dumps(json_body)
return response
@pecan.expose()
def token(self):
"""Token endpoint."""
grant_type = request.params.get("grant_type")
if grant_type == "authorization_code":
# Serve an access token having an authorization code
return self._access_token_by_code()
if grant_type == "refresh_token":
# Serve an access token having a refresh token
return self._access_token_by_refresh_token()

47
radar/api/v1/base.py Normal file
View File

@ -0,0 +1,47 @@
from datetime import datetime
from wsme import types as wtypes
class APIBase(wtypes.Base):
id = int
"""This is a unique identifier used as a primary key in all Database
models.
"""
created_at = datetime
"""The time when an object was added to the Database. This field is
managed by SqlAlchemy automatically.
"""
updated_at = datetime
"""The time when the object was updated to it's actual state. This
field is also managed by SqlAlchemy.
"""
@classmethod
def from_db_model(cls, db_model, skip_fields=None):
"""Returns the object from a given database representation."""
skip_fields = skip_fields or []
data = dict((k, v) for k, v in db_model.as_dict().items()
if k not in skip_fields)
return cls(**data)
def as_dict(self, omit_unset=False):
"""Converts this object into dictionary."""
attribute_names = [a.name for a in self._wsme_attributes]
if omit_unset:
attribute_names = [n for n in attribute_names
if getattr(self, n) != wtypes.Unset]
values = dict((name, self._lookup(name)) for name in attribute_names)
return values
def _lookup(self, key):
"""Looks up a key, translating WSME's Unset into Python's None.
:return: value of the given attribute; None if it is not set
"""
value = getattr(self, key)
if value == wtypes.Unset:
value = None
return value

209
radar/api/v1/operator.py Normal file
View File

@ -0,0 +1,209 @@
# Copyright (c) 2014 Triniplex.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo.config import cfg
from pecan import expose
from pecan import request
from pecan import response
from pecan import rest
from pecan.secure import secure
from wsme.exc import ClientSideError
import wsmeext.pecan as wsme_pecan
from radar.api.auth import authorization_checks as checks
from radar.api.v1.search import search_engine
from radar.api.v1 import wmodels
from radar.db.api import operators as operators_api
from radar.db.api import systems
CONF = cfg.CONF
SEARCH_ENGINE = search_engine.get_engine()
class OperatorsController(rest.RestController):
"""Manages operations on operators."""
_custom_actions = {"search": ["GET"]}
@secure(checks.guest)
@wsme_pecan.wsexpose(wmodels.Operator, int)
def get_one_by_id(self, operator_id):
"""Retrieve details about one operator.
:param operator_id: An ID of the operator.
"""
operator = operators_api.operator_get(operator_id)
if operator:
return wmodels.Operator.from_db_model(operator)
else:
raise ClientSideError("Operator %s not found" % operator_id,
status_code=404)
@secure(checks.guest)
@wsme_pecan.wsexpose(wmodels.Operator, unicode)
def get_one_by_name(self, operator_name):
"""Retrieve information about the given project.
:param name: project name.
"""
operator = operators_api.operator_get_by_name(operator_name)
if operator:
return wmodels.Operator.from_db_model(operator)
else:
raise ClientSideError("Operator %s not found" % operator_name,
status_code=404)
@secure(checks.guest)
@wsme_pecan.wsexpose([wmodels.Operator], int, int, unicode, unicode,
unicode)
def get(self, marker=None, limit=None, operator_name=None, sort_field='id',
sort_dir='asc'):
"""Retrieve definitions of all of the operators.
:param name: A string to filter the name by.
"""
# Boundary check on limit.
if limit is None:
limit = CONF.page_size_default
limit = min(CONF.page_size_maximum, max(1, limit))
# Resolve the marker record.
marker_operator = operators_api.operator_get(marker)
operators = operators_api \
.operator_get_all(marker=marker_operator,
limit=limit,
name=operator_name,
sort_field=sort_field,
sort_dir=sort_dir)
operator_count = operators_api \
.operator_get_count(name=operator_name)
# Apply the query response headers.
response.headers['X-Limit'] = str(limit)
response.headers['X-Total'] = str(operator_count)
if marker_operator:
response.headers['X-Marker'] = str(marker_operator.id)
if operators:
return [wmodels.Operator.from_db_model(o) for o in operators]
else:
raise ClientSideError("Could not retrieve operators list",
status_code=404)
@secure(checks.guest)
@wsme_pecan.wsexpose(wmodels.Operator, int, body=wmodels.Operator)
def post(self, system_id, operator):
"""Create a new operator.
:param operator: a operator within the request body.
"""
operator_dict = operator.as_dict()
created_operator = operators_api.operator_create(operator_dict)
created_operator = operators_api.operator_add_system(created_operator.id, system_id)
return wmodels.Operator.from_db_model(created_operator)
@secure(checks.guest)
@wsme_pecan.wsexpose(wmodels.Operator, int, body=wmodels.Operator)
def put(self, operator_id, operator):
"""Modify this operator.
:param operator_id: An ID of the operator.
:param operator: a operator within the request body.
"""
updated_operator = operators_api.operator_update(
operator_id,
operator.as_dict(omit_unset=True))
if updated_operator:
return wmodels.Operator.from_db_model(updated_operator)
else:
raise ClientSideError("Operator %s not found" % operator_id,
status_code=404)
@secure(checks.superuser)
@wsme_pecan.wsexpose(wmodels.Operator, int)
def delete(self, operator_id):
"""Delete this operator.
:param operator_id: An ID of the operator.
"""
operators_api.operator_delete(operator_id)
response.status_code = 204
def _is_int(self, s):
try:
int(s)
return True
except ValueError:
return False
@secure(checks.guest)
@wsme_pecan.wsexpose([wmodels.Operator], unicode, unicode, int, int)
def search(self, q="", marker=None, limit=None):
"""The search endpoint for operators.
:param q: The query string.
:return: List of Operators matching the query.
"""
operators = SEARCH_ENGINE.operators_query(q=q,
marker=marker,
limit=limit)
return [wmodels.Operator.from_db_model(operator) for operator in operators]
@wsme_pecan.wsexpose(long, unicode)
def count(self, args):
operators = operators_api.count()
if operators:
return operators
else:
raise ClientSideError("Cannot return operator count for %s"
% kwargs, status_code=404)
@expose()
def _route(self, args, request):
if request.method == 'GET' and len(args) > 0:
# It's a request by a name or id
something = args[0]
if something == "search":
# Request to a search endpoint
return self.search, args
if something == "count" and len(args) == 2:
return self.count, args
if self._is_int(something):
# Get by id
return self.get_one_by_id, args
else:
# Get by name
return self.get_one_by_name, ["/".join(args)]
return super(OperatorsController, self)._route(args, request)

0
radar/api/v1/search/__init__.py Normal file
View File

21
radar/api/v1/search/impls.py Normal file
View File

@ -0,0 +1,21 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from radar.api.v1.search.sqlalchemy_impl import SqlAlchemySearchImpl
ENGINE_IMPLS = {
"sqlalchemy": SqlAlchemySearchImpl
}

60
radar/api/v1/search/search_engine.py Normal file
View File

@ -0,0 +1,60 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import abc
from oslo.config import cfg
from radar.db import models
CONF = cfg.CONF
SEARCH_OPTS = [
cfg.StrOpt('search_engine',
default='sqlalchemy',
help='Search engine implementation.'
' The only supported type is "sqlalchemy".')
]
CONF.register_opts(SEARCH_OPTS)
class SearchEngine(object):
"""This is an interface that should be implemented by search engines.
"""
searchable_fields = {
models.System: ["name"],
models.Operator: ["operator_name", "operator_email"],
models.SystemEvent: ["event_type", "event_info"],
}
@abc.abstractmethod
def systems_query(self, q, name=None,
marker=None, limit=None, **kwargs):
pass
ENGINE = None
def get_engine():
global ENGINE
return ENGINE
def set_engine(impl):
global ENGINE
ENGINE = impl

51
radar/api/v1/search/sqlalchemy_impl.py Normal file
View File

@ -0,0 +1,51 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo.db.sqlalchemy import utils
from sqlalchemy_fulltext import FullTextSearch
import sqlalchemy_fulltext.modes as FullTextMode
from radar.api.v1.search import search_engine
from radar.db.api import base as api_base
from radar.db import models
class SqlAlchemySearchImpl(search_engine.SearchEngine):
def _build_fulltext_search(self, model_cls, query, q):
query = query.filter(FullTextSearch(q, model_cls,
mode=FullTextMode.NATURAL))
return query
def _apply_pagination(self, model_cls, query, marker=None, limit=None):
marker_entity = None
if marker:
marker_entity = api_base.entity_get(model_cls, marker, True)
return utils.paginate_query(query=query,
model=model_cls,
limit=limit,
sort_keys=["id"],
marker=marker_entity)
def systems_query(self, q, marker=None, limit=None, **kwargs):
session = api_base.get_session()
query = api_base.model_query(models.System, session)
query = self._build_fulltext_search(models.System, query, q)
query = self._apply_pagination(models.System, query, marker, limit)
return query.all()

191
radar/api/v1/subscription.py Normal file
View File

@ -0,0 +1,191 @@
# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo.config import cfg
from pecan import abort
from pecan import request
from pecan import response
from pecan import rest
from pecan.secure import secure
from wsme import types as wtypes
import wsmeext.pecan as wsme_pecan
from radar.api.auth import authorization_checks as checks
from radar.api.v1 import base
from radar.db.api import subscriptions as subscription_api
from radar.db.api import users as user_api
from radar.openstack.common.gettextutils import _ # noqa
CONF = cfg.CONF
class Subscription(base.APIBase):
"""A model that describes a resource subscription.
"""
user_id = int
"""The owner of this subscription.
"""
target_type = wtypes.text
"""The type of resource that the user is subscribed to.
"""
target_id = int
"""The database ID of the resource that the user is subscribed to.
"""
@classmethod
def sample(cls):
return cls(
user_id=1,
target_type="subscription",
target_id=1)
class SubscriptionsController(rest.RestController):
"""REST controller for Subscriptions.
Provides Create, Delete, and search methods for resource subscriptions.
"""
@secure(checks.authenticated)
@wsme_pecan.wsexpose(Subscription, int)
def get_one(self, subscription_id):
"""Retrieve a specific subscription record.
:param subscription_id: The unique id of this subscription.
"""
subscription = subscription_api.subscription_get(subscription_id)
current_user = user_api.user_get(request.current_user_id)
if subscription.user_id != request.current_user_id \
and not current_user.is_superuser:
abort(403, _("You do not have access to this record."))
return Subscription.from_db_model(subscription)
@secure(checks.authenticated)
@wsme_pecan.wsexpose([Subscription], int, int, [unicode], int, int,
unicode, unicode)
def get(self, marker=None, limit=None, target_type=None, target_id=None,
user_id=None, sort_field='id', sort_dir='asc'):
"""Retrieve a list of subscriptions.
:param marker: The resource id where the page should begin.
:param limit: The number of subscriptions to retrieve.
:param target_type: The type of resource to search by.
:param target_id: The unique ID of the resource to search by.
:param user_id: The unique ID of the user to search by.
:param sort_field: The name of the field to sort on.
:param sort_dir: sort direction for results (asc, desc).
"""
# Boundary check on limit.
if limit is None:
limit = CONF.page_size_default
limit = min(CONF.page_size_maximum, max(1, limit))
# Sanity check on user_id
current_user = user_api.user_get(request.current_user_id)
if user_id != request.current_user_id \
and not current_user.is_superuser:
user_id = request.current_user_id
# Resolve the marker record.
marker_sub = subscription_api.subscription_get(marker)
subscriptions = subscription_api.subscription_get_all(
marker=marker_sub,
limit=limit,
target_type=target_type,
target_id=target_id,
user_id=user_id,
sort_field=sort_field,
sort_dir=sort_dir)
subscription_count = subscription_api.subscription_get_count(
target_type=target_type,
target_id=target_id,
user_id=user_id)
# Apply the query response headers.
response.headers['X-Limit'] = str(limit)
response.headers['X-Total'] = str(subscription_count)
if marker_sub:
response.headers['X-Marker'] = str(marker_sub.id)
return [Subscription.from_db_model(s) for s in subscriptions]
@secure(checks.authenticated)
@wsme_pecan.wsexpose(Subscription, body=Subscription)
def post(self, subscription):
"""Create a new subscription.
:param subscription: A subscription within the request body.
"""
# Data sanity check - are all fields set?
if not subscription.target_type or not subscription.target_id:
abort(400, _('You are missing either the target_type or the'
' target_id'))
# Sanity check on user_id
current_user = user_api.user_get(request.current_user_id)
if not subscription.user_id:
subscription.user_id = request.current_user_id
elif subscription.user_id != request.current_user_id \
and not current_user.is_superuser:
abort(403, _("You can only subscribe to resources on your own."))
# Data sanity check: The resource must exist.
resource = subscription_api.subscription_get_resource(
target_type=subscription.target_type,
target_id=subscription.target_id)
if not resource:
abort(400, _('You cannot subscribe to a nonexistent resource.'))
# Data sanity check: The subscription cannot be duplicated for this
# user.
existing = subscription_api.subscription_get_all(
target_type=[subscription.target_type, ],
target_id=subscription.target_id,
user_id=subscription.user_id)
if existing:
abort(409, _('You are already subscribed to this resource.'))
result = subscription_api.subscription_create(subscription.as_dict())
return Subscription.from_db_model(result)
@secure(checks.authenticated)
@wsme_pecan.wsexpose(None, int)
def delete(self, subscription_id):
"""Delete a specific subscription.
:param subscription_id: The unique id of the subscription to delete.
"""
subscription = subscription_api.subscription_get(subscription_id)
# Sanity check on user_id
current_user = user_api.user_get(request.current_user_id)
if subscription.user_id != request.current_user_id \
and not current_user.is_superuser:
abort(403, _("You can only remove your own subscriptions."))
subscription_api.subscription_delete(subscription_id)
response.status_code = 204

210
radar/api/v1/system.py Normal file
View File

@ -0,0 +1,210 @@
# Copyright (c) 2014 Triniplex.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo.config import cfg
from pecan import expose
from pecan import request
from pecan import response
from pecan import rest
from pecan.secure import secure
from wsme.exc import ClientSideError
import wsmeext.pecan as wsme_pecan
from radar.api.auth import authorization_checks as checks
from radar.api.v1.search import search_engine
from radar.api.v1 import wmodels
from radar.db.api import systems as systems_api
CONF = cfg.CONF
SEARCH_ENGINE = search_engine.get_engine()
class SystemsController(rest.RestController):
"""Manages operations on systems."""
_custom_actions = {"search": ["GET"],
"count": ["GET"]}
@secure(checks.guest)
@wsme_pecan.wsexpose(wmodels.System, int)
def get_one_by_id(self, system_id):
"""Retrieve details about one system.
:param system_id: An ID of the system.
"""
system = systems_api.system_get_by_id(system_id)
if system:
return wmodels.System.from_db_model(system)
else:
raise ClientSideError("System %s not found" % system_id,
status_code=404)
@secure(checks.guest)
@wsme_pecan.wsexpose(wmodels.System, unicode)
def get_one_by_name(self, system_name):
"""Retrieve information about the given project.
:param name: project name.
"""
system = systems_api.system_get_by_name(system_name)
if system:
return wmodels.System.from_db_model(system)
else:
raise ClientSideError("System %s not found" % system_name,
status_code=404)
@secure(checks.guest)
@wsme_pecan.wsexpose([wmodels.System], int, int, unicode, unicode,
unicode)
def get(self, marker=None, limit=None, name=None, sort_field='id',
sort_dir='asc'):
"""Retrieve definitions of all of the systems.
:param name: A string to filter the name by.
"""
# Boundary check on limit.
if limit is None:
limit = CONF.page_size_default
limit = min(CONF.page_size_maximum, max(1, limit))
# Resolve the marker record.
marker_system = systems_api.system_get_by_id(marker)
systems = systems_api \
.system_get_all(marker=marker_system,
limit=limit,
name=name,
sort_field=sort_field,
sort_dir=sort_dir)
system_count = systems_api \
.system_get_count(name=name)
# Apply the query response headers.
response.headers['X-Limit'] = str(limit)
response.headers['X-Total'] = str(system_count)
if marker_system:
response.headers['X-Marker'] = str(marker_system.id)
if systems:
return [wmodels.System.from_db_model(s) for s in systems]
else:
raise ClientSideError("Could not retrieve system list",
status_code=404)
@secure(checks.guest)
@wsme_pecan.wsexpose(wmodels.System, body=wmodels.System)
def post(self, system):
"""Create a new system.
:param system: a system within the request body.
"""
system_dict = system.as_dict()
created_system = systems_api.system_create(system_dict)
if created_system:
return wmodels.System.from_db_model(created_system)
else:
raise ClientSideError("Unable to create system %s" % system,
status_code=404)
@secure(checks.guest)
@wsme_pecan.wsexpose(wmodels.System, int, body=wmodels.System)
def put(self, system_id, system):
"""Modify this system.
:param system_id: An ID of the system.
:param system: a system within the request body.
"""
updated_system = systems_api.system_update(
system_id,
system.as_dict(omit_unset=True))
if updated_system:
return wmodels.System.from_db_model(updated_system)
else:
raise ClientSideError("System %s not found" % system_id,
status_code=404)
@secure(checks.superuser)
@wsme_pecan.wsexpose(wmodels.System, int)
def delete(self, system_id):
"""Delete this system.
:param system_id: An ID of the system.
"""
systems_api.system_delete(system_id)
response.status_code = 204
def _is_int(self, s):
try:
int(s)
return True
except ValueError:
return False
@secure(checks.guest)
@wsme_pecan.wsexpose([wmodels.System], unicode, int, int)
def search(self, q="", marker=None, limit=None):
"""The search endpoint for systems.
:param q: The query string.
:return: List of Systems matching the query.
"""
systems = SEARCH_ENGINE.systems_query(q=q,
marker=marker,
limit=limit)
return [wmodels.System.from_db_model(system) for system in systems]
@secure(checks.guest)
@wsme_pecan.wsexpose(long, unicode)
def count(self, args):
systems = systems_api.count()
if systems:
return systems
else:
raise ClientSideError("Cannot return system count for %s"
% kwargs, status_code=404)
@expose()
def _route(self, args, request):
if request.method == 'GET' and len(args) > 0:
# It's a request by a name or id
something = args[0]
if something == "search":
# Request to a search endpoint
return super(SystemsController, self)._route(args, request)
if self._is_int(something):
# Get by id
return self.get_one_by_id, args
else:
# Get by name
if something == "count" and len(args) == 2:
return self.count, args
else:
return self.get_one_by_name, ["/".join(args)]
return super(SystemsController, self)._route(args, request)

179
radar/api/v1/user.py Normal file
View File

@ -0,0 +1,179 @@
# Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo.config import cfg
from pecan import expose
from pecan import request
from pecan import response
from pecan import rest
from pecan.secure import secure
from wsme.exc import ClientSideError
import wsmeext.pecan as wsme_pecan
from radar.api.auth import authorization_checks as checks
from radar.api.v1.search import search_engine
from radar.api.v1.user_preference import UserPreferencesController
from radar.api.v1.user_token import UserTokensController
from radar.api.v1 import wmodels
from radar.db.api import users as users_api
from radar.openstack.common.gettextutils import _ # noqa
CONF = cfg.CONF
SEARCH_ENGINE = search_engine.get_engine()
class UsersController(rest.RestController):
"""Manages users."""
# Import the user preferences.
preferences = UserPreferencesController()
# Import user token management.
tokens = UserTokensController()
_custom_actions = {"search": ["GET"]}
@secure(checks.guest)
@wsme_pecan.wsexpose([wmodels.User], int, int, unicode, unicode, unicode,
unicode)
def get(self, marker=None, limit=None, username=None, full_name=None,
sort_field='id', sort_dir='asc'):
"""Page and filter the users in radar.
:param marker: The resource id where the page should begin.
:param limit The number of users to retrieve.
:param username A string of characters to filter the username with.
:param full_name A string of characters to filter the full_name with.
:param sort_field: The name of the field to sort on.
:param sort_dir: sort direction for results (asc, desc).
"""
# Boundary check on limit.
if limit is None:
limit = CONF.page_size_default
limit = min(CONF.page_size_maximum, max(1, limit))
# Resolve the marker record.
marker_user = users_api.user_get(marker)
users = users_api.user_get_all(marker=marker_user, limit=limit,
username=username, full_name=full_name,
filter_non_public=True,
sort_field=sort_field,
sort_dir=sort_dir)
user_count = users_api.user_get_count(username=username,
full_name=full_name)
# Apply the query response headers.
response.headers['X-Limit'] = str(limit)
response.headers['X-Total'] = str(user_count)
if marker_user:
response.headers['X-Marker'] = str(marker_user.id)
return [wmodels.User.from_db_model(u) for u in users]
@secure(checks.guest)
@wsme_pecan.wsexpose(wmodels.User, int)
def get_one(self, user_id):
"""Retrieve details about one user.
:param user_id: The unique id of this user
"""
filter_non_public = True
if user_id == request.current_user_id:
filter_non_public = False
user = users_api.user_get(user_id, filter_non_public)
if not user:
raise ClientSideError(_("User %s not found") % user_id,
status_code=404)
return user
@secure(checks.superuser)
@wsme_pecan.wsexpose(wmodels.User, body=wmodels.User)
def post(self, user):
"""Create a new user.
:param user: a user within the request body.
"""
created_user = users_api.user_create(user.as_dict())
return wmodels.User.from_db_model(created_user)
@secure(checks.authenticated)
@wsme_pecan.wsexpose(wmodels.User, int, body=wmodels.User)
def put(self, user_id, user):
"""Modify this user.
:param user_id: unique id to identify the user.
:param user: a user within the request body.
"""
current_user = users_api.user_get(request.current_user_id)
if not user or not user.id or not current_user:
response.status_code = 404
response.body = _("Not found")
return response
# Only owners and superadmins are allowed to modify users.
if request.current_user_id != user.id \
and not current_user.is_superuser:
response.status_code = 403
response.body = _("You are not allowed to update this user.")
return response
# Strip out values that you're not allowed to change.
user_dict = user.as_dict()
# You cannot modify the openid field.
del user_dict['openid']
if not current_user.is_superuser:
# Only superuser may create superusers or modify login permissions.
del user_dict['enable_login']
del user_dict['is_superuser']
updated_user = users_api.user_update(user_id, user_dict)
return wmodels.User.from_db_model(updated_user)
@secure(checks.guest)
@wsme_pecan.wsexpose([wmodels.User], unicode, int, int)
def search(self, q="", marker=None, limit=None):
"""The search endpoint for users.
:param q: The query string.
:return: List of Users matching the query.
"""
users = SEARCH_ENGINE.users_query(q=q, marker=marker, limit=limit)
return [wmodels.User.from_db_model(u) for u in users]
@expose()
def _route(self, args, request):
if request.method == 'GET' and len(args) == 1:
# It's a request by a name or id
something = args[0]
if something == "search":
# Request to a search endpoint
return self.search, args
else:
return self.get_one, args
return super(UsersController, self)._route(args, request)

58
radar/api/v1/user_preference.py Normal file
View File

@ -0,0 +1,58 @@
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo.config import cfg
from pecan import abort
from pecan import request
from pecan import rest
from pecan.secure import secure
import wsme.types as types
import wsmeext.pecan as wsme_pecan
from radar.api.auth import authorization_checks as checks
import radar.db.api.users as user_api
from radar.openstack.common import log
CONF = cfg.CONF
LOG = log.getLogger(__name__)
class UserPreferencesController(rest.RestController):
@secure(checks.authenticated)
@wsme_pecan.wsexpose(types.DictType(unicode, unicode), int)
def get_all(self, user_id):
"""Return all preferences for the current user.
"""
if request.current_user_id != user_id:
abort(403)
return
return user_api.user_get_preferences(user_id)
@secure(checks.authenticated)
@wsme_pecan.wsexpose(types.DictType(unicode, unicode), int,
body=types.DictType(unicode, unicode))
def post(self, user_id, body):
"""Allow a user to update their preferences. Note that a user must
explicitly set a preference value to Null/None to have it deleted.
:param user_id The ID of the user whose preferences we're updating.
:param body A dictionary of preference values.
"""
if request.current_user_id != user_id:
abort(403)
return user_api.user_update_preferences(user_id, body)

184
radar/api/v1/user_token.py Normal file
View File

@ -0,0 +1,184 @@
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import uuid
from oslo.config import cfg
from pecan import abort
from pecan import request
from pecan import response
from pecan import rest
from pecan.secure import secure
import wsmeext.pecan as wsme_pecan
from radar.api.auth import authorization_checks as checks
import radar.api.v1.wmodels as wmodels
import radar.db.api.access_tokens as token_api
import radar.db.api.users as user_api
from radar.openstack.common.gettextutils import _ # noqa
from radar.openstack.common import log
CONF = cfg.CONF
LOG = log.getLogger(__name__)
class UserTokensController(rest.RestController):
@secure(checks.authenticated)
@wsme_pecan.wsexpose([wmodels.AccessToken], int, int, int, unicode,
unicode)
def get_all(self, user_id, marker=None, limit=None, sort_field='id',
sort_dir='asc'):
"""Returns all the access tokens for the provided user.
:param user_id: The ID of the user.
:param marker: The marker record at which to start the page.
:param limit: The number of records to return.
:param sort_field: The field on which to sort.
:param sort_dir: The direction to sort.
:return: A list of access tokens for the given user.
"""
self._assert_can_access(user_id)
# Boundary check on limit.
if limit is None:
limit = CONF.page_size_default
limit = min(CONF.page_size_maximum, max(1, limit))
# Resolve the marker record.
marker_token = token_api.access_token_get(marker)
tokens = token_api.access_token_get_all(marker=marker_token,
limit=limit,
user_id=user_id,
filter_non_public=True,
sort_field=sort_field,
sort_dir=sort_dir)
token_count = token_api.access_token_get_count(user_id=user_id)
# Apply the query response headers.
response.headers['X-Limit'] = str(limit)
response.headers['X-Total'] = str(token_count)
if marker_token:
response.headers['X-Marker'] = str(marker_token.id)
return [wmodels.AccessToken.from_db_model(t) for t in tokens]
@secure(checks.authenticated)
@wsme_pecan.wsexpose(wmodels.AccessToken, int, int)
def get(self, user_id, access_token_id):
"""Returns a specific access token for the given user.
:param user_id: The ID of the user.
:param access_token_id: The ID of the access token.
:return: The requested access token.
"""
access_token = token_api.access_token_get(access_token_id)
self._assert_can_access(user_id, access_token)
if not access_token:
abort(404)
return wmodels.AccessToken.from_db_model(access_token)
@secure(checks.authenticated)
@wsme_pecan.wsexpose(wmodels.AccessToken, int, body=wmodels.AccessToken)
def post(self, user_id, body):
"""Create a new access token for the given user.
:param user_id: The user ID of the user.
:param body: The access token.
:return: The created access token.
"""
self._assert_can_access(user_id, body)
# Generate a random token if one was not provided.
if not body.access_token:
body.access_token = str(uuid.uuid4())
# Token duplication check.
dupes = token_api.access_token_get_all(access_token=body.access_token)
if dupes:
abort(409, _('This token already exists.'))
token = token_api.access_token_create(body.as_dict())
return wmodels.AccessToken.from_db_model(token)
@secure(checks.authenticated)
@wsme_pecan.wsexpose(wmodels.AccessToken, int, int,
body=wmodels.AccessToken)
def put(self, user_id, access_token_id, body):
"""Update an access token for the given user.
:param user_id: The user ID of the user.
:param access_token_id: The ID of the access token.
:param body: The access token.
:return: The created access token.
"""
target_token = token_api.access_token_get(access_token_id)
self._assert_can_access(user_id, body)
self._assert_can_access(user_id, target_token)
if not target_token:
abort(404)
# We only allow updating the expiration date.
target_token.expires_in = body.expires_in
result_token = token_api.access_token_update(access_token_id,
target_token.as_dict())
return wmodels.AccessToken.from_db_model(result_token)
@secure(checks.authenticated)
@wsme_pecan.wsexpose(wmodels.AccessToken, int, int)
def delete(self, user_id, access_token_id):
"""Deletes an access token for the given user.
:param user_id: The user ID of the user.
:param access_token_id: The ID of the access token.
:return: Empty body, or error response.
"""
access_token = token_api.access_token_get(access_token_id)
self._assert_can_access(user_id, access_token)
if not access_token:
abort(404)
token_api.access_token_delete(access_token_id)
response.status_code = 204
def _assert_can_access(self, user_id, token_entity=None):
current_user = user_api.user_get(request.current_user_id)
if not user_id:
abort(400)
# The user must be logged in.
if not current_user:
abort(401)
# If the impacted user is not the current user, the current user must
# be an admin.
if not current_user.is_superuser and current_user.id != user_id:
abort(403)
# The path-based impacted user and the user found in the entity must
# be identical. No PUT /users/1/tokens { user_id: 2 }
if token_entity and token_entity.user_id != user_id:
abort(403)

18
radar/api/v1/v1_controller.py Normal file
View File

@ -0,0 +1,18 @@
import pecan
from pecan import rest
from radar.api.v1.auth import AuthController
from radar.api.v1.subscription import SubscriptionsController
from radar.api.v1.system import SystemsController
from radar.api.v1.operator import OperatorsController
from radar.api.v1.user import UsersController
class V1Controller(rest.RestController):
systems = SystemsController()
operators = OperatorsController()
users = UsersController()
subscriptions = SubscriptionsController()
openid = AuthController()

75
radar/api/v1/wmodels.py Normal file
View File

@ -0,0 +1,75 @@
from datetime import datetime
from wsme import types as wtypes
from api.v1 import base
class System(base.APIBase):
"""Represents the ci systems for the dashboard
"""
name = wtypes.text
"""The system name"""
class SystemEvent(base.APIBase):
event_type = wtypes.text
event_info = wtypes.text
class Operator(base.APIBase):
operator_name = wtypes.text
operator_email = wtypes.text
class User(base.APIBase):
"""Represents a user."""
username = wtypes.text
"""A short unique name, beginning with a lower-case letter or number, and
containing only letters, numbers, dots, hyphens, or plus signs"""
full_name = wtypes.text
"""Full (Display) name."""
openid = wtypes.text
"""The unique identifier, returned by an OpneId provider"""
email = wtypes.text
"""Email Address."""
# Todo(nkonovalov): use teams to define superusers
is_superuser = bool
last_login = datetime
"""Date of the last login."""
enable_login = bool
"""Whether this user is permitted to log in."""
@classmethod
def sample(cls):
return cls(
username="elbarto",
full_name="Bart Simpson",
openid="https://login.launchpad.net/+id/Abacaba",
email="skinnerstinks@springfield.net",
is_staff=False,
is_active=True,
is_superuser=True,
last_login=datetime(2014, 1, 1, 16, 42))
class AccessToken(base.APIBase):
"""Represents a user access token."""
user_id = int
"""The ID of User to whom this token belongs."""
access_token = wtypes.text
"""The access token."""
expires_in = int
"""The number of seconds after creation when this token expires."""
@classmethod
def sample(cls):
return cls(
user_id=1,
access_token="a_unique_access_token",
expires_in=3600)

0
radar/common/__init__.py Normal file
View File

31
radar/common/custom_types.py Normal file
View File

@ -0,0 +1,31 @@
# Copyright (c) 2014 Triniplex
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from wsme import types
class NameType(types.StringType):
"""This type should be applied to the name fields. Currently this type
should be applied to Projects and Project Groups.
This type allows alphanumeric characters with . - and / separators inside
the name. The name should be at least 3 symbols long.
"""
_name_regex = r'^[a-zA-Z0-9]+([\-\./]?[a-zA-Z0-9]+)*$'
def __init__(self):
super(NameType, self).__init__(min_length=3, pattern=self._name_regex)

29
radar/common/exception.py Normal file
View File

@ -0,0 +1,29 @@
class DashboardException(Exception):
"""Base Exception for the project
To correctly use this class, inherit from it and define
the 'message' property.
"""
message = "An unknown exception occurred"
def __str__(self):
return self.message
def __init__(self):
super(DashboardException, self).__init__(self.message)
class NotFound(DashboardException):
message = "Object not found"
def __init__(self, message=None):
if message:
self.message = message
class DuplicateEntry(DashboardException):
message = "Database object already exists"
def __init__(self, message=None):
if message:
self.message = message

0
radar/db/__init__.py Normal file
View File

0
radar/db/api/__init__.py Normal file
View File

114
radar/db/api/access_tokens.py Normal file
View File

@ -0,0 +1,114 @@
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import datetime
from oslo.db.sqlalchemy.utils import InvalidSortKey
from wsme.exc import ClientSideError
from radar.db.api import base as api_base
from radar.db import models
from radar.openstack.common.gettextutils import _ # noqa
def access_token_get(access_token_id):
return api_base.entity_get(models.AccessToken, access_token_id)
def access_token_get_by_token(access_token):
results = api_base.entity_get_all(models.AccessToken,
access_token=access_token)
if not results:
return None
else:
return results[0]
def access_token_get_all(marker=None, limit=None, sort_field=None,
sort_dir=None, **kwargs):
# Sanity checks, in case someone accidentally explicitly passes in 'None'
if not sort_field:
sort_field = 'id'
if not sort_dir:
sort_dir = 'asc'
# Construct the query
query = access_token_build_query(**kwargs)
try:
query = api_base.paginate_query(query=query,
model=models.AccessToken,
limit=limit,
sort_keys=[sort_field],
marker=marker,
sort_dir=sort_dir)
except InvalidSortKey:
raise ClientSideError(_("Invalid sort_field [%s]") % (sort_field,),
status_code=400)
except ValueError as ve:
raise ClientSideError(_("%s") % (ve,), status_code=400)
# Execute the query
return query.all()
def access_token_get_count(**kwargs):
# Construct the query
query = access_token_build_query(**kwargs)
return query.count()
def access_token_create(values):
# Update the expires_at date.
values['created_at'] = datetime.datetime.utcnow()
values['expires_at'] = datetime.datetime.utcnow() + datetime.timedelta(
seconds=values['expires_in'])
return api_base.entity_create(models.AccessToken, values)
def access_token_update(access_token_id, values):
values['expires_at'] = values['created_at'] + datetime.timedelta(
seconds=values['expires_in'])
return api_base.entity_update(models.AccessToken, access_token_id, values)
def access_token_build_query(**kwargs):
# Construct the query
query = api_base.model_query(models.AccessToken)
# Apply the filters
query = api_base.apply_query_filters(query=query,
model=models.AccessToken,
**kwargs)
return query
def access_token_delete_by_token(access_token):
access_token = access_token_get_by_token(access_token)
if access_token:
api_base.entity_hard_delete(models.AccessToken, access_token.id)
def access_token_delete(access_token_id):
access_token = access_token_get(access_token_id)
if access_token:
api_base.entity_hard_delete(models.AccessToken, access_token_id)

66
radar/db/api/auth.py Normal file
View File

@ -0,0 +1,66 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from radar.db.api import base as api_base
from radar.db import models
def authorization_code_get(code):
query = api_base.model_query(models.AuthorizationCode,
api_base.get_session())
return query.filter_by(code=code).first()
def authorization_code_save(values):
return api_base.entity_create(models.AuthorizationCode, values)
def authorization_code_delete(code):
del_code = authorization_code_get(code)
if del_code:
api_base.entity_hard_delete(models.AuthorizationCode, del_code.id)
def access_token_get(access_token):
query = api_base.model_query(models.AccessToken, api_base.get_session())
return query.filter_by(access_token=access_token).first()
def access_token_save(values):
return api_base.entity_create(models.AccessToken, values)
def access_token_delete(access_token):
del_token = access_token_get(access_token)
if del_token:
api_base.entity_hard_delete(models.AccessToken, del_token.id)
def refresh_token_get(refresh_token):
query = api_base.model_query(models.RefreshToken, api_base.get_session())
return query.filter_by(refresh_token=refresh_token).first()
def refresh_token_save(values):
return api_base.entity_create(models.RefreshToken, values)
def refresh_token_delete(refresh_token):
del_token = refresh_token_get(refresh_token)
if del_token:
api_base.entity_hard_delete(models.RefreshToken, del_token.id)

214
radar/db/api/base.py Normal file
View File

@ -0,0 +1,214 @@
import copy
from oslo.config import cfg
from oslo.db import exception as db_exc
from oslo.db.sqlalchemy import session as db_session
from oslo.db.sqlalchemy.utils import InvalidSortKey
from oslo.db.sqlalchemy.utils import paginate_query
import six
import sqlalchemy.types as types
from wsme.exc import ClientSideError
from radar.common import exception as exc
from radar.db import models
from radar.openstack.common import log
CONF = cfg.CONF
LOG = log.getLogger(__name__)
_FACADE = None
BASE = models.Base
def _get_facade_instance():
"""Generate an instance of the DB Facade.
"""
global _FACADE
if _FACADE is None:
_FACADE = db_session.EngineFacade.from_config(CONF)
return _FACADE
def _destroy_facade_instance():
"""Destroys the db facade instance currently in use.
"""
global _FACADE
_FACADE = None
def apply_query_filters(query, model, **kwargs):
"""Parses through a list of kwargs to determine which exist on the model,
which should be filtered as ==, and which should be filtered as LIKE
"""
for k, v in kwargs.iteritems():
if v and hasattr(model, k):
column = getattr(model, k)
if column.is_attribute:
if isinstance(column.type, types.Enum):
query = query.filter(column.in_(v))
elif isinstance(column.type, types.String):
# Filter strings with LIKE
query = query.filter(column.like("%" + v + "%"))
else:
# Everything else is a strict equal
query = query.filter(column == v)
return query
def get_engine():
"""Returns the global instance of our database engine.
"""
facade = _get_facade_instance()
return facade.get_engine(use_slave=True)
def get_session(autocommit=True, expire_on_commit=False, **kwargs):
"""Returns a database session from our facade.
"""
facade = _get_facade_instance()
return facade.get_session(autocommit=autocommit,
expire_on_commit=expire_on_commit, **kwargs)
def cleanup():
"""Manually clean up our database engine.
"""
_destroy_facade_instance()
def model_query(model, session=None):
"""Query helper.
:param model: base model to query
"""
session = session or get_session()
query = session.query(model)
return query
def __entity_get(kls, entity_id, session):
query = model_query(kls, session)
return query.filter_by(id=entity_id).first()
def entity_get(kls, entity_id, filter_non_public=False, session=None):
if not session:
session = get_session()
entity = __entity_get(kls, entity_id, session)
if filter_non_public:
entity = _filter_non_public_fields(entity, entity._public_fields)
return entity
def entity_get_all(kls, filter_non_public=False, marker=None, limit=None,
sort_field='id', sort_dir='asc', **kwargs):
# Sanity checks, in case someone accidentally explicitly passes in 'None'
if not sort_field:
sort_field = 'id'
if not sort_dir:
sort_dir = 'asc'
# Construct the query
query = model_query(kls)
# Sanity check on input parameters
query = apply_query_filters(query=query, model=kls, **kwargs)
# Construct the query
try:
query = paginate_query(query=query,
model=kls,
limit=limit,
sort_keys=[sort_field],
marker=marker,
sort_dir=sort_dir)
except InvalidSortKey:
raise ClientSideError("Invalid sort_field [%s]" % (sort_field,),
status_code=400)
except ValueError as ve:
raise ClientSideError("%s" % (ve,), status_code=400)
# Execute the query
entities = query.all()
if len(entities) > 0 and filter_non_public:
sample_entity = entities[0] if len(entities) > 0 else None
public_fields = getattr(sample_entity, "_public_fields", [])
entities = [_filter_non_public_fields(entity, public_fields)
for entity in entities]
return entities
def entity_get_count(kls, **kwargs):
# Construct the query
query = model_query(kls)
# Sanity check on input parameters
query = apply_query_filters(query=query, model=kls, **kwargs)
count = query.count()
return count
def _filter_non_public_fields(entity, public_list=list()):
ent_copy = copy.copy(entity)
for attr_name, val in six.iteritems(entity.__dict__):
if attr_name.startswith("_"):
continue
if attr_name not in public_list:
delattr(ent_copy, attr_name)
return ent_copy
def entity_create(kls, values):
entity = kls()
entity.update(values.copy())
session = get_session()
with session.begin():
try:
session.add(entity)
except db_exc.DBDuplicateEntry:
raise exc.DuplicateEntry("Duplicate entry for : %s"
% kls.__name__)
return entity
def entity_update(kls, entity_id, values):
session = get_session()
with session.begin():
entity = __entity_get(kls, entity_id, session)
if entity is None:
raise exc.NotFound("%s %s not found" % (kls.__name__, entity_id))
values_copy = values.copy()
values_copy["id"] = entity_id
entity.update(values_copy)
session.add(entity)
session = get_session()
entity = __entity_get(kls, entity_id, session)
return entity
def entity_hard_delete(kls, entity_id):
session = get_session()
with session.begin():
query = model_query(kls, session)
entity = query.filter_by(id=entity_id).first()
if entity is None:
raise exc.NotFound("%s %s not found" % (kls.__name__, entity_id))
session.delete(entity)

118
radar/db/api/operators.py Normal file
View File

@ -0,0 +1,118 @@
# Copyright (c) 2014 Triniplex.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo.db.sqlalchemy.utils import InvalidSortKey
from sqlalchemy.orm import subqueryload
from wsme.exc import ClientSideError
from radar.common import exception as exc
from radar.db.api import base as api_base
from radar.db.api import systems
from radar.db import models
def operator_get(operator_id, session=None):
return api_base.model_query(models.Operator, session) \
.filter_by(id=operator_id).first()
def operator_get_by_name(name, session=None):
query = api_base.model_query(models.Operator, session)
return query.filter_by(operator_name=name).first()
def count(session=None):
return api_base.model_query(models.Operator, session) \
.count()
def operator_get_all(marker=None, limit=None, name=None, sort_field=None,
sort_dir=None, **kwargs):
# Sanity checks, in case someone accidentally explicitly passes in 'None'
if not sort_field:
sort_field = 'id'
if not sort_dir:
sort_dir = 'asc'
query = _operator_build_query(name=name, **kwargs)
try:
query = api_base.paginate_query(query=query,
model=models.Operator,
limit=limit,
sort_keys=[sort_field],
marker=marker,
sort_dir=sort_dir)
except InvalidSortKey:
raise ClientSideError("Invalid sort_field [%s]" % (sort_field,),
status_code=400)
except ValueError as ve:
raise ClientSideError("%s" % (ve,), status_code=400)
# Execute the query
return query.all()
def operator_create(values):
return api_base.entity_create(models.Operator, values)
def operator_add_system(operator_id, system_id):
session = api_base.get_session()
with session.begin():
operator = _entity_get(operator_id, session)
if operator is None:
raise exc.NotFound("%s %s not found"
% ("Operator", operator_id))
system = systems.system_get_by_id(system_id, session)
if system is None:
raise exc.NotFound("%s %s not found"
% ("System", system_id))
if system_id in [s.id for s in operator.systems]:
raise ClientSideError("The System %d is already associated with"
"Operator %d" %
(system_id, operator_id))
operator.systems.append(system)
session.add(operator)
return operator
def operator_update(operator_id, values):
return api_base.entity_update(models.Operator, operator_id, values)
def operator_delete(operator_id):
operator = operator_get(operator_id)
if operator:
api_base.entity_hard_delete(models.Operator, operator_id)
def _entity_get(id, session=None):
if not session:
session = api_base.get_session()
query = session.query(models.Operator)\
.filter_by(id=id)
return query.first()
def operator_get_count(**kwargs):
query = _operator_build_query(**kwargs)
return query.count()
def _operator_build_query(**kwargs):
query = api_base.model_query(models.Operator)
query = api_base.apply_query_filters(query=query,
model=models.Operator,
**kwargs)
return query

59
radar/db/api/subscriptions.py Normal file
View File

@ -0,0 +1,59 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from radar.db.api import base as api_base
from radar.db import models
SUPPORTED_TYPES = {
'system': models.System,
'operator': models.Operator
}
def subscription_get(subscription_id):
return api_base.entity_get(models.Subscription, subscription_id)
def subscription_get_all(**kwargs):
return api_base.entity_get_all(models.Subscription,
**kwargs)
def subscription_get_all_by_target(target_type, target_id):
return api_base.entity_get_all(models.Subscription,
target_type=target_type,
target_id=target_id)
def subscription_get_resource(target_type, target_id):
if target_type not in SUPPORTED_TYPES:
return None
return api_base.entity_get(SUPPORTED_TYPES[target_type], target_id)
def subscription_get_count(**kwargs):
return api_base.entity_get_count(models.Subscription, **kwargs)
def subscription_create(values):
return api_base.entity_create(models.Subscription, values)
def subscription_delete(subscription_id):
subscription = subscription_get(subscription_id)
if subscription:
api_base.entity_hard_delete(models.Subscription, subscription_id)

85
radar/db/api/systems.py Normal file
View File

@ -0,0 +1,85 @@
# Copyright (c) 2014 Triniplex.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo.db.sqlalchemy.utils import InvalidSortKey
from sqlalchemy.orm import subqueryload
from wsme.exc import ClientSideError
from radar.db.api import base as api_base
from radar.db import models
def system_get_by_id(system_id, session=None):
return api_base.model_query(models.System, session) \
.filter_by(id=system_id).first()
def count(session=None):
return api_base.model_query(models.System, session) \
.count()
def system_get_by_name(name, session=None):
query = api_base.model_query(models.System, session)
return query.filter_by(name=name).first()
def system_get_all(marker=None, limit=None, name=None, sort_field=None,
sort_dir=None, **kwargs):
# Sanity checks, in case someone accidentally explicitly passes in 'None'
if not sort_field:
sort_field = 'id'
if not sort_dir:
sort_dir = 'asc'
query = _system_build_query(name=name, **kwargs)
try:
query = api_base.paginate_query(query=query,
model=models.System,
limit=limit,
sort_keys=[sort_field],
marker=marker,
sort_dir=sort_dir)
except InvalidSortKey:
raise ClientSideError("Invalid sort_field [%s]" % (sort_field,),
status_code=400)
except ValueError as ve:
raise ClientSideError("%s" % (ve,), status_code=400)
# Execute the query
return query.all()
def system_create(values):
return api_base.entity_create(models.System, values)
def system_update(system_id, values):
return api_base.entity_update(models.System, system_id, values)
def system_delete(system_id):
system = system_get(system_id)
if system:
api_base.entity_hard_delete(models.System, system_id)
def system_get_count(**kwargs):
query = _system_build_query(**kwargs)
return query.count()
def _system_build_query(**kwargs):
query = api_base.model_query(models.System)
query = api_base.apply_query_filters(query=query,
model=models.System,
**kwargs)
return query

118
radar/db/api/users.py Normal file
View File

@ -0,0 +1,118 @@
# Copyright (c) 2014 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo.db import exception as db_exc
from radar.common import exception as exc
from radar.db.api import base as api_base
from radar.db import models
from radar.openstack.common.gettextutils import _ # noqa
from radar.plugin.user_preferences import PREFERENCE_DEFAULTS
def user_get(user_id, filter_non_public=False):
entity = api_base.entity_get(models.User, user_id,
filter_non_public=filter_non_public)
return entity
def user_get_all(marker=None, limit=None, filter_non_public=False,
sort_field=None, sort_dir=None, **kwargs):
return api_base.entity_get_all(models.User,
marker=marker,
limit=limit,
filter_non_public=filter_non_public,
sort_field=sort_field,
sort_dir=sort_dir,
**kwargs)
def user_get_count(**kwargs):
return api_base.entity_get_count(models.User, **kwargs)
def user_get_by_openid(openid):
query = api_base.model_query(models.User, api_base.get_session())
return query.filter_by(openid=openid).first()
def user_create(values):
user = models.User()
user.update(values.copy())
session = api_base.get_session()
with session.begin():
try:
user.save(session=session)
except db_exc.DBDuplicateEntry as e:
raise exc.DuplicateEntry(_("Duplicate entry for User: %s")
% e.columns)
return user
def user_update(user_id, values):
return api_base.entity_update(models.User, user_id, values)
def user_get_preferences(user_id):
preferences = api_base.entity_get_all(models.UserPreference,
user_id=user_id)
pref_dict = dict()
for pref in preferences:
pref_dict[pref.key] = pref.cast_value
# Decorate with plugin defaults.
for key in PREFERENCE_DEFAULTS:
if key not in pref_dict:
pref_dict[key] = PREFERENCE_DEFAULTS[key]
return pref_dict
def user_update_preferences(user_id, preferences):
for key in preferences:
value = preferences[key]
prefs = api_base.entity_get_all(models.UserPreference,
user_id=user_id,
key=key)
if prefs:
pref = prefs[0]
else:
pref = None
# If the preference exists and it's null.
if pref and value is None:
api_base.entity_hard_delete(models.UserPreference, pref.id)
continue
# If the preference exists and has a new value.
if pref and value and pref.cast_value != value:
pref.cast_value = value
api_base.entity_update(models.UserPreference, pref.id, dict(pref))
continue
# If the preference does not exist and a new value exists.
if not pref and value:
api_base.entity_create(models.UserPreference, {
'user_id': user_id,
'key': key,
'cast_value': value
})
return user_get_preferences(user_id)

0
radar/db/migration/__init__.py Normal file
View File

52
radar/db/migration/alembic.ini Normal file
View File

@ -0,0 +1,52 @@
# A generic, single database configuration.
[alembic]
# path to migration scripts
script_location = %(here)s/alembic_migrations
# template used to generate migration files
# file_template = %%(rev)s_%%(slug)s
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
# default to an empty string because the radar migration cli will
# extract the correct value and set it programatically before alembic is fully
# invoked.
sqlalchemy.url =
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S

1
radar/db/migration/alembic_migrations/README Normal file
View File

@ -0,0 +1 @@
Generic single-database configuration.

0
radar/db/migration/alembic_migrations/__init__.py Normal file
View File

81
radar/db/migration/alembic_migrations/env.py Normal file
View File

@ -0,0 +1,81 @@
# Copyright 2012 New Dream Network, LLC (DreamHost)
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# from logging.config import fileConfig
from alembic import context
from sqlalchemy import create_engine, pool
from radar.db import models
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
radar_config = config.radar_config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
# TODO(mordred): enable this once we're doing something with logging
# fileConfig(config.config_file_name)
# set the target for 'autogenerate' support
target_metadata = models.Base.metadata
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
context.configure(url=radar_config.database.connection)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
engine = create_engine(
radar_config.database.connection,
poolclass=pool.NullPool)
connection = engine.connect()
context.configure(
connection=connection,
target_metadata=target_metadata
)
try:
with context.begin_transaction():
context.run_migrations()
finally:
connection.close()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()

24
radar/db/migration/alembic_migrations/script.py.mako Normal file
View File

@ -0,0 +1,24 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
# revision identifiers, used by Alembic.
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
branch_labels = ${repr(branch_labels)}
depends_on = ${repr(depends_on)}
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
def upgrade():
${upgrades if upgrades else "pass"}
def downgrade():
${downgrades if downgrades else "pass"}

75
radar/db/migration/alembic_migrations/versions/12f5a539f16f_initial_tables.py Normal file
View File

@ -0,0 +1,75 @@
"""initial_tables
Revision ID: 12f5a539f16f
Revises:
Create Date: 2014-12-08 20:56:38.468330
"""
# revision identifiers, used by Alembic.
revision = '12f5a539f16f'
down_revision = None
branch_labels = None
depends_on = None
from alembic import op
import sqlalchemy as sa
MYSQL_ENGINE = 'MyISAM'
MYSQL_CHARSET = 'utf8'
def upgrade():
op.create_table(
'systems_operators',
sa.Column('system_id', sa.Integer(), nullable=True),
sa.Column('operator_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['system_id'], ['systems.id'],),
sa.ForeignKeyConstraint(['operator_id'], ['operators.id'], ),
sa.PrimaryKeyConstraint(),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET
)
op.create_table(
'systems',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('name', sa.String(length=50), nullable=True),
sa.UniqueConstraint('name', name='uniq_systems_name'),
sa.PrimaryKeyConstraint('id'),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET
)
op.create_table(
'system_events',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('event_type', sa.Unicode(length=100), nullable=False),
sa.Column('event_info', sa.UnicodeText(), nullable=True),
sa.Column('system_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['system_id'], ['systems.id'], ),
sa.PrimaryKeyConstraint('id'),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET)
op.create_table(
'operators',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('operator_name', sa.String(length=50), nullable=True),
sa.Column('operator_email', sa.String(length=50), nullable=True),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('operator_name', name='uniq_operator_name'),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET)
def downgrade():
op.drop_table('systems_operators')
op.drop_table('systems')
op.drop_table('system_events')
op.drop_table('operators')

68
radar/db/migration/alembic_migrations/versions/135e9f8aeb9c_add_users_permissions.py Normal file
View File

@ -0,0 +1,68 @@
"""add users permissions
Revision ID: 135e9f8aeb9c
Revises: 4d5b6d924547
Create Date: 2014-12-19 04:28:35.739935
"""
# revision identifiers, used by Alembic.
revision = '135e9f8aeb9c'
down_revision = '4d5b6d924547'
branch_labels = None
depends_on = None
from alembic import op
import sqlalchemy as sa
MYSQL_ENGINE = 'MyISAM'
MYSQL_CHARSET = 'utf8'
def upgrade():
op.create_table(
'users',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('username', sa.Unicode(length=30), nullable=True),
sa.Column('full_name', sa.Unicode(length=255), nullable=True),
sa.Column('email', sa.String(length=255), nullable=True),
sa.Column('openid', sa.String(length=255), nullable=True),
sa.Column('is_staff', sa.Boolean(), nullable=True),
sa.Column('is_active', sa.Boolean(), nullable=True),
sa.Column('is_superuser', sa.Boolean(), nullable=True),
sa.Column('last_login', sa.DateTime(), nullable=True),
sa.Column('enable_login', sa.Boolean(), nullable=False),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('email', name='uniq_user_email'),
sa.UniqueConstraint('username', name='uniq_user_username'),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET
)
op.create_table(
'permissions',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('name', sa.Unicode(length=50), nullable=True),
sa.Column('codename', sa.Unicode(length=255), nullable=True),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name', name='uniq_permission_name'),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET
)
op.create_table(
'user_permissions',
sa.Column('user_id', sa.Integer(), nullable=True),
sa.Column('permission_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['permission_id'], ['permissions.id'], ),
sa.ForeignKeyConstraint(['user_id'], ['users.id'], ),
sa.PrimaryKeyConstraint(),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET
)
def downgrade():
op.drop_table('users')
op.drop_table('user_permissions')
op.drop_table('permissions')

70
radar/db/migration/alembic_migrations/versions/1e10d235df14_add_authorization_models.py Normal file
View File

@ -0,0 +1,70 @@
"""add authorization models
Revision ID: 1e10d235df14
Revises: 135e9f8aeb9c
Create Date: 2014-12-19 05:16:31.019506
"""
# revision identifiers, used by Alembic.
revision = '1e10d235df14'
down_revision = '135e9f8aeb9c'
branch_labels = None
depends_on = None
from alembic import op
import sqlalchemy as sa
MYSQL_ENGINE = 'MyISAM'
MYSQL_CHARSET = 'utf8'
def upgrade(active_plugins=None, options=None):
op.create_table(
'authorization_codes',
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('code', sa.Unicode(100), nullable=False),
sa.Column('state', sa.Unicode(100), nullable=False),
sa.Column('user_id', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['user_id'], ['users.id'], ),
sa.PrimaryKeyConstraint('id'),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET
)
op.create_table(
'accesstokens',
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('user_id', sa.Integer(), nullable=False),
sa.Column('access_token', sa.Unicode(length=100), nullable=False),
sa.Column('expires_in', sa.Integer(), nullable=False),
sa.Column('expires_at', sa.DateTime(), nullable=False),
sa.ForeignKeyConstraint(['user_id'], ['users.id'], ),
sa.PrimaryKeyConstraint('id'),
mysql_default_charset=MYSQL_CHARSET,
mysql_engine=MYSQL_ENGINE)
op.create_table(
'refreshtokens',
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('user_id', sa.Integer(), nullable=False),
sa.Column('refresh_token', sa.Unicode(length=100), nullable=False),
sa.Column('expires_at', sa.DateTime(),nullable=False),
sa.Column('expires_in', sa.Integer(), nullable=False),
sa.ForeignKeyConstraint(['user_id'], ['users.id'], ),
sa.PrimaryKeyConstraint('id'),
mysql_default_charset=MYSQL_CHARSET,
mysql_engine=MYSQL_ENGINE)
def downgrade(active_plugins=None, options=None):
op.drop_table('refreshtokens')
op.drop_table('accesstokens')
op.drop_table('authorization_codes')

28
radar/db/migration/alembic_migrations/versions/4d5b6d924547_add_fulltext_indexes.py Normal file
View File

@ -0,0 +1,28 @@
"""add fulltext indexes
Revision ID: 4d5b6d924547
Revises: 12f5a539f16f
Create Date: 2014-12-19 03:52:41.910419
"""
# revision identifiers, used by Alembic.
revision = '4d5b6d924547'
down_revision = '12f5a539f16f'
branch_labels = None
depends_on = None
from alembic import op
import sqlalchemy as sa
def upgrade():
op.execute("ALTER TABLE systems "
"ADD FULLTEXT systems_name_fti (name)")
op.execute("ALTER TABLE operators "
"ADD FULLTEXT operators_fti (operator_name, operator_email)")
def downgrade():
op.drop_index("systems_name_fti", table_name='systems')
op.drop_index("operators_fti", table_name='operators')

39
radar/db/migration/alembic_migrations/versions/842a5f411f2_create_subscriptions_table.py Normal file
View File

@ -0,0 +1,39 @@
"""create subscriptions table
Revision ID: 842a5f411f2
Revises: 1e10d235df14
Create Date: 2014-12-19 21:41:15.172502
"""
# revision identifiers, used by Alembic.
revision = '842a5f411f2'
down_revision = '1e10d235df14'
branch_labels = None
depends_on = None
from alembic import op
import sqlalchemy as sa
MYSQL_ENGINE = 'MyISAM'
MYSQL_CHARSET = 'utf8'
target_type_enum = sa.Enum('system', 'operator')
def upgrade():
op.create_table(
'subscriptions',
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('user_id', sa.Integer(), nullable=True),
sa.Column('target_type', target_type_enum, nullable=True),
sa.Column('target_id', sa.Integer(), nullable=True),
sa.PrimaryKeyConstraint('id'),
mysql_engine=MYSQL_ENGINE,
mysql_charset=MYSQL_CHARSET
)
def downgrade():
op.drop_table('subscriptions')

121
radar/db/migration/cli.py Normal file
View File

@ -0,0 +1,121 @@
# -*- encoding: utf-8 -*-
#
# Copyright 2013 Hewlett-Packard Development Company, L.P.
# Copyright 2012 New Dream Network, LLC (DreamHost)
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import gettext
import os
from alembic import command as alembic_command
from alembic import config as alembic_config
from alembic import util as alembic_util
from oslo.config import cfg
from oslo.db import options
gettext.install('radar', unicode=1)
CONF = cfg.CONF
def do_alembic_command(config, cmd, *args, **kwargs):
try:
getattr(alembic_command, cmd)(config, *args, **kwargs)
except alembic_util.CommandError as e:
alembic_util.err(str(e))
def do_check_migration(config, cmd):
do_alembic_command(config, 'branches')
def do_upgrade_downgrade(config, cmd):
if not CONF.command.revision and not CONF.command.delta:
raise SystemExit(_('You must provide a revision or relative delta'))
revision = CONF.command.revision
if CONF.command.delta:
sign = '+' if CONF.command.name == 'upgrade' else '-'
revision = sign + str(CONF.command.delta)
else:
revision = CONF.command.revision
do_alembic_command(config, cmd, revision, sql=CONF.command.sql)
def do_stamp(config, cmd):
do_alembic_command(config, cmd,
CONF.command.revision,
sql=CONF.command.sql)
def do_revision(config, cmd):
do_alembic_command(config, cmd,
message=CONF.command.message,
autogenerate=CONF.command.autogenerate,
sql=CONF.command.sql)
def add_command_parsers(subparsers):
for name in ['current', 'history', 'branches']:
parser = subparsers.add_parser(name)
parser.set_defaults(func=do_alembic_command)
parser = subparsers.add_parser('check_migration')
parser.set_defaults(func=do_check_migration)
for name in ['upgrade', 'downgrade']:
parser = subparsers.add_parser(name)
parser.add_argument('--delta', type=int)
parser.add_argument('--sql', action='store_true')
parser.add_argument('revision', nargs='?')
parser.set_defaults(func=do_upgrade_downgrade)
parser = subparsers.add_parser('stamp')
parser.add_argument('--sql', action='store_true')
parser.add_argument('revision')
parser.set_defaults(func=do_stamp)
parser = subparsers.add_parser('revision')
parser.add_argument('-m', '--message')
parser.add_argument('--autogenerate', action='store_true')
parser.add_argument('--sql', action='store_true')
parser.set_defaults(func=do_revision)
command_opt = cfg.SubCommandOpt('command',
title='Command',
help=_('Available commands'),
handler=add_command_parsers)
CONF.register_cli_opt(command_opt)
CONF.register_opts(options.database_opts, 'database')
def get_alembic_config():
print os.path.join(os.path.dirname(__file__), 'alembic.ini')
config = alembic_config.Config(
os.path.join(os.path.dirname(__file__), 'alembic.ini'))
config.set_main_option('script_location',
'radar.db.migration:alembic_migrations')
return config
def main():
config = get_alembic_config()
# attach the radar conf to the Alembic conf
config.radar_config = CONF
CONF(project='radar')
CONF.command.func(config, CONF.command.name)

221
radar/db/models.py Normal file
View File

@ -0,0 +1,221 @@
"""
SQLAlchemy Models
"""
from oslo.config import cfg
from oslo.db.sqlalchemy import models
from sqlalchemy import Boolean
from sqlalchemy import Column
from sqlalchemy import DateTime
from sqlalchemy.dialects.mysql import MEDIUMTEXT
from sqlalchemy import Enum
from sqlalchemy.ext import declarative
from sqlalchemy import ForeignKey
from sqlalchemy import Integer
from sqlalchemy.orm import relationship
from sqlalchemy import schema
from sqlalchemy import select
import sqlalchemy.sql.expression as expr
import sqlalchemy.sql.functions as func
from sqlalchemy import String
from sqlalchemy import Table
from sqlalchemy import Unicode
from sqlalchemy import UnicodeText
from sqlalchemy_fulltext import FullText
import six.moves.urllib.parse as urlparse
CONF = cfg.CONF
def table_args():
engine_name = urlparse.urlparse(cfg.CONF.database_connection).scheme
if engine_name == 'mysql':
return {'mysql_engine': cfg.CONF.mysql_engine,
'mysql_charset': "utf8"}
return None
## CUSTOM TYPES
# A mysql medium text type.
MYSQL_MEDIUM_TEXT = UnicodeText().with_variant(MEDIUMTEXT(), 'mysql')
class IdMixin(object):
id = Column(Integer, primary_key=True)
class RadarBase(models.TimestampMixin,
IdMixin,
models.ModelBase):
metadata = None
@declarative.declared_attr
def __tablename__(cls):
return cls.__name__.lower() + 's'
def as_dict(self):
d = {}
for c in self.__table__.columns:
d[c.name] = self[c.name]
return d
Base = declarative.declarative_base(cls=RadarBase)
class ModelBuilder(object):
def __init__(self, **kwargs):
super(ModelBuilder, self).__init__()
if kwargs:
for key in kwargs:
if key in self:
self[key] = kwargs[key]
class AuthorizationCode(ModelBuilder, Base):
__tablename__ = "authorization_codes"
code = Column(Unicode(100), nullable=False)
state = Column(Unicode(100), nullable=False)
user_id = Column(Integer, ForeignKey('users.id'), nullable=False)
class AccessToken(ModelBuilder, Base):
__tablename__ = "accesstokens"
user_id = Column(Integer, ForeignKey('users.id'), nullable=False)
access_token = Column(Unicode(100), nullable=False)
expires_in = Column(Integer, nullable=False)
expires_at = Column(DateTime, nullable=False)
class RefreshToken(ModelBuilder, Base):
__tablename__ = "refreshtokens"
user_id = Column(Integer, ForeignKey('users.id'), nullable=False)
refresh_token = Column(Unicode(100), nullable=False)
expires_in = Column(Integer, nullable=False)
expires_at = Column(DateTime, nullable=False)
user_permissions = Table(
'user_permissions', Base.metadata,
Column('user_id', Integer, ForeignKey('users.id')),
Column('permission_id', Integer, ForeignKey('permissions.id')),
)
systems_operators = Table(
'systems_operators', Base.metadata,
Column('system_id', Integer, ForeignKey('systems.id')),
Column('operator_id', Integer, ForeignKey('operators.id')),
)
class System(FullText, ModelBuilder, Base):
__tablename__ = "systems"
__fulltext_columns__ = ['name']
name = Column(Unicode(50))
events = relationship('SystemEvent', backref='system')
operators = relationship("Operator", secondary="systems_operators")
_public_fields = ["id", "name", "events", "operators"]
class SystemEvent(ModelBuilder, Base):
__tablename__ = 'system_events'
__fulltext_columns__ = ['event_type', 'event_info']
system_id = Column(Integer, ForeignKey('systems.id'))
event_type = Column(Unicode(100), nullable=False)
event_info = Column(UnicodeText(), nullable=True)
_public_fields = ["id", "system_id", "event_type", "event_info"]
class Operator(ModelBuilder, Base):
__tablename__ = "operators"
__fulltext_columns__ = ['operator_name', 'operator_email']
operator_name = Column(Unicode(50))
operator_email = Column(Unicode(50))
systems = relationship('System', secondary="systems_operators")
_public_fields = ["id", "operator_name", "operator_email", "systems"]
class User(FullText, ModelBuilder, Base):
__table_args__ = (
schema.UniqueConstraint('email', name='uniq_user_email'),
)
__fulltext_columns__ = ['username', 'full_name', 'email']
username = Column(Unicode(30))
full_name = Column(Unicode(255), nullable=True)
email = Column(String(255))
openid = Column(String(255))
is_staff = Column(Boolean, default=False)
is_active = Column(Boolean, default=True)
is_superuser = Column(Boolean, default=False)
last_login = Column(DateTime)
permissions = relationship("Permission", secondary="user_permissions")
enable_login = Column(Boolean, default=True)
preferences = relationship("UserPreference")
_public_fields = ["id", "openid", "full_name", "username", "last_login",
"enable_login"]
class Permission(ModelBuilder, Base):
__table_args__ = (
schema.UniqueConstraint('name', name='uniq_permission_name'),
)
name = Column(Unicode(50))
codename = Column(Unicode(255))
class UserPreference(ModelBuilder, Base):
__tablename__ = 'user_preferences'
_TASK_TYPES = ('string', 'int', 'bool', 'float')
user_id = Column(Integer, ForeignKey('users.id'))
key = Column(Unicode(100))
value = Column(Unicode(255))
type = Column(Enum(*_TASK_TYPES), default='string')
@property
def cast_value(self):
try:
cast_func = {
'float': lambda x: float(x),
'int': lambda x: int(x),
'bool': lambda x: bool(x),
'string': lambda x: str(x)
}[self.type]
return cast_func(self.value)
except ValueError:
return self.value
@cast_value.setter
def cast_value(self, value):
if isinstance(value, bool):
self.type = 'bool'
elif isinstance(value, int):
self.type = 'int'
elif isinstance(value, float):
self.type = 'float'
else:
self.type = 'string'
self.value = str(value)
_public_fields = ["id", "key", "value", "type"]
class Subscription(ModelBuilder, Base):
_SUBSCRIPTION_TARGETS = ('system')
user_id = Column(Integer, ForeignKey('users.id'))
target_type = Column(Enum(*_SUBSCRIPTION_TARGETS))
# Cant use foreign key here as it depends on the type
target_id = Column(Integer)
_public_fields = ["id", "target_type", "target_id", "user_id"]

0
radar/notifications/__init__.py Normal file
View File

43
radar/notifications/conf.py Normal file
View File

@ -0,0 +1,43 @@
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo.config import cfg
CONF = cfg.CONF
NOTIFICATION_OPTS = [
cfg.StrOpt("rabbit_exchange_name", default="radar",
help="The name of the topic exchange which radar will "
"use to broadcast its events."),
cfg.StrOpt("rabbit_event_queue_name", default="radar_events",
help="The name of the queue that will be created for "
"API events."),
cfg.StrOpt("rabbit_application_name", default="radar",
help="The rabbit application identifier for radar's "
"connection."),
cfg.StrOpt("rabbit_host", default="localhost",
help="Host of the rabbitmq server."),
cfg.StrOpt("rabbit_login_method", default="AMQPLAIN",
help="The RabbitMQ login method."),
cfg.StrOpt("rabbit_userid", default="radar",
help="The RabbitMQ userid."),
cfg.StrOpt("rabbit_password", default="radar",
help="The RabbitMQ password."),
cfg.IntOpt("rabbit_port", default=5672,
help="The RabbitMQ broker port where a single node is used."),
cfg.StrOpt("rabbit_virtual_host", default="/",
help="The virtual host within which our queues and exchanges "
"live."),
]

153
radar/notifications/connection_service.py Normal file
View File

@ -0,0 +1,153 @@
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from threading import Timer
import pika
from oslo.config import cfg
from radar.openstack.common import log
from radar.openstack.common.gettextutils import _, _LI # noqa
CONF = cfg.CONF
LOG = log.getLogger(__name__)
class ConnectionService(object):
"""A generic amqp connection agent that handles unexpected
interactions with RabbitMQ such as channel and connection closures,
by reconnecting on failure.
"""
def __init__(self, conf):
"""Setup the connection instance based on our configuration.
:param conf A configuration object.
"""
self._connection = None
self._channel = None
self._open = False
self.started = False
self._timer = None
self._closing = False
self._open_hooks = set()
self._exchange_name = conf.rabbit_exchange_name
self._application_id = conf.rabbit_application_name
self._properties = pika.BasicProperties(
app_id='radar', content_type='application/json')
self._connection_credentials = pika.PlainCredentials(
conf.rabbit_userid,
conf.rabbit_password)
self._connection_parameters = pika.ConnectionParameters(
conf.rabbit_host,
conf.rabbit_port,
conf.rabbit_virtual_host,
self._connection_credentials)
def _connect(self):
"""This method connects to RabbitMQ, establishes a channel, declares
the radar exchange if it doesn't yet exist, and executes any
post-connection hooks that an extending class may have registered.
"""
# If the closing flag is set, just exit.
if self._closing:
return
# If a timer is set, kill it.
if self._timer:
LOG.debug(_('Clearing timer...'))
self._timer.cancel()
self._timer = None
# Create the connection
LOG.info(_LI('Connecting to %s'), self._connection_parameters.host)
self._connection = pika.BlockingConnection(self._connection_parameters)
# Create a channel
LOG.debug(_('Creating a new channel'))
self._channel = self._connection.channel()
self._channel.confirm_delivery()
# Declare the exchange
LOG.debug(_('Declaring exchange %s'), self._exchange_name)
self._channel.exchange_declare(exchange=self._exchange_name,
exchange_type='topic',
durable=True,
auto_delete=False)
# Set the open flag and execute any connection hooks.
self._open = True
self._execute_open_hooks()
def _reconnect(self):
"""Reconnect to rabbit.
"""
# Sanity check - if we're closing, do nothing.
if self._closing:
return
# If a timer is already there, assume it's doing its thing...
if self._timer:
return
LOG.debug(_('Scheduling reconnect in 5 seconds...'))
self._timer = Timer(5, self._connect)
self._timer.start()
def _close(self):
"""This method closes the connection to RabbitMQ."""
LOG.info(_LI('Closing connection'))
self._open = False
if self._channel:
self._channel.close()
self._channel = None
if self._connection:
self._connection.close()
self._connection = None
self._closing = False
LOG.debug(_('Connection Closed'))
def _execute_open_hooks(self):
"""Executes all hooks that have been registered to run on open.
"""
for hook in self._open_hooks:
hook()
def start(self):
"""Start the publisher, opening a connection to RabbitMQ. This method
must be explicitly invoked, otherwise any messages will simply be
cached for later broadcast.
"""
# Create the connection.
self.started = True
self._closing = False
self._connect()
def stop(self):
"""Stop the publisher by closing the channel and the connection.
"""
self.started = False
self._closing = True
self._close()
def add_open_hook(self, hook):
"""Add a method that will be executed whenever a connection is
established.
"""
self._open_hooks.add(hook)

82
radar/notifications/notification_hook.py Normal file
View File

@ -0,0 +1,82 @@
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import re
from pecan import hooks
from radar.notifications.publisher import publish
class NotificationHook(hooks.PecanHook):
def __init__(self):
super(NotificationHook, self).__init__()
def after(self, state):
# Ignore get methods, we only care about changes.
if state.request.method not in ['POST', 'PUT', 'DELETE']:
return
request = state.request
req_method = request.method
req_author_id = request.current_user_id
req_path = request.path
req_resource_grp = self._parse(req_path)
if not req_resource_grp:
return
resource = req_resource_grp[0]
if req_resource_grp[1]:
resource_id = req_resource_grp[1]
else:
# When a resource is created..
response_str = state.response.body
response = json.loads(response_str)
if response:
resource_id = response.get('id')
else:
resource_id = None
# when adding/removing projects to project_groups..
if req_resource_grp[3]:
sub_resource_id = req_resource_grp[3]
payload = {
"author_id": req_author_id,
"method": req_method,
"resource": resource,
"resource_id": resource_id,
"sub_resource_id": sub_resource_id
}
else:
payload = {
"author_id": req_author_id,
"method": req_method,
"resource": resource,
"resource_id": resource_id
}
publish(resource, payload)
def _parse(self, s):
url_pattern = re.match("^\/v1\/([a-z_]+)\/?([0-9]+)?"
"\/?([a-z]+)?\/?([0-9]+)?$", s)
if url_pattern and url_pattern.groups()[0] != "openid":
return url_pattern.groups()
else:
return

149
radar/notifications/publisher.py Normal file
View File

@ -0,0 +1,149 @@
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
from oslo.config import cfg
from pika.exceptions import ConnectionClosed
from radar.notifications.conf import NOTIFICATION_OPTS
from radar.notifications.connection_service import ConnectionService
from radar.openstack.common import log
from radar.openstack.common.gettextutils import _, _LW, _LE # noqa
CONF = cfg.CONF
LOG = log.getLogger(__name__)
PUBLISHER = None
class Publisher(ConnectionService):
"""A generic message publisher that uses delivery confirmation to ensure
that messages are delivered, and will keep a running cache of unsent
messages while the publisher is attempting to reconnect.
"""
def __init__(self, conf):
"""Setup the publisher instance based on our configuration.
:param conf A configuration object.
"""
super(Publisher, self).__init__(conf)
self._pending = list()
self.add_open_hook(self._publish_pending)
def _publish_pending(self):
"""Publishes any pending messages that were broadcast while the
publisher was connecting.
"""
# Shallow copy, so we can iterate over it without having it be modified
# out of band.
pending = list(self._pending)
for payload in pending:
self._publish(payload)
def _publish(self, payload):
"""Publishes a payload to the passed exchange. If it encounters a
failure, will store the payload for later.
:param Payload payload: The payload to send.
"""
LOG.debug(_("Sending message to %(name)s [%(topic)s]") %
{'name': self._exchange_name, 'topic': payload.topic})
# First check, are we closing?
if self._closing:
LOG.warning(_LW("Cannot send message, publisher is closing."))
if payload not in self._pending:
self._pending.append(payload)
return
# Second check, are we open?
if not self._open:
LOG.debug(_("Cannot send message, publisher is connecting."))
if payload not in self._pending:
self._pending.append(payload)
self._reconnect()
return
# Third check, are we in a sane state? This should never happen,
# but just in case...
if not self._connection or not self._channel:
LOG.error(_LE("Cannot send message, publisher is "
"an unexpected state."))
if payload not in self._pending:
self._pending.append(payload)
self._reconnect()
return
# Try to send a message. If we fail, schedule a reconnect and store
# the message.
try:
self._channel.basic_publish(self._exchange_name,
payload.topic,
json.dumps(payload.payload,
ensure_ascii=False),
self._properties)
if payload in self._pending:
self._pending.remove(payload)
return True
except ConnectionClosed as cc:
LOG.warning(_LW("Attempted to send message on closed connection."))
LOG.debug(cc)
self._open = False
if payload not in self._pending:
self._pending.append(payload)
self._reconnect()
return False
def publish_message(self, topic, payload):
"""Publishes a message to RabbitMQ.
"""
self._publish(Payload(topic, payload))
class Payload(object):
def __init__(self, topic, payload):
"""Setup the example publisher object, passing in the URL we will use
to connect to RabbitMQ.
:param topic string The exchange topic to broadcast on.
:param payload string The message payload to send.
"""
self.topic = topic
self.payload = payload
def publish(topic, payload):
"""Send a message with a given topic and payload to the radar
exchange. The message will be automatically JSON encoded.
:param topic: The RabbitMQ topic.
:param payload: The JSON-serializable payload.
:return:
"""
global PUBLISHER
if not PUBLISHER:
CONF.register_opts(NOTIFICATION_OPTS, "notifications")
PUBLISHER = Publisher(CONF.notifications)
PUBLISHER.start()
PUBLISHER.publish_message(topic, payload)

135
radar/notifications/subscriber.py Normal file
View File

@ -0,0 +1,135 @@
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import time
from oslo.config import cfg
from pika.exceptions import ConnectionClosed
from stevedore import enabled
from radar.notifications.conf import NOTIFICATION_OPTS
from radar.notifications.connection_service import ConnectionService
from radar.openstack.common import log
from radar.openstack.common.gettextutils import _, _LW # noqa
CONF = cfg.CONF
LOG = log.getLogger(__name__)
def subscribe():
log.setup('radar')
CONF(project='radar')
CONF.register_opts(NOTIFICATION_OPTS, "notifications")
subscriber = Subscriber(CONF.notifications)
subscriber.start()
manager = enabled.EnabledExtensionManager(
namespace='radar.worker.task',
check_func=check_enabled,
invoke_on_load=True,
invoke_args=(CONF,)
)
while subscriber.started:
(method, properties, body) = subscriber.get()
if not method or not properties:
LOG.debug(_("No messages available, sleeping for 5 seconds."))
time.sleep(5)
continue
manager.map(handle_event, body)
# Ack the message
subscriber.ack(method.delivery_tag)
def handle_event(ext, body):
"""Handle an event from the queue.
:param ext: The extension that's handling this event.
:param body: The body of the event.
:return: The result of the handler.
"""
return ext.obj.handle(body)
def check_enabled(ext):
"""Check to see whether an extension should be enabled.
:param ext: The extension instance to check.
:return: True if it should be enabled. Otherwise false.
"""
return ext.obj.enabled()
class Subscriber(ConnectionService):
def __init__(self, conf):
"""Setup the subscriber instance based on our configuration.
:param conf A configuration object.
"""
super(Subscriber, self).__init__(conf)
self._queue_name = conf.rabbit_event_queue_name
self._binding_keys = ['systems']
self.add_open_hook(self._declare_queue)
def _declare_queue(self):
"""Declare the subscription queue against our exchange.
"""
self._channel.queue_declare(queue=self._queue_name,
durable=True)
# Set up the queue bindings.
for binding_key in self._binding_keys:
self._channel.queue_bind(exchange=self._exchange_name,
queue=self._queue_name,
routing_key=binding_key)
def ack(self, delivery_tag):
"""Acknowledge receipt and processing of the message.
"""
self._channel.basic_ack(delivery_tag)
def get(self):
"""Get a single message from the queue. If the subscriber is currently
waiting to reconnect, it will return None. Note that you must
manually ack the message after it has been successfully processed.
:rtype: (None, None, None)|(spec.Basic.Get,
spec.Basic.Properties,
str or unicode)
"""
# Sanity check one, are we closing?
if self._closing:
return None, None, None
# Sanity check two, are we open, or reconnecting?
if not self._open:
return None, None, None
try:
return self._channel.basic_get(queue=self._queue_name,
no_ack=False)
except ConnectionClosed as cc:
LOG.warning(_LW("Attempted to get message on closed connection."))
LOG.debug(cc)
self._open = False
self._reconnect()
return None, None, None

0
radar/openstack/__init__.py Normal file
View File

2
radar/openstack/common/__init__.py Normal file
View File

@ -0,0 +1,2 @@
import six
six.add_move(six.MovedModule('mox', 'mox', 'mox3.mox'))

474
radar/openstack/common/gettextutils.py Normal file
View File

@ -0,0 +1,474 @@
# Copyright 2012 Red Hat, Inc.
# Copyright 2013 IBM Corp.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
gettext for openstack-common modules.
Usual usage in an openstack.common module:
from radar.openstack.common.gettextutils import _
"""
import copy
import functools
import gettext
import locale
from logging import handlers
import os
import re
from babel import localedata
import six
_localedir = os.environ.get('radar'.upper() + '_LOCALEDIR')
_t = gettext.translation('radar', localedir=_localedir, fallback=True)
# We use separate translation catalogs for each log level, so set up a
# mapping between the log level name and the translator. The domain
# for the log level is project_name + "-log-" + log_level so messages
# for each level end up in their own catalog.
_t_log_levels = dict(
(level, gettext.translation('radar' + '-log-' + level,
localedir=_localedir,
fallback=True))
for level in ['info', 'warning', 'error', 'critical']
)
_AVAILABLE_LANGUAGES = {}
USE_LAZY = False
def enable_lazy():
"""Convenience function for configuring _() to use lazy gettext
Call this at the start of execution to enable the gettextutils._
function to use lazy gettext functionality. This is useful if
your project is importing _ directly instead of using the
gettextutils.install() way of importing the _ function.
"""
global USE_LAZY
USE_LAZY = True
def _(msg):
if USE_LAZY:
return Message(msg, domain='radar')
else:
if six.PY3:
return _t.gettext(msg)
return _t.ugettext(msg)
def _log_translation(msg, level):
"""Build a single translation of a log message
"""
if USE_LAZY:
return Message(msg, domain='radar' + '-log-' + level)
else:
translator = _t_log_levels[level]
if six.PY3:
return translator.gettext(msg)
return translator.ugettext(msg)
# Translators for log levels.
#
# The abbreviated names are meant to reflect the usual use of a short
# name like '_'. The "L" is for "log" and the other letter comes from
# the level.
_LI = functools.partial(_log_translation, level='info')
_LW = functools.partial(_log_translation, level='warning')
_LE = functools.partial(_log_translation, level='error')
_LC = functools.partial(_log_translation, level='critical')
def install(domain, lazy=False):
"""Install a _() function using the given translation domain.
Given a translation domain, install a _() function using gettext's
install() function.
The main difference from gettext.install() is that we allow
overriding the default localedir (e.g. /usr/share/locale) using
a translation-domain-specific environment variable (e.g.
NOVA_LOCALEDIR).
:param domain: the translation domain
:param lazy: indicates whether or not to install the lazy _() function.
The lazy _() introduces a way to do deferred translation
of messages by installing a _ that builds Message objects,
instead of strings, which can then be lazily translated into
any available locale.
"""
if lazy:
# NOTE(mrodden): Lazy gettext functionality.
#
# The following introduces a deferred way to do translations on
# messages in OpenStack. We override the standard _() function
# and % (format string) operation to build Message objects that can
# later be translated when we have more information.
def _lazy_gettext(msg):
"""Create and return a Message object.
Lazy gettext function for a given domain, it is a factory method
for a project/module to get a lazy gettext function for its own
translation domain (i.e. nova, glance, cinder, etc.)
Message encapsulates a string so that we can translate
it later when needed.
"""
return Message(msg, domain=domain)
from six import moves
moves.builtins.__dict__['_'] = _lazy_gettext
else:
localedir = '%s_LOCALEDIR' % domain.upper()
if six.PY3:
gettext.install(domain,
localedir=os.environ.get(localedir))
else:
gettext.install(domain,
localedir=os.environ.get(localedir),
unicode=True)
class Message(six.text_type):
"""A Message object is a unicode object that can be translated.
Translation of Message is done explicitly using the translate() method.
For all non-translation intents and purposes, a Message is simply unicode,
and can be treated as such.
"""
def __new__(cls, msgid, msgtext=None, params=None,
domain='radar', *args):
"""Create a new Message object.
In order for translation to work gettext requires a message ID, this
msgid will be used as the base unicode text. It is also possible
for the msgid and the base unicode text to be different by passing
the msgtext parameter.
"""
# If the base msgtext is not given, we use the default translation
# of the msgid (which is in English) just in case the system locale is
# not English, so that the base text will be in that locale by default.
if not msgtext:
msgtext = Message._translate_msgid(msgid, domain)
# We want to initialize the parent unicode with the actual object that
# would have been plain unicode if 'Message' was not enabled.
msg = super(Message, cls).__new__(cls, msgtext)
msg.msgid = msgid
msg.domain = domain
msg.params = params
return msg
def translate(self, desired_locale=None):
"""Translate this message to the desired locale.
:param desired_locale: The desired locale to translate the message to,
if no locale is provided the message will be
translated to the system's default locale.
:returns: the translated message in unicode
"""
translated_message = Message._translate_msgid(self.msgid,
self.domain,
desired_locale)
if self.params is None:
# No need for more translation
return translated_message
# This Message object may have been formatted with one or more
# Message objects as substitution arguments, given either as a single
# argument, part of a tuple, or as one or more values in a dictionary.
# When translating this Message we need to translate those Messages too
translated_params = _translate_args(self.params, desired_locale)
translated_message = translated_message % translated_params
return translated_message
@staticmethod
def _translate_msgid(msgid, domain, desired_locale=None):
if not desired_locale:
system_locale = locale.getdefaultlocale()
# If the system locale is not available to the runtime use English
if not system_locale[0]:
desired_locale = 'en_US'
else:
desired_locale = system_locale[0]
locale_dir = os.environ.get(domain.upper() + '_LOCALEDIR')
lang = gettext.translation(domain,
localedir=locale_dir,
languages=[desired_locale],
fallback=True)
if six.PY3:
translator = lang.gettext
else:
translator = lang.ugettext
translated_message = translator(msgid)
return translated_message
def __mod__(self, other):
# When we mod a Message we want the actual operation to be performed
# by the parent class (i.e. unicode()), the only thing we do here is
# save the original msgid and the parameters in case of a translation
params = self._sanitize_mod_params(other)
unicode_mod = super(Message, self).__mod__(params)
modded = Message(self.msgid,
msgtext=unicode_mod,
params=params,
domain=self.domain)
return modded
def _sanitize_mod_params(self, other):
"""Sanitize the object being modded with this Message.
- Add support for modding 'None' so translation supports it
- Trim the modded object, which can be a large dictionary, to only
those keys that would actually be used in a translation
- Snapshot the object being modded, in case the message is
translated, it will be used as it was when the Message was created
"""
if other is None:
params = (other,)
elif isinstance(other, dict):
params = self._trim_dictionary_parameters(other)
else:
params = self._copy_param(other)
return params
def _trim_dictionary_parameters(self, dict_param):
"""Return a dict that only has matching entries in the msgid."""
# NOTE(luisg): Here we trim down the dictionary passed as parameters
# to avoid carrying a lot of unnecessary weight around in the message
# object, for example if someone passes in Message() % locals() but
# only some params are used, and additionally we prevent errors for
# non-deepcopyable objects by unicoding() them.
# Look for %(param) keys in msgid;
# Skip %% and deal with the case where % is first character on the line
keys = re.findall('(?:[^%]|^)?%\((\w*)\)[a-z]', self.msgid)
# If we don't find any %(param) keys but have a %s
if not keys and re.findall('(?:[^%]|^)%[a-z]', self.msgid):
# Apparently the full dictionary is the parameter
params = self._copy_param(dict_param)
else:
params = {}
# Save our existing parameters as defaults to protect
# ourselves from losing values if we are called through an
# (erroneous) chain that builds a valid Message with
# arguments, and then does something like "msg % kwds"
# where kwds is an empty dictionary.
src = {}
if isinstance(self.params, dict):
src.update(self.params)
src.update(dict_param)
for key in keys:
params[key] = self._copy_param(src[key])
return params
def _copy_param(self, param):
try:
return copy.deepcopy(param)
except TypeError:
# Fallback to casting to unicode this will handle the
# python code-like objects that can't be deep-copied
return six.text_type(param)
def __add__(self, other):
msg = _('Message objects do not support addition.')
raise TypeError(msg)
def __radd__(self, other):
return self.__add__(other)
def __str__(self):
# NOTE(luisg): Logging in python 2.6 tries to str() log records,
# and it expects specifically a UnicodeError in order to proceed.
msg = _('Message objects do not support str() because they may '
'contain non-ascii characters. '
'Please use unicode() or translate() instead.')
raise UnicodeError(msg)
def get_available_languages(domain):
"""Lists the available languages for the given translation domain.
:param domain: the domain to get languages for
"""
if domain in _AVAILABLE_LANGUAGES:
return copy.copy(_AVAILABLE_LANGUAGES[domain])
localedir = '%s_LOCALEDIR' % domain.upper()
find = lambda x: gettext.find(domain,
localedir=os.environ.get(localedir),
languages=[x])
# NOTE(mrodden): en_US should always be available (and first in case
# order matters) since our in-line message strings are en_US
language_list = ['en_US']
# NOTE(luisg): Babel <1.0 used a function called list(), which was
# renamed to locale_identifiers() in >=1.0, the requirements master list
# requires >=0.9.6, uncapped, so defensively work with both. We can remove
# this check when the master list updates to >=1.0, and update all projects
list_identifiers = (getattr(localedata, 'list', None) or
getattr(localedata, 'locale_identifiers'))
locale_identifiers = list_identifiers()
for i in locale_identifiers:
if find(i) is not None:
language_list.append(i)
# NOTE(luisg): Babel>=1.0,<1.3 has a bug where some OpenStack supported
# locales (e.g. 'zh_CN', and 'zh_TW') aren't supported even though they
# are perfectly legitimate locales:
# https://github.com/mitsuhiko/babel/issues/37
# In Babel 1.3 they fixed the bug and they support these locales, but
# they are still not explicitly "listed" by locale_identifiers().
# That is why we add the locales here explicitly if necessary so that
# they are listed as supported.
aliases = {'zh': 'zh_CN',
'zh_Hant_HK': 'zh_HK',
'zh_Hant': 'zh_TW',
'fil': 'tl_PH'}
for (locale, alias) in six.iteritems(aliases):
if locale in language_list and alias not in language_list:
language_list.append(alias)
_AVAILABLE_LANGUAGES[domain] = language_list
return copy.copy(language_list)
def translate(obj, desired_locale=None):
"""Gets the translated unicode representation of the given object.
If the object is not translatable it is returned as-is.
If the locale is None the object is translated to the system locale.
:param obj: the object to translate
:param desired_locale: the locale to translate the message to, if None the
default system locale will be used
:returns: the translated object in unicode, or the original object if
it could not be translated
"""
message = obj
if not isinstance(message, Message):
# If the object to translate is not already translatable,
# let's first get its unicode representation
message = six.text_type(obj)
if isinstance(message, Message):
# Even after unicoding() we still need to check if we are
# running with translatable unicode before translating
return message.translate(desired_locale)
return obj
def _translate_args(args, desired_locale=None):
"""Translates all the translatable elements of the given arguments object.
This method is used for translating the translatable values in method
arguments which include values of tuples or dictionaries.
If the object is not a tuple or a dictionary the object itself is
translated if it is translatable.
If the locale is None the object is translated to the system locale.
:param args: the args to translate
:param desired_locale: the locale to translate the args to, if None the
default system locale will be used
:returns: a new args object with the translated contents of the original
"""
if isinstance(args, tuple):
return tuple(translate(v, desired_locale) for v in args)
if isinstance(args, dict):
translated_dict = {}
for (k, v) in six.iteritems(args):
translated_v = translate(v, desired_locale)
translated_dict[k] = translated_v
return translated_dict
return translate(args, desired_locale)
class TranslationHandler(handlers.MemoryHandler):
"""Handler that translates records before logging them.
The TranslationHandler takes a locale and a target logging.Handler object
to forward LogRecord objects to after translating them. This handler
depends on Message objects being logged, instead of regular strings.
The handler can be configured declaratively in the logging.conf as follows:
[handlers]
keys = translatedlog, translator
[handler_translatedlog]
class = handlers.WatchedFileHandler
args = ('/var/log/api-localized.log',)
formatter = context
[handler_translator]
class = openstack.common.log.TranslationHandler
target = translatedlog
args = ('zh_CN',)
If the specified locale is not available in the system, the handler will
log in the default locale.
"""
def __init__(self, locale=None, target=None):
"""Initialize a TranslationHandler
:param locale: locale to use for translating messages
:param target: logging.Handler object to forward
LogRecord objects to after translation
"""
# NOTE(luisg): In order to allow this handler to be a wrapper for
# other handlers, such as a FileHandler, and still be able to
# configure it using logging.conf, this handler has to extend
# MemoryHandler because only the MemoryHandlers' logging.conf
# parsing is implemented such that it accepts a target handler.
handlers.MemoryHandler.__init__(self, capacity=0, target=target)
self.locale = locale
def setFormatter(self, fmt):
self.target.setFormatter(fmt)
def emit(self, record):
# We save the message from the original record to restore it
# after translation, so other handlers are not affected by this
original_msg = record.msg
original_args = record.args
try:
self._translate_and_log_record(record)
finally:
record.msg = original_msg
record.args = original_args
def _translate_and_log_record(self, record):
record.msg = translate(record.msg, self.locale)
# In addition to translating the message, we also need to translate
# arguments that were passed to the log method that were not part
# of the main message e.g., log.info(_('Some message %s'), this_one))
record.args = _translate_args(record.args, self.locale)
self.target.emit(record)

73
radar/openstack/common/importutils.py Normal file
View File

@ -0,0 +1,73 @@
# Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Import related utilities and helper functions.
"""
import sys
import traceback
def import_class(import_str):
"""Returns a class from a string including module and class."""
mod_str, _sep, class_str = import_str.rpartition('.')
try:
__import__(mod_str)
return getattr(sys.modules[mod_str], class_str)
except (ValueError, AttributeError):
raise ImportError('Class %s cannot be found (%s)' %
(class_str,
traceback.format_exception(*sys.exc_info())))
def import_object(import_str, *args, **kwargs):
"""Import a class and return an instance of it."""
return import_class(import_str)(*args, **kwargs)
def import_object_ns(name_space, import_str, *args, **kwargs):
"""Tries to import object from default namespace.
Imports a class and return an instance of it, first by trying
to find the class in a default namespace, then failing back to
a full path if not found in the default namespace.
"""
import_value = "%s.%s" % (name_space, import_str)
try:
return import_class(import_value)(*args, **kwargs)
except ImportError:
return import_class(import_str)(*args, **kwargs)
def import_module(import_str):
"""Import a module."""
__import__(import_str)
return sys.modules[import_str]
def import_versioned_module(version, submodule=None):
module = 'radar.v%s' % version
if submodule:
module = '.'.join((module, submodule))
return import_module(module)
def try_import(import_str, default=None):
"""Try to import a module and if it fails return default."""
try:
return import_module(import_str)
except ImportError:
return default

174
radar/openstack/common/jsonutils.py Normal file
View File

@ -0,0 +1,174 @@
# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# Copyright 2011 Justin Santa Barbara
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
'''
JSON related utilities.
This module provides a few things:
1) A handy function for getting an object down to something that can be
JSON serialized. See to_primitive().
2) Wrappers around loads() and dumps(). The dumps() wrapper will
automatically use to_primitive() for you if needed.
3) This sets up anyjson to use the loads() and dumps() wrappers if anyjson
is available.
'''
import datetime
import functools
import inspect
import itertools
import json
import six
import six.moves.xmlrpc_client as xmlrpclib
from radar.openstack.common import gettextutils
from radar.openstack.common import importutils
from radar.openstack.common import timeutils
netaddr = importutils.try_import("netaddr")
_nasty_type_tests = [inspect.ismodule, inspect.isclass, inspect.ismethod,
inspect.isfunction, inspect.isgeneratorfunction,
inspect.isgenerator, inspect.istraceback, inspect.isframe,
inspect.iscode, inspect.isbuiltin, inspect.isroutine,
inspect.isabstract]
_simple_types = (six.string_types + six.integer_types
+ (type(None), bool, float))
def to_primitive(value, convert_instances=False, convert_datetime=True,
level=0, max_depth=3):
"""Convert a complex object into primitives.
Handy for JSON serialization. We can optionally handle instances,
but since this is a recursive function, we could have cyclical
data structures.
To handle cyclical data structures we could track the actual objects
visited in a set, but not all objects are hashable. Instead we just
track the depth of the object inspections and don't go too deep.
Therefore, convert_instances=True is lossy ... be aware.
"""
# handle obvious types first - order of basic types determined by running
# full tests on nova project, resulting in the following counts:
# 572754 <type 'NoneType'>
# 460353 <type 'int'>
# 379632 <type 'unicode'>
# 274610 <type 'str'>
# 199918 <type 'dict'>
# 114200 <type 'datetime.datetime'>
# 51817 <type 'bool'>
# 26164 <type 'list'>
# 6491 <type 'float'>
# 283 <type 'tuple'>
# 19 <type 'long'>
if isinstance(value, _simple_types):
return value
if isinstance(value, datetime.datetime):
if convert_datetime:
return timeutils.strtime(value)
else:
return value
# value of itertools.count doesn't get caught by nasty_type_tests
# and results in infinite loop when list(value) is called.
if type(value) == itertools.count:
return six.text_type(value)
# FIXME(vish): Workaround for LP bug 852095. Without this workaround,
# tests that raise an exception in a mocked method that
# has a @wrap_exception with a notifier will fail. If
# we up the dependency to 0.5.4 (when it is released) we
# can remove this workaround.
if getattr(value, '__module__', None) == 'mox':
return 'mock'
if level > max_depth:
return '?'
# The try block may not be necessary after the class check above,
# but just in case ...
try:
recursive = functools.partial(to_primitive,
convert_instances=convert_instances,
convert_datetime=convert_datetime,
level=level,
max_depth=max_depth)
if isinstance(value, dict):
return dict((k, recursive(v)) for k, v in six.iteritems(value))
elif isinstance(value, (list, tuple)):
return [recursive(lv) for lv in value]
# It's not clear why xmlrpclib created their own DateTime type, but
# for our purposes, make it a datetime type which is explicitly
# handled
if isinstance(value, xmlrpclib.DateTime):
value = datetime.datetime(*tuple(value.timetuple())[:6])
if convert_datetime and isinstance(value, datetime.datetime):
return timeutils.strtime(value)
elif isinstance(value, gettextutils.Message):
return value.data
elif hasattr(value, 'iteritems'):
return recursive(dict(value.iteritems()), level=level + 1)
elif hasattr(value, '__iter__'):
return recursive(list(value))
elif convert_instances and hasattr(value, '__dict__'):
# Likely an instance of something. Watch for cycles.
# Ignore class member vars.
return recursive(value.__dict__, level=level + 1)
elif netaddr and isinstance(value, netaddr.IPAddress):
return six.text_type(value)
else:
if any(test(value) for test in _nasty_type_tests):
return six.text_type(value)
return value
except TypeError:
# Class objects are tricky since they may define something like
# __iter__ defined but it isn't callable as list().
return six.text_type(value)
def dumps(value, default=to_primitive, **kwargs):
return json.dumps(value, default=default, **kwargs)
def loads(s):
return json.loads(s)
def load(s):
return json.load(s)
try:
import anyjson
except ImportError:
pass
else:
anyjson._modules.append((__name__, 'dumps', TypeError,
'loads', ValueError, 'load'))
anyjson.force_implementation(__name__)

45
radar/openstack/common/local.py Normal file
View File

@ -0,0 +1,45 @@
# Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Local storage of variables using weak references"""
import threading
import weakref
class WeakLocal(threading.local):
def __getattribute__(self, attr):
rval = super(WeakLocal, self).__getattribute__(attr)
if rval:
# NOTE(mikal): this bit is confusing. What is stored is a weak
# reference, not the value itself. We therefore need to lookup
# the weak reference and return the inner value here.
rval = rval()
return rval
def __setattr__(self, attr, value):
value = weakref.ref(value)
return super(WeakLocal, self).__setattr__(attr, value)
# NOTE(mikal): the name "store" should be deprecated in the future
store = WeakLocal()
# A "weak" store uses weak references and allows an object to fall out of scope
# when it falls out of scope in the code that uses the thread local storage. A
# "strong" store will hold a reference to the object so that it never falls out
# of scope.
weak_store = WeakLocal()
strong_store = threading.local()

712
radar/openstack/common/log.py Normal file
View File

@ -0,0 +1,712 @@
# Copyright 2011 OpenStack Foundation.
# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""OpenStack logging handler.
This module adds to logging functionality by adding the option to specify
a context object when calling the various log methods. If the context object
is not specified, default formatting is used. Additionally, an instance uuid
may be passed as part of the log message, which is intended to make it easier
for admins to find messages related to a specific instance.
It also allows setting of formatting information through conf.
"""
import inspect
import itertools
import logging
import logging.config
import logging.handlers
import os
import re
import sys
import traceback
from oslo.config import cfg
import six
from six import moves
from radar.openstack.common.gettextutils import _
from radar.openstack.common import importutils
from radar.openstack.common import jsonutils
from radar.openstack.common import local
_DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password']
# NOTE(ldbragst): Let's build a list of regex objects using the list of
# _SANITIZE_KEYS we already have. This way, we only have to add the new key
# to the list of _SANITIZE_KEYS and we can generate regular expressions
# for XML and JSON automatically.
_SANITIZE_PATTERNS = []
_FORMAT_PATTERNS = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])',
r'(<%(key)s>).*?(</%(key)s>)',
r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])',
r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])']
for key in _SANITIZE_KEYS:
for pattern in _FORMAT_PATTERNS:
reg_ex = re.compile(pattern % {'key': key}, re.DOTALL)
_SANITIZE_PATTERNS.append(reg_ex)
common_cli_opts = [
cfg.BoolOpt('debug',
short='d',
default=False,
help='Print debugging output (set logging level to '
'DEBUG instead of default WARNING level).'),
cfg.BoolOpt('verbose',
short='v',
default=False,
help='Print more verbose output (set logging level to '
'INFO instead of default WARNING level).'),
]
logging_cli_opts = [
cfg.StrOpt('log-config-append',
metavar='PATH',
deprecated_name='log-config',
help='The name of logging configuration file. It does not '
'disable existing loggers, but just appends specified '
'logging configuration to any other existing logging '
'options. Please see the Python logging module '
'documentation for details on logging configuration '
'files.'),
cfg.StrOpt('log-format',
default=None,
metavar='FORMAT',
help='DEPRECATED. '
'A logging.Formatter log message format string which may '
'use any of the available logging.LogRecord attributes. '
'This option is deprecated. Please use '
'logging_context_format_string and '
'logging_default_format_string instead.'),
cfg.StrOpt('log-date-format',
default=_DEFAULT_LOG_DATE_FORMAT,
metavar='DATE_FORMAT',
help='Format string for %%(asctime)s in log records. '
'Default: %(default)s'),
cfg.StrOpt('log-file',
metavar='PATH',
deprecated_name='logfile',
help='(Optional) Name of log file to output to. '
'If no default is set, logging will go to stdout.'),
cfg.StrOpt('log-dir',
deprecated_name='logdir',
help='(Optional) The base directory used for relative '
'--log-file paths'),
cfg.BoolOpt('use-syslog',
default=False,
help='Use syslog for logging. '
'Existing syslog format is DEPRECATED during I, '
'and then will be changed in J to honor RFC5424'),
cfg.BoolOpt('use-syslog-rfc-format',
# TODO(bogdando) remove or use True after existing
# syslog format deprecation in J
default=False,
help='(Optional) Use syslog rfc5424 format for logging. '
'If enabled, will add APP-NAME (RFC5424) before the '
'MSG part of the syslog message. The old format '
'without APP-NAME is deprecated in I, '
'and will be removed in J.'),
cfg.StrOpt('syslog-log-facility',
default='LOG_USER',
help='Syslog facility to receive log lines')
]
generic_log_opts = [
cfg.BoolOpt('use_stderr',
default=True,
help='Log output to standard error')
]
log_opts = [
cfg.StrOpt('logging_context_format_string',
default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s '
'%(name)s [%(request_id)s %(user_identity)s] '
'%(instance)s%(message)s',
help='Format string to use for log messages with context'),
cfg.StrOpt('logging_default_format_string',
default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s '
'%(name)s [-] %(instance)s%(message)s',
help='Format string to use for log messages without context'),
cfg.StrOpt('logging_debug_format_suffix',
default='%(funcName)s %(pathname)s:%(lineno)d',
help='Data to append to log format when level is DEBUG'),
cfg.StrOpt('logging_exception_prefix',
default='%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s '
'%(instance)s',
help='Prefix each line of exception output with this format'),
cfg.ListOpt('default_log_levels',
default=[
'amqp=WARN',
'amqplib=WARN',
'boto=WARN',
'qpid=WARN',
'sqlalchemy=WARN',
'suds=INFO',
'iso8601=WARN',
'requests.packages.urllib3.connectionpool=WARN'
],
help='List of logger=LEVEL pairs'),
cfg.BoolOpt('publish_errors',
default=False,
help='Publish error events'),
cfg.BoolOpt('fatal_deprecations',
default=False,
help='Make deprecations fatal'),
# NOTE(mikal): there are two options here because sometimes we are handed
# a full instance (and could include more information), and other times we
# are just handed a UUID for the instance.
cfg.StrOpt('instance_format',
default='[instance: %(uuid)s] ',
help='If an instance is passed with the log message, format '
'it like this'),
cfg.StrOpt('instance_uuid_format',
default='[instance: %(uuid)s] ',
help='If an instance UUID is passed with the log message, '
'format it like this'),
]
CONF = cfg.CONF
CONF.register_cli_opts(common_cli_opts)
CONF.register_cli_opts(logging_cli_opts)
CONF.register_opts(generic_log_opts)
CONF.register_opts(log_opts)
# our new audit level
# NOTE(jkoelker) Since we synthesized an audit level, make the logging
# module aware of it so it acts like other levels.
logging.AUDIT = logging.INFO + 1
logging.addLevelName(logging.AUDIT, 'AUDIT')
try:
NullHandler = logging.NullHandler
except AttributeError: # NOTE(jkoelker) NullHandler added in Python 2.7
class NullHandler(logging.Handler):
def handle(self, record):
pass
def emit(self, record):
pass
def createLock(self):
self.lock = None
def _dictify_context(context):
if context is None:
return None
if not isinstance(context, dict) and getattr(context, 'to_dict', None):
context = context.to_dict()
return context
def _get_binary_name():
return os.path.basename(inspect.stack()[-1][1])
def _get_log_file_path(binary=None):
logfile = CONF.log_file
logdir = CONF.log_dir
if logfile and not logdir:
return logfile
if logfile and logdir:
return os.path.join(logdir, logfile)
if logdir:
binary = binary or _get_binary_name()
return '%s.log' % (os.path.join(logdir, binary),)
return None
def mask_password(message, secret="***"):
"""Replace password with 'secret' in message.
:param message: The string which includes security information.
:param secret: value with which to replace passwords.
:returns: The unicode value of message with the password fields masked.
For example:
>>> mask_password("'adminPass' : 'aaaaa'")
"'adminPass' : '***'"
>>> mask_password("'admin_pass' : 'aaaaa'")
"'admin_pass' : '***'"
>>> mask_password('"password" : "aaaaa"')
'"password" : "***"'
>>> mask_password("'original_password' : 'aaaaa'")
"'original_password' : '***'"
>>> mask_password("u'original_password' : u'aaaaa'")
"u'original_password' : u'***'"
"""
message = six.text_type(message)
# NOTE(ldbragst): Check to see if anything in message contains any key
# specified in _SANITIZE_KEYS, if not then just return the message since
# we don't have to mask any passwords.
if not any(key in message for key in _SANITIZE_KEYS):
return message
secret = r'\g<1>' + secret + r'\g<2>'
for pattern in _SANITIZE_PATTERNS:
message = re.sub(pattern, secret, message)
return message
class BaseLoggerAdapter(logging.LoggerAdapter):
def audit(self, msg, *args, **kwargs):
self.log(logging.AUDIT, msg, *args, **kwargs)
class LazyAdapter(BaseLoggerAdapter):
def __init__(self, name='unknown', version='unknown'):
self._logger = None
self.extra = {}
self.name = name
self.version = version
@property
def logger(self):
if not self._logger:
self._logger = getLogger(self.name, self.version)
return self._logger
class ContextAdapter(BaseLoggerAdapter):
warn = logging.LoggerAdapter.warning
def __init__(self, logger, project_name, version_string):
self.logger = logger
self.project = project_name
self.version = version_string
self._deprecated_messages_sent = dict()
@property
def handlers(self):
return self.logger.handlers
def deprecated(self, msg, *args, **kwargs):
"""Call this method when a deprecated feature is used.
If the system is configured for fatal deprecations then the message
is logged at the 'critical' level and :class:`DeprecatedConfig` will
be raised.
Otherwise, the message will be logged (once) at the 'warn' level.
:raises: :class:`DeprecatedConfig` if the system is configured for
fatal deprecations.
"""
stdmsg = _("Deprecated: %s") % msg
if CONF.fatal_deprecations:
self.critical(stdmsg, *args, **kwargs)
raise DeprecatedConfig(msg=stdmsg)
# Using a list because a tuple with dict can't be stored in a set.
sent_args = self._deprecated_messages_sent.setdefault(msg, list())
if args in sent_args:
# Already logged this message, so don't log it again.
return
sent_args.append(args)
self.warn(stdmsg, *args, **kwargs)
def process(self, msg, kwargs):
# NOTE(mrodden): catch any Message/other object and
# coerce to unicode before they can get
# to the python logging and possibly
# cause string encoding trouble
if not isinstance(msg, six.string_types):
msg = six.text_type(msg)
if 'extra' not in kwargs:
kwargs['extra'] = {}
extra = kwargs['extra']
context = kwargs.pop('context', None)
if not context:
context = getattr(local.store, 'context', None)
if context:
extra.update(_dictify_context(context))
instance = kwargs.pop('instance', None)
instance_uuid = (extra.get('instance_uuid') or
kwargs.pop('instance_uuid', None))
instance_extra = ''
if instance:
instance_extra = CONF.instance_format % instance
elif instance_uuid:
instance_extra = (CONF.instance_uuid_format
% {'uuid': instance_uuid})
extra['instance'] = instance_extra
extra.setdefault('user_identity', kwargs.pop('user_identity', None))
extra['project'] = self.project
extra['version'] = self.version
extra['extra'] = extra.copy()
return msg, kwargs
class JSONFormatter(logging.Formatter):
def __init__(self, fmt=None, datefmt=None):
# NOTE(jkoelker) we ignore the fmt argument, but its still there
# since logging.config.fileConfig passes it.
self.datefmt = datefmt
def formatException(self, ei, strip_newlines=True):
lines = traceback.format_exception(*ei)
if strip_newlines:
lines = [moves.filter(
lambda x: x,
line.rstrip().splitlines()) for line in lines]
lines = list(itertools.chain(*lines))
return lines
def format(self, record):
message = {'message': record.getMessage(),
'asctime': self.formatTime(record, self.datefmt),
'name': record.name,
'msg': record.msg,
'args': record.args,
'levelname': record.levelname,
'levelno': record.levelno,
'pathname': record.pathname,
'filename': record.filename,
'module': record.module,
'lineno': record.lineno,
'funcname': record.funcName,
'created': record.created,
'msecs': record.msecs,
'relative_created': record.relativeCreated,
'thread': record.thread,
'thread_name': record.threadName,
'process_name': record.processName,
'process': record.process,
'traceback': None}
if hasattr(record, 'extra'):
message['extra'] = record.extra
if record.exc_info:
message['traceback'] = self.formatException(record.exc_info)
return jsonutils.dumps(message)
def _create_logging_excepthook(product_name):
def logging_excepthook(exc_type, value, tb):
extra = {}
if CONF.verbose or CONF.debug:
extra['exc_info'] = (exc_type, value, tb)
getLogger(product_name).critical(
"".join(traceback.format_exception_only(exc_type, value)),
**extra)
return logging_excepthook
class LogConfigError(Exception):
message = _('Error loading logging config %(log_config)s: %(err_msg)s')
def __init__(self, log_config, err_msg):
self.log_config = log_config
self.err_msg = err_msg
def __str__(self):
return self.message % dict(log_config=self.log_config,
err_msg=self.err_msg)
def _load_log_config(log_config_append):
try:
logging.config.fileConfig(log_config_append,
disable_existing_loggers=False)
except moves.configparser.Error as exc:
raise LogConfigError(log_config_append, str(exc))
def setup(product_name, version='unknown'):
"""Setup logging."""
if CONF.log_config_append:
_load_log_config(CONF.log_config_append)
else:
_setup_logging_from_conf(product_name, version)
sys.excepthook = _create_logging_excepthook(product_name)
def set_defaults(logging_context_format_string):
cfg.set_defaults(log_opts,
logging_context_format_string=
logging_context_format_string)
def _find_facility_from_conf():
facility_names = logging.handlers.SysLogHandler.facility_names
facility = getattr(logging.handlers.SysLogHandler,
CONF.syslog_log_facility,
None)
if facility is None and CONF.syslog_log_facility in facility_names:
facility = facility_names.get(CONF.syslog_log_facility)
if facility is None:
valid_facilities = facility_names.keys()
consts = ['LOG_AUTH', 'LOG_AUTHPRIV', 'LOG_CRON', 'LOG_DAEMON',
'LOG_FTP', 'LOG_KERN', 'LOG_LPR', 'LOG_MAIL', 'LOG_NEWS',
'LOG_AUTH', 'LOG_SYSLOG', 'LOG_USER', 'LOG_UUCP',
'LOG_LOCAL0', 'LOG_LOCAL1', 'LOG_LOCAL2', 'LOG_LOCAL3',
'LOG_LOCAL4', 'LOG_LOCAL5', 'LOG_LOCAL6', 'LOG_LOCAL7']
valid_facilities.extend(consts)
raise TypeError(_('syslog facility must be one of: %s') %
', '.join("'%s'" % fac
for fac in valid_facilities))
return facility
class RFCSysLogHandler(logging.handlers.SysLogHandler):
def __init__(self, *args, **kwargs):
self.binary_name = _get_binary_name()
super(RFCSysLogHandler, self).__init__(*args, **kwargs)
def format(self, record):
msg = super(RFCSysLogHandler, self).format(record)
msg = self.binary_name + ' ' + msg
return msg
def _setup_logging_from_conf(project, version):
log_root = getLogger(None).logger
for handler in log_root.handlers:
log_root.removeHandler(handler)
if CONF.use_syslog:
facility = _find_facility_from_conf()
# TODO(bogdando) use the format provided by RFCSysLogHandler
# after existing syslog format deprecation in J
if CONF.use_syslog_rfc_format:
syslog = RFCSysLogHandler(address='/dev/log',
facility=facility)
else:
syslog = logging.handlers.SysLogHandler(address='/dev/log',
facility=facility)
log_root.addHandler(syslog)
logpath = _get_log_file_path()
if logpath:
filelog = logging.handlers.WatchedFileHandler(logpath)
log_root.addHandler(filelog)
if CONF.use_stderr:
streamlog = ColorHandler()
log_root.addHandler(streamlog)
elif not logpath:
# pass sys.stdout as a positional argument
# python2.6 calls the argument strm, in 2.7 it's stream
streamlog = logging.StreamHandler(sys.stdout)
log_root.addHandler(streamlog)
if CONF.publish_errors:
handler = importutils.import_object(
"radar.openstack.common.log_handler.PublishErrorsHandler",
logging.ERROR)
log_root.addHandler(handler)
datefmt = CONF.log_date_format
for handler in log_root.handlers:
# NOTE(alaski): CONF.log_format overrides everything currently. This
# should be deprecated in favor of context aware formatting.
if CONF.log_format:
handler.setFormatter(logging.Formatter(fmt=CONF.log_format,
datefmt=datefmt))
log_root.info('Deprecated: log_format is now deprecated and will '
'be removed in the next release')
else:
handler.setFormatter(ContextFormatter(project=project,
version=version,
datefmt=datefmt))
if CONF.debug:
log_root.setLevel(logging.DEBUG)
elif CONF.verbose:
log_root.setLevel(logging.INFO)
else:
log_root.setLevel(logging.WARNING)
for pair in CONF.default_log_levels:
mod, _sep, level_name = pair.partition('=')
level = logging.getLevelName(level_name)
logger = logging.getLogger(mod)
logger.setLevel(level)
_loggers = {}
def getLogger(name='unknown', version='unknown'):
if name not in _loggers:
_loggers[name] = ContextAdapter(logging.getLogger(name),
name,
version)
return _loggers[name]
def getLazyLogger(name='unknown', version='unknown'):
"""Returns lazy logger.
Creates a pass-through logger that does not create the real logger
until it is really needed and delegates all calls to the real logger
once it is created.
"""
return LazyAdapter(name, version)
class WritableLogger(object):
"""A thin wrapper that responds to `write` and logs."""
def __init__(self, logger, level=logging.INFO):
self.logger = logger
self.level = level
def write(self, msg):
self.logger.log(self.level, msg.rstrip())
class ContextFormatter(logging.Formatter):
"""A context.RequestContext aware formatter configured through flags.
The flags used to set format strings are: logging_context_format_string
and logging_default_format_string. You can also specify
logging_debug_format_suffix to append extra formatting if the log level is
debug.
For information about what variables are available for the formatter see:
http://docs.python.org/library/logging.html#formatter
If available, uses the context value stored in TLS - local.store.context
"""
def __init__(self, *args, **kwargs):
"""Initialize ContextFormatter instance
Takes additional keyword arguments which can be used in the message
format string.
:keyword project: project name
:type project: string
:keyword version: project version
:type version: string
"""
self.project = kwargs.pop('project', 'unknown')
self.version = kwargs.pop('version', 'unknown')
logging.Formatter.__init__(self, *args, **kwargs)
def format(self, record):
"""Uses contextstring if request_id is set, otherwise default."""
# store project info
record.project = self.project
record.version = self.version
# store request info
context = getattr(local.store, 'context', None)
if context:
d = _dictify_context(context)
for k, v in d.items():
setattr(record, k, v)
# NOTE(sdague): default the fancier formatting params
# to an empty string so we don't throw an exception if
# they get used
for key in ('instance', 'color'):
if key not in record.__dict__:
record.__dict__[key] = ''
if record.__dict__.get('request_id'):
self._fmt = CONF.logging_context_format_string
else:
self._fmt = CONF.logging_default_format_string
if (record.levelno == logging.DEBUG and
CONF.logging_debug_format_suffix):
self._fmt += " " + CONF.logging_debug_format_suffix
# Cache this on the record, Logger will respect our formatted copy
if record.exc_info:
record.exc_text = self.formatException(record.exc_info, record)
return logging.Formatter.format(self, record)
def formatException(self, exc_info, record=None):
"""Format exception output with CONF.logging_exception_prefix."""
if not record:
return logging.Formatter.formatException(self, exc_info)
stringbuffer = moves.StringIO()
traceback.print_exception(exc_info[0], exc_info[1], exc_info[2],
None, stringbuffer)
lines = stringbuffer.getvalue().split('\n')
stringbuffer.close()
if CONF.logging_exception_prefix.find('%(asctime)') != -1:
record.asctime = self.formatTime(record, self.datefmt)
formatted_lines = []
for line in lines:
pl = CONF.logging_exception_prefix % record.__dict__
fl = '%s%s' % (pl, line)
formatted_lines.append(fl)
return '\n'.join(formatted_lines)
class ColorHandler(logging.StreamHandler):
LEVEL_COLORS = {
logging.DEBUG: '\033[00;32m', # GREEN
logging.INFO: '\033[00;36m', # CYAN
logging.AUDIT: '\033[01;36m', # BOLD CYAN
logging.WARN: '\033[01;33m', # BOLD YELLOW
logging.ERROR: '\033[01;31m', # BOLD RED
logging.CRITICAL: '\033[01;31m', # BOLD RED
}
def format(self, record):
record.color = self.LEVEL_COLORS[record.levelno]
return logging.StreamHandler.format(self, record)
class DeprecatedConfig(Exception):
message = _("Fatal call to deprecated config: %(msg)s")
def __init__(self, msg):
super(Exception, self).__init__(self.message % dict(msg=msg))

210
radar/openstack/common/timeutils.py Normal file
View File

@ -0,0 +1,210 @@
# Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Time related utilities and helper functions.
"""
import calendar
import datetime
import time
import iso8601
import six
# ISO 8601 extended time format with microseconds
_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f'
_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
PERFECT_TIME_FORMAT = _ISO8601_TIME_FORMAT_SUBSECOND
def isotime(at=None, subsecond=False):
"""Stringify time in ISO 8601 format."""
if not at:
at = utcnow()
st = at.strftime(_ISO8601_TIME_FORMAT
if not subsecond
else _ISO8601_TIME_FORMAT_SUBSECOND)
tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC'
st += ('Z' if tz == 'UTC' else tz)
return st
def parse_isotime(timestr):
"""Parse time from ISO 8601 format."""
try:
return iso8601.parse_date(timestr)
except iso8601.ParseError as e:
raise ValueError(six.text_type(e))
except TypeError as e:
raise ValueError(six.text_type(e))
def strtime(at=None, fmt=PERFECT_TIME_FORMAT):
"""Returns formatted utcnow."""
if not at:
at = utcnow()
return at.strftime(fmt)
def parse_strtime(timestr, fmt=PERFECT_TIME_FORMAT):
"""Turn a formatted time back into a datetime."""
return datetime.datetime.strptime(timestr, fmt)
def normalize_time(timestamp):
"""Normalize time in arbitrary timezone to UTC naive object."""
offset = timestamp.utcoffset()
if offset is None:
return timestamp
return timestamp.replace(tzinfo=None) - offset
def is_older_than(before, seconds):
"""Return True if before is older than seconds."""
if isinstance(before, six.string_types):
before = parse_strtime(before).replace(tzinfo=None)
else:
before = before.replace(tzinfo=None)
return utcnow() - before > datetime.timedelta(seconds=seconds)
def is_newer_than(after, seconds):
"""Return True if after is newer than seconds."""
if isinstance(after, six.string_types):
after = parse_strtime(after).replace(tzinfo=None)
else:
after = after.replace(tzinfo=None)
return after - utcnow() > datetime.timedelta(seconds=seconds)
def utcnow_ts():
"""Timestamp version of our utcnow function."""
if utcnow.override_time is None:
# NOTE(kgriffs): This is several times faster
# than going through calendar.timegm(...)
return int(time.time())
return calendar.timegm(utcnow().timetuple())
def utcnow():
"""Overridable version of utils.utcnow."""
if utcnow.override_time:
try:
return utcnow.override_time.pop(0)
except AttributeError:
return utcnow.override_time
return datetime.datetime.utcnow()
def iso8601_from_timestamp(timestamp):
"""Returns a iso8601 formatted date from timestamp."""
return isotime(datetime.datetime.utcfromtimestamp(timestamp))
utcnow.override_time = None
def set_time_override(override_time=None):
"""Overrides utils.utcnow.
Make it return a constant time or a list thereof, one at a time.
:param override_time: datetime instance or list thereof. If not
given, defaults to the current UTC time.
"""
utcnow.override_time = override_time or datetime.datetime.utcnow()
def advance_time_delta(timedelta):
"""Advance overridden time using a datetime.timedelta."""
assert(not utcnow.override_time is None)
try:
for dt in utcnow.override_time:
dt += timedelta
except TypeError:
utcnow.override_time += timedelta
def advance_time_seconds(seconds):
"""Advance overridden time by seconds."""
advance_time_delta(datetime.timedelta(0, seconds))
def clear_time_override():
"""Remove the overridden time."""
utcnow.override_time = None
def marshall_now(now=None):
"""Make an rpc-safe datetime with microseconds.
Note: tzinfo is stripped, but not required for relative times.
"""
if not now:
now = utcnow()
return dict(day=now.day, month=now.month, year=now.year, hour=now.hour,
minute=now.minute, second=now.second,
microsecond=now.microsecond)
def unmarshall_time(tyme):
"""Unmarshall a datetime dict."""
return datetime.datetime(day=tyme['day'],
month=tyme['month'],
year=tyme['year'],
hour=tyme['hour'],
minute=tyme['minute'],
second=tyme['second'],
microsecond=tyme['microsecond'])
def delta_seconds(before, after):
"""Return the difference between two timing objects.
Compute the difference in seconds between two date, time, or
datetime objects (as a float, to microsecond resolution).
"""
delta = after - before
return total_seconds(delta)
def total_seconds(delta):
"""Return the total seconds of datetime.timedelta object.
Compute total seconds of datetime.timedelta, datetime.timedelta
doesn't have method total_seconds in Python2.6, calculate it manually.
"""
try:
return delta.total_seconds()
except AttributeError:
return ((delta.days * 24 * 3600) + delta.seconds +
float(delta.microseconds) / (10 ** 6))
def is_soon(dt, window):
"""Determines if time is going to happen in the next window seconds.
:param dt: the time
:param window: minimum seconds to remain to consider the time not soon
:return: True if expiration is within the given duration
"""
soon = (utcnow() + datetime.timedelta(seconds=window))
return normalize_time(dt) <= soon

0
radar/plugin/__init__.py Normal file
View File

67
radar/plugin/base.py Normal file
View File

@ -0,0 +1,67 @@
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing permissions and
# limitations under the License.
import abc
import six
from oslo.config import cfg
from stevedore.enabled import EnabledExtensionManager
CONF = cfg.CONF
def is_enabled(ext):
"""Check to see whether a plugin should be enabled. Assumes that the
plugin extends PluginBase.
:param ext: The extension instance to check.
:return: True if it should be enabled. Otherwise false.
"""
return ext.obj.enabled()
@six.add_metaclass(abc.ABCMeta)
class PluginBase(object):
"""Base class for all radar plugins.
Every radar plugin will be provided an instance of the application
configuration, and will then be asked whether it should be enabled. Each
plugin should decide, given the configuration and the environment,
whether it has the necessary resources to operate properly.
"""
def __init__(self, config):
self.config = config
@abc.abstractmethod
def enabled(self):
"""A method which indicates whether this plugin is properly
configured and should be enabled. If it's ready to go, return True.
Otherwise, return False.
"""
class RadarPluginLoader(EnabledExtensionManager):
"""The radar plugin loader, a stevedore abstraction that formalizes
our plugin contract.
"""
def __init__(self, namespace, on_load_failure_callback=None):
super(RadarPluginLoader, self) \
.__init__(namespace=namespace,
check_func=is_enabled,
invoke_on_load=True,
invoke_args=(CONF,),
on_load_failure_callback=on_load_failure_callback)

68
radar/plugin/user_preferences.py Normal file
View File

@ -0,0 +1,68 @@
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing permissions and
# limitations under the License.
import abc
import six
from radar.openstack.common import log
from radar.plugin.base import PluginBase
from radar.plugin.base import RadarPluginLoader
LOG = log.getLogger(__name__)
PREFERENCE_DEFAULTS = dict()
def initialize_user_preferences():
"""Initialize any plugins that were installed via pip. This will parse
out all the default preference values into one dictionary for later
use in the API.
"""
manager = RadarPluginLoader(
namespace='radar.plugin.user_preferences')
if manager.extensions:
manager.map(load_preferences, PREFERENCE_DEFAULTS)
def load_preferences(ext, defaults):
"""Load all plugin default preferences into our cache.
:param ext: The extension that's handling this event.
:param defaults: The current dict of default preferences.
"""
plugin_defaults = ext.obj.get_default_preferences()
for key in plugin_defaults:
if key in defaults:
# Let's not error out here.
LOG.error("Duplicate preference key %s found." % (key,))
else:
defaults[key] = plugin_defaults[key]
@six.add_metaclass(abc.ABCMeta)
class UserPreferencesPluginBase(PluginBase):
"""Base class for a plugin that provides a set of expected user
preferences and their default values. By extending this plugin, you can
add preferences for your own radar plugins and workers, and have
them be manageable via your web client (Your client may need to be
customized).
"""
@abc.abstractmethod
def get_default_preferences(self):
"""Return a dictionary of preferences and their default values."""

0
radar/tasks/__init__.py Normal file
View File

54
radar/tasks/update_systems.py Normal file
View File

@ -0,0 +1,54 @@
# Copyright (c) 2014 Triniplex.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import time
from oslo.config import cfg
from pika.exceptions import ConnectionClosed
from stevedore import enabled
from radar.openstack.common import log
from radar.worker.task.process_update import ProcessCISystems
CONF = cfg.CONF
LOG = log.getLogger(__name__)
def update():
log.setup('radar')
CONF(project='radar')
updater = Updater()
updater.start()
while updater.started:
LOG.info("processing systems")
updater.systems.do_update()
LOG.info("done processing systems. Sleeping for 5 minutes.")
time.sleep(300)
continue
class Updater():
def __init__(self):
self.started = False
self.systems = ProcessCISystems()
def start(self):
self.started = True
def stop(self):
self.started = False

0
radar/worker/__init__.py Normal file
View File

148
radar/worker/daemon.py Normal file
View File

@ -0,0 +1,148 @@
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing permissions and
# limitations under the License.
import signal
from multiprocessing import Process
from threading import Timer
from oslo.config import cfg
from radar.tasks.update_systems import update
from radar.openstack.common import log
CONF = cfg.CONF
LOG = log.getLogger(__name__)
MANAGER = None
IMPORT_OPTS = [
cfg.IntOpt("worker-count",
default="1",
help="The number of workers to spawn and manage.")
]
def run():
"""Start the daemon manager.
"""
global MANAGER
log.setup('radar')
CONF.register_cli_opts(IMPORT_OPTS)
CONF(project='radar')
signal.signal(signal.SIGTERM, terminate)
signal.signal(signal.SIGINT, terminate)
MANAGER = DaemonManager(daemon_method=update,
child_process_count=CONF.worker_count)
MANAGER.start()
def terminate(signal, frame):
# This assumes that all the child processes will terminate gracefully
# on a SIGINT
global MANAGER
MANAGER.stop()
# Raise SIGINT to all child processes.
signal.default_int_handler()
class DaemonManager():
"""A Daemon manager to handle multiple subprocesses.
"""
def __init__(self, child_process_count, daemon_method):
"""Create a new daemon manager with N processes running the passed
method. Once start() is called, The daemon method will be spawned N
times and continually checked/restarted until the process is
interrupted either by a system exit or keyboard interrupt.
:param child_process_count: The number of child processes to spawn.
:param daemon_method: The method to run in the child process.
"""
# Number of child procs.
self._child_process_count = child_process_count
# Process management threads.
self._procs = list()
# Save the daemon method
self._daemon_method = daemon_method
# Health check timer
self._timer = PerpetualTimer(1, self._health_check)
def _health_check(self):
processes = list(self._procs)
dead_processes = 0
for process in processes:
if not process.is_alive():
LOG.warning("Dead Process found [exit code:%d]" %
(process.exitcode,))
dead_processes += 1
self._procs.remove(process)
for i in range(dead_processes):
self._add_process()
def start(self):
"""Start the daemon manager and spawn child processes.
"""
LOG.info("Spawning %s child processes" % (self._child_process_count,))
self._timer.start()
for i in range(self._child_process_count):
self._add_process()
def stop(self):
self._timer.cancel()
processes = list(self._procs)
for process in processes:
if process.is_alive():
process.terminate()
process.join()
self._procs.remove(process)
def _add_process(self):
process = Process(target=self._daemon_method)
process.start()
self._procs.append(process)
class PerpetualTimer():
"""A timer wrapper class that repeats itself.
"""
def __init__(self, t, handler):
self.t = t
self.handler = handler
self.thread = Timer(self.t, self.handle_function)
def handle_function(self):
self.handler()
self.thread = Timer(self.t, self.handle_function)
self.thread.setDaemon(True)
self.thread.start()
def start(self):
self.thread.start()
def cancel(self):
self.thread.cancel()
if __name__ == "__main__":
run()

0
radar/worker/task/__init__.py Normal file
View File

37
radar/worker/task/base.py Normal file
View File

@ -0,0 +1,37 @@
# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing permissions and
# limitations under the License.
import abc
class WorkerTaskBase(object):
"""Base class for a worker that listens to events that occur within the
API.
"""
__metaclass__ = abc.ABCMeta
def __init__(self, config):
self.config = config
@abc.abstractmethod
def enabled(self):
"""A method which indicates whether this worker task is properly
configured and should be enabled. If it's ready to go, return True.
Otherwise, return False.
"""
@abc.abstractmethod
def handle(self, body):
"""Handle an event."""

198
radar/worker/task/process_update.py Normal file
View File

@ -0,0 +1,198 @@
import sys
import os
import urllib2
import httplib
import json
import pprint
import re
import requests
from urllib import urlencode
from datetime import datetime
from ConfigParser import ConfigParser
class ProcessCISystems():
def __init__(self):
self._requests = list()
self._responses = list()
self._uri = "https://review.openstack.org"
self._pp = pprint.PrettyPrinter(indent=4)
self._systems_resource = '/systems'
self._operators_resource = '/operators'
self.default_headers = {}
def get_credentials(self):
dashboardconfig = ConfigParser()
dashboardconfig.readfp(open("/opt/.dashboardconfig"))
username = dashboardconfig.get("review", "user")
password = dashboardconfig.get("review", "password")
passwordmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
passwordmgr.add_password(None, self._uri, username, password)
handler = urllib2.HTTPDigestAuthHandler(passwordmgr)
opener = urllib2.build_opener(handler)
urllib2.install_opener(opener)
# Make the request
self._requests.append(urllib2.Request("%s/a/groups/95d633d37a5d6b06df758e57b1370705ec071a57/members/" % (self._uri)))
self._responses.append(urllib2.urlopen(self._requests[0]))
def process_systems(self):
processed=0
for index, line in enumerate(self._responses[0]):
if index > 0:
if line.find("account_id") > -1:
cis_account_id = line[line.rfind(":")+2:line.rfind(",")]
if line.find("\"name\"") > -1:
cis_system_name = line[line.rfind(":")+3:line.rfind(",")-1]
if line.find("\"email\"") > -1:
cis_operator_email = line[line.rfind(":")+3:line.rfind(",")-1]
if line.find("\"username\"") > -1:
cis_operator_username = line[line.rfind(":")+3:line.rfind(",")-1]
print "username: %s" % cis_operator_username
system = {"name": cis_system_name}
print "Attempting to import %s" % cis_system_name
if not self.system_exists(self._systems_resource, system):
self._responses.append(self.post_json(self._systems_resource, system))
thesystem = json.loads(self._responses[1].text)
success=True
system_id=''
try:
system_id = thesystem['id']
url = "%s/%d" % (self._systems_resource, system_id)
thesystem = self.get_json(url)
except KeyError as ke:
print "System %s has already been imported" % cis_system_name
success=False
finally:
self._responses.remove(self._responses[1])
if success:
operator = {"system_id": system_id,
"operator_name": cis_operator_username,
"operator_email": cis_operator_email}
print 'updating operator: %s' % operator
self._responses.append(self.post_json(self._operators_resource, operator))
theoperator = json.loads(self._responses[1].text)
try:
operator_id = theoperator['id']
url = "%s/%d" % (self._operators_resource, operator_id)
theoperator = self.get_json(url)
except KeyError as ke:
pass
finally:
self._responses.remove(self._responses[1])
else:
system_id=''
success=True
try:
response = self.get_json(self._systems_resource + "/" + cis_system_name)
thesystem = json.loads(response.text)
system_id = thesystem['id']
except KeyError as ke:
print "Unable to retrieve system %s" % cis_system_name
success=False
if success:
print "put request for system_id: %s and system %s" % (system_id, cis_system_name)
system = { "system_id": system_id,
"name": cis_system_name,
"updated_at": datetime.utcnow().isoformat()}
self._responses.append(self.put_json(self._systems_resource, system))
thesystem = json.loads(self._responses[1].text)
success=True
system_id=''
try:
system_id = thesystem['id']
url = "%s/%d" % (self._systems_resource, system_id)
thesystem = self.get_json(url)
except KeyError as ke:
print "System %s update failed" % cis_system_name
success=False
finally:
self._responses.remove(self._responses[1])
if success:
print "system %s updated successfully" % system_id
success=True
operator_id=''
try:
response = self.get_json(self._operators_resource + "/" + cis_operator_username)
theoperator = json.loads(response.text)
operator_id = theoperator['id']
except KeyError as ke:
print "Unable to retrieve operator %s" % cis_operator_username
success=False
if success:
success=True
operator = {"operator_id": operator_id,
"operator_name": cis_operator_username,
"operator_email": cis_operator_email,
"updated_at": datetime.utcnow().isoformat()}
self._responses.append(self.put_json(self._operators_resource, operator))
theoperator = json.loads(self._responses[1].text)
try:
operator_id = theoperator['id']
operator_name = theoperator['operator_name']
url = "%s/%d" % (self._operators_resource, operator_id)
response = self.get_json(url)
except KeyError as ke:
success=False
finally:
self._responses.remove(self._responses[1])
if success:
theoperator = json.loads(response.text)
print "operator %s was updated successfully" % theoperator['operator_name']
def _request_json(self, path, params, headers=None, method="post",
status=None, path_prefix="http://10.211.55.29:8080/v1"):
merged_headers = self.default_headers.copy()
if headers:
merged_headers.update(headers)
full_path = path_prefix + path
if not headers:
headers = {'content-type': 'application/json'}
if method is "post":
response = requests.post(str(full_path), data=json.dumps(params), headers=headers)
elif method is "put":
response = requests.put(str(full_path), data=json.dumps(params), headers=headers)
else:
response = requests.get(str(full_path))
return response
def put_json(self, path, params, headers=None, status=None):
return self._request_json(path=path, params=params, headers=headers,
status=status, method="put")
def post_json(self, path, params, headers=None, status=None):
return self._request_json(path=path, params=params,
headers=headers,
status=status, method="post")
def get_json(self, path, headers=None, status=None):
return self._request_json(path=path, params=None,
headers=headers,
status=status, method="get")
def system_exists(self, path, system, headers=None, status=None):
try:
response = self.get_json(path + "/" + system['name'])
if response.status_code == requests.codes.ok:
print "%s has already been imported" % system['name']
return True
else:
return False
except KeyError as ke:
return False
def do_update(self):
self.__init__()
self.get_credentials()
self.process_systems()
if __name__ == "__main__":
process = ProcessCISystems()
process.do_update()

22
requirements.txt Normal file
View File

@ -0,0 +1,22 @@
pbr>=0.6,!=0.7,<1.0
argparse
alembic>=0.4.1
Babel>=1.3
iso8601>=0.1.9
oauthlib>=0.6
oslo.config>=1.2.1
pecan>=0.4.5
oslo.db>=0.2.0
pika>=0.9.14
python-openid
PyYAML>=3.1.0
requests>=1.1
six>=1.7.0
SQLAlchemy>=0.8,<=0.8.99
WSME>=0.6
sqlalchemy-migrate>=0.8.2,!=0.8.4
SQLAlchemy-FullText-Search
eventlet>=0.13.0
stevedore>=1.0.0
python-crontab>=1.8.1
tzlocal>=1.1.2

0
analyse.py → scripts/analyse.py
View File

0
conf.py → scripts/conf.py
View File

0
feedutils.py → scripts/feedutils.py
View File

0
mirror_fetchers.py → scripts/mirror_fetchers.py
View File

0
report.py → scripts/report.py
View File

0
reports/reviewsummary.html → scripts/reports/reviewsummary.html
View File

0
reports/reviewsummaryfeed.py → scripts/reports/reviewsummaryfeed.py
View File

0
scrapereviews.py → scripts/scrapereviews.py
View File

0
sql.py → scripts/sql.py
View File

0
templates/report.pt → scripts/templates/report.pt
View File

0
utility.py → scripts/utility.py
View File

37
setup.cfg Normal file
View File

@ -0,0 +1,37 @@
[metadata]
name = radar
summary = OpenStack Third Party Dashboard
description-file =
README.rst
author = OpenStack
author-email = openstack-dev@lists.openstack.org
home-page = http://www.openstack.org/
classifier =
Environment :: OpenStack
Framework :: Pecan/WSME
Intended Audience :: Developers
Intended Audience :: Information Technology
Intended Audience :: System Administrators
License :: OSI Approved :: Apache Software License
Operating System :: OS Independent
Operating System :: POSIX :: Linux
Programming Language :: Python
Programming Language :: Python :: 2
Programming Language :: Python :: 2.7
Programming Language :: Python :: 3
Programming Language :: Python :: 3.3
Topic :: Internet :: WWW/HTTP
[files]
packages =
radar
data_files =
etc/radar =
etc/radar.conf.sample
[entry_points]
console_scripts =
radar-api = radar.api.app:start
radar-db-manage = radar.db.migration.cli:main
radar-update-daemon = radar.worker.daemon:run

22
setup.py Normal file
View File

@ -0,0 +1,22 @@
#!/usr/bin/env python
# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
import setuptools
setuptools.setup(
setup_requires=['pbr'],
pbr=True)

204
webclient/Gruntfile.js Normal file
View File

@ -0,0 +1,204 @@
var proxySnippet = require('grunt-connect-proxy/lib/utils').proxyRequest;
var config = {
livereload: {
port: 35729
}
};
var lrSnippet = require('connect-livereload')(config.livereload);
module.exports = function(grunt) {
var mountFolder = function (connect, dir) {
'use strict';
return connect.static(require('path').resolve(dir));
};
var proxySnippet = require('grunt-connect-proxy/lib/utils').proxyRequest;
var dir = {
source: './src',
theme: './src/theme',
test: './test',
build: './build',
report: './reports',
bower: './bower_components'
};
var proxies = {
localhost: {
context: '/api/v1',
host: '0.0.0.0',
port: 8080,
https: false,
rewrite: {
'^/api/v1': '/v1'
}
}
};
grunt.initConfig({
pkg: grunt.file.readJSON('package.json'),
concat: {
dist: {
src: [
dir.source + '/app/**/module.js',
dir.source + '/app/**/*.js'
],
dest: dir.build + '/js/dashboard.js'
}
},
copy: {
build: {
files: [
{
expand: true,
dot: true,
cwd: dir.source,
dest: dir.build,
src: [
'*.html',
'robots.txt',
'config.json'
]
},
{
expand: true,
dot: true,
cwd: dir.source + '/theme',
dest: dir.build,
src: [
'**/*.{txt,eot,ttf,woff}'
]
},
{
expand: true,
dot: true,
cwd: dir.source + '/theme/js',
dest: dir.build + '/js',
src: [
'jquery.js'
]
}
]
},
publish: {
files: [
{
expand: true,
dot: true,
cwd: dir.build,
dest: dir.publish,
src: [
'**/*.*'
]
}
]
}
},
html2js: {
options: {
module: 'db.templates',
base: dir.source
},
main: {
src: [dir.source + '/app/*/template/**/**.html'],
dest: dir.build + '/js/templates.js'
}
},
cssmin: {
build: {
options: {
report: "min"
},
src: [dir.source + '/theme/css/**/*.css'],
dest: dir.build + '/styles/main.css',
}
},
useminPrepare: {
html: [dir.source + '/index.html'],
options: {
dest: dir.build,
flow: {
steps: {
'js': ['concat'],
},
post: []
}
}
},
usemin: {
html: [
dir.build + '/index.html'
],
options: {
dirs: [dir.output]
}
},
connect: {
options: {
hostname: '0.0.0.0'
},
livereload: {
options: {
port: 9000,
middleware: function (connect) {
return [
lrSnippet,
mountFolder(connect, dir.build),
proxySnippet
];
}
},
proxies: [proxies.localhost]
},
dist: {
options: {
port: 9000,
keepalive: true,
middleware: function (connect) {
return [
mountFolder(connect, dir.build),
proxySnippet
];
}
},
proxies: [proxies.localhost]
}
},
open: {
server: {
url: 'http://0.0.0.0:9000'
}
},
watch: {
livereload: {
options: {
livereload: config.livereload.port
},
files: [
dir.source + '/**/*.*'
]
}
}
});
grunt.loadNpmTasks('grunt-contrib-uglify');
grunt.loadNpmTasks('grunt-angular-templates');
grunt.loadNpmTasks('grunt-html2js');
grunt.loadNpmTasks('grunt-contrib-cssmin');
grunt.loadNpmTasks('grunt-usemin');
grunt.loadNpmTasks('grunt-contrib-concat');
grunt.loadNpmTasks('grunt-contrib-copy');
grunt.loadNpmTasks('grunt-contrib-connect');
grunt.loadNpmTasks('grunt-connect-proxy');
grunt.loadNpmTasks('grunt-open');
grunt.loadNpmTasks('grunt-contrib-watch');
grunt.registerTask('default', ['html2js','cssmin','useminPrepare','concat','copy','usemin','copy:publish','serve']);
grunt.registerTask('publish', ['copy']);
grunt.registerTask('serve', function (target) {
grunt.task.run([
'configureProxies:livereload',
'connect:livereload',
'open', 'watch']);
});
};

Some files were not shown because too many files have changed in this diff Show More