diff --git a/.gitignore b/.gitignore index 075e764..56388a7 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ .testrepository .project .pydevproject +.stestr/* build dist ord.egg-info/ @@ -14,4 +15,4 @@ ranger_agent.egg-info localrc # Files created by releasenotes build -releasenotes/build \ No newline at end of file +releasenotes/build diff --git a/.testr.conf b/.testr.conf deleted file mode 100644 index ba6dfd4..0000000 --- a/.testr.conf +++ /dev/null @@ -1,8 +0,0 @@ -[DEFAULT] -test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ - OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \ - OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-160} \ - ${PYTHON:-python} -m subunit.run discover -t ./ ${OS_TEST_PATH:-./ord/tests} $LISTOPT $IDOPTION - -test_id_option=--load-list $IDFILE -test_list_option=--list diff --git a/AUTHORS b/AUTHORS index 59d19f0..6eab799 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1 +1,15 @@ +Chi Lo +Ian Wienand +Michael Glaser +Michael Glaser +MikeG451 +Nguyen Hung Phuong +Tin Lam hosingh000 +jh629g +raigax9 +ranadheer <7ranadheer@gmail.com> +st6218 +stewie925 +wangqi +zhouxinyong diff --git a/ChangeLog b/ChangeLog new file mode 100644 index 0000000..f3da922 --- /dev/null +++ b/ChangeLog @@ -0,0 +1,70 @@ +CHANGES +======= + +* Create editable Ranger-Agent Configuration +* create new tagging method in ranger-agent makefile +* Dockerfile fix modular variable +* Update Image build process +* Minor fix to correct missing argument issue +* Ranger add git dependency back +* Ranger Agent add tag and push of latest +* Ranger-Agent Add Image Build & Publish +* Port from python2: Specify exchange for Ranger-agent pods +* Ranger-Agent: Update heat send logic +* Add unit test cases for Ranger-agent health check +* Add health check for Ranger and Ranger-agent +* Add pip install upgrade to Dockerfile +* Update SQLAlchemy version +* Increase Ranger-agent-engine retries interval +* Minor fix - Catch DBConnectionError when update target data +* Catch DBConnectionError when update target data +* OpenDev Migration Patch +* Fix update template issue in Ranger-Agent +* Add Security Headers into Ranger-Agent +* Replace openstack.org git:// URLs with https:// +* Update Dockerfile +* Update User variable for specific component +* Docker image creation by using component specific user +* Bandit Scans for the ranger-agent +* Ranger Agent - Configurable log levels (446126) +* Update ranger-agent +* Passing project\_name to heat client and glance +* Update ranger-agent requirements.txt for heat +* ranger agent https verify +* Fix link in HACKING.rst +* Fix Ranger-Agent to allow Token Scope Authorization +* Ranger-Agent minor fix +* Allow user to set use\_stderr In case no log\_dir or log\_fle set from oslo\_conf function should allow to use stdout/stderr +* Add reno +* Fix bad path to Docker file causing image build +* Add variable for proxy to Makefile +* Add conditional proxy values to Makefile +* Recent change causing image build to fail +* Replace os.makedirs to avoid process race +* ranger-agent needs restart on github issue +* Update ranger-agent to include latest helm chart toolkit changes +* Remove file added in error +* remove unused conf +* add enable flag for rds +* oslo messenger warns of possible hang with wait() +* cleanup Docker File +* remove template file before worker thread killed +* Update ranger-agent to use Ocata global requirements +* change script and configuration file name to ranger-agent +* upgrade to use keystone v3 +* Remove pbr warnerrors in favor of sphinx check +* fix git repo and ssh key issue +* change database conf name +* Minor change to README.rst +* revert docker file without ssh config +* ranger-agent cleanup +* docker file and ubuntu 16.04 package changes +* merge latestest downstream Changes +* Updated ranger-agent README and added tempest and debian directories +* Fix dev tools +* Added files to run ranger-agent +* Add coverage to ranger-agent +* ranger-agent - fix pep8 errors +* initial code cleanup for openstack/ranger-agent +* Intial Commit +* Added .gitreview diff --git a/ord/api/app.py b/ord/api/app.py index c58d300..32b80b4 100644 --- a/ord/api/app.py +++ b/ord/api/app.py @@ -30,26 +30,7 @@ import pecan from werkzeug import serving LOG = log.getLogger(__name__) - -CONF = cfg.CONF - -OPTS = [ - cfg.StrOpt('api_paste_config', - default="api-paste.ini", - help="Configuration file for WSGI definition of API." - ), - cfg.IntOpt('api_workers', default=1, - help='Number of workers for ORD API server.'), -] - -API_OPTS = [ - cfg.BoolOpt('pecan_debug', - default=False, - help='Toggle Pecan Debug Middleware.'), -] - -CONF.register_opts(OPTS) -CONF.register_opts(API_OPTS, group='api') +CONF = service.CONF def get_pecan_config(): @@ -69,7 +50,7 @@ def setup_app(pecan_config=None, extra_hooks=None): pecan.configuration.set_config(dict(pecan_config), overwrite=True) # NOTE(sileht): pecan debug won't work in multi-process environment - pecan_debug = CONF.api.pecan_debug + pecan_debug = CONF.DEFAULT.pecan_debug if service.get_workers('api') != 1 and pecan_debug: pecan_debug = False LOG.warning(_LW('pecan_debug cannot be enabled, if workers is > 1, ' @@ -84,10 +65,10 @@ def setup_app(pecan_config=None, extra_hooks=None): guess_content_type_from_ext=False ) - transport = messaging.get_rpc_transport(cfg.CONF) + transport = messaging.get_rpc_transport(CONF) target = messaging.Target(topic='ord-listener-q', exchange='ranger-agent', - server=cfg.CONF.host) + server=CONF.DEFAULT.host) endpoints = [api.ListenerQueueHandler()] server = messaging.get_rpc_server(transport, target, @@ -116,14 +97,14 @@ class VersionSelectorApplication(object): def load_app(): # Build the WSGI app cfg_file = None - cfg_path = cfg.CONF.api_paste_config + cfg_path = CONF.DEFAULT.api_paste_config if not os.path.isabs(cfg_path): cfg_file = CONF.find_file(cfg_path) elif os.path.exists(cfg_path): cfg_file = cfg_path if not cfg_file: - raise cfg.ConfigFilesNotFoundError([cfg.CONF.api_paste_config]) + raise cfg.ConfigFilesNotFoundError([CONF.DEFAULT.api_paste_config]) LOG.info("Full WSGI config used: %s" % cfg_file) return deploy.loadapp("config:" + cfg_file) @@ -131,11 +112,11 @@ def load_app(): def build_server(): app = load_app() # Create the WSGI server and start it - host, port = cfg.CONF.api.host, cfg.CONF.api.port + host, port = CONF.api.host, CONF.api.port LOG.info(_('Starting server in PID %s') % os.getpid()) LOG.info(_("Configuration:")) - cfg.CONF.log_opt_values(LOG, logging.INFO) + CONF.log_opt_values(LOG, logging.INFO) if host == '0.0.0.0': # nosec LOG.info(_( @@ -146,7 +127,7 @@ def build_server(): {'host': host, 'port': port})) workers = service.get_workers('api') - serving.run_simple(cfg.CONF.api.host, cfg.CONF.api.port, + serving.run_simple(CONF.api.host, CONF.api.port, app, processes=workers) diff --git a/ord/api/controllers/v1/api.py b/ord/api/controllers/v1/api.py index 8b837d7..38050f0 100644 --- a/ord/api/controllers/v1/api.py +++ b/ord/api/controllers/v1/api.py @@ -23,6 +23,7 @@ from ord.common.utils import ErrorCode from ord.db import api as db_api from ord.i18n import _ from ord.openstack.common import log +from ord import service from oslo_config import cfg from pecan import expose from pecan import request as pecan_req @@ -41,12 +42,7 @@ import webob.exc LOG = log.getLogger(__name__) -CONF = cfg.CONF -orm_opts = [ - cfg.StrOpt('rds_listener_endpoint', - help='Endpoint to rds_listener ') - -] +CONF = service.CONF opts = [ cfg.StrOpt('region', @@ -58,12 +54,6 @@ opts = [ CONF.register_opts(opts) -opt_group = cfg.OptGroup(name='orm', - title='Options for the orm service') - -CONF.register_group(opt_group) -CONF.register_opts(orm_opts, opt_group) - class ListenerQueueHandler(object): @@ -75,15 +65,21 @@ class ListenerQueueHandler(object): LOG.debug(" Payload: %s \n ctxt: %s " % (str(payload), str(ctxt))) LOG.debug(" -------------------------------") listener_response_body = {} + + rds_endpoint = \ + db_api.retrieve_configuration(region=CONF.DEFAULT.region) + + rds_endpoint = (rds_endpoint['rds_listener_endpoint'] + if rds_endpoint is not None else + CONF.orm.rds_listener_endpoint) try: listener_response_body = json.loads(payload) LOG.debug(" Payload to RDS Listener %s " % listener_response_body) headers = {'Content-type': 'application/json'} - rds_url = CONF.orm.rds_listener_endpoint # Python3 urllib: convert listener_response_body to bytes response_body_bytes = \ json.dumps(listener_response_body).encode("utf-8") - req = urllib.request.Request(rds_url, # nosec + req = urllib.request.Request(rds_endpoint, # nosec response_body_bytes, headers, unverifiable=False) @@ -100,7 +96,7 @@ class ListenerQueueHandler(object): ['ord-notifier-id']) status_code = None try: - LOG.info('Connecting to RDS at %s' % rds_url) + LOG.info('Connecting to RDS at %s' % rds_endpoint) resp = request.urlopen(req) # nosec status = utils.STATUS_RDS_SUCCESS if resp is not None: @@ -114,9 +110,11 @@ class ListenerQueueHandler(object): except ValueError as e: status = utils.STATUS_RDS_ERROR LOG.error('Error while parsing input payload %r', e) + status_code = None except Exception as ex: status = utils.STATUS_RDS_ERROR LOG.error('Error while calling RDS Listener %r', ex) + status_code = None finally: LOG.info('RDS Listener status %s ' % status) LOG.info('RDS Listener status code %s ' % status_code) @@ -134,7 +132,6 @@ class NotifierController(object): def __init__(self): super(NotifierController, self).__init__() - self._rpcapi = rpcapi.RpcAPI() self._set_keystone_client() def _set_keystone_client(cls): @@ -212,6 +209,10 @@ class NotifierController(object): token = pecan_req.headers['X-Auth-Token'] self.kc.tokens.validate(token) + @expose(generic=True) + def ord_configuration(self, **args): + raise webob.exc.HTTPNotFound + @expose(generic=True) def ord_notifier(self, **args): raise webob.exc.HTTPNotFound @@ -220,6 +221,17 @@ class NotifierController(object): def health_check(self, **args): raise webob.exc.HTTPNotFound + @ord_configuration.when(method='POST', template='json') + def ord_configuration_update(self, **args): + if CONF.auth_enabled: + self._validate_token() + else: + LOG.debug("Authentication is disabled. We don't recommend this.") + LOG.debug('Updating ranger-agent configuration') + db_api.update_configuration(**args) + + return {'Ranger-Agent': 'Update request processed'} + @health_check.when(method='GET', template='json') def ord_health_status(self): LOG.debug('Received health message via api endpoint') @@ -322,7 +334,8 @@ class NotifierController(object): try: ctxt = {'request_id': kwargs.get('request_id')} heat_template = base64.b64decode(file_info.file.read()) - self._rpcapi.invoke_notifier_rpc(ctxt, payload, heat_template) + rpcapi.RpcAPI().\ + invoke_notifier_rpc(ctxt, payload, heat_template) except messaging.MessageDeliveryFailure: LOG.error("Fail to deliver message") diff --git a/ord/db/api.py b/ord/db/api.py index 7e6f710..67cb0a6 100644 --- a/ord/db/api.py +++ b/ord/db/api.py @@ -53,6 +53,14 @@ def retrieve_target(request_id): return IMPL.retrieve_target(request_id) +def retrieve_configuration(region): + return IMPL.retrieve_configuration(region) + + +def update_configuration(**vals): + return IMPL.update_configuration(**vals) + + def retrieve_target_by_status(template_status_id): return IMPL.retrieve_target(template_status_id) diff --git a/ord/db/sqlalchemy/api.py b/ord/db/sqlalchemy/api.py index bf5472f..85470a7 100644 --- a/ord/db/sqlalchemy/api.py +++ b/ord/db/sqlalchemy/api.py @@ -20,33 +20,12 @@ import threading from ord.db.sqlalchemy import models from oslo_config import cfg -from oslo_db import options as oslo_db_options from oslo_db.sqlalchemy import session as db_session from oslo_db.sqlalchemy import utils as sqlalchemyutils from oslo_log import log as logging CONF = cfg.CONF -api_db_opts = [ - cfg.StrOpt('connection', - help='The SQLAlchemy connection string to use to connect to ' - 'the ORD database.', - secret=True), - cfg.StrOpt('mysql_sql_mode', - default='TRADITIONAL', - help='The SQL mode to be used for MySQL sessions. ' - 'This option, including the default, overrides any ' - 'server-set SQL mode. To use whatever SQL mode ' - 'is set by the server configuration, ' - 'set this to no value. Example: mysql_sql_mode='), -] - - -opt_group = cfg.OptGroup(name='database', - title='Options for the database service') -CONF.register_group(opt_group) -CONF.register_opts(oslo_db_options.database_opts, opt_group) - LOG = logging.getLogger(__name__) _ENGINE_FACADE = {'ord': None} @@ -196,6 +175,29 @@ def retrieve_target_by_status(template_status_id): return query.first() +# retrieve ranger configuration from database +def retrieve_configuration(region): + LOG.debug('Retrieve ranger-agent configuration of %s', region) + session = get_session() + query = model_query(models.Ord_Configuration, session=session) + query = query.filter_by(region=region) + + return query.first() + + +def update_configuration(**vals): + LOG.debug('Update ranger-agent configuration in database') + session = get_session() + with session.begin(): + query = model_query(models.Ord_Configuration, session=session) + query = query.filter_by(region=vals.get('region')) + if query.first() is None: + ord_conf = models.Ord_Configuration(**vals) + session.add(ord_conf) + else: + query.update(vals) + + def retrieve_target(request_id): LOG.debug('Retrieve Target data %s', request_id) session = get_session() diff --git a/ord/db/sqlalchemy/migrate_repo/versions/003_ord.py b/ord/db/sqlalchemy/migrate_repo/versions/003_ord.py new file mode 100644 index 0000000..72eb5f1 --- /dev/null +++ b/ord/db/sqlalchemy/migrate_repo/versions/003_ord.py @@ -0,0 +1,110 @@ +# Copyright 2012 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log as logging +from sqlalchemy import Column +from sqlalchemy import dialects +from sqlalchemy import MetaData, String, Table, Boolean +from sqlalchemy import Text + +LOG = logging.getLogger(__name__) + + +# Note on the autoincrement flag: this is defaulted for primary key columns +# of integral type, so is no longer set explicitly in such cases. + +def MediumText(): + return Text().with_variant(dialects.mysql.MEDIUMTEXT(), 'mysql') + + +def upgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + + ord_configuration = Table('ord_configuration', meta, + Column('region', String(length=30), + primary_key=True, nullable=False), + Column('api_workers', String(length=10), + nullable=False), + Column('debug_level', String(length=10), + nullable=False), + Column('pecan_debug', Boolean, + nullable=False), + Column('resource_creation_timeout_min', + String(length=10), nullable=False), + Column('resource_creation_timeout_max', + String(length=10), nullable=False), + Column('log_dir', String(length=80), + nullable=False), + Column('resource_status_check_wait', + String(length=10), nullable=False), + Column('api_paste_config', String(length=80), + nullable=False), + Column('transport_url', String(length=300), + nullable=False), + Column('enable_rds_callback_check', + Boolean, nullable=False), + Column('host', String(length=80), + nullable=False), + Column('port', String(length=10), + nullable=False), + Column('auth_type', String(length=20), + nullable=False), + Column('auth_url', String(length=80), + nullable=False), + Column('auth_version', String(length=10), + nullable=False), + Column('password', String(length=80), + nullable=False), + Column('project_domain_name', String(length=30), + nullable=False), + Column('project_name', String(length=80), + nullable=False), + Column('region_name', String(length=30), + nullable=False), + Column('user_domain_name', String(length=30), + nullable=False), + Column('username', String(length=80), + nullable=False), + Column('connection', String(length=240), + nullable=False), + Column('max_retries', String(length=10), + nullable=False), + Column('rds_listener_endpoint', + String(length=120), + nullable=False), + mysql_engine='InnoDB', + mysql_charset='utf8') + + tables = [ord_configuration] + + for table in tables: + try: + table.create() + except Exception: + LOG.info(repr(table)) + LOG.exception('Exception while creating table.') + raise + if migrate_engine.name == 'mysql': + # In Folsom we explicitly converted migrate_version to UTF8. + migrate_engine.execute( + 'ALTER TABLE migrate_version CONVERT TO CHARACTER SET utf8') + # Set default DB charset to UTF8. + migrate_engine.execute( + 'ALTER DATABASE %s DEFAULT CHARACTER SET utf8' % + migrate_engine.url.database) + + +def downgrade(migrate_engine): + raise NotImplementedError('Downgrade is not implemented.') diff --git a/ord/db/sqlalchemy/models.py b/ord/db/sqlalchemy/models.py index aff8461..af82d87 100644 --- a/ord/db/sqlalchemy/models.py +++ b/ord/db/sqlalchemy/models.py @@ -19,7 +19,7 @@ SQLAlchemy models for ranger-agent data. import datetime import uuid -from sqlalchemy import (Column, DateTime, String, Integer) +from sqlalchemy import (Column, DateTime, String, Integer, Boolean) from sqlalchemy import ForeignKey, Text from sqlalchemy import orm @@ -27,12 +27,35 @@ from sqlalchemy.dialects.mysql import MEDIUMTEXT from sqlalchemy.ext.declarative import declarative_base from oslo_config import cfg +from oslo_db import options as oslo_db_options from oslo_db.sqlalchemy import models CONF = cfg.CONF BASE = declarative_base() +database_opts = [ + cfg.StrOpt('connection', + help='The SQLAlchemy connection string to use' + ' to connect to the ORD database.', + secret=True), + cfg.StrOpt('max_retries', + default='5', + help='max attempts to connect to database allowed'), + cfg.StrOpt('mysql_sql_mode', + default='TRADITIONAL', + help='The SQL mode to be used for MySQL sessions. ' + 'This option, including the default, overrides any ' + 'server-set SQL mode. To use whatever SQL mode ' + 'is set by the server configuration, ' + 'set this to no value. Example: mysql_sql_mode') +] + +opt_group = cfg.OptGroup(name='database', + title='Options for the database service') +CONF.register_group(opt_group) +CONF.register_opts(oslo_db_options.database_opts, opt_group) + def MediumText(): return Text().with_variant(MEDIUMTEXT(), 'mysql') @@ -87,6 +110,90 @@ class ORDBase(models.ModelBase): id(self), ', '.join(items)) +# Used to override deployed ranger-agent configuration values +class Ord_Configuration(BASE, ORDBase): + __tablename__ = 'ord_configuration' + + api_workers = Column( + String(10), + nullable=False) + log_dir = Column( + String(60), + nullable=True) + debug_level = Column( + String(10), + nullable=False) + pecan_debug = Column( + String(10), + nullable=False) + region = Column( + String(30), + primary_key=True, + nullable=False) + resource_creation_timeout_min = Column( + String(10), nullable=False) + resource_creation_timeout_max = Column( + String(10), + nullable=False) + resource_status_check_wait = Column( + String(10), + nullable=False) + api_paste_config = Column( + String(80), + nullable=False) + transport_url = Column( + String(300), + nullable=False) + enable_rds_callback_check = Column( + Boolean) + + host = Column( + String(80), + nullable=False) + port = Column( + String(10), + nullable=False) + + auth_type = Column( + String(20), + nullable=False) + auth_url = Column( + String(80), + nullable=False) + auth_version = Column( + String(10), + nullable=False) + password = Column( + String(80), + nullable=False) + project_domain_name = Column( + String(30), + nullable=False) + project_name = Column( + String(80), + nullable=False) + region_name = Column( + String(30), + nullable=False) + user_domain_name = Column( + String(30), + nullable=False) + username = Column( + String(80), + nullable=False) + + connection = Column( + String(240), + nullable=False) + max_retries = Column( + String(10), + nullable=False) + + rds_listener_endpoint = Column( + String(80), + nullable=False) + + class Ord_Notification(BASE, ORDBase): __tablename__ = 'ord_notification' diff --git a/ord/engine/app.py b/ord/engine/app.py index 8fbe25b..8b022f0 100644 --- a/ord/engine/app.py +++ b/ord/engine/app.py @@ -16,23 +16,24 @@ from ord.engine.engine import Engine from ord.engine.engine import QueueHandler from ord.openstack.common import log as logging -from oslo_config import cfg +from ord import service import oslo_messaging as messaging import time LOG = logging.getLogger(__name__) +CONF = service.CONF def start(): engine = Engine() # start Notify message listener - transport = messaging.get_rpc_transport(cfg.CONF) + transport = messaging.get_rpc_transport(CONF) target = messaging.Target(topic='ord-notifier-q', exchange='ranger-agent', - server=cfg.CONF.host) + server=CONF.DEFAULT.host) endpoints = [QueueHandler(engine)] diff --git a/ord/engine/workerfactory.py b/ord/engine/workerfactory.py index f8a4dcf..1bb3b79 100755 --- a/ord/engine/workerfactory.py +++ b/ord/engine/workerfactory.py @@ -309,7 +309,8 @@ class WorkerThread(threading.Thread): max_range = int(CONF.retry_limits) self._rpcengine. \ - invoke_listener_rpc(res_ctxt, json.dumps(rds_payload)) + invoke_listener_rpc(res_ctxt, + json.dumps(rds_payload)) check_wait = CONF.resource_status_check_wait # increase the polling interval after the initial wait time @@ -327,8 +328,10 @@ class WorkerThread(threading.Thread): rds_payload.get('rds-listener')['status'] = status_original # if image_payload: # rds_payload.get('rds-listener')['status'] = image_payload + self._rpcengine. \ - invoke_listener_rpc(res_ctxt, json.dumps(rds_payload)) + invoke_listener_rpc(res_ctxt, + json.dumps(rds_payload)) if status != utils.STATUS_RDS_SUCCESS: LOG.info("Retrying for api response") diff --git a/ord/service.py b/ord/service.py index e32f79a..6e63611 100644 --- a/ord/service.py +++ b/ord/service.py @@ -26,6 +26,8 @@ from ord.i18n import _ from ord.openstack.common import log +LOG = log.getLogger(__name__) + OPTS = [ cfg.StrOpt('host', default=socket.gethostname(), @@ -33,10 +35,95 @@ OPTS = [ 'key. Can be an opaque identifier. For ZeroMQ only, must ' 'be a valid host name, FQDN, or IP address.'), ] -cfg.CONF.register_opts(OPTS) +cfg.CONF.register_opts(OPTS, group='DEFAULT') +default_opts = [ + cfg.StrOpt('api_paste_config', default='/etc/ranger-agent/api-paste.ini', + help=""), + cfg.IntOpt('api_workers', default=1, + help="Number of worker threads to be used by API"), + cfg.StrOpt('debug', default='true', + help="Enables debug output in logging"), + cfg.StrOpt('debug_level', default='ERROR', + help='Determines level of debug content' + ' output: Error/Warning/Debug'), + cfg.StrOpt('enable_heat_health_check', default='true', help=""), + cfg.BoolOpt('pecan_debug', default=True, help=""), + cfg.StrOpt('region', default='', + help="name of site ranger-agent is deployed on"), + cfg.StrOpt('resource_creation_timeout_max', default='14400', + help="Max allotment of time for resource creation"), + cfg.StrOpt('resource_creation_timeout_min', default='1200', + help='Min allotment of time before a timeout' + ' error can be returned'), + cfg.StrOpt('resource_status_check_wait', default='15', + help='Allotment of time between checks' + ' during resource creation'), + cfg.IntOpt('retry_limits', default=5, + help="Max allotment of tries for resource creation"), + cfg.StrOpt('transport_url', default='', + help="Messaging queue url", secret=True), + cfg.StrOpt('use_stderr', default='true', help=""), + cfg.StrOpt('verbose', default='false', help=""), + cfg.BoolOpt('enable_rds_callback_check', + default=True, + help='validate rds api is reachable') +] -LOG = log.getLogger(__name__) +auth_opts = [ + cfg.StrOpt('project_name', default='service', + help="project name used to stack heat resources"), + cfg.StrOpt('auth_type', default='password', + help="type of credentials used for authentication"), + cfg.StrOpt('auth_url', default='', + help='auth url used by ranger agent to' + ' invoke keystone apis'), + cfg.StrOpt('username', default='', + help='user name used by ranger agent to' + ' invoke keystone apis'), + cfg.StrOpt('password', default='', secret=True, + help='password used by ranger agent to' + ' invoke keystone apis'), + cfg.StrOpt('project_domain_name', default='default', + help='default project domain ' + 'used by ranger agent to invoke keystone apis'), + cfg.StrOpt('auth_version', default='v3', help="Keystone version"), + cfg.StrOpt("user_domain_name", default='default', + help='default project domain ' + 'used by ranger agent to invoke keystone apis'), + cfg.StrOpt("https_cacert", default=None, + help="Path to CA server certificate for SSL"), + + cfg.StrOpt('region_name', default='', help='Region'), + cfg.StrOpt('auth_enabled', default='True', + help='check if authentication turned on') +] + +api_opts = [ + cfg.IntOpt('port', + default=9010, + help='The port for the ORD API server.', + ), + cfg.StrOpt('host', + default='0.0.0.0', # nosec + help='The listen IP for the ORD API server.', + ) + +] + +orm_opts = [ + cfg.StrOpt('rds_listener_endpoint', default='', + help='The rds endpoint of ranger deployment'), + cfg.StrOpt('retry_limits', default='5', + help='Max attempts to contact Ranger rds endpoint') +] + +cfg.CONF.register_opts(default_opts, group='DEFAULT') +cfg.CONF.register_opts(auth_opts, group='keystone_authtoken') +cfg.CONF.register_opts(api_opts, group='api') +cfg.CONF.register_opts(orm_opts, group='orm') + +CONF = cfg.CONF class WorkerException(Exception): @@ -44,8 +131,7 @@ class WorkerException(Exception): def get_workers(name): - workers = (cfg.CONF.get('%s_workers' % name) or - utils.cpu_count()) + workers = (CONF.DEFAULT.api_workers or utils.cpu_count()) if workers and workers < 1: msg = (_("%(worker_name)s value of %(workers)s is invalid, " "must be greater than 0") % diff --git a/ord/tests/unit/api/controllers/v1/test_api.py b/ord/tests/unit/api/controllers/v1/test_api.py index c399d4a..f14ec9a 100644 --- a/ord/tests/unit/api/controllers/v1/test_api.py +++ b/ord/tests/unit/api/controllers/v1/test_api.py @@ -17,15 +17,14 @@ Unit Tests for ord.api.test_api """ -import base64 from cgi import FieldStorage -import mock from mox3.mox import stubout from ord.api.controllers.v1 import api from ord.db import api as db_api from ord.tests import base from oslo_config import cfg import requests +from unittest import mock from urllib import request import webob @@ -42,7 +41,6 @@ class OrdApiTestCase(base.BaseTestCase): self.addCleanup(self.stubs.SmartUnsetAll) def test_api_notifier(self): - kwargs = { 'request_id': '1', 'resource_id': 'qwe1234', @@ -80,40 +78,21 @@ class OrdApiTestCase(base.BaseTestCase): CONF.set_default('region', 'local') - def fake_keystone_client(*args, **kwds): - return + api.rpcapi = mock.MagicMock() + ord_notifier = api.NotifierController + ord_notifier._set_keystone_client = mock.MagicMock() + ord_notifier._validate_token = mock.MagicMock() + ord_notifier._persist_notification_record = \ + mock.MagicMock(return_value=db_response) - self.stubs.Set(api.NotifierController, '_set_keystone_client', - fake_keystone_client) - ord_notifier = api.NotifierController() - - def fake_validate_token(*args): - return - - def fake_persist_notification_record(*args, **kwds): - return db_response - - def fake_b64decode(*args, **kwds): - return "heat_template" - - def fake_invoke_notifier_rpc(*args, **kwds): - return payload - - self.stubs.Set(ord_notifier, "_validate_token", fake_validate_token) - self.stubs.Set(ord_notifier, "_persist_notification_record", - fake_persist_notification_record) - self.stubs.Set(base64, "b64decode", fake_b64decode) - self.stubs.Set(ord_notifier._rpcapi, "invoke_notifier_rpc", - fake_invoke_notifier_rpc) - - response = ord_notifier.ord_notifier_POST(**params) + response = ord_notifier().ord_notifier_POST(**params) expect_response = response['ord-notifier-response']['status'] self.assertEqual(expect_response, 'Submitted') def test_api_listener(self): ctxt = {'request_id': '1'} - api_listener = api.ListenerQueueHandler() + api_listener = api.ListenerQueueHandler kwargs = '{"request_id": "1",'\ ' "resource_id": "qwe1234","resource-type": "image"}' payload = str(kwargs) @@ -122,25 +101,15 @@ class OrdApiTestCase(base.BaseTestCase): 'error_code': '', 'error_msg': ''} - def mock_url_open(mock_response): - mock_response = mock.Mock() - mock_response.getcode.return_value = 200 - - def urlrequest_mock_method(url, payload, headers, unverifiable=False): - return "Failure" - - def fake_update_target(*args, **kwds): - return db_template_target - - self.stubs.Set(request, 'urlopen', mock_url_open) - self.stubs.Set(db_api, "update_target_data", - fake_update_target) - self.stubs.Set(request, 'Request', urlrequest_mock_method) - api_listener.invoke_listener_rpc(ctxt, payload) + request.urlopen = mock.MagicMock() + request.Request = mock.MagicMock() + db_api.update_target_data = mock.MagicMock() + db_api.retrieve_configuration = mock.MagicMock() + api_listener().invoke_listener_rpc(ctxt, payload) def test_rds_listener_failure(self): ctxt = {'request_id': '1'} - api_listener = api.ListenerQueueHandler() + api_listener = api.ListenerQueueHandler kwargs = '{"rds-listener": { "ord-notifier-id": "2",'\ '"status": "error","resource-type": "image",'\ @@ -153,28 +122,18 @@ class OrdApiTestCase(base.BaseTestCase): payload = str(kwargs) output_status = 'STATUS_RDS_SUCCESS' + http_error = requests.exceptions.HTTPError() + request.urlopen = mock.MagicMock(side_effect=http_error) + request.Request = mock.MagicMock() + db_api.update_target_data = mock.MagicMock() + db_api.retrieve_configuration = mock.MagicMock() - def mock_method(url, payload, headers, unverifiable=False): - return "Failure" - self.stubs.Set(request, 'Request', mock_method) - - def mock_url_open(mock_response): - mock_response = mock.Mock() - http_error = requests.exceptions.HTTPError() - mock_response.raise_for_status.side_effect = http_error - - def fake_update_target(*args, **kwds): - return db_template_target - - self.stubs.Set(request, 'urlopen', mock_url_open) - self.stubs.Set(db_api, "update_target_data", - fake_update_target) - api_listener.invoke_listener_rpc(ctxt, payload) + api_listener().invoke_listener_rpc(ctxt, payload) self.assertEqual(output_status, db_template_target['status']) def test_rds_listener_success(self): ctxt = {'request_id': '1'} - api_listener = api.ListenerQueueHandler() + api_listener = api.ListenerQueueHandler kwargs = '{"rds-listener": { "ord-notifier-id": "2",'\ '"status": "error","resource-type": "image",'\ @@ -188,21 +147,12 @@ class OrdApiTestCase(base.BaseTestCase): payload = str(kwargs) output_status = 'Error_RDS_Dispatch' - def mock_method(url, payload, headers, unverifiable=False): - return "Success" - self.stubs.Set(request, 'Request', mock_method) + request.Request = mock.MagicMock() + request.urlopen = mock.MagicMock() + db_api.update_target_data = mock.MagicMock() + db_api.retrieve_configuration = mock.MagicMock() - def mock_url_open(mock_response): - mock_response = mock.Mock() - mock_response.getcode.return_value = 200 - - def fake_update_target(*args, **kwds): - return db_template_target - - self.stubs.Set(request, 'urlopen', mock_url_open) - self.stubs.Set(db_api, "update_target_data", - fake_update_target) - api_listener.invoke_listener_rpc(ctxt, payload) + api_listener().invoke_listener_rpc(ctxt, payload) self.assertEqual(output_status, db_template_target['status']) @@ -223,19 +173,12 @@ class OrdApiTestCase(base.BaseTestCase): }' } - def fake_keystone_client(*args, **kwds): - return + ord_notifier = api.NotifierController + ord_notifier._set_keystone_client = mock.MagicMock() + ord_notifier._validate_token = mock.MagicMock() - self.stubs.Set(api.NotifierController, '_set_keystone_client', - fake_keystone_client) - ord_notifier = api.NotifierController() - - def fake_validate_token(*args): - return - - self.stubs.Set(ord_notifier, "_validate_token", fake_validate_token) self.assertRaises(webob.exc.HTTPBadRequest, - ord_notifier.ord_notifier_POST, + ord_notifier().ord_notifier_POST, **params) def test_api_notifier_for_invalid_region(self): @@ -258,19 +201,12 @@ class OrdApiTestCase(base.BaseTestCase): CONF.set_default('region', 'local') - def fake_keystone_client(*args, **kwds): - return + ord_notifier = api.NotifierController + ord_notifier._set_keystone_client = mock.MagicMock() + ord_notifier._validate_token = mock.MagicMock() - self.stubs.Set(api.NotifierController, '_set_keystone_client', - fake_keystone_client) - ord_notifier = api.NotifierController() - - def fake_validate_token(*args): - return - - self.stubs.Set(ord_notifier, "_validate_token", fake_validate_token) self.assertRaises(webob.exc.HTTPBadRequest, - ord_notifier.ord_notifier_POST, + ord_notifier().ord_notifier_POST, **params) def test_api_notifier_for_invalid_payload(self): @@ -293,19 +229,12 @@ class OrdApiTestCase(base.BaseTestCase): CONF.set_default('region', 'local') - def fake_keystone_client(*args, **kwds): - return + ord_notifier = api.NotifierController + ord_notifier._set_keystone_client = mock.MagicMock() + ord_notifier._validate_token = mock.MagicMock() - self.stubs.Set(api.NotifierController, '_set_keystone_client', - fake_keystone_client) - ord_notifier = api.NotifierController() - - def fake_validate_token(*args): - return - - self.stubs.Set(ord_notifier, "_validate_token", fake_validate_token) self.assertRaises(webob.exc.HTTPBadRequest, - ord_notifier.ord_notifier_POST, + ord_notifier().ord_notifier_POST, **params) def test_api_ord_notifier_status(self): @@ -345,23 +274,50 @@ class OrdApiTestCase(base.BaseTestCase): 'error-msg': 'stack fail'} } - def fake_keystone_client(*args, **kwds): - return + ord_notifier = api.NotifierController - self.stubs.Set(api.NotifierController, '_set_keystone_client', - fake_keystone_client) - ord_notifier = api.NotifierController() + ord_notifier._set_keystone_client = mock.MagicMock() + db_api.retrieve_template = mock.MagicMock(return_value=db_template) + db_api.retrieve_target = \ + mock.MagicMock(return_value=db_template_target) - def fake_retrieve_template(*args, **kwds): - return db_template - - def fake_retrieve_target(*args, **kwds): - return db_template_target - - self.stubs.Set(db_api, "retrieve_template", - fake_retrieve_template) - self.stubs.Set(db_api, "retrieve_target", - fake_retrieve_target) - - notification_status = ord_notifier.ord_notifier_status(**request_id) + notification_status = ord_notifier().ord_notifier_status(**request_id) self.assertEqual(payload, notification_status) + + def test_update_configuration(self): + payload = { + "api_workers": 1, + "debug_level": "DEBUG", + "pecan_debug": True, + "region": "local", + "resource_creation_timeout_min": 1200, + "resource_creation_timeout_max": 14400, + "resource_status_check_wait": 15, + "api_paste_config": "/etc/ranger-agent/api-paste.ini", + "transport_url": + "rabbit://stackrabbit:stackqueue@192.168.56.135:5672", + "enable_rds_callback_check": True, + "host": "0.0.0.0", + "port": 9010, + "auth_type": "password", + "auth_url": "http://192.168.56.135/identity/v3", + "auth_version": "v3", + "password": "secret", + "project_domain_name": "Default", + "project_name": "service", + "region_name": "RegionOne", + "user_domain_name": "Default", + "username": "admin", + "connection": "mysql+pymysql://root:stackdb@127.0.0.1:3306/ord", + "max_retries": 5, + "rds_listener_endpoint": "http://192.168.56.127:8777/v1/rds/status" + } + + mock_notifierController = api.NotifierController + mock_notifierController._set_keystone_client = mock.MagicMock() + mock_notifierController._validate_token = mock.MagicMock() + db_api.update_configuration = mock.MagicMock() + + resp = mock_notifierController().ord_configuration_update(**payload) + + self.assertEqual(resp, {"Ranger-Agent": "Update request processed"}) diff --git a/test-requirements.txt b/test-requirements.txt index 252c939..830a597 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -10,11 +10,10 @@ python-subunit>=0.0.18 sphinx>>=1.2.1,!=1.3b1,<1.4 # BSD oslosphinx>=4.7.0 # Apache-2.0 oslotest>=1.10.0 # Apache-2.0 -testrepository>=0.0.18 testscenarios>=0.4 +stestr>=1.0.0 # Apache-2.0 testtools>=1.4.0 mock>=2.0 discover mox3>=0.27.0 -psycopg2>=2.5 reno>=1.8.0 # Apache-2.0 diff --git a/tox.ini b/tox.ini index 1ee82f7..4f07855 100644 --- a/tox.ini +++ b/tox.ini @@ -12,7 +12,7 @@ setenv = PYTHONWARNINGS=default::DeprecationWarning deps = -r {toxinidir}/requirements.txt -r {toxinidir}/test-requirements.txt -commands = python setup.py test --slowest --testr-args='{posargs}' +commands = stestr run [testenv:bandit] deps = .[bandit]