From a376a02e2e4578d6ef85cbbe9776e5b1cae26755 Mon Sep 17 00:00:00 2001 From: jh629g Date: Fri, 1 May 2020 16:44:52 -0500 Subject: [PATCH] Create editable Ranger-Agent Configuration When ranger-agent is deployed in kubernetes, the configuration becomes uneditable without editing secrets and restarting the pod. This patchset will add configuration to the database so that values can be overriden as needed to serve development needs. This includes such needs as altering logging level and changing the ranger site which ranger-agent points at. Change-Id: Id8b9f16668914e3c071639359d33aba0eee076c2 --- .gitignore | 3 +- .testr.conf | 8 - AUTHORS | 14 ++ ChangeLog | 70 ++++++ ord/api/app.py | 37 +-- ord/api/controllers/v1/api.py | 47 ++-- ord/db/api.py | 8 + ord/db/sqlalchemy/api.py | 44 ++-- .../migrate_repo/versions/003_ord.py | 110 +++++++++ ord/db/sqlalchemy/models.py | 109 ++++++++- ord/engine/app.py | 7 +- ord/engine/workerfactory.py | 7 +- ord/service.py | 94 +++++++- ord/tests/unit/api/controllers/v1/test_api.py | 210 +++++++----------- test-requirements.txt | 3 +- tox.ini | 2 +- 16 files changed, 558 insertions(+), 215 deletions(-) delete mode 100644 .testr.conf create mode 100644 ChangeLog create mode 100644 ord/db/sqlalchemy/migrate_repo/versions/003_ord.py diff --git a/.gitignore b/.gitignore index 075e764..56388a7 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ .testrepository .project .pydevproject +.stestr/* build dist ord.egg-info/ @@ -14,4 +15,4 @@ ranger_agent.egg-info localrc # Files created by releasenotes build -releasenotes/build \ No newline at end of file +releasenotes/build diff --git a/.testr.conf b/.testr.conf deleted file mode 100644 index ba6dfd4..0000000 --- a/.testr.conf +++ /dev/null @@ -1,8 +0,0 @@ -[DEFAULT] -test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ - OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \ - OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-160} \ - ${PYTHON:-python} -m subunit.run discover -t ./ ${OS_TEST_PATH:-./ord/tests} $LISTOPT $IDOPTION - -test_id_option=--load-list $IDFILE -test_list_option=--list diff --git a/AUTHORS b/AUTHORS index 59d19f0..6eab799 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1 +1,15 @@ +Chi Lo +Ian Wienand +Michael Glaser +Michael Glaser +MikeG451 +Nguyen Hung Phuong +Tin Lam hosingh000 +jh629g +raigax9 +ranadheer <7ranadheer@gmail.com> +st6218 +stewie925 +wangqi +zhouxinyong diff --git a/ChangeLog b/ChangeLog new file mode 100644 index 0000000..f3da922 --- /dev/null +++ b/ChangeLog @@ -0,0 +1,70 @@ +CHANGES +======= + +* Create editable Ranger-Agent Configuration +* create new tagging method in ranger-agent makefile +* Dockerfile fix modular variable +* Update Image build process +* Minor fix to correct missing argument issue +* Ranger add git dependency back +* Ranger Agent add tag and push of latest +* Ranger-Agent Add Image Build & Publish +* Port from python2: Specify exchange for Ranger-agent pods +* Ranger-Agent: Update heat send logic +* Add unit test cases for Ranger-agent health check +* Add health check for Ranger and Ranger-agent +* Add pip install upgrade to Dockerfile +* Update SQLAlchemy version +* Increase Ranger-agent-engine retries interval +* Minor fix - Catch DBConnectionError when update target data +* Catch DBConnectionError when update target data +* OpenDev Migration Patch +* Fix update template issue in Ranger-Agent +* Add Security Headers into Ranger-Agent +* Replace openstack.org git:// URLs with https:// +* Update Dockerfile +* Update User variable for specific component +* Docker image creation by using component specific user +* Bandit Scans for the ranger-agent +* Ranger Agent - Configurable log levels (446126) +* Update ranger-agent +* Passing project\_name to heat client and glance +* Update ranger-agent requirements.txt for heat +* ranger agent https verify +* Fix link in HACKING.rst +* Fix Ranger-Agent to allow Token Scope Authorization +* Ranger-Agent minor fix +* Allow user to set use\_stderr In case no log\_dir or log\_fle set from oslo\_conf function should allow to use stdout/stderr +* Add reno +* Fix bad path to Docker file causing image build +* Add variable for proxy to Makefile +* Add conditional proxy values to Makefile +* Recent change causing image build to fail +* Replace os.makedirs to avoid process race +* ranger-agent needs restart on github issue +* Update ranger-agent to include latest helm chart toolkit changes +* Remove file added in error +* remove unused conf +* add enable flag for rds +* oslo messenger warns of possible hang with wait() +* cleanup Docker File +* remove template file before worker thread killed +* Update ranger-agent to use Ocata global requirements +* change script and configuration file name to ranger-agent +* upgrade to use keystone v3 +* Remove pbr warnerrors in favor of sphinx check +* fix git repo and ssh key issue +* change database conf name +* Minor change to README.rst +* revert docker file without ssh config +* ranger-agent cleanup +* docker file and ubuntu 16.04 package changes +* merge latestest downstream Changes +* Updated ranger-agent README and added tempest and debian directories +* Fix dev tools +* Added files to run ranger-agent +* Add coverage to ranger-agent +* ranger-agent - fix pep8 errors +* initial code cleanup for openstack/ranger-agent +* Intial Commit +* Added .gitreview diff --git a/ord/api/app.py b/ord/api/app.py index c58d300..32b80b4 100644 --- a/ord/api/app.py +++ b/ord/api/app.py @@ -30,26 +30,7 @@ import pecan from werkzeug import serving LOG = log.getLogger(__name__) - -CONF = cfg.CONF - -OPTS = [ - cfg.StrOpt('api_paste_config', - default="api-paste.ini", - help="Configuration file for WSGI definition of API." - ), - cfg.IntOpt('api_workers', default=1, - help='Number of workers for ORD API server.'), -] - -API_OPTS = [ - cfg.BoolOpt('pecan_debug', - default=False, - help='Toggle Pecan Debug Middleware.'), -] - -CONF.register_opts(OPTS) -CONF.register_opts(API_OPTS, group='api') +CONF = service.CONF def get_pecan_config(): @@ -69,7 +50,7 @@ def setup_app(pecan_config=None, extra_hooks=None): pecan.configuration.set_config(dict(pecan_config), overwrite=True) # NOTE(sileht): pecan debug won't work in multi-process environment - pecan_debug = CONF.api.pecan_debug + pecan_debug = CONF.DEFAULT.pecan_debug if service.get_workers('api') != 1 and pecan_debug: pecan_debug = False LOG.warning(_LW('pecan_debug cannot be enabled, if workers is > 1, ' @@ -84,10 +65,10 @@ def setup_app(pecan_config=None, extra_hooks=None): guess_content_type_from_ext=False ) - transport = messaging.get_rpc_transport(cfg.CONF) + transport = messaging.get_rpc_transport(CONF) target = messaging.Target(topic='ord-listener-q', exchange='ranger-agent', - server=cfg.CONF.host) + server=CONF.DEFAULT.host) endpoints = [api.ListenerQueueHandler()] server = messaging.get_rpc_server(transport, target, @@ -116,14 +97,14 @@ class VersionSelectorApplication(object): def load_app(): # Build the WSGI app cfg_file = None - cfg_path = cfg.CONF.api_paste_config + cfg_path = CONF.DEFAULT.api_paste_config if not os.path.isabs(cfg_path): cfg_file = CONF.find_file(cfg_path) elif os.path.exists(cfg_path): cfg_file = cfg_path if not cfg_file: - raise cfg.ConfigFilesNotFoundError([cfg.CONF.api_paste_config]) + raise cfg.ConfigFilesNotFoundError([CONF.DEFAULT.api_paste_config]) LOG.info("Full WSGI config used: %s" % cfg_file) return deploy.loadapp("config:" + cfg_file) @@ -131,11 +112,11 @@ def load_app(): def build_server(): app = load_app() # Create the WSGI server and start it - host, port = cfg.CONF.api.host, cfg.CONF.api.port + host, port = CONF.api.host, CONF.api.port LOG.info(_('Starting server in PID %s') % os.getpid()) LOG.info(_("Configuration:")) - cfg.CONF.log_opt_values(LOG, logging.INFO) + CONF.log_opt_values(LOG, logging.INFO) if host == '0.0.0.0': # nosec LOG.info(_( @@ -146,7 +127,7 @@ def build_server(): {'host': host, 'port': port})) workers = service.get_workers('api') - serving.run_simple(cfg.CONF.api.host, cfg.CONF.api.port, + serving.run_simple(CONF.api.host, CONF.api.port, app, processes=workers) diff --git a/ord/api/controllers/v1/api.py b/ord/api/controllers/v1/api.py index 8b837d7..38050f0 100644 --- a/ord/api/controllers/v1/api.py +++ b/ord/api/controllers/v1/api.py @@ -23,6 +23,7 @@ from ord.common.utils import ErrorCode from ord.db import api as db_api from ord.i18n import _ from ord.openstack.common import log +from ord import service from oslo_config import cfg from pecan import expose from pecan import request as pecan_req @@ -41,12 +42,7 @@ import webob.exc LOG = log.getLogger(__name__) -CONF = cfg.CONF -orm_opts = [ - cfg.StrOpt('rds_listener_endpoint', - help='Endpoint to rds_listener ') - -] +CONF = service.CONF opts = [ cfg.StrOpt('region', @@ -58,12 +54,6 @@ opts = [ CONF.register_opts(opts) -opt_group = cfg.OptGroup(name='orm', - title='Options for the orm service') - -CONF.register_group(opt_group) -CONF.register_opts(orm_opts, opt_group) - class ListenerQueueHandler(object): @@ -75,15 +65,21 @@ class ListenerQueueHandler(object): LOG.debug(" Payload: %s \n ctxt: %s " % (str(payload), str(ctxt))) LOG.debug(" -------------------------------") listener_response_body = {} + + rds_endpoint = \ + db_api.retrieve_configuration(region=CONF.DEFAULT.region) + + rds_endpoint = (rds_endpoint['rds_listener_endpoint'] + if rds_endpoint is not None else + CONF.orm.rds_listener_endpoint) try: listener_response_body = json.loads(payload) LOG.debug(" Payload to RDS Listener %s " % listener_response_body) headers = {'Content-type': 'application/json'} - rds_url = CONF.orm.rds_listener_endpoint # Python3 urllib: convert listener_response_body to bytes response_body_bytes = \ json.dumps(listener_response_body).encode("utf-8") - req = urllib.request.Request(rds_url, # nosec + req = urllib.request.Request(rds_endpoint, # nosec response_body_bytes, headers, unverifiable=False) @@ -100,7 +96,7 @@ class ListenerQueueHandler(object): ['ord-notifier-id']) status_code = None try: - LOG.info('Connecting to RDS at %s' % rds_url) + LOG.info('Connecting to RDS at %s' % rds_endpoint) resp = request.urlopen(req) # nosec status = utils.STATUS_RDS_SUCCESS if resp is not None: @@ -114,9 +110,11 @@ class ListenerQueueHandler(object): except ValueError as e: status = utils.STATUS_RDS_ERROR LOG.error('Error while parsing input payload %r', e) + status_code = None except Exception as ex: status = utils.STATUS_RDS_ERROR LOG.error('Error while calling RDS Listener %r', ex) + status_code = None finally: LOG.info('RDS Listener status %s ' % status) LOG.info('RDS Listener status code %s ' % status_code) @@ -134,7 +132,6 @@ class NotifierController(object): def __init__(self): super(NotifierController, self).__init__() - self._rpcapi = rpcapi.RpcAPI() self._set_keystone_client() def _set_keystone_client(cls): @@ -212,6 +209,10 @@ class NotifierController(object): token = pecan_req.headers['X-Auth-Token'] self.kc.tokens.validate(token) + @expose(generic=True) + def ord_configuration(self, **args): + raise webob.exc.HTTPNotFound + @expose(generic=True) def ord_notifier(self, **args): raise webob.exc.HTTPNotFound @@ -220,6 +221,17 @@ class NotifierController(object): def health_check(self, **args): raise webob.exc.HTTPNotFound + @ord_configuration.when(method='POST', template='json') + def ord_configuration_update(self, **args): + if CONF.auth_enabled: + self._validate_token() + else: + LOG.debug("Authentication is disabled. We don't recommend this.") + LOG.debug('Updating ranger-agent configuration') + db_api.update_configuration(**args) + + return {'Ranger-Agent': 'Update request processed'} + @health_check.when(method='GET', template='json') def ord_health_status(self): LOG.debug('Received health message via api endpoint') @@ -322,7 +334,8 @@ class NotifierController(object): try: ctxt = {'request_id': kwargs.get('request_id')} heat_template = base64.b64decode(file_info.file.read()) - self._rpcapi.invoke_notifier_rpc(ctxt, payload, heat_template) + rpcapi.RpcAPI().\ + invoke_notifier_rpc(ctxt, payload, heat_template) except messaging.MessageDeliveryFailure: LOG.error("Fail to deliver message") diff --git a/ord/db/api.py b/ord/db/api.py index 7e6f710..67cb0a6 100644 --- a/ord/db/api.py +++ b/ord/db/api.py @@ -53,6 +53,14 @@ def retrieve_target(request_id): return IMPL.retrieve_target(request_id) +def retrieve_configuration(region): + return IMPL.retrieve_configuration(region) + + +def update_configuration(**vals): + return IMPL.update_configuration(**vals) + + def retrieve_target_by_status(template_status_id): return IMPL.retrieve_target(template_status_id) diff --git a/ord/db/sqlalchemy/api.py b/ord/db/sqlalchemy/api.py index bf5472f..85470a7 100644 --- a/ord/db/sqlalchemy/api.py +++ b/ord/db/sqlalchemy/api.py @@ -20,33 +20,12 @@ import threading from ord.db.sqlalchemy import models from oslo_config import cfg -from oslo_db import options as oslo_db_options from oslo_db.sqlalchemy import session as db_session from oslo_db.sqlalchemy import utils as sqlalchemyutils from oslo_log import log as logging CONF = cfg.CONF -api_db_opts = [ - cfg.StrOpt('connection', - help='The SQLAlchemy connection string to use to connect to ' - 'the ORD database.', - secret=True), - cfg.StrOpt('mysql_sql_mode', - default='TRADITIONAL', - help='The SQL mode to be used for MySQL sessions. ' - 'This option, including the default, overrides any ' - 'server-set SQL mode. To use whatever SQL mode ' - 'is set by the server configuration, ' - 'set this to no value. Example: mysql_sql_mode='), -] - - -opt_group = cfg.OptGroup(name='database', - title='Options for the database service') -CONF.register_group(opt_group) -CONF.register_opts(oslo_db_options.database_opts, opt_group) - LOG = logging.getLogger(__name__) _ENGINE_FACADE = {'ord': None} @@ -196,6 +175,29 @@ def retrieve_target_by_status(template_status_id): return query.first() +# retrieve ranger configuration from database +def retrieve_configuration(region): + LOG.debug('Retrieve ranger-agent configuration of %s', region) + session = get_session() + query = model_query(models.Ord_Configuration, session=session) + query = query.filter_by(region=region) + + return query.first() + + +def update_configuration(**vals): + LOG.debug('Update ranger-agent configuration in database') + session = get_session() + with session.begin(): + query = model_query(models.Ord_Configuration, session=session) + query = query.filter_by(region=vals.get('region')) + if query.first() is None: + ord_conf = models.Ord_Configuration(**vals) + session.add(ord_conf) + else: + query.update(vals) + + def retrieve_target(request_id): LOG.debug('Retrieve Target data %s', request_id) session = get_session() diff --git a/ord/db/sqlalchemy/migrate_repo/versions/003_ord.py b/ord/db/sqlalchemy/migrate_repo/versions/003_ord.py new file mode 100644 index 0000000..72eb5f1 --- /dev/null +++ b/ord/db/sqlalchemy/migrate_repo/versions/003_ord.py @@ -0,0 +1,110 @@ +# Copyright 2012 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log as logging +from sqlalchemy import Column +from sqlalchemy import dialects +from sqlalchemy import MetaData, String, Table, Boolean +from sqlalchemy import Text + +LOG = logging.getLogger(__name__) + + +# Note on the autoincrement flag: this is defaulted for primary key columns +# of integral type, so is no longer set explicitly in such cases. + +def MediumText(): + return Text().with_variant(dialects.mysql.MEDIUMTEXT(), 'mysql') + + +def upgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + + ord_configuration = Table('ord_configuration', meta, + Column('region', String(length=30), + primary_key=True, nullable=False), + Column('api_workers', String(length=10), + nullable=False), + Column('debug_level', String(length=10), + nullable=False), + Column('pecan_debug', Boolean, + nullable=False), + Column('resource_creation_timeout_min', + String(length=10), nullable=False), + Column('resource_creation_timeout_max', + String(length=10), nullable=False), + Column('log_dir', String(length=80), + nullable=False), + Column('resource_status_check_wait', + String(length=10), nullable=False), + Column('api_paste_config', String(length=80), + nullable=False), + Column('transport_url', String(length=300), + nullable=False), + Column('enable_rds_callback_check', + Boolean, nullable=False), + Column('host', String(length=80), + nullable=False), + Column('port', String(length=10), + nullable=False), + Column('auth_type', String(length=20), + nullable=False), + Column('auth_url', String(length=80), + nullable=False), + Column('auth_version', String(length=10), + nullable=False), + Column('password', String(length=80), + nullable=False), + Column('project_domain_name', String(length=30), + nullable=False), + Column('project_name', String(length=80), + nullable=False), + Column('region_name', String(length=30), + nullable=False), + Column('user_domain_name', String(length=30), + nullable=False), + Column('username', String(length=80), + nullable=False), + Column('connection', String(length=240), + nullable=False), + Column('max_retries', String(length=10), + nullable=False), + Column('rds_listener_endpoint', + String(length=120), + nullable=False), + mysql_engine='InnoDB', + mysql_charset='utf8') + + tables = [ord_configuration] + + for table in tables: + try: + table.create() + except Exception: + LOG.info(repr(table)) + LOG.exception('Exception while creating table.') + raise + if migrate_engine.name == 'mysql': + # In Folsom we explicitly converted migrate_version to UTF8. + migrate_engine.execute( + 'ALTER TABLE migrate_version CONVERT TO CHARACTER SET utf8') + # Set default DB charset to UTF8. + migrate_engine.execute( + 'ALTER DATABASE %s DEFAULT CHARACTER SET utf8' % + migrate_engine.url.database) + + +def downgrade(migrate_engine): + raise NotImplementedError('Downgrade is not implemented.') diff --git a/ord/db/sqlalchemy/models.py b/ord/db/sqlalchemy/models.py index aff8461..af82d87 100644 --- a/ord/db/sqlalchemy/models.py +++ b/ord/db/sqlalchemy/models.py @@ -19,7 +19,7 @@ SQLAlchemy models for ranger-agent data. import datetime import uuid -from sqlalchemy import (Column, DateTime, String, Integer) +from sqlalchemy import (Column, DateTime, String, Integer, Boolean) from sqlalchemy import ForeignKey, Text from sqlalchemy import orm @@ -27,12 +27,35 @@ from sqlalchemy.dialects.mysql import MEDIUMTEXT from sqlalchemy.ext.declarative import declarative_base from oslo_config import cfg +from oslo_db import options as oslo_db_options from oslo_db.sqlalchemy import models CONF = cfg.CONF BASE = declarative_base() +database_opts = [ + cfg.StrOpt('connection', + help='The SQLAlchemy connection string to use' + ' to connect to the ORD database.', + secret=True), + cfg.StrOpt('max_retries', + default='5', + help='max attempts to connect to database allowed'), + cfg.StrOpt('mysql_sql_mode', + default='TRADITIONAL', + help='The SQL mode to be used for MySQL sessions. ' + 'This option, including the default, overrides any ' + 'server-set SQL mode. To use whatever SQL mode ' + 'is set by the server configuration, ' + 'set this to no value. Example: mysql_sql_mode') +] + +opt_group = cfg.OptGroup(name='database', + title='Options for the database service') +CONF.register_group(opt_group) +CONF.register_opts(oslo_db_options.database_opts, opt_group) + def MediumText(): return Text().with_variant(MEDIUMTEXT(), 'mysql') @@ -87,6 +110,90 @@ class ORDBase(models.ModelBase): id(self), ', '.join(items)) +# Used to override deployed ranger-agent configuration values +class Ord_Configuration(BASE, ORDBase): + __tablename__ = 'ord_configuration' + + api_workers = Column( + String(10), + nullable=False) + log_dir = Column( + String(60), + nullable=True) + debug_level = Column( + String(10), + nullable=False) + pecan_debug = Column( + String(10), + nullable=False) + region = Column( + String(30), + primary_key=True, + nullable=False) + resource_creation_timeout_min = Column( + String(10), nullable=False) + resource_creation_timeout_max = Column( + String(10), + nullable=False) + resource_status_check_wait = Column( + String(10), + nullable=False) + api_paste_config = Column( + String(80), + nullable=False) + transport_url = Column( + String(300), + nullable=False) + enable_rds_callback_check = Column( + Boolean) + + host = Column( + String(80), + nullable=False) + port = Column( + String(10), + nullable=False) + + auth_type = Column( + String(20), + nullable=False) + auth_url = Column( + String(80), + nullable=False) + auth_version = Column( + String(10), + nullable=False) + password = Column( + String(80), + nullable=False) + project_domain_name = Column( + String(30), + nullable=False) + project_name = Column( + String(80), + nullable=False) + region_name = Column( + String(30), + nullable=False) + user_domain_name = Column( + String(30), + nullable=False) + username = Column( + String(80), + nullable=False) + + connection = Column( + String(240), + nullable=False) + max_retries = Column( + String(10), + nullable=False) + + rds_listener_endpoint = Column( + String(80), + nullable=False) + + class Ord_Notification(BASE, ORDBase): __tablename__ = 'ord_notification' diff --git a/ord/engine/app.py b/ord/engine/app.py index 8fbe25b..8b022f0 100644 --- a/ord/engine/app.py +++ b/ord/engine/app.py @@ -16,23 +16,24 @@ from ord.engine.engine import Engine from ord.engine.engine import QueueHandler from ord.openstack.common import log as logging -from oslo_config import cfg +from ord import service import oslo_messaging as messaging import time LOG = logging.getLogger(__name__) +CONF = service.CONF def start(): engine = Engine() # start Notify message listener - transport = messaging.get_rpc_transport(cfg.CONF) + transport = messaging.get_rpc_transport(CONF) target = messaging.Target(topic='ord-notifier-q', exchange='ranger-agent', - server=cfg.CONF.host) + server=CONF.DEFAULT.host) endpoints = [QueueHandler(engine)] diff --git a/ord/engine/workerfactory.py b/ord/engine/workerfactory.py index f8a4dcf..1bb3b79 100755 --- a/ord/engine/workerfactory.py +++ b/ord/engine/workerfactory.py @@ -309,7 +309,8 @@ class WorkerThread(threading.Thread): max_range = int(CONF.retry_limits) self._rpcengine. \ - invoke_listener_rpc(res_ctxt, json.dumps(rds_payload)) + invoke_listener_rpc(res_ctxt, + json.dumps(rds_payload)) check_wait = CONF.resource_status_check_wait # increase the polling interval after the initial wait time @@ -327,8 +328,10 @@ class WorkerThread(threading.Thread): rds_payload.get('rds-listener')['status'] = status_original # if image_payload: # rds_payload.get('rds-listener')['status'] = image_payload + self._rpcengine. \ - invoke_listener_rpc(res_ctxt, json.dumps(rds_payload)) + invoke_listener_rpc(res_ctxt, + json.dumps(rds_payload)) if status != utils.STATUS_RDS_SUCCESS: LOG.info("Retrying for api response") diff --git a/ord/service.py b/ord/service.py index e32f79a..6e63611 100644 --- a/ord/service.py +++ b/ord/service.py @@ -26,6 +26,8 @@ from ord.i18n import _ from ord.openstack.common import log +LOG = log.getLogger(__name__) + OPTS = [ cfg.StrOpt('host', default=socket.gethostname(), @@ -33,10 +35,95 @@ OPTS = [ 'key. Can be an opaque identifier. For ZeroMQ only, must ' 'be a valid host name, FQDN, or IP address.'), ] -cfg.CONF.register_opts(OPTS) +cfg.CONF.register_opts(OPTS, group='DEFAULT') +default_opts = [ + cfg.StrOpt('api_paste_config', default='/etc/ranger-agent/api-paste.ini', + help=""), + cfg.IntOpt('api_workers', default=1, + help="Number of worker threads to be used by API"), + cfg.StrOpt('debug', default='true', + help="Enables debug output in logging"), + cfg.StrOpt('debug_level', default='ERROR', + help='Determines level of debug content' + ' output: Error/Warning/Debug'), + cfg.StrOpt('enable_heat_health_check', default='true', help=""), + cfg.BoolOpt('pecan_debug', default=True, help=""), + cfg.StrOpt('region', default='', + help="name of site ranger-agent is deployed on"), + cfg.StrOpt('resource_creation_timeout_max', default='14400', + help="Max allotment of time for resource creation"), + cfg.StrOpt('resource_creation_timeout_min', default='1200', + help='Min allotment of time before a timeout' + ' error can be returned'), + cfg.StrOpt('resource_status_check_wait', default='15', + help='Allotment of time between checks' + ' during resource creation'), + cfg.IntOpt('retry_limits', default=5, + help="Max allotment of tries for resource creation"), + cfg.StrOpt('transport_url', default='', + help="Messaging queue url", secret=True), + cfg.StrOpt('use_stderr', default='true', help=""), + cfg.StrOpt('verbose', default='false', help=""), + cfg.BoolOpt('enable_rds_callback_check', + default=True, + help='validate rds api is reachable') +] -LOG = log.getLogger(__name__) +auth_opts = [ + cfg.StrOpt('project_name', default='service', + help="project name used to stack heat resources"), + cfg.StrOpt('auth_type', default='password', + help="type of credentials used for authentication"), + cfg.StrOpt('auth_url', default='', + help='auth url used by ranger agent to' + ' invoke keystone apis'), + cfg.StrOpt('username', default='', + help='user name used by ranger agent to' + ' invoke keystone apis'), + cfg.StrOpt('password', default='', secret=True, + help='password used by ranger agent to' + ' invoke keystone apis'), + cfg.StrOpt('project_domain_name', default='default', + help='default project domain ' + 'used by ranger agent to invoke keystone apis'), + cfg.StrOpt('auth_version', default='v3', help="Keystone version"), + cfg.StrOpt("user_domain_name", default='default', + help='default project domain ' + 'used by ranger agent to invoke keystone apis'), + cfg.StrOpt("https_cacert", default=None, + help="Path to CA server certificate for SSL"), + + cfg.StrOpt('region_name', default='', help='Region'), + cfg.StrOpt('auth_enabled', default='True', + help='check if authentication turned on') +] + +api_opts = [ + cfg.IntOpt('port', + default=9010, + help='The port for the ORD API server.', + ), + cfg.StrOpt('host', + default='0.0.0.0', # nosec + help='The listen IP for the ORD API server.', + ) + +] + +orm_opts = [ + cfg.StrOpt('rds_listener_endpoint', default='', + help='The rds endpoint of ranger deployment'), + cfg.StrOpt('retry_limits', default='5', + help='Max attempts to contact Ranger rds endpoint') +] + +cfg.CONF.register_opts(default_opts, group='DEFAULT') +cfg.CONF.register_opts(auth_opts, group='keystone_authtoken') +cfg.CONF.register_opts(api_opts, group='api') +cfg.CONF.register_opts(orm_opts, group='orm') + +CONF = cfg.CONF class WorkerException(Exception): @@ -44,8 +131,7 @@ class WorkerException(Exception): def get_workers(name): - workers = (cfg.CONF.get('%s_workers' % name) or - utils.cpu_count()) + workers = (CONF.DEFAULT.api_workers or utils.cpu_count()) if workers and workers < 1: msg = (_("%(worker_name)s value of %(workers)s is invalid, " "must be greater than 0") % diff --git a/ord/tests/unit/api/controllers/v1/test_api.py b/ord/tests/unit/api/controllers/v1/test_api.py index c399d4a..f14ec9a 100644 --- a/ord/tests/unit/api/controllers/v1/test_api.py +++ b/ord/tests/unit/api/controllers/v1/test_api.py @@ -17,15 +17,14 @@ Unit Tests for ord.api.test_api """ -import base64 from cgi import FieldStorage -import mock from mox3.mox import stubout from ord.api.controllers.v1 import api from ord.db import api as db_api from ord.tests import base from oslo_config import cfg import requests +from unittest import mock from urllib import request import webob @@ -42,7 +41,6 @@ class OrdApiTestCase(base.BaseTestCase): self.addCleanup(self.stubs.SmartUnsetAll) def test_api_notifier(self): - kwargs = { 'request_id': '1', 'resource_id': 'qwe1234', @@ -80,40 +78,21 @@ class OrdApiTestCase(base.BaseTestCase): CONF.set_default('region', 'local') - def fake_keystone_client(*args, **kwds): - return + api.rpcapi = mock.MagicMock() + ord_notifier = api.NotifierController + ord_notifier._set_keystone_client = mock.MagicMock() + ord_notifier._validate_token = mock.MagicMock() + ord_notifier._persist_notification_record = \ + mock.MagicMock(return_value=db_response) - self.stubs.Set(api.NotifierController, '_set_keystone_client', - fake_keystone_client) - ord_notifier = api.NotifierController() - - def fake_validate_token(*args): - return - - def fake_persist_notification_record(*args, **kwds): - return db_response - - def fake_b64decode(*args, **kwds): - return "heat_template" - - def fake_invoke_notifier_rpc(*args, **kwds): - return payload - - self.stubs.Set(ord_notifier, "_validate_token", fake_validate_token) - self.stubs.Set(ord_notifier, "_persist_notification_record", - fake_persist_notification_record) - self.stubs.Set(base64, "b64decode", fake_b64decode) - self.stubs.Set(ord_notifier._rpcapi, "invoke_notifier_rpc", - fake_invoke_notifier_rpc) - - response = ord_notifier.ord_notifier_POST(**params) + response = ord_notifier().ord_notifier_POST(**params) expect_response = response['ord-notifier-response']['status'] self.assertEqual(expect_response, 'Submitted') def test_api_listener(self): ctxt = {'request_id': '1'} - api_listener = api.ListenerQueueHandler() + api_listener = api.ListenerQueueHandler kwargs = '{"request_id": "1",'\ ' "resource_id": "qwe1234","resource-type": "image"}' payload = str(kwargs) @@ -122,25 +101,15 @@ class OrdApiTestCase(base.BaseTestCase): 'error_code': '', 'error_msg': ''} - def mock_url_open(mock_response): - mock_response = mock.Mock() - mock_response.getcode.return_value = 200 - - def urlrequest_mock_method(url, payload, headers, unverifiable=False): - return "Failure" - - def fake_update_target(*args, **kwds): - return db_template_target - - self.stubs.Set(request, 'urlopen', mock_url_open) - self.stubs.Set(db_api, "update_target_data", - fake_update_target) - self.stubs.Set(request, 'Request', urlrequest_mock_method) - api_listener.invoke_listener_rpc(ctxt, payload) + request.urlopen = mock.MagicMock() + request.Request = mock.MagicMock() + db_api.update_target_data = mock.MagicMock() + db_api.retrieve_configuration = mock.MagicMock() + api_listener().invoke_listener_rpc(ctxt, payload) def test_rds_listener_failure(self): ctxt = {'request_id': '1'} - api_listener = api.ListenerQueueHandler() + api_listener = api.ListenerQueueHandler kwargs = '{"rds-listener": { "ord-notifier-id": "2",'\ '"status": "error","resource-type": "image",'\ @@ -153,28 +122,18 @@ class OrdApiTestCase(base.BaseTestCase): payload = str(kwargs) output_status = 'STATUS_RDS_SUCCESS' + http_error = requests.exceptions.HTTPError() + request.urlopen = mock.MagicMock(side_effect=http_error) + request.Request = mock.MagicMock() + db_api.update_target_data = mock.MagicMock() + db_api.retrieve_configuration = mock.MagicMock() - def mock_method(url, payload, headers, unverifiable=False): - return "Failure" - self.stubs.Set(request, 'Request', mock_method) - - def mock_url_open(mock_response): - mock_response = mock.Mock() - http_error = requests.exceptions.HTTPError() - mock_response.raise_for_status.side_effect = http_error - - def fake_update_target(*args, **kwds): - return db_template_target - - self.stubs.Set(request, 'urlopen', mock_url_open) - self.stubs.Set(db_api, "update_target_data", - fake_update_target) - api_listener.invoke_listener_rpc(ctxt, payload) + api_listener().invoke_listener_rpc(ctxt, payload) self.assertEqual(output_status, db_template_target['status']) def test_rds_listener_success(self): ctxt = {'request_id': '1'} - api_listener = api.ListenerQueueHandler() + api_listener = api.ListenerQueueHandler kwargs = '{"rds-listener": { "ord-notifier-id": "2",'\ '"status": "error","resource-type": "image",'\ @@ -188,21 +147,12 @@ class OrdApiTestCase(base.BaseTestCase): payload = str(kwargs) output_status = 'Error_RDS_Dispatch' - def mock_method(url, payload, headers, unverifiable=False): - return "Success" - self.stubs.Set(request, 'Request', mock_method) + request.Request = mock.MagicMock() + request.urlopen = mock.MagicMock() + db_api.update_target_data = mock.MagicMock() + db_api.retrieve_configuration = mock.MagicMock() - def mock_url_open(mock_response): - mock_response = mock.Mock() - mock_response.getcode.return_value = 200 - - def fake_update_target(*args, **kwds): - return db_template_target - - self.stubs.Set(request, 'urlopen', mock_url_open) - self.stubs.Set(db_api, "update_target_data", - fake_update_target) - api_listener.invoke_listener_rpc(ctxt, payload) + api_listener().invoke_listener_rpc(ctxt, payload) self.assertEqual(output_status, db_template_target['status']) @@ -223,19 +173,12 @@ class OrdApiTestCase(base.BaseTestCase): }' } - def fake_keystone_client(*args, **kwds): - return + ord_notifier = api.NotifierController + ord_notifier._set_keystone_client = mock.MagicMock() + ord_notifier._validate_token = mock.MagicMock() - self.stubs.Set(api.NotifierController, '_set_keystone_client', - fake_keystone_client) - ord_notifier = api.NotifierController() - - def fake_validate_token(*args): - return - - self.stubs.Set(ord_notifier, "_validate_token", fake_validate_token) self.assertRaises(webob.exc.HTTPBadRequest, - ord_notifier.ord_notifier_POST, + ord_notifier().ord_notifier_POST, **params) def test_api_notifier_for_invalid_region(self): @@ -258,19 +201,12 @@ class OrdApiTestCase(base.BaseTestCase): CONF.set_default('region', 'local') - def fake_keystone_client(*args, **kwds): - return + ord_notifier = api.NotifierController + ord_notifier._set_keystone_client = mock.MagicMock() + ord_notifier._validate_token = mock.MagicMock() - self.stubs.Set(api.NotifierController, '_set_keystone_client', - fake_keystone_client) - ord_notifier = api.NotifierController() - - def fake_validate_token(*args): - return - - self.stubs.Set(ord_notifier, "_validate_token", fake_validate_token) self.assertRaises(webob.exc.HTTPBadRequest, - ord_notifier.ord_notifier_POST, + ord_notifier().ord_notifier_POST, **params) def test_api_notifier_for_invalid_payload(self): @@ -293,19 +229,12 @@ class OrdApiTestCase(base.BaseTestCase): CONF.set_default('region', 'local') - def fake_keystone_client(*args, **kwds): - return + ord_notifier = api.NotifierController + ord_notifier._set_keystone_client = mock.MagicMock() + ord_notifier._validate_token = mock.MagicMock() - self.stubs.Set(api.NotifierController, '_set_keystone_client', - fake_keystone_client) - ord_notifier = api.NotifierController() - - def fake_validate_token(*args): - return - - self.stubs.Set(ord_notifier, "_validate_token", fake_validate_token) self.assertRaises(webob.exc.HTTPBadRequest, - ord_notifier.ord_notifier_POST, + ord_notifier().ord_notifier_POST, **params) def test_api_ord_notifier_status(self): @@ -345,23 +274,50 @@ class OrdApiTestCase(base.BaseTestCase): 'error-msg': 'stack fail'} } - def fake_keystone_client(*args, **kwds): - return + ord_notifier = api.NotifierController - self.stubs.Set(api.NotifierController, '_set_keystone_client', - fake_keystone_client) - ord_notifier = api.NotifierController() + ord_notifier._set_keystone_client = mock.MagicMock() + db_api.retrieve_template = mock.MagicMock(return_value=db_template) + db_api.retrieve_target = \ + mock.MagicMock(return_value=db_template_target) - def fake_retrieve_template(*args, **kwds): - return db_template - - def fake_retrieve_target(*args, **kwds): - return db_template_target - - self.stubs.Set(db_api, "retrieve_template", - fake_retrieve_template) - self.stubs.Set(db_api, "retrieve_target", - fake_retrieve_target) - - notification_status = ord_notifier.ord_notifier_status(**request_id) + notification_status = ord_notifier().ord_notifier_status(**request_id) self.assertEqual(payload, notification_status) + + def test_update_configuration(self): + payload = { + "api_workers": 1, + "debug_level": "DEBUG", + "pecan_debug": True, + "region": "local", + "resource_creation_timeout_min": 1200, + "resource_creation_timeout_max": 14400, + "resource_status_check_wait": 15, + "api_paste_config": "/etc/ranger-agent/api-paste.ini", + "transport_url": + "rabbit://stackrabbit:stackqueue@192.168.56.135:5672", + "enable_rds_callback_check": True, + "host": "0.0.0.0", + "port": 9010, + "auth_type": "password", + "auth_url": "http://192.168.56.135/identity/v3", + "auth_version": "v3", + "password": "secret", + "project_domain_name": "Default", + "project_name": "service", + "region_name": "RegionOne", + "user_domain_name": "Default", + "username": "admin", + "connection": "mysql+pymysql://root:stackdb@127.0.0.1:3306/ord", + "max_retries": 5, + "rds_listener_endpoint": "http://192.168.56.127:8777/v1/rds/status" + } + + mock_notifierController = api.NotifierController + mock_notifierController._set_keystone_client = mock.MagicMock() + mock_notifierController._validate_token = mock.MagicMock() + db_api.update_configuration = mock.MagicMock() + + resp = mock_notifierController().ord_configuration_update(**payload) + + self.assertEqual(resp, {"Ranger-Agent": "Update request processed"}) diff --git a/test-requirements.txt b/test-requirements.txt index 252c939..830a597 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -10,11 +10,10 @@ python-subunit>=0.0.18 sphinx>>=1.2.1,!=1.3b1,<1.4 # BSD oslosphinx>=4.7.0 # Apache-2.0 oslotest>=1.10.0 # Apache-2.0 -testrepository>=0.0.18 testscenarios>=0.4 +stestr>=1.0.0 # Apache-2.0 testtools>=1.4.0 mock>=2.0 discover mox3>=0.27.0 -psycopg2>=2.5 reno>=1.8.0 # Apache-2.0 diff --git a/tox.ini b/tox.ini index 1ee82f7..4f07855 100644 --- a/tox.ini +++ b/tox.ini @@ -12,7 +12,7 @@ setenv = PYTHONWARNINGS=default::DeprecationWarning deps = -r {toxinidir}/requirements.txt -r {toxinidir}/test-requirements.txt -commands = python setup.py test --slowest --testr-args='{posargs}' +commands = stestr run [testenv:bandit] deps = .[bandit]