openstack
/
trove
Code Issues Proposed changes

Configuration Groups for MongoDB 

Also includes:

* Use pymongo to determine database status.
* JsonCodec for writing/reading JSON files.

Implements blueprint: mongodb-configuration-groups
Change-Id: I64f3fdd05b2c320613cbd1c394e6d3a88e09363b
This commit is contained in:
Petr Malik 2015-06-14 21:47:18 -04:00
parent 67b68a9bcc
commit 60876617bf
19 changed files with 851 additions and 484 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
trove/common/configurations.py
View File

@ -39,3 +39,14 @@ class MySQLConfParser(object):
config_dict = self.CODEC.deserialize(self.config) config_dict = self.CODEC.deserialize(self.config)
mysqld_section_dict = config_dict['mysqld'] mysqld_section_dict = config_dict['mysqld']
return mysqld_section_dict.items() return mysqld_section_dict.items()
class MongoDBConfParser(object):
CODEC = stream_codecs.SafeYamlCodec(default_flow_style=False)
def __init__(self, config):
self.config = config
def parse(self):
return self.CODEC.deserialize(self.config).items()

10
trove/common/stream_codecs.py
View File

@ -16,6 +16,7 @@
import abc import abc
import ast import ast
import csv import csv
import json
import six import six
import StringIO import StringIO
import yaml import yaml
@ -344,3 +345,12 @@ class PropertiesCodec(StreamCodec):
container.append(item) container.append(item)
return container return container
class JsonCodec(StreamCodec):
def serialize(self, dict_data):
return json.dumps(dict_data)
def deserialize(self, stream):
return json.load(StringIO.StringIO(stream))

1
trove/common/template.py
View File

@ -31,6 +31,7 @@ ENV = utils.ENV
# TODO(cp16net) Maybe this should be moved to a config dict # TODO(cp16net) Maybe this should be moved to a config dict
SERVICE_PARSERS = { SERVICE_PARSERS = {
'mongodb': configurations.MongoDBConfParser,
'mysql': configurations.MySQLConfParser, 'mysql': configurations.MySQLConfParser,
'percona': configurations.MySQLConfParser, 'percona': configurations.MySQLConfParser,
'redis': configurations.RedisConfParser, 'redis': configurations.RedisConfParser,

1
trove/guestagent/common/configuration.py
View File

@ -443,6 +443,7 @@ class OneFileOverrideStrategy(ConfigurationOverrideStrategy):
self._regenerate_base_configuration() self._regenerate_base_configuration()
def remove(self, group_name, change_id=None): def remove(self, group_name, change_id=None):
if self._import_strategy.has_revisions:
self._import_strategy.remove(group_name, change_id=change_id) self._import_strategy.remove(group_name, change_id=change_id)
self._regenerate_base_configuration() self._regenerate_base_configuration()
if not self._import_strategy.has_revisions: if not self._import_strategy.has_revisions:

43
trove/guestagent/common/guestagent_utils.py
View File

@ -40,6 +40,49 @@ def update_dict(updates, target):
return target return target
def expand_dict(target, namespace_sep='.'):
"""Expand a flat dict to a nested one.
This is an inverse of 'flatten_dict'.
:seealso: flatten_dict
"""
nested = {}
for k, v in target.items():
sub = nested
keys = k.split(namespace_sep)
for key in keys[:-1]:
sub = sub.setdefault(key, {})
sub[keys[-1]] = v
return nested
def flatten_dict(target, namespace_sep='.'):
"""Flatten a nested dict.
Return a one-level dict with all sub-level keys joined by a namespace
separator.
The following nested dict:
{'ns1': {'ns2a': {'ns3a': True, 'ns3b': False}, 'ns2b': 10}}
would be flattened to:
{'ns1.ns2a.ns3a': True, 'ns1.ns2a.ns3b': False, 'ns1.ns2b': 10}
"""
def flatten(target, keys, namespace_sep):
flattened = {}
if isinstance(target, collections.Mapping):
for k, v in target.items():
flattened.update(
flatten(v, keys + [k], namespace_sep))
else:
ns = namespace_sep.join(keys)
flattened[ns] = target
return flattened
return flatten(target, [], namespace_sep)
def build_file_path(base_dir, base_name, *extensions): def build_file_path(base_dir, base_name, *extensions):
"""Build a path to a file in a given directory. """Build a path to a file in a given directory.
The file may have an extension(s). The file may have an extension(s).

110
trove/guestagent/datastore/experimental/mongodb/manager.py
View File

@ -17,7 +17,6 @@ import os
from oslo_log import log as logging from oslo_log import log as logging
from oslo_service import periodic_task from oslo_service import periodic_task
from oslo_utils import netutils
from trove.common import cfg from trove.common import cfg
from trove.common import exception from trove.common import exception
@ -39,14 +38,13 @@ MANAGER = CONF.datastore_manager
class Manager(periodic_task.PeriodicTasks): class Manager(periodic_task.PeriodicTasks):
def __init__(self): def __init__(self):
self.status = service.MongoDBAppStatus() self.app = service.MongoDBApp()
self.app = service.MongoDBApp(self.status)
super(Manager, self).__init__(CONF) super(Manager, self).__init__(CONF)
@periodic_task.periodic_task @periodic_task.periodic_task
def update_status(self, context): def update_status(self, context):
"""Update the status of the MongoDB service.""" """Update the status of the MongoDB service."""
self.status.update() self.app.status.update()
def rpc_ping(self, context): def rpc_ping(self, context):
LOG.debug("Responding to RPC ping.") LOG.debug("Responding to RPC ping.")
@ -60,7 +58,7 @@ class Manager(periodic_task.PeriodicTasks):
LOG.debug("Preparing MongoDB instance.") LOG.debug("Preparing MongoDB instance.")
self.status.begin_install() self.app.status.begin_install()
self.app.install_if_needed(packages) self.app.install_if_needed(packages)
self.app.wait_for_start() self.app.wait_for_start()
self.app.stop_db() self.app.stop_db()
@ -81,69 +79,46 @@ class Manager(periodic_task.PeriodicTasks):
LOG.debug("Mounted the volume %(path)s as %(mount)s." % LOG.debug("Mounted the volume %(path)s as %(mount)s." %
{'path': device_path, "mount": mount_point}) {'path': device_path, "mount": mount_point})
self.app.secure(cluster_config) if config_contents:
conf_changes = self.get_config_changes(cluster_config, mount_point) # Save resolved configuration template first.
config_contents = self.app.update_config_contents( self.app.configuration_manager.save_configuration(config_contents)
config_contents, conf_changes)
if cluster_config is None: # Apply guestagent specific configuration changes.
self.app.start_db_with_conf_changes(config_contents) self.app.apply_initial_guestagent_configuration(
if backup_info: cluster_config, mount_point)
self._perform_restore(backup_info, context,
mount_point, self.app) if not cluster_config:
# Create the Trove admin user.
self.app.secure()
# Don't start mongos until add_config_servers is invoked.
if not self.app.is_query_router:
self.app.start_db(update_db=False)
if not cluster_config and backup_info:
self._perform_restore(backup_info, context, mount_point, self.app)
if service.MongoDBAdmin().is_root_enabled(): if service.MongoDBAdmin().is_root_enabled():
self.status.report_root('root') self.app.status.report_root('root')
elif root_password:
if not cluster_config and root_password:
LOG.debug('Root password provided. Enabling root.') LOG.debug('Root password provided. Enabling root.')
service.MongoDBAdmin().enable_root(root_password) service.MongoDBAdmin().enable_root(root_password)
if not cluster_config:
if databases: if databases:
self.create_database(context, databases) self.create_database(context, databases)
if users: if users:
self.create_user(context, users) self.create_user(context, users)
if cluster_config:
self.app.status.set_status(
ds_instance.ServiceStatuses.BUILD_PENDING)
else: else:
if cluster_config['instance_type'] == "query_router": self.app.status.set_status(
self.app.reset_configuration({'config_contents': ds_instance.ServiceStatuses.RUNNING)
config_contents})
self.app.write_mongos_upstart()
self.app.status.is_query_router = True
# don't start mongos until add_config_servers is invoked
elif cluster_config['instance_type'] == "config_server":
self.app.status.is_config_server = True
self.app.start_db_with_conf_changes(config_contents)
elif cluster_config['instance_type'] == "member":
self.app.start_db_with_conf_changes(config_contents)
else:
LOG.error(_("Bad cluster configuration; instance type "
"given as %s.") % cluster_config['instance_type'])
self.status.set_status(ds_instance.ServiceStatuses.FAILED)
return
self.status.set_status(ds_instance.ServiceStatuses.BUILD_PENDING)
LOG.info(_('Completed setup of MongoDB database instance.')) LOG.info(_('Completed setup of MongoDB database instance.'))
def get_config_changes(self, cluster_config, mount_point=None):
LOG.debug("Getting configuration changes.")
config_changes = {}
# todo mvandijk: uncomment the following when auth is being enabled
# config_changes['auth'] = 'true'
config_changes['bind_ip'] = ','.join([netutils.get_my_ipv4(),
'127.0.0.1'])
if cluster_config is not None:
# todo mvandijk: uncomment the following when auth is being enabled
# config_changes['keyFile'] = self.app.get_key_file()
if cluster_config["instance_type"] == "config_server":
config_changes["configsvr"] = "true"
elif cluster_config["instance_type"] == "member":
config_changes["replSet"] = cluster_config["replica_set_name"]
if (mount_point is not None and
(cluster_config is None or
cluster_config['instance_type'] != "query_router")):
config_changes['dbpath'] = mount_point
return config_changes
def restart(self, context): def restart(self, context):
LOG.debug("Restarting MongoDB.") LOG.debug("Restarting MongoDB.")
self.app.restart() self.app.restart()
@ -260,13 +235,14 @@ class Manager(periodic_task.PeriodicTasks):
def update_overrides(self, context, overrides, remove=False): def update_overrides(self, context, overrides, remove=False):
LOG.debug("Updating overrides.") LOG.debug("Updating overrides.")
raise exception.DatastoreOperationNotSupported( if remove:
operation='update_overrides', datastore=MANAGER) self.app.remove_overrides()
else:
self.app.update_overrides(context, overrides, remove)
def apply_overrides(self, context, overrides): def apply_overrides(self, context, overrides):
LOG.debug("Applying overrides.") LOG.debug("Overrides will be applied after restart.")
raise exception.DatastoreOperationNotSupported( pass
operation='apply_overrides', datastore=MANAGER)
def get_replication_snapshot(self, context, snapshot_info, def get_replication_snapshot(self, context, snapshot_info,
replica_source_config=None): replica_source_config=None):
@ -320,7 +296,7 @@ class Manager(periodic_task.PeriodicTasks):
self.app.add_members(members) self.app.add_members(members)
LOG.debug("add_members call has finished.") LOG.debug("add_members call has finished.")
except Exception: except Exception:
self.status.set_status(ds_instance.ServiceStatuses.FAILED) self.app.status.set_status(ds_instance.ServiceStatuses.FAILED)
raise raise
def add_config_servers(self, context, config_servers): def add_config_servers(self, context, config_servers):
@ -330,7 +306,7 @@ class Manager(periodic_task.PeriodicTasks):
self.app.add_config_servers(config_servers) self.app.add_config_servers(config_servers)
LOG.debug("add_config_servers call has finished.") LOG.debug("add_config_servers call has finished.")
except Exception: except Exception:
self.status.set_status(ds_instance.ServiceStatuses.FAILED) self.app.status.set_status(ds_instance.ServiceStatuses.FAILED)
raise raise
def add_shard(self, context, replica_set_name, replica_set_member): def add_shard(self, context, replica_set_name, replica_set_member):
@ -341,14 +317,14 @@ class Manager(periodic_task.PeriodicTasks):
self.app.add_shard(replica_set_name, replica_set_member) self.app.add_shard(replica_set_name, replica_set_member)
LOG.debug("add_shard call has finished.") LOG.debug("add_shard call has finished.")
except Exception: except Exception:
self.status.set_status(ds_instance.ServiceStatuses.FAILED) self.app.status.set_status(ds_instance.ServiceStatuses.FAILED)
raise raise
def cluster_complete(self, context): def cluster_complete(self, context):
# Now that cluster creation is complete, start status checks # Now that cluster creation is complete, start status checks
LOG.debug("Cluster creation complete, starting status checks.") LOG.debug("Cluster creation complete, starting status checks.")
status = self.status._get_actual_db_status() status = self.app.status._get_actual_db_status()
self.status.set_status(status) self.app.status.set_status(status)
def get_key(self, context): def get_key(self, context):
# Return the cluster key # Return the cluster key

388
trove/guestagent/datastore/experimental/mongodb/service.py
View File

@ -13,10 +13,7 @@
# License for the specific language governing permissions and limitations # License for the specific language governing permissions and limitations
# under the License. # under the License.
import json
import os import os
import re
import tempfile
from oslo_log import log as logging from oslo_log import log as logging
from oslo_utils import netutils from oslo_utils import netutils
@ -28,7 +25,11 @@ from trove.common.exception import ProcessExecutionError
from trove.common.i18n import _ from trove.common.i18n import _
from trove.common import instance as ds_instance from trove.common import instance as ds_instance
from trove.common import pagination from trove.common import pagination
from trove.common.stream_codecs import JsonCodec, SafeYamlCodec
from trove.common import utils as utils from trove.common import utils as utils
from trove.guestagent.common.configuration import ConfigurationManager
from trove.guestagent.common.configuration import OneFileOverrideStrategy
from trove.guestagent.common import guestagent_utils
from trove.guestagent.common import operating_system from trove.guestagent.common import operating_system
from trove.guestagent.datastore.experimental.mongodb import system from trove.guestagent.datastore.experimental.mongodb import system
from trove.guestagent.datastore import service from trove.guestagent.datastore import service
@ -39,6 +40,10 @@ LOG = logging.getLogger(__name__)
CONF = cfg.CONF CONF = cfg.CONF
CONFIG_FILE = (operating_system. CONFIG_FILE = (operating_system.
file_discovery(system.CONFIG_CANDIDATES)) file_discovery(system.CONFIG_CANDIDATES))
# Configuration group for clustering-related settings.
CNF_CLUSTER = 'clustering'
MONGODB_PORT = CONF.mongodb.mongodb_port MONGODB_PORT = CONF.mongodb.mongodb_port
CONFIGSVR_PORT = CONF.mongodb.configsvr_port CONFIGSVR_PORT = CONF.mongodb.configsvr_port
IGNORED_DBS = CONF.mongodb.ignore_dbs IGNORED_DBS = CONF.mongodb.ignore_dbs
@ -48,9 +53,34 @@ IGNORED_USERS = CONF.mongodb.ignore_users
class MongoDBApp(object): class MongoDBApp(object):
"""Prepares DBaaS on a Guest container.""" """Prepares DBaaS on a Guest container."""
def __init__(self, status): @classmethod
def _init_overrides_dir(cls):
"""Initialize a directory for configuration overrides.
"""
revision_dir = guestagent_utils.build_file_path(
os.path.dirname(system.MONGO_USER),
ConfigurationManager.DEFAULT_STRATEGY_OVERRIDES_SUB_DIR)
if not os.path.exists(revision_dir):
operating_system.create_directory(
revision_dir,
user=system.MONGO_USER, group=system.MONGO_USER,
force=True, as_root=True)
return revision_dir
def __init__(self):
self.state_change_wait_time = CONF.state_change_wait_time self.state_change_wait_time = CONF.state_change_wait_time
self.status = status
revision_dir = self._init_overrides_dir()
self.configuration_manager = ConfigurationManager(
CONFIG_FILE, system.MONGO_USER, system.MONGO_USER,
SafeYamlCodec(default_flow_style=False),
requires_root=True,
override_strategy=OneFileOverrideStrategy(revision_dir))
self.is_query_router = False
self.status = MongoDBAppStatus()
def install_if_needed(self, packages): def install_if_needed(self, packages):
"""Prepare the guest machine with a MongoDB installation.""" """Prepare the guest machine with a MongoDB installation."""
@ -61,7 +91,7 @@ class MongoDBApp(object):
LOG.info(_("Finished installing MongoDB server.")) LOG.info(_("Finished installing MongoDB server."))
def _get_service(self): def _get_service(self):
if self.status._is_query_router() is True: if self.is_query_router:
return (operating_system. return (operating_system.
service_discovery(system.MONGOS_SERVICE_CANDIDATES)) service_discovery(system.MONGOS_SERVICE_CANDIDATES))
else: else:
@ -151,75 +181,141 @@ class MongoDBApp(object):
raise RuntimeError("Could not start MongoDB.") raise RuntimeError("Could not start MongoDB.")
LOG.debug('MongoDB started successfully.') LOG.debug('MongoDB started successfully.')
def update_overrides(self, context, overrides, remove=False):
if overrides:
self.configuration_manager.apply_user_override(overrides)
def remove_overrides(self):
self.configuration_manager.remove_user_override()
def start_db_with_conf_changes(self, config_contents): def start_db_with_conf_changes(self, config_contents):
LOG.info(_("Starting MongoDB with configuration changes.")) LOG.info(_('Starting MongoDB with configuration changes.'))
LOG.info(_("Configuration contents:\n %s.") % config_contents)
if self.status.is_running: if self.status.is_running:
LOG.error(_("Cannot start MongoDB with configuration changes. " format = 'Cannot start_db_with_conf_changes because status is %s.'
"MongoDB state == %s.") % self.status) LOG.debug(format, self.status)
raise RuntimeError("MongoDB is not stopped.") raise RuntimeError(format % self.status)
self._write_config(config_contents) LOG.info(_("Initiating config."))
self.configuration_manager.save_configuration(config_contents)
# The configuration template has to be updated with
# guestagent-controlled settings.
self.apply_initial_guestagent_configuration(
None, mount_point=system.MONGODB_MOUNT_POINT)
self.start_db(True) self.start_db(True)
def reset_configuration(self, configuration): def reset_configuration(self, configuration):
config_contents = configuration['config_contents']
LOG.info(_("Resetting configuration.")) LOG.info(_("Resetting configuration."))
self._write_config(config_contents) config_contents = configuration['config_contents']
self.configuration_manager.save_configuration(config_contents)
def update_config_contents(self, config_contents, parameters): def apply_initial_guestagent_configuration(
LOG.info(_("Updating configuration contents.")) self, cluster_config, mount_point=None):
if not config_contents: LOG.debug("Applying initial configuration.")
config_contents = self._read_config()
contents = self._delete_config_parameters(config_contents, # todo mvandijk: enable authorization.
parameters.keys()) # 'security.authorization': True
for param, value in parameters.items(): self.configuration_manager.apply_system_override(
if param and value: {'processManagement.fork': False,
contents = self._add_config_parameter(contents, 'processManagement.pidFilePath': system.MONGO_PID_FILE,
param, value) 'systemLog.destination': 'file',
return contents 'systemLog.path': system.MONGO_LOG_FILE,
'systemLog.logAppend': True
})
def _write_config(self, config_contents): if mount_point:
""" self.configuration_manager.apply_system_override(
Update contents of MongoDB configuration file {'storage.dbPath': mount_point})
"""
LOG.info(_("Updating MongoDB config."))
if config_contents:
LOG.info(_("Writing %s.") % system.TMP_CONFIG)
try:
with open(system.TMP_CONFIG, 'w') as t:
t.write(config_contents)
LOG.info(_("Moving %(a)s to %(b)s.") if cluster_config is not None:
% {'a': system.TMP_CONFIG, 'b': CONFIG_FILE}) self._configure_as_cluster_instance(cluster_config)
operating_system.move(system.TMP_CONFIG, CONFIG_FILE,
as_root=True)
except Exception:
os.unlink(system.TMP_CONFIG)
raise
else: else:
LOG.debug("Empty config_contents. Do nothing.") self._configure_network(MONGODB_PORT)
def _read_config(self): def _configure_as_cluster_instance(self, cluster_config):
try: """Configure this guest as a cluster instance and return its
with open(CONFIG_FILE, 'r') as f: new status.
return f.read() """
except IOError: if cluster_config['instance_type'] == "query_router":
LOG.info(_("Config file %s not found.") % CONFIG_FILE) self._configure_as_query_router()
return '' elif cluster_config["instance_type"] == "config_server":
self._configure_as_config_server()
elif cluster_config["instance_type"] == "member":
self._configure_as_cluster_member(
cluster_config['replica_set_name'])
else:
LOG.error(_("Bad cluster configuration; instance type "
"given as %s.") % cluster_config['instance_type'])
return ds_instance.ServiceStatuses.FAILED
def _delete_config_parameters(self, config_contents, parameters): if 'key' in cluster_config:
if not config_contents: self._configure_cluster_security(cluster_config['key'])
return None
params_as_string = '|'.join(parameters) def _configure_as_query_router(self):
p = re.compile("\\s*#?\\s*(%s)\\s*=" % params_as_string) LOG.info(_("Configuring instance as a cluster query router."))
contents_as_list = config_contents.splitlines() self.is_query_router = True
filtered = filter(lambda line: not p.match(line), contents_as_list)
return '\n'.join(filtered)
def _add_config_parameter(self, config_contents, parameter, value): # Write the 'mongos' upstart script.
return (config_contents or '') + "\n%s = %s" % (parameter, value) # FIXME(pmalik): The control script should really be written in the
# elements.
# The guestagent will choose the right daemon ('mongod' or 'mongos')
# based on the 'cluster_config' values.
upstart_contents = (system.MONGOS_UPSTART_CONTENTS.
format(config_file_placeholder=CONFIG_FILE))
operating_system.write_file(system.MONGOS_UPSTART, upstart_contents,
as_root=True)
# FIXME(pmalik): We should really have a separate configuration
# template for the 'mongos' process.
# Remove all storage configurations from the template.
# They apply only to 'mongod' processes.
# Already applied overrides will be integrated into the base file and
# their current groups removed.
config = guestagent_utils.expand_dict(
self.configuration_manager.parse_configuration())
if 'storage' in config:
LOG.debug("Removing 'storage' directives from the configuration "
"template.")
del config['storage']
self.configuration_manager.save_configuration(
guestagent_utils.flatten_dict(config))
# Apply 'mongos' configuration.
self._configure_network(MONGODB_PORT)
self.configuration_manager.apply_system_override(
{'sharding.configDB': ''}, CNF_CLUSTER)
def _configure_as_config_server(self):
LOG.info(_("Configuring instance as a cluster config server."))
self._configure_network(CONFIGSVR_PORT)
self.configuration_manager.apply_system_override(
{'sharding.clusterRole': 'configsvr'}, CNF_CLUSTER)
def _configure_as_cluster_member(self, replica_set_name):
LOG.info(_("Configuring instance as a cluster member."))
self._configure_network(MONGODB_PORT)
self.configuration_manager.apply_system_override(
{'replication.replSetName': replica_set_name}, CNF_CLUSTER)
def _configure_cluster_security(self, key_value):
"""Force cluster key-file-based authentication.
"""
# Store the cluster member authentication key.
self.store_key(key_value)
self.configuration_manager.apply_system_override(
{'security.clusterAuthMode': 'keyFile',
'security.keyFile': self.get_key_file()}, CNF_CLUSTER)
def _configure_network(self, port=None):
"""Make the service accessible at a given (or default if not) port.
"""
instance_ip = netutils.get_my_ipv4()
bind_interfaces_string = ','.join([instance_ip, '127.0.0.1'])
options = {'net.bindIp': bind_interfaces_string}
if port is not None:
guestagent_utils.update_dict({'net.port': port}, options)
self.configuration_manager.apply_system_override(options)
self.status.set_host(instance_ip, port=port)
def clear_storage(self): def clear_storage(self):
mount_point = "/var/lib/mongodb/*" mount_point = "/var/lib/mongodb/*"
@ -229,41 +325,24 @@ class MongoDBApp(object):
except exception.ProcessExecutionError: except exception.ProcessExecutionError:
LOG.exception(_("Error clearing storage.")) LOG.exception(_("Error clearing storage."))
def _has_config_db(self):
value_string = self.configuration_manager.get_value(
'sharding', {}).get('configDB')
return value_string is not None
# FIXME(pmalik): This method should really be called 'set_config_servers'.
# The current name suggests it adds more config servers, but it
# rather replaces the existing ones.
def add_config_servers(self, config_server_hosts): def add_config_servers(self, config_server_hosts):
"""Set config servers on a query router (mongos) instance.
""" """
This method is used by query router (mongos) instances. config_servers_string = ','.join(['%s:27019' % host
"""
config_contents = self._read_config()
configdb_contents = ','.join(['%(host)s:%(port)s'
% {'host': host, 'port': CONFIGSVR_PORT}
for host in config_server_hosts]) for host in config_server_hosts])
LOG.debug("Config server list %s." % configdb_contents) LOG.info(_("Setting config servers: %s") % config_servers_string)
# remove db path from config and update configdb self.configuration_manager.apply_system_override(
contents = self._delete_config_parameters(config_contents, {'sharding.configDB': config_servers_string}, CNF_CLUSTER)
["dbpath", "nojournal", self.start_db(True)
"smallfiles", "journal",
"noprealloc", "configdb"])
contents = self._add_config_parameter(contents,
"configdb", configdb_contents)
LOG.info(_("Rewriting configuration."))
self.start_db_with_conf_changes(contents)
def write_mongos_upstart(self):
upstart_contents = (system.MONGOS_UPSTART_CONTENTS.
format(config_file_placeholder=CONFIG_FILE))
LOG.info(_("Writing %s.") % system.TMP_MONGOS_UPSTART)
with open(system.TMP_MONGOS_UPSTART, 'w') as t:
t.write(upstart_contents)
LOG.info(_("Moving %(a)s to %(b)s.")
% {'a': system.TMP_MONGOS_UPSTART,
'b': system.MONGOS_UPSTART})
operating_system.move(system.TMP_MONGOS_UPSTART, system.MONGOS_UPSTART,
as_root=True)
operating_system.remove('/etc/init/mongodb.conf', force=True,
as_root=True)
def add_shard(self, replica_set_name, replica_set_member): def add_shard(self, replica_set_name, replica_set_member):
""" """
@ -330,6 +409,15 @@ class MongoDBApp(object):
# TODO(ramashri) see if hardcoded values can be removed # TODO(ramashri) see if hardcoded values can be removed
utils.poll_until(check_rs_status, sleep_time=60, time_out=100) utils.poll_until(check_rs_status, sleep_time=60, time_out=100)
def _set_localhost_auth_bypass(self, enabled):
"""When active, the localhost exception allows connections from the
localhost interface to create the first user on the admin database.
The exception applies only when there are no users created in the
MongoDB instance.
"""
self.configuration_manager.apply_system_override(
{'setParameter': {'enableLocalhostAuthBypass': enabled}})
def list_all_dbs(self): def list_all_dbs(self):
return MongoDBAdmin().list_database_names() return MongoDBAdmin().list_database_names()
@ -349,11 +437,7 @@ class MongoDBApp(object):
def store_key(self, key): def store_key(self, key):
"""Store the cluster key.""" """Store the cluster key."""
LOG.debug('Storing key for MongoDB cluster.') LOG.debug('Storing key for MongoDB cluster.')
with tempfile.NamedTemporaryFile() as f: operating_system.write_file(system.MONGO_KEY_FILE, key, as_root=True)
f.write(key)
f.flush()
operating_system.copy(f.name, system.MONGO_KEY_FILE,
force=True, as_root=True)
operating_system.chmod(system.MONGO_KEY_FILE, operating_system.chmod(system.MONGO_KEY_FILE,
operating_system.FileMode.SET_USR_RO, operating_system.FileMode.SET_USR_RO,
as_root=True) as_root=True)
@ -375,68 +459,58 @@ class MongoDBApp(object):
user = models.MongoDBUser(name='admin.%s' % creds.username, user = models.MongoDBUser(name='admin.%s' % creds.username,
password=creds.password) password=creds.password)
user.roles = system.MONGO_ADMIN_ROLES user.roles = system.MONGO_ADMIN_ROLES
with MongoDBClient(user, auth=False) as client: with MongoDBClient(None) as client:
MongoDBAdmin().create_user(user, client=client) MongoDBAdmin().create_user(user, client=client)
LOG.debug('Created admin user.') LOG.debug('Created admin user.')
def secure(self, cluster_config=None): def secure(self):
# Secure the server by storing the cluster key if this is a cluster """Create the Trove admin user.
# or creating the admin user if this is a single instance.
LOG.debug('Securing MongoDB instance.') The service should not be running at this point.
if cluster_config: """
self.store_key(cluster_config['key']) if self.status.is_running:
else: raise RuntimeError(_("Cannot secure the instance. "
LOG.debug('Generating admin password.') "The service is still running."))
try:
self._set_localhost_auth_bypass(True)
self.start_db(update_db=False)
password = utils.generate_random_password() password = utils.generate_random_password()
self.start_db()
self.create_admin_user(password) self.create_admin_user(password)
LOG.debug("MongoDB secure complete.")
finally:
self._set_localhost_auth_bypass(False)
self.stop_db() self.stop_db()
LOG.debug('MongoDB secure complete.')
def get_configuration_property(self, name, default=None):
"""Return the value of a MongoDB configuration property.
"""
return self.configuration_manager.get_value(name, default)
class MongoDBAppStatus(service.BaseDbStatus): class MongoDBAppStatus(service.BaseDbStatus):
is_config_server = None def __init__(self, host='localhost', port=None):
is_query_router = None super(MongoDBAppStatus, self).__init__()
self.set_host(host, port=port)
def _is_config_server(self): def set_host(self, host, port=None):
if self.is_config_server is None: # This forces refresh of the 'pymongo' engine cached in the
try: # MongoDBClient class.
cmd = ("grep '^configsvr[ \t]*=[ \t]*true$' %s" # Authentication is not required to check the server status.
% CONFIG_FILE) MongoDBClient(None, host=host, port=port)
utils.execute_with_timeout(cmd, shell=True)
self.is_config_server = True
except exception.ProcessExecutionError:
self.is_config_server = False
return self.is_config_server
def _is_query_router(self):
if self.is_query_router is None:
try:
cmd = ("grep '^configdb[ \t]*=.*$' %s"
% CONFIG_FILE)
utils.execute_with_timeout(cmd, shell=True)
self.is_query_router = True
except exception.ProcessExecutionError:
self.is_query_router = False
return self.is_query_router
def _get_actual_db_status(self): def _get_actual_db_status(self):
try: try:
port = CONFIGSVR_PORT if self._is_config_server() else MONGODB_PORT with MongoDBClient(None) as client:
out, err = utils.execute_with_timeout( client.server_info()
'mongostat', '--host', str(netutils.get_my_ipv4()),
'--port', str(port), '-n', str(1), check_exit_code=[0, 1]
)
if not err:
return ds_instance.ServiceStatuses.RUNNING return ds_instance.ServiceStatuses.RUNNING
else: except (pymongo.errors.ServerSelectionTimeoutError,
pymongo.errors.AutoReconnect):
return ds_instance.ServiceStatuses.SHUTDOWN return ds_instance.ServiceStatuses.SHUTDOWN
except exception.ProcessExecutionError as e: except Exception:
LOG.exception(_("Process execution %s.") % e) LOG.exception(_("Error getting MongoDB status."))
return ds_instance.ServiceStatuses.SHUTDOWN
except OSError as e:
LOG.exception(_("OS Error %s.") % e)
return ds_instance.ServiceStatuses.SHUTDOWN return ds_instance.ServiceStatuses.SHUTDOWN
@ -667,13 +741,11 @@ class MongoDBClient(object):
# engine information is cached by making it a class attribute # engine information is cached by making it a class attribute
engine = {} engine = {}
def __init__(self, user, host=None, port=None, def __init__(self, user, host=None, port=None):
auth=True):
"""Get the client. Specifying host and/or port updates cached values. """Get the client. Specifying host and/or port updates cached values.
:param user: (required) MongoDBUser instance :param user: MongoDBUser instance used to authenticate
:param host: server address, defaults to localhost :param host: server address, defaults to localhost
:param port: server port, defaults to 27017 :param port: server port, defaults to 27017
:param auth: set to False to disable authentication, default True
:return: :return:
""" """
new_client = False new_client = False
@ -688,7 +760,7 @@ class MongoDBClient(object):
if host: if host:
type(self).engine['host'] = host type(self).engine['host'] = host
if port: if port:
type(self).engine['host'] = port type(self).engine['port'] = port
new_client = True new_client = True
if new_client: if new_client:
host = type(self).engine['host'] host = type(self).engine['host']
@ -699,7 +771,7 @@ class MongoDBClient(object):
port=port, port=port,
connect=False) connect=False)
self.session = type(self).engine['client'] self.session = type(self).engine['client']
if auth: if user:
db_name = user.database.name db_name = user.database.name
LOG.debug("Authentication MongoDB client on %s." % db_name) LOG.debug("Authentication MongoDB client on %s." % db_name)
self._db = self.session[db_name] self._db = self.session[db_name]
@ -724,25 +796,13 @@ class MongoDBCredentials(object):
self.password = password self.password = password
def read(self, filename): def read(self, filename):
with open(filename) as f: credentials = operating_system.read_file(filename, codec=JsonCodec())
credentials = json.load(f)
self.username = credentials['username'] self.username = credentials['username']
self.password = credentials['password'] self.password = credentials['password']
def write(self, filename): def write(self, filename):
self.clear_file(filename)
with open(filename, 'w') as f:
credentials = {'username': self.username, credentials = {'username': self.username,
'password': self.password} 'password': self.password}
json.dump(credentials, f)
@staticmethod operating_system.write_file(filename, credentials, codec=JsonCodec())
def clear_file(filename): operating_system.chmod(filename, operating_system.FileMode.SET_USR_RW)
LOG.debug("Creating clean file %s" % filename)
if operating_system.file_discovery([filename]):
operating_system.remove(filename)
# force file creation by just opening it
open(filename, 'wb')
operating_system.chmod(filename,
operating_system.FileMode.SET_USR_RW,
as_root=True)

4
trove/guestagent/datastore/experimental/mongodb/system.py
View File

@ -21,11 +21,11 @@ from trove.guestagent import pkg
OS_NAME = operating_system.get_os() OS_NAME = operating_system.get_os()
MONGODB_MOUNT_POINT = "/var/lib/mongodb" MONGODB_MOUNT_POINT = "/var/lib/mongodb"
MONGO_PID_FILE = '/var/run/mongodb.pid'
MONGO_LOG_FILE = '/var/log/mongodb/mongod.log'
TMP_CONFIG = "/tmp/mongodb.conf.tmp"
CONFIG_CANDIDATES = ["/etc/mongodb.conf", "/etc/mongod.conf"] CONFIG_CANDIDATES = ["/etc/mongodb.conf", "/etc/mongod.conf"]
MONGOS_UPSTART = "/etc/init/mongos.conf" MONGOS_UPSTART = "/etc/init/mongos.conf"
TMP_MONGOS_UPSTART = "/tmp/mongos.conf.tmp"
MONGO_ADMIN_NAME = 'os_admin' MONGO_ADMIN_NAME = 'os_admin'
MONGO_ADMIN_ROLES = [{'db': 'admin', 'role': 'userAdminAnyDatabase'}, MONGO_ADMIN_ROLES = [{'db': 'admin', 'role': 'userAdminAnyDatabase'},
{'db': 'admin', 'role': 'dbAdminAnyDatabase'}, {'db': 'admin', 'role': 'dbAdminAnyDatabase'},

4
trove/guestagent/strategies/backup/experimental/mongo_impl.py
View File

@ -43,9 +43,7 @@ class MongoDump(base.BackupRunner):
backup_cmd = 'mongodump --out ' + MONGO_DUMP_DIR backup_cmd = 'mongodump --out ' + MONGO_DUMP_DIR
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
self.status = mongo_service.MongoDBAppStatus() self.app = mongo_service.MongoDBApp()
self.app = mongo_service.MongoDBApp(self.status)
self.admin = mongo_service.MongoDBApp(self.status)
super(MongoDump, self).__init__(*args, **kwargs) super(MongoDump, self).__init__(*args, **kwargs)
def _run_pre_backup(self): def _run_pre_backup(self):

3
trove/guestagent/strategies/restore/experimental/mongo_impl.py
View File

@ -39,8 +39,7 @@ class MongoDump(base.RestoreRunner):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
super(MongoDump, self).__init__(*args, **kwargs) super(MongoDump, self).__init__(*args, **kwargs)
self.status = mongo_service.MongoDBAppStatus() self.app = mongo_service.MongoDBApp()
self.app = mongo_service.MongoDBApp(self.status)
def post_restore(self): def post_restore(self):
""" """

94
trove/templates/mongodb/config.template
View File

@ -1,94 +1,4 @@
# mongodb.conf # mongodb.conf
smallfiles = false storage.mmapv1.smallFiles: false
storage.journal.enabled: true
# Where to store the data.
dbpath=/var/lib/mongodb
#where to log
logpath=/var/log/mongodb/mongodb.log
logappend=true
#port = 27017
# Enable journaling, http://www.mongodb.org/display/DOCS/Journaling
journal=true
# Enables periodic logging of CPU utilization and I/O wait
#cpu = true
# Turn on/off security. Off is currently the default
#noauth = true
#auth = true
# Verbose logging output.
#verbose = true
# Inspect all client data for validity on receipt (useful for
# developing drivers)
#objcheck = true
# Enable db quota management
#quota = true
# Set oplogging level where n is
# 0=off (default)
# 1=W
# 2=R
# 3=both
# 7=W+some reads
#oplog = 0
# Diagnostic/debugging option
#nocursors = true
# Ignore query hints
#nohints = true
# Disable the HTTP interface (Defaults to localhost:27018).
#nohttpinterface = true
# Turns off server-side scripting. This will result in greatly limited
# functionality
#noscripting = true
# Turns off table scans. Any query that would do a table scan fails.
#notablescan = true
# Disable data file preallocation.
#noprealloc = true
# Specify .ns file size for new databases.
# nssize = <size>
# Accout token for Mongo monitoring server.
#mms-token = <token>
# Server name for Mongo monitoring server.
#mms-name = <server-name>
# Ping interval for Mongo monitoring server.
#mms-interval = <seconds>
# Replication Options
# in replicated mongo databases, specify here whether this is a slave or master
#slave = true
#source = master.example.com
# Slave only: specify a single database to replicate
#only = master.example.com
# or
#master = true
#source = slave.example.com
# Address of a server to pair with.
#pairwith = <server:port>
# Address of arbiter server.
#arbiter = <server:port>
# Automatically resync if slave data is stale
#autoresync
# Custom size for replication operation log.
#oplogSize = <MB>
# Size limit for in-memory storage of op ids.
#opIdMem = <bytes>

3
trove/templates/mongodb/override.config.template
View File

@ -0,0 +1,3 @@
{% for key, value in overrides.iteritems() -%}
{{key}}: {{value}}
{% endfor %}

333
trove/templates/mongodb/validation-rules.json Normal file
View File

@ -0,0 +1,333 @@
{
"configuration-parameters": [
{
"name": "systemLog.verbosity",
"restart_required": true,
"min": 0,
"max": 5,
"type": "integer"
},
{
"name": "systemLog.component.accessControl.verbosity",
"restart_required": true,
"min": 0,
"max": 5,
"type": "integer"
},
{
"name": "systemLog.component.command.verbosity",
"restart_required": true,
"min": 0,
"max": 5,
"type": "integer"
},
{
"name": "systemLog.component.control.verbosity",
"restart_required": true,
"min": 0,
"max": 5,
"type": "integer"
},
{
"name": "systemLog.component.geo.verbosity",
"restart_required": true,
"min": 0,
"max": 5,
"type": "integer"
},
{
"name": "systemLog.component.index.verbosity",
"restart_required": true,
"min": 0,
"max": 5,
"type": "integer"
},
{
"name": "systemLog.component.network.verbosity",
"restart_required": true,
"min": 0,
"max": 5,
"type": "integer"
},
{
"name": "systemLog.component.query.verbosity",
"restart_required": true,
"min": 0,
"max": 5,
"type": "integer"
},
{
"name": "systemLog.component.replication.verbosity",
"restart_required": true,
"min": 0,
"max": 5,
"type": "integer"
},
{
"name": "systemLog.component.sharding.verbosity",
"restart_required": true,
"min": 0,
"max": 5,
"type": "integer"
},
{
"name": "systemLog.component.storage.verbosity",
"restart_required": true,
"min": 0,
"max": 5,
"type": "integer"
},
{
"name": "systemLog.component.storage.journal.verbosity",
"restart_required": true,
"min": 0,
"max": 5,
"type": "integer"
},
{
"name": "systemLog.component.write.verbosity",
"restart_required": true,
"min": 0,
"max": 5,
"type": "integer"
},
{
"name": "systemLog.quiet",
"restart_required": true,
"type": "boolean"
},
{
"name": "systemLog.traceAllExceptions",
"restart_required": true,
"type": "boolean"
},
{
"name": "systemLog.logAppend",
"restart_required": true,
"type": "boolean"
},
{
"name": "systemLog.logRotate",
"restart_required": true,
"type": "string"
},
{
"name": "systemLog.timeStampFormat",
"restart_required": true,
"type": "string"
},
{
"name": "net.maxIncomingConnections",
"restart_required": true,
"min": 0,
"type": "integer"
},
{
"name": "net.wireObjectCheck",
"restart_required": true,
"type": "boolean"
},
{
"name": "net.ipv6",
"restart_required": true,
"type": "boolean"
},
{
"name": "net.http.enabled",
"restart_required": true,
"type": "boolean"
},
{
"name": "net.http.JSONPEnabled",
"restart_required": true,
"type": "boolean"
},
{
"name": "net.http.RESTInterfaceEnabled",
"restart_required": true,
"type": "boolean"
},
{
"name": "security.authorization",
"restart_required": true,
"type": "boolean"
},
{
"name": "security.sasl.hostName",
"restart_required": true,
"type": "string"
},
{
"name": "security.sasl.serviceName",
"restart_required": true,
"type": "string"
},
{
"name": "security.sasl.saslauthdSocketPath",
"restart_required": true,
"type": "string"
},
{
"name": "security.javascriptEnabled",
"restart_required": true,
"type": "boolean"
},
{
"name": "operationProfiling.slowOpThresholdMs",
"restart_required": true,
"min": 0,
"type": "integer"
},
{
"name": "operationProfiling.mode",
"restart_required": true,
"type": "string"
},
{
"name": "storage.indexBuildRetry",
"restart_required": true,
"type": "boolean"
},
{
"name": "storage.journal.enabled",
"restart_required": true,
"type": "boolean"
},
{
"name": "storage.directoryPerDB",
"restart_required": true,
"type": "boolean"
},
{
"name": "storage.syncPeriodSecs",
"restart_required": true,
"min": 0,
"type": "integer"
},
{
"name": "storage.engine",
"restart_required": true,
"type": "string"
},
{
"name": "storage.mmapv1.nsSize",
"restart_required": true,
"min": 0,
"max": 2047,
"type": "integer"
},
{
"name": "storage.mmapv1.quota.enforced",
"restart_required": true,
"type": "boolean"
},
{
"name": "storage.mmapv1.quota.maxFilesPerDB",
"restart_required": true,
"min": 0,
"type": "integer"
},
{
"name": "storage.mmapv1.smallFiles",
"restart_required": true,
"type": "boolean"
},
{
"name": "storage.mmapv1.journal.debugFlags",
"restart_required": true,
"type": "integer"
},
{
"name": "storage.mmapv1.journal.commitIntervalMs",
"restart_required": true,
"min": 2,
"max": 300,
"type": "integer"
},
{
"name": "storage.wiredTiger.engineConfig.cacheSizeGB",
"restart_required": true,
"min": 0,
"type": "integer"
},
{
"name": "storage.wiredTiger.engineConfig.statisticsLogDelaySecs",
"restart_required": true,
"min": 0,
"type": "integer"
},
{
"name": "storage.wiredTiger.engineConfig.journalCompressor",
"restart_required": true,
"type": "string"
},
{
"name": "storage.wiredTiger.collectionConfig.blockCompressor",
"restart_required": true,
"type": "string"
},
{
"name": "storage.wiredTiger.indexConfig.prefixCompression",
"restart_required": true,
"type": "boolean"
},
{
"name": "replication.oplogSizeMB",
"restart_required": true,
"min": 0,
"type": "integer"
},
{
"name": "replication.secondaryIndexPrefetch",
"restart_required": true,
"type": "string"
},
{
"name": "sharding.clusterRole",
"restart_required": true,
"type": "string"
},
{
"name": "auditLog.format",
"restart_required": true,
"type": "string"
},
{
"name": "auditLog.filter",
"restart_required": true,
"type": "string"
},
{
"name": "snmp.subagent",
"restart_required": true,
"type": "boolean"
},
{
"name": "snmp.master",
"restart_required": true,
"type": "boolean"
},
{
"name": "replication.localPingThresholdMs",
"restart_required": true,
"min": 0,
"type": "integer"
},
{
"name": "sharding.autoSplit",
"restart_required": true,
"type": "boolean"
},
{
"name": "sharding.chunkSize",
"restart_required": true,
"min": 0,
"type": "integer"
},
{
"name": "setParameter",
"restart_required": true,
"type": "string"
}
]
}

16
trove/tests/unittests/guestagent/test_backups.py
View File

@ -17,6 +17,7 @@ from testtools.testcase import ExpectedException
from trove.common import exception from trove.common import exception
from trove.common import utils from trove.common import utils
from trove.guestagent.common.operating_system import FileMode from trove.guestagent.common.operating_system import FileMode
from trove.guestagent.datastore.experimental.mongodb.service import MongoDBApp
from trove.guestagent.strategies.backup import base as backupBase from trove.guestagent.strategies.backup import base as backupBase
from trove.guestagent.strategies.backup import mysql_impl from trove.guestagent.strategies.backup import mysql_impl
from trove.guestagent.strategies.restore import base as restoreBase from trove.guestagent.strategies.restore import base as restoreBase
@ -327,7 +328,8 @@ class GuestAgentBackupTest(trove_testtools.TestCase):
# (see bug/1423759). # (see bug/1423759).
remove.assert_called_once_with(ANY, force=True, as_root=True) remove.assert_called_once_with(ANY, force=True, as_root=True)
def test_backup_encrypted_mongodump_command(self): @mock.patch.object(MongoDBApp, '_init_overrides_dir')
def test_backup_encrypted_mongodump_command(self, _):
backupBase.BackupRunner.is_encrypted = True backupBase.BackupRunner.is_encrypted = True
backupBase.BackupRunner.encrypt_key = CRYPTO_KEY backupBase.BackupRunner.encrypt_key = CRYPTO_KEY
RunnerClass = utils.import_class(BACKUP_MONGODUMP_CLS) RunnerClass = utils.import_class(BACKUP_MONGODUMP_CLS)
@ -338,7 +340,8 @@ class GuestAgentBackupTest(trove_testtools.TestCase):
MONGODUMP_CMD + PIPE + ZIP + PIPE + ENCRYPT, bkp.command) MONGODUMP_CMD + PIPE + ZIP + PIPE + ENCRYPT, bkp.command)
self.assertIn("gz.enc", bkp.manifest) self.assertIn("gz.enc", bkp.manifest)
def test_backup_not_encrypted_mongodump_command(self): @mock.patch.object(MongoDBApp, '_init_overrides_dir')
def test_backup_not_encrypted_mongodump_command(self, _):
backupBase.BackupRunner.is_encrypted = False backupBase.BackupRunner.is_encrypted = False
backupBase.BackupRunner.encrypt_key = CRYPTO_KEY backupBase.BackupRunner.encrypt_key = CRYPTO_KEY
RunnerClass = utils.import_class(BACKUP_MONGODUMP_CLS) RunnerClass = utils.import_class(BACKUP_MONGODUMP_CLS)
@ -348,7 +351,8 @@ class GuestAgentBackupTest(trove_testtools.TestCase):
self.assertEqual(MONGODUMP_CMD + PIPE + ZIP, bkp.command) self.assertEqual(MONGODUMP_CMD + PIPE + ZIP, bkp.command)
self.assertIn("gz", bkp.manifest) self.assertIn("gz", bkp.manifest)
def test_restore_decrypted_mongodump_command(self): @mock.patch.object(MongoDBApp, '_init_overrides_dir')
def test_restore_decrypted_mongodump_command(self, _):
restoreBase.RestoreRunner.is_zipped = True restoreBase.RestoreRunner.is_zipped = True
restoreBase.RestoreRunner.is_encrypted = False restoreBase.RestoreRunner.is_encrypted = False
RunnerClass = utils.import_class(RESTORE_MONGODUMP_CLS) RunnerClass = utils.import_class(RESTORE_MONGODUMP_CLS)
@ -356,7 +360,8 @@ class GuestAgentBackupTest(trove_testtools.TestCase):
location="filename", checksum="md5") location="filename", checksum="md5")
self.assertEqual(restr.restore_cmd, UNZIP + PIPE + MONGODUMP_RESTORE) self.assertEqual(restr.restore_cmd, UNZIP + PIPE + MONGODUMP_RESTORE)
def test_restore_encrypted_mongodump_command(self): @mock.patch.object(MongoDBApp, '_init_overrides_dir')
def test_restore_encrypted_mongodump_command(self, _):
restoreBase.RestoreRunner.is_zipped = True restoreBase.RestoreRunner.is_zipped = True
restoreBase.RestoreRunner.is_encrypted = True restoreBase.RestoreRunner.is_encrypted = True
restoreBase.RestoreRunner.decrypt_key = CRYPTO_KEY restoreBase.RestoreRunner.decrypt_key = CRYPTO_KEY
@ -488,7 +493,8 @@ class MongodbBackupTests(trove_testtools.TestCase):
class MongodbRestoreTests(trove_testtools.TestCase): class MongodbRestoreTests(trove_testtools.TestCase):
def setUp(self): @patch.object(MongoDBApp, '_init_overrides_dir')
def setUp(self, _):
super(MongodbRestoreTests, self).setUp() super(MongodbRestoreTests, self).setUp()
self.restore_runner = utils.import_class( self.restore_runner = utils.import_class(

41
trove/tests/unittests/guestagent/test_dbaas.py
View File

@ -2530,7 +2530,8 @@ class MongoDBAppTest(testtools.TestCase):
'cmd_disable': 'disable' 'cmd_disable': 'disable'
} }
def setUp(self): @patch.object(mongo_service.MongoDBApp, '_init_overrides_dir')
def setUp(self, _):
super(MongoDBAppTest, self).setUp() super(MongoDBAppTest, self).setUp()
self.orig_utils_execute_with_timeout = (mongo_service. self.orig_utils_execute_with_timeout = (mongo_service.
utils.execute_with_timeout) utils.execute_with_timeout)
@ -2545,9 +2546,10 @@ class MongoDBAppTest(testtools.TestCase):
self.FAKE_ID = str(uuid4()) self.FAKE_ID = str(uuid4())
InstanceServiceStatus.create(instance_id=self.FAKE_ID, InstanceServiceStatus.create(instance_id=self.FAKE_ID,
status=rd_instance.ServiceStatuses.NEW) status=rd_instance.ServiceStatuses.NEW)
self.appStatus = FakeAppStatus(self.FAKE_ID,
self.mongoDbApp = mongo_service.MongoDBApp()
self.mongoDbApp.status = FakeAppStatus(self.FAKE_ID,
rd_instance.ServiceStatuses.NEW) rd_instance.ServiceStatuses.NEW)
self.mongoDbApp = mongo_service.MongoDBApp(self.appStatus)
time.sleep = Mock() time.sleep = Mock()
os.unlink = Mock() os.unlink = Mock()
@ -2568,7 +2570,7 @@ class MongoDBAppTest(testtools.TestCase):
def test_stopdb(self): def test_stopdb(self):
mongo_service.utils.execute_with_timeout = Mock() mongo_service.utils.execute_with_timeout = Mock()
self.appStatus.set_next_status( self.mongoDbApp.status.set_next_status(
rd_instance.ServiceStatuses.SHUTDOWN) rd_instance.ServiceStatuses.SHUTDOWN)
self.mongoDbApp.stop_db() self.mongoDbApp.stop_db()
@ -2577,7 +2579,7 @@ class MongoDBAppTest(testtools.TestCase):
def test_stop_db_with_db_update(self): def test_stop_db_with_db_update(self):
mongo_service.utils.execute_with_timeout = Mock() mongo_service.utils.execute_with_timeout = Mock()
self.appStatus.set_next_status( self.mongoDbApp.status.set_next_status(
rd_instance.ServiceStatuses.SHUTDOWN) rd_instance.ServiceStatuses.SHUTDOWN)
self.mongoDbApp.stop_db(True) self.mongoDbApp.stop_db(True)
@ -2587,13 +2589,15 @@ class MongoDBAppTest(testtools.TestCase):
def test_stop_db_error(self): def test_stop_db_error(self):
mongo_service.utils.execute_with_timeout = Mock() mongo_service.utils.execute_with_timeout = Mock()
self.appStatus.set_next_status(rd_instance.ServiceStatuses.RUNNING) self.mongoDbApp.status.set_next_status(
rd_instance.ServiceStatuses.RUNNING)
self.mongoDbApp.state_change_wait_time = 1 self.mongoDbApp.state_change_wait_time = 1
self.assertRaises(RuntimeError, self.mongoDbApp.stop_db) self.assertRaises(RuntimeError, self.mongoDbApp.stop_db)
def test_restart(self): def test_restart(self):
self.appStatus.set_next_status(rd_instance.ServiceStatuses.RUNNING) self.mongoDbApp.status.set_next_status(
rd_instance.ServiceStatuses.RUNNING)
self.mongoDbApp.stop_db = Mock() self.mongoDbApp.stop_db = Mock()
self.mongoDbApp.start_db = Mock() self.mongoDbApp.start_db = Mock()
@ -2611,7 +2615,8 @@ class MongoDBAppTest(testtools.TestCase):
def test_start_db(self): def test_start_db(self):
mongo_service.utils.execute_with_timeout = Mock() mongo_service.utils.execute_with_timeout = Mock()
self.appStatus.set_next_status(rd_instance.ServiceStatuses.RUNNING) self.mongoDbApp.status.set_next_status(
rd_instance.ServiceStatuses.RUNNING)
self.mongoDbApp.start_db() self.mongoDbApp.start_db()
self.assert_reported_status(rd_instance.ServiceStatuses.NEW) self.assert_reported_status(rd_instance.ServiceStatuses.NEW)
@ -2619,7 +2624,8 @@ class MongoDBAppTest(testtools.TestCase):
def test_start_db_with_update(self): def test_start_db_with_update(self):
mongo_service.utils.execute_with_timeout = Mock() mongo_service.utils.execute_with_timeout = Mock()
self.appStatus.set_next_status(rd_instance.ServiceStatuses.RUNNING) self.mongoDbApp.status.set_next_status(
rd_instance.ServiceStatuses.RUNNING)
self.mongoDbApp.start_db(True) self.mongoDbApp.start_db(True)
self.assertTrue(conductor_api.API.heartbeat.called_once_with( self.assertTrue(conductor_api.API.heartbeat.called_once_with(
@ -2631,7 +2637,8 @@ class MongoDBAppTest(testtools.TestCase):
return_value=["ubuntu 17036 0.0 0.1 618960 " return_value=["ubuntu 17036 0.0 0.1 618960 "
"29232 pts/8 Sl+ Jan29 0:07 mongod", ""]) "29232 pts/8 Sl+ Jan29 0:07 mongod", ""])
self.mongoDbApp.state_change_wait_time = 1 self.mongoDbApp.state_change_wait_time = 1
self.appStatus.set_next_status(rd_instance.ServiceStatuses.SHUTDOWN) self.mongoDbApp.status.set_next_status(
rd_instance.ServiceStatuses.SHUTDOWN)
self.assertRaises(RuntimeError, self.mongoDbApp.start_db) self.assertRaises(RuntimeError, self.mongoDbApp.start_db)
self.assertTrue(conductor_api.API.heartbeat.called_once_with( self.assertTrue(conductor_api.API.heartbeat.called_once_with(
@ -2645,23 +2652,11 @@ class MongoDBAppTest(testtools.TestCase):
self.assertRaises(RuntimeError, self.mongoDbApp.start_db) self.assertRaises(RuntimeError, self.mongoDbApp.start_db)
def test_mongodb_error_in_write_config_verify_unlink(self):
configuration = {'config_contents': 'some junk'}
with patch.object(os.path, 'isfile', return_value=True):
with patch.object(operating_system, 'move',
side_effect=ProcessExecutionError):
self.assertRaises(ProcessExecutionError,
self.mongoDbApp.reset_configuration,
configuration=configuration)
self.assertEqual(1, operating_system.move.call_count)
self.assertEqual(1, os.unlink.call_count)
def test_start_db_with_conf_changes_db_is_running(self): def test_start_db_with_conf_changes_db_is_running(self):
self.mongoDbApp.start_db = Mock() self.mongoDbApp.start_db = Mock()
self.appStatus.status = rd_instance.ServiceStatuses.RUNNING self.mongoDbApp.status.status = rd_instance.ServiceStatuses.RUNNING
self.assertRaises(RuntimeError, self.assertRaises(RuntimeError,
self.mongoDbApp.start_db_with_conf_changes, self.mongoDbApp.start_db_with_conf_changes,
Mock()) Mock())

13
trove/tests/unittests/guestagent/test_guestagent_utils.py
View File

@ -92,3 +92,16 @@ class TestGuestagentUtils(trove_testtools.TestCase):
'base_dir/base_name.ext1.ext2', 'base_dir/base_name.ext1.ext2',
guestagent_utils.build_file_path( guestagent_utils.build_file_path(
'base_dir', 'base_name', 'ext1', 'ext2')) 'base_dir', 'base_name', 'ext1', 'ext2'))
def test_flatten_expand_dict(self):
self._assert_flatten_expand_dict({}, {})
self._assert_flatten_expand_dict({'ns1': 1}, {'ns1': 1})
self._assert_flatten_expand_dict(
{'ns1': {'ns2a': {'ns3a': True, 'ns3b': False}, 'ns2b': 10}},
{'ns1.ns2a.ns3a': True, 'ns1.ns2a.ns3b': False, 'ns1.ns2b': 10})
def _assert_flatten_expand_dict(self, nested_dict, flattened_dict):
self.assertEqual(
flattened_dict, guestagent_utils.flatten_dict(nested_dict))
self.assertEqual(
nested_dict, guestagent_utils.expand_dict(flattened_dict))

187
trove/tests/unittests/guestagent/test_mongodb_cluster_manager.py
View File

@ -20,18 +20,24 @@ import pymongo
import trove.common.context as context import trove.common.context as context
import trove.common.instance as ds_instance import trove.common.instance as ds_instance
import trove.common.utils as utils import trove.common.utils as utils
from trove.guestagent.common import operating_system
import trove.guestagent.datastore.experimental.mongodb.manager as manager import trove.guestagent.datastore.experimental.mongodb.manager as manager
import trove.guestagent.datastore.experimental.mongodb.service as service import trove.guestagent.datastore.experimental.mongodb.service as service
import trove.guestagent.datastore.experimental.mongodb.system as system
import trove.guestagent.volume as volume import trove.guestagent.volume as volume
import trove.tests.unittests.trove_testtools as trove_testtools import trove.tests.unittests.trove_testtools as trove_testtools
class GuestAgentMongoDBClusterManagerTest(trove_testtools.TestCase): class GuestAgentMongoDBClusterManagerTest(trove_testtools.TestCase):
def setUp(self): @mock.patch.object(service.MongoDBApp, '_init_overrides_dir')
def setUp(self, _):
super(GuestAgentMongoDBClusterManagerTest, self).setUp() super(GuestAgentMongoDBClusterManagerTest, self).setUp()
self.context = context.TroveContext() self.context = context.TroveContext()
self.manager = manager.Manager() self.manager = manager.Manager()
self.manager.app.configuration_manager = mock.MagicMock()
self.manager.app.status = mock.MagicMock()
self.conf_mgr = self.manager.app.configuration_manager
self.pymongo_patch = mock.patch.object( self.pymongo_patch = mock.patch.object(
pymongo, 'MongoClient' pymongo, 'MongoClient'
@ -42,14 +48,14 @@ class GuestAgentMongoDBClusterManagerTest(trove_testtools.TestCase):
def tearDown(self): def tearDown(self):
super(GuestAgentMongoDBClusterManagerTest, self).tearDown() super(GuestAgentMongoDBClusterManagerTest, self).tearDown()
@mock.patch.object(service.MongoDBAppStatus, 'set_status')
@mock.patch.object(service.MongoDBApp, 'add_members', @mock.patch.object(service.MongoDBApp, 'add_members',
side_effect=RuntimeError("Boom!")) side_effect=RuntimeError("Boom!"))
def test_add_members_failure(self, mock_add_members, mock_set_status): def test_add_members_failure(self, mock_add_members):
members = ["test1", "test2"] members = ["test1", "test2"]
self.assertRaises(RuntimeError, self.manager.add_members, self.assertRaises(RuntimeError, self.manager.add_members,
self.context, members) self.context, members)
mock_set_status.assert_called_with(ds_instance.ServiceStatuses.FAILED) self.manager.app.status.set_status.assert_called_with(
ds_instance.ServiceStatuses.FAILED)
@mock.patch.object(utils, 'poll_until') @mock.patch.object(utils, 'poll_until')
@mock.patch.object(utils, 'generate_random_password', return_value='pwd') @mock.patch.object(utils, 'generate_random_password', return_value='pwd')
@ -64,127 +70,118 @@ class GuestAgentMongoDBClusterManagerTest(trove_testtools.TestCase):
mock_initiate.assert_any_call() mock_initiate.assert_any_call()
mock_add.assert_any_call(["test1", "test2"]) mock_add.assert_any_call(["test1", "test2"])
@mock.patch.object(service.MongoDBAppStatus, 'set_status')
@mock.patch.object(service.MongoDBApp, 'add_shard', @mock.patch.object(service.MongoDBApp, 'add_shard',
side_effect=RuntimeError("Boom!")) side_effect=RuntimeError("Boom!"))
def test_add_shard_failure(self, mock_add_shard, mock_set_status): def test_add_shard_failure(self, mock_add_shard):
self.assertRaises(RuntimeError, self.manager.add_shard, self.assertRaises(RuntimeError, self.manager.add_shard,
self.context, "rs", "rs_member") self.context, "rs", "rs_member")
mock_set_status.assert_called_with(ds_instance.ServiceStatuses.FAILED) self.manager.app.status.set_status.assert_called_with(
ds_instance.ServiceStatuses.FAILED)
@mock.patch.object(service.MongoDBAdmin, 'add_shard') @mock.patch.object(service.MongoDBAdmin, 'add_shard')
def test_add_shard(self, mock_add_shard): def test_add_shard(self, mock_add_shard):
self.manager.add_shard(self.context, "rs", "rs_member") self.manager.add_shard(self.context, "rs", "rs_member")
mock_add_shard.assert_called_with("rs/rs_member:27017") mock_add_shard.assert_called_with("rs/rs_member:27017")
@mock.patch.object(service.MongoDBAppStatus, 'set_status')
@mock.patch.object(service.MongoDBApp, 'add_config_servers', @mock.patch.object(service.MongoDBApp, 'add_config_servers',
side_effect=RuntimeError("Boom!")) side_effect=RuntimeError("Boom!"))
def test_add_config_server_failure(self, mock_add_config, def test_add_config_server_failure(self, mock_add_config):
mock_set_status):
self.assertRaises(RuntimeError, self.manager.add_config_servers, self.assertRaises(RuntimeError, self.manager.add_config_servers,
self.context, self.context,
["cfg_server1", "cfg_server2"]) ["cfg_server1", "cfg_server2"])
mock_set_status.assert_called_with(ds_instance.ServiceStatuses.FAILED) self.manager.app.status.set_status.assert_called_with(
ds_instance.ServiceStatuses.FAILED)
@mock.patch.object(service.MongoDBApp, 'start_db_with_conf_changes') @mock.patch.object(service.MongoDBApp, 'start_db')
@mock.patch.object(service.MongoDBApp, '_add_config_parameter', def test_add_config_servers(self, mock_start_db):
return_value="")
@mock.patch.object(service.MongoDBApp, '_delete_config_parameters',
return_value="")
@mock.patch.object(service.MongoDBApp, '_read_config', return_value="")
def test_add_config_servers(self, mock_read, mock_delete,
mock_add, mock_start):
self.manager.add_config_servers(self.context, self.manager.add_config_servers(self.context,
["cfg_server1", ["cfg_server1",
"cfg_server2"]) "cfg_server2"])
mock_read.assert_called_with() self.conf_mgr.apply_system_override.assert_called_once_with(
mock_delete.assert_called_with("", ["dbpath", "nojournal", {'sharding.configDB': "cfg_server1:27019,cfg_server2:27019"},
"smallfiles", "journal", 'clustering')
"noprealloc", "configdb"]) mock_start_db.assert_called_with(True)
mock_add.assert_called_with("", "configdb",
"cfg_server1:27019,cfg_server2:27019")
mock_start.assert_called_with("")
@mock.patch.object(service.MongoDBAppStatus, 'set_status')
@mock.patch.object(service.MongoDBApp, 'write_mongos_upstart')
@mock.patch.object(service.MongoDBApp, 'reset_configuration')
@mock.patch.object(service.MongoDBApp, 'update_config_contents')
@mock.patch.object(service.MongoDBApp, 'secure')
@mock.patch.object(service.MongoDBApp, 'get_key_file',
return_value="/test/key/file")
@mock.patch.object(netutils, 'get_my_ipv4', return_value="10.0.0.2")
def test_prepare_mongos(self, mock_ip_address, mock_key_file,
mock_secure, mock_update, mock_reset,
mock_upstart, mock_set_status):
@mock.patch.object(service.MongoDBApp, '_configure_as_query_router')
@mock.patch.object(service.MongoDBApp, '_configure_cluster_security')
def test_prepare_mongos(self, mock_secure, mock_config):
self._prepare_method("test-id-1", "query_router", None) self._prepare_method("test-id-1", "query_router", None)
mock_update.assert_called_with(None, {'bind_ip': '10.0.0.2,127.0.0.1', mock_config.assert_called_once_with()
# 'keyFile': '/test/key/file'}) mock_secure.assert_called_once_with(None)
}) self.manager.app.status.set_status.assert_called_with(
self.assertTrue(self.manager.app.status.is_query_router)
mock_set_status.assert_called_with(
ds_instance.ServiceStatuses.BUILD_PENDING) ds_instance.ServiceStatuses.BUILD_PENDING)
@mock.patch.object(service.MongoDBAppStatus, 'set_status') @mock.patch.object(service.MongoDBApp, '_configure_as_config_server')
@mock.patch.object(utils, 'poll_until') @mock.patch.object(service.MongoDBApp, '_configure_cluster_security')
@mock.patch.object(service.MongoDBApp, 'start_db_with_conf_changes') def test_prepare_config_server(self, mock_secure, mock_config):
@mock.patch.object(service.MongoDBApp, 'update_config_contents')
@mock.patch.object(service.MongoDBApp, 'secure')
@mock.patch.object(service.MongoDBApp, 'get_key_file',
return_value="/test/key/file")
@mock.patch.object(netutils, 'get_my_ipv4', return_value="10.0.0.3")
def test_prepare_config_server(self, mock_ip_address, mock_key_file,
mock_secure, mock_update, mock_start,
mock_poll, mock_set_status):
self._prepare_method("test-id-2", "config_server", None) self._prepare_method("test-id-2", "config_server", None)
mock_update.assert_called_with(None, {'configsvr': 'true', mock_config.assert_called_once_with()
'bind_ip': '10.0.0.3,127.0.0.1', mock_secure.assert_called_once_with(None)
# 'keyFile': '/test/key/file', self.manager.app.status.set_status.assert_called_with(
'dbpath': '/var/lib/mongodb'})
self.assertTrue(self.manager.app.status.is_config_server)
mock_set_status.assert_called_with(
ds_instance.ServiceStatuses.BUILD_PENDING) ds_instance.ServiceStatuses.BUILD_PENDING)
@mock.patch.object(service.MongoDBAppStatus, 'set_status') @mock.patch.object(service.MongoDBApp, '_configure_as_cluster_member')
@mock.patch.object(utils, 'poll_until') @mock.patch.object(service.MongoDBApp, '_configure_cluster_security')
@mock.patch.object(service.MongoDBApp, 'start_db_with_conf_changes') def test_prepare_member(self, mock_secure, mock_config):
@mock.patch.object(service.MongoDBApp, 'update_config_contents')
@mock.patch.object(service.MongoDBApp, 'secure')
@mock.patch.object(service.MongoDBApp, 'get_key_file',
return_value="/test/key/file")
@mock.patch.object(netutils, 'get_my_ipv4', return_value="10.0.0.4")
def test_prepare_member(self, mock_ip_address, mock_key_file,
mock_secure, mock_update, mock_start,
mock_poll, mock_set_status):
self._prepare_method("test-id-3", "member", None) self._prepare_method("test-id-3", "member", None)
mock_update.assert_called_with(None, mock_config.assert_called_once_with('rs1')
{'bind_ip': '10.0.0.4,127.0.0.1', mock_secure.assert_called_once_with(None)
# 'keyFile': '/test/key/file', self.manager.app.status.set_status.assert_called_with(
'dbpath': '/var/lib/mongodb',
'replSet': 'rs1'})
mock_set_status.assert_called_with(
ds_instance.ServiceStatuses.BUILD_PENDING) ds_instance.ServiceStatuses.BUILD_PENDING)
@mock.patch.object(service.MongoDBAppStatus, 'set_status') @mock.patch.object(operating_system, 'write_file')
@mock.patch.object(utils, 'poll_until') @mock.patch.object(service.MongoDBApp, '_configure_network')
@mock.patch.object(service.MongoDBApp, 'start_db_with_conf_changes') def test_configure_as_query_router(self, net_conf, os_write_file):
@mock.patch.object(service.MongoDBApp, 'update_config_contents') self.conf_mgr.parse_configuration = mock.Mock(
@mock.patch.object(service.MongoDBApp, 'secure') return_value={'storage.mmapv1.smallFiles': False,
@mock.patch.object(netutils, 'get_my_ipv4', return_value="10.0.0.4") 'storage.journal.enabled': True})
def test_prepare_secure(self, mock_ip_address, mock_secure, self.manager.app._configure_as_query_router()
mock_update, mock_start, mock_poll, os_write_file.assert_called_once_with(system.MONGOS_UPSTART, mock.ANY,
mock_set_status): as_root=True)
key = "test_key" self.conf_mgr.save_configuration.assert_called_once_with({})
self._prepare_method("test-id-4", "member", key) net_conf.assert_called_once_with(service.MONGODB_PORT)
mock_secure.assert_called_with( self.conf_mgr.apply_system_override.assert_called_once_with(
{"id": "test-id-4", {'sharding.configDB': ''}, 'clustering')
"shard_id": "test_shard_id", self.assertTrue(self.manager.app.is_query_router)
"instance_type": 'member',
"replica_set_name": "rs1",
"key": key}
) @mock.patch.object(service.MongoDBApp, '_configure_network')
def test_configure_as_config_server(self, net_conf):
self.manager.app._configure_as_config_server()
net_conf.assert_called_once_with(service.CONFIGSVR_PORT)
self.conf_mgr.apply_system_override.assert_called_once_with(
{'sharding.clusterRole': 'configsvr'}, 'clustering')
@mock.patch.object(service.MongoDBApp, '_configure_network')
def test_configure_as_cluster_member(self, net_conf):
self.manager.app._configure_as_cluster_member('rs1')
net_conf.assert_called_once_with(service.MONGODB_PORT)
self.conf_mgr.apply_system_override.assert_called_once_with(
{'replication.replSetName': 'rs1'}, 'clustering')
@mock.patch.object(service.MongoDBApp, 'store_key')
@mock.patch.object(service.MongoDBApp, 'get_key_file',
return_value='/var/keypath')
def test_configure_cluster_security(self, get_key_mock, store_key_mock):
self.manager.app._configure_cluster_security('key')
store_key_mock.assert_called_once_with('key')
self.conf_mgr.apply_system_override.assert_called_once_with(
{'security.clusterAuthMode': 'keyFile',
'security.keyFile': '/var/keypath'}, 'clustering')
@mock.patch.object(netutils, 'get_my_ipv4', return_value="10.0.0.2")
def test_configure_network(self, ip_mock):
self.manager.app._configure_network()
self.conf_mgr.apply_system_override.assert_called_once_with(
{'net.bindIp': '10.0.0.2,127.0.0.1'})
self.manager.app.status.set_host.assert_called_once_with(
'10.0.0.2', port=None)
self.manager.app._configure_network(10000)
self.conf_mgr.apply_system_override.assert_called_with(
{'net.bindIp': '10.0.0.2,127.0.0.1',
'net.port': 10000})
self.manager.app.status.set_host.assert_called_with(
'10.0.0.2', port=10000)
@mock.patch.object(volume.VolumeDevice, 'mount_points', return_value=[]) @mock.patch.object(volume.VolumeDevice, 'mount_points', return_value=[])
@mock.patch.object(volume.VolumeDevice, 'mount', return_value=None) @mock.patch.object(volume.VolumeDevice, 'mount', return_value=None)

19
trove/tests/unittests/guestagent/test_mongodb_manager.py
View File

@ -27,7 +27,8 @@ import trove.tests.unittests.trove_testtools as trove_testtools
class GuestAgentMongoDBManagerTest(trove_testtools.TestCase): class GuestAgentMongoDBManagerTest(trove_testtools.TestCase):
def setUp(self): @mock.patch.object(service.MongoDBApp, '_init_overrides_dir')
def setUp(self, _):
super(GuestAgentMongoDBManagerTest, self).setUp() super(GuestAgentMongoDBManagerTest, self).setUp()
self.context = context.TroveContext() self.context = context.TroveContext()
self.manager = manager.Manager() self.manager = manager.Manager()
@ -50,9 +51,9 @@ class GuestAgentMongoDBManagerTest(trove_testtools.TestCase):
super(GuestAgentMongoDBManagerTest, self).tearDown() super(GuestAgentMongoDBManagerTest, self).tearDown()
def test_update_status(self): def test_update_status(self):
with mock.patch.object(self.manager, 'status') as status: self.manager.app.status = mock.MagicMock()
self.manager.update_status(self.context) self.manager.update_status(self.context)
status.update.assert_any_call() self.manager.app.status.update.assert_any_call()
def _prepare_method(self, packages=['packages'], databases=None, def _prepare_method(self, packages=['packages'], databases=None,
memory_mb='2048', users=None, device_path=None, memory_mb='2048', users=None, device_path=None,
@ -61,8 +62,7 @@ class GuestAgentMongoDBManagerTest(trove_testtools.TestCase):
overrides=None, cluster_config=None,): overrides=None, cluster_config=None,):
"""self.manager.app must be correctly mocked before calling.""" """self.manager.app must be correctly mocked before calling."""
self.manager.status = mock.Mock() self.manager.app.status = mock.Mock()
self.manager.get_config_changes = mock.Mock()
self.manager.prepare(self.context, packages, self.manager.prepare(self.context, packages,
databases, memory_mb, users, databases, memory_mb, users,
@ -74,12 +74,13 @@ class GuestAgentMongoDBManagerTest(trove_testtools.TestCase):
overrides=overrides, overrides=overrides,
cluster_config=cluster_config) cluster_config=cluster_config)
self.manager.status.begin_install.assert_any_call() self.manager.app.status.begin_install.assert_any_call()
self.manager.app.install_if_needed.assert_called_with(packages) self.manager.app.install_if_needed.assert_called_with(packages)
self.manager.app.stop_db.assert_any_call() self.manager.app.stop_db.assert_any_call()
self.manager.app.clear_storage.assert_any_call() self.manager.app.clear_storage.assert_any_call()
self.manager.get_config_changes.assert_called_with(cluster_config,
self.mount_point) (self.manager.app.apply_initial_guestagent_configuration.
assert_called_once_with(cluster_config, self.mount_point))
@mock.patch.object(volume, 'VolumeDevice') @mock.patch.object(volume, 'VolumeDevice')
@mock.patch('os.path.exists') @mock.patch('os.path.exists')
@ -103,7 +104,7 @@ class GuestAgentMongoDBManagerTest(trove_testtools.TestCase):
self._prepare_method() self._prepare_method()
mock_secure.assert_called_with(None) mock_secure.assert_called_with()
@mock.patch.object(backup, 'restore') @mock.patch.object(backup, 'restore')
@mock.patch.object(service.MongoDBAdmin, 'is_root_enabled') @mock.patch.object(service.MongoDBAdmin, 'is_root_enabled')

12
trove/tests/unittests/guestagent/test_operating_system.py
View File

@ -25,7 +25,7 @@ from testtools import ExpectedException
from trove.common import exception from trove.common import exception
from trove.common.stream_codecs import ( from trove.common.stream_codecs import (
IdentityCodec, IniCodec, PropertiesCodec, YamlCodec) IdentityCodec, IniCodec, JsonCodec, PropertiesCodec, YamlCodec)
from trove.common import utils from trove.common import utils
from trove.guestagent.common import guestagent_utils from trove.guestagent.common import guestagent_utils
from trove.guestagent.common import operating_system from trove.guestagent.common import operating_system
@ -95,6 +95,16 @@ class TestOperatingSystem(trove_testtools.TestCase):
self._test_file_codec(data, PropertiesCodec( self._test_file_codec(data, PropertiesCodec(
string_mappings={'yes': True, 'no': False, "''": None})) string_mappings={'yes': True, 'no': False, "''": None}))
def test_json_file_codec(self):
data = {"Section1": 's1v1',
"Section2": {"s2k1": '1',
"s2k2": 'True'},
"Section3": {"Section4": {"s4k1": '3.1415926535',
"s4k2": None}}
}
self._test_file_codec(data, JsonCodec())
def _test_file_codec(self, data, read_codec, write_codec=None, def _test_file_codec(self, data, read_codec, write_codec=None,
expected_data=None, expected_data=None,
expected_exception=None): expected_exception=None):