From b6829d9d8bc3f7ab9c544bc223c4ec5253b3f83f Mon Sep 17 00:00:00 2001 From: Sergey Lukjanov Date: Tue, 21 May 2013 00:26:28 +0400 Subject: [PATCH] Initial version of Savanna v0.2 * pluggable provisioning mechanism * a lot of old code has been removed * new version of REST API (1.0) * image registry draft implemented as novaclient extension * oslo updated * using oslo.db instead of flask-sqlalchemy * some hacking fixes * using alembic for db migrations Partially implements blueprint pluggable-cluster-provisioning. Partially implements blueprint savanna-rest-api-1-0. Implements blueprint hadoop-image-registry. Implements blueprint fulfill-openstack-requirements. Implements blueprint db-migrate-support. Change-Id: I5df80d67e25c2f4f8367f78f67fb9e9e76fc3647 --- bin/savanna-api | 2 +- bin/savanna-db-manage | 27 + bin/savanna-manage | 44 - etc/savanna/savanna.conf.sample | 5 + etc/savanna/savanna.conf.sample-full | 22 +- openstack-common.conf | 2 +- savanna/api/v02.py | 109 -- savanna/api/v10.py | 168 +++ savanna/cli.py | 89 -- savanna/config.py | 18 +- savanna/context.py | 63 + savanna/{storage => db}/__init__.py | 0 savanna/db/api.py | 70 + savanna/db/migration/README | 50 + .../integration => db/migration}/__init__.py | 0 savanna/db/migration/alembic.ini | 66 + .../db/migration/alembic_migrations/env.py | 62 + .../alembic_migrations/script.py.mako | 41 + .../alembic_migrations/versions/README | 14 + .../versions/v02_initial.py | 167 +++ savanna/db/migration/cli.py | 116 ++ savanna/db/model_base.py | 143 ++ savanna/db/models.py | 225 +++ savanna/db/storage.py | 104 ++ savanna/exceptions.py | 99 -- savanna/main.py | 20 +- savanna/openstack/common/db/__init__.py | 16 + savanna/openstack/common/db/api.py | 106 ++ savanna/openstack/common/db/exception.py | 45 + .../common/db/sqlalchemy/__init__.py | 16 + .../openstack/common/db/sqlalchemy/models.py | 105 ++ .../openstack/common/db/sqlalchemy/session.py | 698 +++++++++ .../openstack/common/db/sqlalchemy/utils.py | 132 ++ savanna/openstack/common/fileutils.py | 35 + savanna/openstack/common/jsonutils.py | 8 +- savanna/openstack/common/local.py | 4 +- savanna/openstack/common/lockutils.py | 278 ++++ savanna/openstack/common/log.py | 71 +- savanna/openstack/common/loopingcall.py | 147 ++ savanna/openstack/common/threadgroup.py | 121 ++ savanna/plugins/__init__.py | 0 savanna/plugins/base.py | 167 +++ savanna/plugins/provisioning.py | 118 ++ savanna/plugins/vanilla/__init__.py | 0 savanna/plugins/vanilla/plugin.py | 59 + savanna/resources/core-default.xml | 580 -------- savanna/resources/hdfs-default.xml | 547 ------- savanna/resources/mapred-default.xml | 1282 ----------------- savanna/resources/setup-general.sh.template | 20 - savanna/resources/setup-master.sh.template | 21 - savanna/service/api.py | 266 +--- savanna/service/cluster_ops.py | 510 ------- savanna/service/validation.py | 296 ---- savanna/storage/db.py | 40 - savanna/storage/defaults.py | 119 -- savanna/storage/models.py | 232 --- savanna/storage/storage.py | 163 --- savanna/tests/integration/README.rst | 17 - savanna/tests/integration/config.py.sample | 24 - savanna/tests/integration/db.py | 244 ---- savanna/tests/integration/parameters.py | 33 - savanna/tests/integration/script.sh | 170 --- savanna/tests/integration/test_clusters.py | 44 - savanna/tests/integration/test_hadoop.py | 235 --- .../tests/integration/test_node_templates.py | 48 - savanna/tests/unit/base.py | 190 --- savanna/tests/unit/db/__init__.py | 0 savanna/tests/unit/db/models/__init__.py | 0 savanna/tests/unit/db/models/base.py | 63 + savanna/tests/unit/db/models/test_clusters.py | 45 + .../tests/unit/db/models/test_templates.py | 98 ++ savanna/tests/unit/test_api_v02.py | 332 ----- savanna/tests/unit/test_cluster_ops.py | 54 - savanna/tests/unit/test_service.py | 244 ---- savanna/tests/unit/test_validation.py | 544 ------- savanna/utils/api.py | 95 +- savanna/utils/openstack/images.py | 101 ++ savanna/utils/openstack/nova.py | 18 +- savanna/utils/patches.py | 12 +- savanna/utils/resources.py | 70 + savanna/utils/sqlatypes.py | 110 ++ setup.py | 4 +- tools/pip-requires | 5 +- tools/run_pyflakes | 3 - tox.ini | 9 +- 85 files changed, 4075 insertions(+), 6665 deletions(-) create mode 100755 bin/savanna-db-manage delete mode 100755 bin/savanna-manage delete mode 100644 savanna/api/v02.py create mode 100644 savanna/api/v10.py delete mode 100644 savanna/cli.py create mode 100644 savanna/context.py rename savanna/{storage => db}/__init__.py (100%) create mode 100644 savanna/db/api.py create mode 100644 savanna/db/migration/README rename savanna/{tests/integration => db/migration}/__init__.py (100%) create mode 100644 savanna/db/migration/alembic.ini create mode 100644 savanna/db/migration/alembic_migrations/env.py create mode 100644 savanna/db/migration/alembic_migrations/script.py.mako create mode 100644 savanna/db/migration/alembic_migrations/versions/README create mode 100644 savanna/db/migration/alembic_migrations/versions/v02_initial.py create mode 100644 savanna/db/migration/cli.py create mode 100644 savanna/db/model_base.py create mode 100644 savanna/db/models.py create mode 100644 savanna/db/storage.py create mode 100644 savanna/openstack/common/db/__init__.py create mode 100644 savanna/openstack/common/db/api.py create mode 100644 savanna/openstack/common/db/exception.py create mode 100644 savanna/openstack/common/db/sqlalchemy/__init__.py create mode 100644 savanna/openstack/common/db/sqlalchemy/models.py create mode 100644 savanna/openstack/common/db/sqlalchemy/session.py create mode 100644 savanna/openstack/common/db/sqlalchemy/utils.py create mode 100644 savanna/openstack/common/fileutils.py create mode 100644 savanna/openstack/common/lockutils.py create mode 100644 savanna/openstack/common/loopingcall.py create mode 100644 savanna/openstack/common/threadgroup.py create mode 100644 savanna/plugins/__init__.py create mode 100644 savanna/plugins/base.py create mode 100644 savanna/plugins/provisioning.py create mode 100644 savanna/plugins/vanilla/__init__.py create mode 100644 savanna/plugins/vanilla/plugin.py delete mode 100644 savanna/resources/core-default.xml delete mode 100644 savanna/resources/hdfs-default.xml delete mode 100644 savanna/resources/mapred-default.xml delete mode 100644 savanna/resources/setup-general.sh.template delete mode 100644 savanna/resources/setup-master.sh.template delete mode 100644 savanna/service/cluster_ops.py delete mode 100644 savanna/service/validation.py delete mode 100644 savanna/storage/db.py delete mode 100644 savanna/storage/defaults.py delete mode 100644 savanna/storage/models.py delete mode 100644 savanna/storage/storage.py delete mode 100644 savanna/tests/integration/README.rst delete mode 100644 savanna/tests/integration/config.py.sample delete mode 100644 savanna/tests/integration/db.py delete mode 100644 savanna/tests/integration/parameters.py delete mode 100644 savanna/tests/integration/script.sh delete mode 100644 savanna/tests/integration/test_clusters.py delete mode 100644 savanna/tests/integration/test_hadoop.py delete mode 100644 savanna/tests/integration/test_node_templates.py delete mode 100644 savanna/tests/unit/base.py create mode 100644 savanna/tests/unit/db/__init__.py create mode 100644 savanna/tests/unit/db/models/__init__.py create mode 100644 savanna/tests/unit/db/models/base.py create mode 100644 savanna/tests/unit/db/models/test_clusters.py create mode 100644 savanna/tests/unit/db/models/test_templates.py delete mode 100644 savanna/tests/unit/test_api_v02.py delete mode 100644 savanna/tests/unit/test_cluster_ops.py delete mode 100644 savanna/tests/unit/test_service.py delete mode 100644 savanna/tests/unit/test_validation.py create mode 100644 savanna/utils/openstack/images.py create mode 100644 savanna/utils/resources.py create mode 100644 savanna/utils/sqlatypes.py delete mode 100755 tools/run_pyflakes diff --git a/bin/savanna-api b/bin/savanna-api index ccde32cd..31c736dc 100755 --- a/bin/savanna-api +++ b/bin/savanna-api @@ -47,7 +47,7 @@ def main(): if os.path.exists(dev_conf): config_files = [dev_conf] - config.parse_args(sys.argv[1:], config_files) + config.parse_configs(sys.argv[1:], config_files) logging.setup("savanna") app = server.make_app() diff --git a/bin/savanna-db-manage b/bin/savanna-db-manage new file mode 100755 index 00000000..fbc2dbff --- /dev/null +++ b/bin/savanna-db-manage @@ -0,0 +1,27 @@ +#!/usr/bin/env python + +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys + + +sys.path.insert(0, os.getcwd()) + +from savanna.db.migration.cli import main + +if __name__ == '__main__': + main() diff --git a/bin/savanna-manage b/bin/savanna-manage deleted file mode 100755 index 34908e13..00000000 --- a/bin/savanna-manage +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2013 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import sys - -# If ../savanna/__init__.py exists, add ../ to Python search path, so that -# it will override what happens to be installed in /usr/(local/)lib/python... -possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), - os.pardir, - os.pardir)) -if os.path.exists(os.path.join(possible_topdir, - 'savanna', - '__init__.py')): - sys.path.insert(0, possible_topdir) - - -from savanna import cli - - -if __name__ == '__main__': - dev_conf = os.path.join(possible_topdir, - 'etc', - 'savanna', - 'savanna.conf') - config_files = None - if os.path.exists(dev_conf): - config_files = [dev_conf] - - cli.main(argv=sys.argv, config_files=config_files) diff --git a/etc/savanna/savanna.conf.sample b/etc/savanna/savanna.conf.sample index dee4b5a6..029be4df 100644 --- a/etc/savanna/savanna.conf.sample +++ b/etc/savanna/savanna.conf.sample @@ -15,6 +15,8 @@ # logging will go to stdout. (string value) #log_file= +plugins=vanilla + [cluster_node] # An existing user on Hadoop image (string value) @@ -35,3 +37,6 @@ # URL for sqlalchemy database (string value) #database_uri=sqlite:////tmp/savanna-server.db + +[plugin:vanilla] +plugin_class=savanna.plugins.vanilla.plugin:VanillaProvider diff --git a/etc/savanna/savanna.conf.sample-full b/etc/savanna/savanna.conf.sample-full index 1e0fd589..bc1ebec4 100644 --- a/etc/savanna/savanna.conf.sample-full +++ b/etc/savanna/savanna.conf.sample-full @@ -58,13 +58,9 @@ # Log output to standard error (boolean value) #use_stderr=true -# Default file mode used when creating log files (string -# value) -#logfile_mode=0644 - # format string to use for log messages with context (string # value) -#logging_context_format_string=%(asctime)s.%(msecs)03d %(levelname)s %(name)s [%(request_id)s %(user)s %(tenant)s] %(instance)s%(message)s +#logging_context_format_string=%(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user)s %(tenant)s] %(instance)s%(message)s # format string to use for log messages without context # (string value) @@ -111,12 +107,12 @@ # %(default)s (string value) #log_date_format=%Y-%m-%d %H:%M:%S -# (Optional) Name of log file to output to. If not set, -# logging will go to stdout. (string value) +# (Optional) Name of log file to output to. If no default is +# set, logging will go to stdout. (string value) #log_file= -# (Optional) The directory to keep log files in (will be -# prepended to --log-file) (string value) +# (Optional) The base directory used for relative --log-file +# paths (string value) #log_dir= # Use syslog for logging. (boolean value) @@ -143,6 +139,14 @@ #default_publisher_id=$host +# +# Options defined in savanna.plugins.base +# + +# List of plugins to be loaded (list value) +#plugins= + + [cluster_node] # diff --git a/openstack-common.conf b/openstack-common.conf index 95e2df1b..632c81bb 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -1,5 +1,5 @@ [DEFAULT] -modules=setup, jsonutils, xmlutils, timeutils, exception, gettextutils, log, local, notifier/api, notifier/log_notifier, notifier/no_op_notifier, notifier/test_notifier, notifier/__init__, importutils, context, uuidutils, version +modules=setup, jsonutils, xmlutils, timeutils, exception, gettextutils, log, local, notifier/api, notifier/log_notifier, notifier/no_op_notifier, notifier/test_notifier, notifier/__init__, importutils, context, uuidutils, version, threadgroup, db, db.sqlalchemy base=savanna # The following code from 'wsgi' is needed: diff --git a/savanna/api/v02.py b/savanna/api/v02.py deleted file mode 100644 index 7f4be0af..00000000 --- a/savanna/api/v02.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright (c) 2013 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from flask import request - -from savanna.openstack.common import log as logging -from savanna.service import api -import savanna.service.validation as v -import savanna.utils.api as api_u - -LOG = logging.getLogger(__name__) - -rest = api_u.Rest('v02', __name__) - - -@rest.get('/node-templates') -def templates_list(): - try: - return api_u.render( - node_templates=[nt.dict for nt in api.get_node_templates()]) - except Exception, e: - return api_u.internal_error(500, - "Exception while listing NodeTemplates", e) - - -@rest.post('/node-templates') -@v.validate(v.validate_node_template_create) -def templates_create(): - data = api_u.request_data() - headers = request.headers - - return api_u.render(api.create_node_template(data, headers).wrapped_dict) - - -@rest.get('/node-templates/') -@v.exists_by_id(api.get_node_template, 'template_id') -def templates_get(template_id): - nt = api.get_node_template(id=template_id) - return api_u.render(nt.wrapped_dict) - - -@rest.put('/node-templates/') -def templates_update(template_id): - return api_u.internal_error(501, NotImplementedError( - "Template update op isn't implemented (id '%s')" - % template_id)) - - -@rest.delete('/node-templates/') -@v.exists_by_id(api.get_node_template, 'template_id') -@v.validate(v.validate_node_template_terminate) -def templates_delete(template_id): - api.terminate_node_template(id=template_id) - return api_u.render() - - -@rest.get('/clusters') -def clusters_list(): - tenant_id = request.headers['X-Tenant-Id'] - try: - return api_u.render( - clusters=[c.dict for c in api.get_clusters(tenant_id=tenant_id)]) - except Exception, e: - return api_u.internal_error(500, 'Exception while listing Clusters', e) - - -@rest.post('/clusters') -@v.validate(v.validate_cluster_create) -def clusters_create(): - data = api_u.request_data() - headers = request.headers - - return api_u.render(api.create_cluster(data, headers).wrapped_dict) - - -@rest.get('/clusters/') -@v.exists_by_id(api.get_cluster, 'cluster_id', tenant_specific=True) -def clusters_get(cluster_id): - tenant_id = request.headers['X-Tenant-Id'] - c = api.get_cluster(id=cluster_id, tenant_id=tenant_id) - return api_u.render(c.wrapped_dict) - - -@rest.put('/clusters/') -def clusters_update(cluster_id): - return api_u.internal_error(501, NotImplementedError( - "Cluster update op isn't implemented (id '%s')" - % cluster_id)) - - -@rest.delete('/clusters/') -@v.exists_by_id(api.get_cluster, 'cluster_id', tenant_specific=True) -def clusters_delete(cluster_id): - headers = request.headers - tenant_id = headers['X-Tenant-Id'] - api.terminate_cluster(headers, id=cluster_id, tenant_id=tenant_id) - - return api_u.render() diff --git a/savanna/api/v10.py b/savanna/api/v10.py new file mode 100644 index 00000000..05386855 --- /dev/null +++ b/savanna/api/v10.py @@ -0,0 +1,168 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from savanna.openstack.common import log as logging +from savanna.service import api +import savanna.utils.api as u +from savanna.utils.openstack.nova import novaclient + +LOG = logging.getLogger(__name__) + +rest = u.Rest('v10', __name__) + + +## Cluster ops + +@rest.get('/clusters') +def clusters_list(ctx): + return u.render(clusters=[c.dict for c in api.get_clusters()]) + + +@rest.post('/clusters') +def clusters_create(data): + return u.render(api.create_cluster(data).wrapped_dict) + + +@rest.get('/clusters/') +def clusters_get(cluster_id): + return u.render(api.get_cluster(id=cluster_id).wrapped_dict) + + +@rest.put('/clusters/') +def clusters_update(cluster_id): + return u.internal_error(501, NotImplementedError( + "Cluster update op isn't implemented (id '%s')" + % cluster_id)) + + +@rest.delete('/clusters/') +def clusters_delete(cluster_id): + api.terminate_cluster(id=cluster_id) + return u.render() + + +## ClusterTemplate ops + +@rest.get('/cluster-templates') +def cluster_templates_list(): + return u.render( + cluster_templates=[t.dict for t in api.get_cluster_templates()]) + + +@rest.post('/cluster-templates') +def cluster_templates_create(data): + return u.render(api.create_cluster_template(data).wrapped_dict) + + +@rest.get('/cluster-templates/') +def cluster_templates_get(cluster_template_id): + return u.render( + api.get_cluster_template(id=cluster_template_id).wrapped_dict) + + +@rest.put('/cluster-templates/') +def cluster_templates_update(_cluster_template_id): + pass + + +@rest.delete('/cluster-templates/') +def cluster_templates_delete(cluster_template_id): + api.terminate_cluster_template(id=cluster_template_id) + return u.render() + + +## NodeGroupTemplate ops + +@rest.get('/node-group-templates') +def node_group_templates_list(): + return u.render( + node_group_template=[t.dict for t in api.get_node_group_templates()]) + + +@rest.post('/node-group-templates') +def node_group_templates_create(data): + return u.render(api.create_node_group_template(data).wrapped_dict) + + +@rest.get('/node-group-templates/') +def node_group_templates_get(node_group_template_id): + return u.render( + api.get_node_group_template(id=node_group_template_id).wrapped_dict) + + +@rest.put('/node-group-templates/') +def node_group_templates_update(_node_group_template_id): + pass + + +@rest.delete('/node-group-templates/') +def node_group_templates_delete(node_group_template_id): + api.terminate_node_group_template(id=node_group_template_id) + return u.render() + + +## Plugins ops + +@rest.get('/plugins') +def plugins_list(): + return u.render(plugins=[p.dict for p in api.get_plugins()]) + + +@rest.get('/plugins/') +def plugins_get(plugin_name): + return u.render(api.get_plugin(plugin_name).wrapped_dict) + + +@rest.get('/plugins//') +def plugins_get_version(plugin_name, version): + return u.render(api.get_plugin(plugin_name, version).wrapped_dict) + + +## Image Registry ops + +@rest.get('/images') +def images_list(): + return u.render( + images=[i.dict for i in novaclient().images.list_registered()]) + + +@rest.get('/images/') +def images_get(image_id): + return u.render(novaclient().images.get(image_id).dict) + + +def _render_image(image_id, nova): + return u.render(nova.images.get(image_id).wrapped_dict) + + +@rest.post('/images/') +def images_set(image_id, data): + nova = novaclient() + nova.images.set_description(image_id, **data) + return _render_image(image_id, nova) + + +@rest.post('/images//tag') +def image_tags_add(image_id, data): + nova = novaclient() + nova.images.tag(image_id, **data) + return _render_image(image_id, nova) + + +@rest.post('/images//untag') +def image_tags_delete(image_id, data): + nova = novaclient() + nova.images.untag(image_id, **data) + return _render_image(image_id, nova) diff --git a/savanna/cli.py b/savanna/cli.py deleted file mode 100644 index 823bedfc..00000000 --- a/savanna/cli.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright (c) 2013 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from flask import Flask - -from oslo.config import cfg -from savanna.openstack.common import log -from savanna.storage.db import DB -from savanna.storage.db import setup_storage -from savanna.storage.defaults import setup_defaults - -CONF = cfg.CONF -LOG = log.getLogger(__name__) - - -class BaseCmd(object): - name = None - - @classmethod - def add_argument_parser(cls, subparsers): - parser = subparsers.add_parser(cls.name, help=cls.__doc__) - parser.set_defaults(cmd_class=cls) - return parser - - -class ResetDbCmd(BaseCmd): - """Reset the database.""" - - name = 'reset-db' - - @classmethod - def add_argument_parser(cls, subparsers): - parser = super(ResetDbCmd, cls).add_argument_parser(subparsers) - parser.add_argument('--with-gen-templates', action='store_true') - return parser - - @staticmethod - def main(): - gen = CONF.command.with_gen_templates - - app = Flask('savanna.manage') - setup_storage(app) - - DB.drop_all() - DB.create_all() - - setup_defaults(True, gen) - - LOG.info("DB has been removed and created from scratch, " - "gen templates: %s", gen) - - -CLI_COMMANDS = [ - ResetDbCmd, -] - - -def add_command_parsers(subparsers): - for cmd in CLI_COMMANDS: - cmd.add_argument_parser(subparsers) - - -command_opt = cfg.SubCommandOpt('command', - title='Commands', - help='Available commands', - handler=add_command_parsers) - - -def main(argv=None, config_files=None): - CONF.register_cli_opt(command_opt) - CONF(args=argv[1:], - project='savanna', - usage='%(prog)s [' + '|'.join( - [cmd.name for cmd in CLI_COMMANDS]) + ']', - default_config_files=config_files) - log.setup("savanna") - CONF.command.cmd_class.main() diff --git a/savanna/config.py b/savanna/config.py index 3ea711e1..b6848266 100644 --- a/savanna/config.py +++ b/savanna/config.py @@ -20,15 +20,21 @@ cli_opts = [ help='set host'), cfg.IntOpt('port', default=8080, help='set port'), - cfg.BoolOpt('allow-cluster-ops', default=True, - help='without that option' - ' the application operates in dry run mode and does not ' - ' send any requests to the OpenStack cluster') ] CONF = cfg.CONF CONF.register_cli_opts(cli_opts) +ARGV = [] -def parse_args(argv, conf_files): - CONF(argv, project='savanna', default_config_files=conf_files) + +def parse_configs(argv=None, conf_files=None): + if argv is not None: + global ARGV + ARGV = argv + try: + CONF(ARGV, project='savanna', default_config_files=conf_files) + except cfg.RequiredOptError as roe: + # todo replace RuntimeError with Savanna-specific exception + raise RuntimeError("Option '%s' is required for config group " + "'%s'" % (roe.opt_name, roe.group.name)) diff --git a/savanna/context.py b/savanna/context.py new file mode 100644 index 00000000..cfbc32b6 --- /dev/null +++ b/savanna/context.py @@ -0,0 +1,63 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import threading + +from savanna.db import api as db_api +from savanna.openstack.common import log as logging + + +LOG = logging.getLogger(__name__) + + +# TODO(slukjanov): it'll be better to use common_context.RequestContext as base +class Context(object): + def __init__(self, user_id, tenant_id, auth_token, headers, **kwargs): + if kwargs: + LOG.warn('Arguments dropped when creating context: %s', kwargs) + + self.user_id = user_id + self.tenant_id = tenant_id + self.auth_token = auth_token + self.headers = headers + self._db_session = None + + @property + def session(self): + if self._db_session is None: + self._db_session = db_api.get_session() + return self._db_session + + +_CTXS = threading.local() + + +def ctx(): + if not hasattr(_CTXS, '_curr_ctx'): + # todo replace with specific error + raise RuntimeError("Context isn't available here") + return _CTXS._curr_ctx + + +def set_ctx(new_ctx): + if not new_ctx and hasattr(_CTXS, '_curr_ctx'): + del _CTXS._curr_ctx + elif new_ctx: + _CTXS._curr_ctx = new_ctx + + +def model_query(model, context=None): + context = context or ctx() + return context.session.query(model) diff --git a/savanna/storage/__init__.py b/savanna/db/__init__.py similarity index 100% rename from savanna/storage/__init__.py rename to savanna/db/__init__.py diff --git a/savanna/db/api.py b/savanna/db/api.py new file mode 100644 index 00000000..e4ea208e --- /dev/null +++ b/savanna/db/api.py @@ -0,0 +1,70 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sqlalchemy as sql + +from savanna.db import model_base +from savanna.openstack.common.db.sqlalchemy import session +from savanna.openstack.common import log as logging + +LOG = logging.getLogger(__name__) + +_DB_ENGINE = None + + +def configure_db(): + """Configure database. + + Establish the database, create an engine if needed, and register + the models. + """ + global _DB_ENGINE + if not _DB_ENGINE: + _DB_ENGINE = session.get_engine(sqlite_fk=True) + register_models() + + +def clear_db(base=model_base.SavannaBase): + global _DB_ENGINE + unregister_models(base) + session.cleanup() + _DB_ENGINE = None + + +def get_session(autocommit=True, expire_on_commit=False): + """Helper method to grab session.""" + return session.get_session(autocommit=autocommit, + expire_on_commit=expire_on_commit, + sqlite_fk=True) + + +def register_models(base=model_base.SavannaBase): + """Register Models and create properties.""" + try: + engine = session.get_engine(sqlite_fk=True) + base.metadata.create_all(engine) + except sql.exc.OperationalError as e: + LOG.info("Database registration exception: %s", e) + return False + return True + + +def unregister_models(base=model_base.SavannaBase): + """Unregister Models, useful clearing out data before testing.""" + try: + engine = session.get_engine(sqlite_fk=True) + base.metadata.drop_all(engine) + except Exception: + LOG.exception("Database exception") diff --git a/savanna/db/migration/README b/savanna/db/migration/README new file mode 100644 index 00000000..73e3fc74 --- /dev/null +++ b/savanna/db/migration/README @@ -0,0 +1,50 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +You can then upgrade to the latest database version via: +$ savanna-db-manage --config-file /path/to/savanna.conf upgrade head + +To check the current database version: +$ savanna-db-manage --config-file /path/to/savanna.conf current + +To create a script to run the migration offline: +$ savanna-db-manage --config-file /path/to/savanna.conf upgrade head --sql + +To run the offline migration between specific migration versions: +$ savanna-db-manage --config-file /path/to/savanna.conf upgrade \ + : --sql + +Upgrade the database incrementally: +$ savanna-db-manage --config-file /path/to/savanna.conf upgrade --delta \ + <# of revs> + +Downgrade the database by a certain number of revisions: +$ savanna-db-manage --config-file /path/to/savanna.conf downgrade --delta \ + <# of revs> + + +Create new revision: +$ savanna-db-manage --config-file /path/to/savanna.conf revision \ + -m "description of revision" --autogenerate + +Create a blank file: +$ savanna-db-manage --config-file /path/to/savanna.conf revision \ + -m "description of revision" + +To verify that the timeline does branch, you can run this command: +$ savanna-db-manage --config-file /path/to/savanna.conf check_migration + +If the migration path does branch, you can find the branch point via: +$ savanna-db-manage --config-file /path/to/savanna.conf history diff --git a/savanna/tests/integration/__init__.py b/savanna/db/migration/__init__.py similarity index 100% rename from savanna/tests/integration/__init__.py rename to savanna/db/migration/__init__.py diff --git a/savanna/db/migration/alembic.ini b/savanna/db/migration/alembic.ini new file mode 100644 index 00000000..a8f53307 --- /dev/null +++ b/savanna/db/migration/alembic.ini @@ -0,0 +1,66 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = %(here)s/alembic + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# default to an empty string because the Savanna migration cli will extract +# the correct value and set it programatically before alembic is fully invoked. +sqlalchemy.url = + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/savanna/db/migration/alembic_migrations/env.py b/savanna/db/migration/alembic_migrations/env.py new file mode 100644 index 00000000..a55dbcd8 --- /dev/null +++ b/savanna/db/migration/alembic_migrations/env.py @@ -0,0 +1,62 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from alembic import context +from logging.config import fileConfig +from savanna.openstack.common import importutils +from sqlalchemy import create_engine, pool + +from savanna.db import model_base + + +importutils.import_module('savanna.db.models') + +config = context.config +savanna_config = config.savanna_config + +fileConfig(config.config_file_name) + +# set the target for 'autogenerate' support +target_metadata = model_base.SavannaBase.metadata + + +def run_migrations_offline(): + context.configure(url=savanna_config.database.connection) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + engine = create_engine(savanna_config.database.connection, + poolclass=pool.NullPool) + + connection = engine.connect() + context.configure( + connection=connection, + target_metadata=target_metadata + ) + + try: + with context.begin_transaction(): + context.run_migrations() + finally: + connection.close() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/savanna/db/migration/alembic_migrations/script.py.mako b/savanna/db/migration/alembic_migrations/script.py.mako new file mode 100644 index 00000000..8cd3245d --- /dev/null +++ b/savanna/db/migration/alembic_migrations/script.py.mako @@ -0,0 +1,41 @@ +# Copyright (c) ${create_date.year} Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +from savanna.utils.sqlatypes import JSONEncoded +sa.JSONEncoded = JSONEncoded + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/savanna/db/migration/alembic_migrations/versions/README b/savanna/db/migration/alembic_migrations/versions/README new file mode 100644 index 00000000..e3f3e8d4 --- /dev/null +++ b/savanna/db/migration/alembic_migrations/versions/README @@ -0,0 +1,14 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/savanna/db/migration/alembic_migrations/versions/v02_initial.py b/savanna/db/migration/alembic_migrations/versions/v02_initial.py new file mode 100644 index 00000000..47e16b40 --- /dev/null +++ b/savanna/db/migration/alembic_migrations/versions/v02_initial.py @@ -0,0 +1,167 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""v02_initial + +Revision ID: 2e1cdcf1dff1 +Revises: None +Create Date: 2013-05-31 11:57:18.181738 + +""" + +# revision identifiers, used by Alembic. +revision = '2e1cdcf1dff1' +down_revision = None + +from alembic import op +import sqlalchemy as sa + +from savanna.utils.sqlatypes import JSONEncoded + +sa.JSONEncoded = JSONEncoded + + +def upgrade(): + op.create_table('NodeGroupTemplate', + sa.Column('created', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('tenant_id', sa.String(length=36), + nullable=True), + sa.Column('plugin_name', sa.String(length=80), + nullable=False), + sa.Column('hadoop_version', sa.String(length=80), + nullable=False), + sa.Column('name', sa.String(length=80), nullable=False), + sa.Column('description', sa.String(length=200), + nullable=True), + sa.Column('flavor_id', sa.String(length=36), + nullable=False), + sa.Column('node_processes', sa.JSONEncoded(), + nullable=True), + sa.Column('node_configs', sa.JSONEncoded(), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name', 'tenant_id')) + + op.create_table('ClusterTemplate', + sa.Column('created', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('tenant_id', sa.String(length=36), + nullable=True), + sa.Column('plugin_name', sa.String(length=80), + nullable=False), + sa.Column('hadoop_version', sa.String(length=80), + nullable=False), + sa.Column('name', sa.String(length=80), nullable=False), + sa.Column('description', sa.String(length=200), + nullable=True), + sa.Column('cluster_configs', sa.JSONEncoded(), + nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name', 'tenant_id')) + + op.create_table('Cluster', + sa.Column('created', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('tenant_id', sa.String(length=36), + nullable=True), + sa.Column('plugin_name', sa.String(length=80), + nullable=False), + sa.Column('hadoop_version', sa.String(length=80), + nullable=False), + sa.Column('extra', sa.JSONEncoded(), nullable=True), + sa.Column('name', sa.String(length=80), nullable=False), + sa.Column('default_image_id', sa.String(length=36), + nullable=True), + sa.Column('cluster_configs', sa.JSONEncoded(), + nullable=True), + sa.Column('status', sa.String(length=80), nullable=True), + sa.Column('status_description', sa.String(length=200), + nullable=True), + sa.Column('base_cluster_template_id', sa.String(length=36), + nullable=True), + sa.ForeignKeyConstraint(['base_cluster_template_id'], + ['ClusterTemplate.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name', 'tenant_id')) + + op.create_table('TemplatesRelation', + sa.Column('created', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('cluster_template_id', sa.String(length=36), + nullable=False), + sa.Column('node_group_template_id', sa.String(length=36), + nullable=False), + sa.Column('node_group_name', sa.String(length=80), + nullable=False), + sa.Column('count', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['cluster_template_id'], + ['ClusterTemplate.id'], ), + sa.ForeignKeyConstraint(['node_group_template_id'], + ['NodeGroupTemplate.id'], ), + sa.PrimaryKeyConstraint('cluster_template_id', + 'node_group_template_id')) + + op.create_table('NodeGroup', + sa.Column('created', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('extra', sa.JSONEncoded(), nullable=True), + sa.Column('cluster_id', sa.String(length=36), + nullable=True), + sa.Column('name', sa.String(length=80), nullable=False), + sa.Column('flavor_id', sa.String(length=36), + nullable=False), + sa.Column('image_id', sa.String(length=36), + nullable=False), + sa.Column('node_processes', sa.JSONEncoded(), + nullable=True), + sa.Column('node_configs', sa.JSONEncoded(), nullable=True), + sa.Column('anti_affinity_group', sa.String(length=36), + nullable=True), + sa.Column('count', sa.Integer(), nullable=False), + sa.Column('base_node_group_template_id', + sa.String(length=36), nullable=True), + sa.ForeignKeyConstraint(['base_node_group_template_id'], + ['NodeGroupTemplate.id'], ), + sa.ForeignKeyConstraint(['cluster_id'], ['Cluster.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name', 'cluster_id')) + + op.create_table('Instance', + sa.Column('created', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('extra', sa.JSONEncoded(), nullable=True), + sa.Column('node_group_id', sa.String(length=36), + nullable=True), + sa.Column('instance_id', sa.String(length=36), + nullable=False), + sa.Column('management_ip', sa.String(length=15), + nullable=False), + sa.ForeignKeyConstraint(['node_group_id'], + ['NodeGroup.id'], ), + sa.PrimaryKeyConstraint('instance_id'), + sa.UniqueConstraint('instance_id', 'node_group_id')) + + +def downgrade(): + op.drop_table('Instance') + op.drop_table('NodeGroup') + op.drop_table('TemplatesRelation') + op.drop_table('Cluster') + op.drop_table('ClusterTemplate') + op.drop_table('NodeGroupTemplate') diff --git a/savanna/db/migration/cli.py b/savanna/db/migration/cli.py new file mode 100644 index 00000000..9b66c6cc --- /dev/null +++ b/savanna/db/migration/cli.py @@ -0,0 +1,116 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Based on Quantum's migration/cli.py + +import os + +from alembic import command as alembic_cmd +from alembic import config as alembic_cfg +from alembic import util as alembic_u +from oslo.config import cfg + + +_db_opts = [ + cfg.StrOpt('connection', default='', help='URL to database'), +] + +CONF = cfg.ConfigOpts() +CONF.register_opts(_db_opts, 'database') + + +def do_alembic_command(config, cmd, *args, **kwargs): + try: + getattr(alembic_cmd, cmd)(config, *args, **kwargs) + except alembic_u.CommandError as e: + alembic_u.err(str(e)) + + +def do_check_migration(config, _cmd): + do_alembic_command(config, 'branches') + + +def do_upgrade_downgrade(config, cmd): + if not CONF.command.revision and not CONF.command.delta: + raise SystemExit('You must provide a revision or relative delta') + + revision = CONF.command.revision + + if CONF.command.delta: + sign = '+' if CONF.command.name == 'upgrade' else '-' + revision = sign + str(CONF.command.delta) + + do_alembic_command(config, cmd, revision, sql=CONF.command.sql) + + +def do_stamp(config, cmd): + do_alembic_command(config, cmd, + CONF.command.revision, + sql=CONF.command.sql) + + +def do_revision(config, cmd): + do_alembic_command(config, cmd, + message=CONF.command.message, + autogenerate=CONF.command.autogenerate, + sql=CONF.command.sql) + + +def add_command_parsers(subparsers): + for name in ['current', 'history', 'branches']: + parser = subparsers.add_parser(name) + parser.set_defaults(func=do_alembic_command) + + parser = subparsers.add_parser('check_migration') + parser.set_defaults(func=do_check_migration) + + for name in ['upgrade', 'downgrade']: + parser = subparsers.add_parser(name) + parser.add_argument('--delta', type=int) + parser.add_argument('--sql', action='store_true') + parser.add_argument('revision', nargs='?') + parser.set_defaults(func=do_upgrade_downgrade) + + parser = subparsers.add_parser('stamp') + parser.add_argument('--sql', action='store_true') + parser.add_argument('revision') + parser.set_defaults(func=do_stamp) + + parser = subparsers.add_parser('revision') + parser.add_argument('-m', '--message') + parser.add_argument('--autogenerate', action='store_true') + parser.add_argument('--sql', action='store_true') + parser.set_defaults(func=do_revision) + + +command_opt = cfg.SubCommandOpt('command', + title='Command', + help='Available commands', + handler=add_command_parsers) + +CONF.register_cli_opt(command_opt) + + +def main(): + config = alembic_cfg.Config( + os.path.join(os.path.dirname(__file__), 'alembic.ini') + ) + config.set_main_option('script_location', + 'savanna.db.migration:alembic_migrations') + # attach the Savanna conf to the Alembic conf + config.savanna_config = CONF + + CONF() + CONF.command.func(config, CONF.command.name) diff --git a/savanna/db/model_base.py b/savanna/db/model_base.py new file mode 100644 index 00000000..1869e2f2 --- /dev/null +++ b/savanna/db/model_base.py @@ -0,0 +1,143 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import re + +import sqlalchemy as sa +from sqlalchemy.ext import declarative +from sqlalchemy import orm + +from savanna.openstack.common import timeutils +from savanna.openstack.common import uuidutils +from savanna.utils.resources import BaseResource +from savanna.utils.sqlatypes import JsonDictType + + +class _SavannaBase(BaseResource): + """Base class for all Savanna Models.""" + + created = sa.Column(sa.DateTime, default=timeutils.utcnow, + nullable=False) + updated = sa.Column(sa.DateTime, default=timeutils.utcnow, + nullable=False, onupdate=timeutils.utcnow) + + __protected_attributes__ = ["created", "updated"] + + @declarative.declared_attr + def __tablename__(cls): + # Table name is equals to the class name + return cls.__name__ + + @property + def __resource_name__(self): + # convert CamelCase class name to camel_case + s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', self.__class__.__name__) + return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() + + def __setitem__(self, key, value): + setattr(self, key, value) + + def __getitem__(self, key): + return getattr(self, key) + + def get(self, key, default=None): + return getattr(self, key, default) + + def __iter__(self): + self._i = iter(orm.object_mapper(self).columns) + return self + + def next(self): + n = self._i.next().name + return n, getattr(self, n) + + def update(self, values): + """Make the model object behave like a dict.""" + for k, v in values.iteritems(): + setattr(self, k, v) + + def iteritems(self): + """Make the model object behave like a dict. + + Includes attributes from joins. + """ + local = dict(self) + joined = dict([(k, v) for k, v in self.__dict__.iteritems() + if not k[0] == '_']) + local.update(joined) + return local.iteritems() + + def keys(self): + return self.__dict__.keys() + + def values(self): + return self.__dict__.values() + + def items(self): + return self.__dict__.items() + + def __repr__(self): + """sqlalchemy based automatic __repr__ method.""" + items = ['%s=%r' % (col.name, getattr(self, col.name)) + for col in self.__table__.columns] + return "<%s.%s[object at %x] {%s}>" % (self.__class__.__module__, + self.__class__.__name__, + id(self), ', '.join(items)) + + def to_dict(self): + """sqlalchemy based automatic to_dict method.""" + d = {} + for col in self.__table__.columns: + if self._filter_field(col.name): + continue + d[col.name] = getattr(self, col.name) + return d + + +SavannaBase = declarative.declarative_base(cls=_SavannaBase) + + +def _generate_unicode_uuid(): + return unicode(uuidutils.generate_uuid()) + + +class IdMixin(object): + """Id mixin, add to subclasses that have an id.""" + + id = sa.Column(sa.String(36), + primary_key=True, + default=_generate_unicode_uuid) + + +class TenantMixin(object): + """Tenant mixin, add to subclasses that have a tenant.""" + + __filter_cols__ = ['tenant_id'] + + tenant_id = sa.Column(sa.String(36)) + + +class PluginSpecificMixin(object): + """Plugin specific info mixin, add to subclass that plugin specific.""" + + plugin_name = sa.Column(sa.String(80), nullable=False) + hadoop_version = sa.Column(sa.String(80), nullable=False) + + +class ExtraMixin(object): + """Extra info mixin, add to subclass that stores extra data w/o schema.""" + + __filter_cols__ = ['extra'] + + extra = sa.Column(JsonDictType()) diff --git a/savanna/db/models.py b/savanna/db/models.py new file mode 100644 index 00000000..4bb3eee4 --- /dev/null +++ b/savanna/db/models.py @@ -0,0 +1,225 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sqlalchemy as sa +from sqlalchemy.ext.associationproxy import association_proxy +from sqlalchemy.orm import relationship + +from savanna.db import model_base as mb +from savanna.utils.openstack.nova import novaclient +from savanna.utils.sqlatypes import JsonDictType +from savanna.utils.sqlatypes import JsonListType + + +CLUSTER_STATUSES = ['Starting', 'Active', 'Stopping', 'Error'] + + +class Cluster(mb.SavannaBase, mb.IdMixin, mb.TenantMixin, + mb.PluginSpecificMixin, mb.ExtraMixin): + """Contains all info about cluster.""" + + __table_args__ = ( + sa.UniqueConstraint('name', 'tenant_id'), + ) + + name = sa.Column(sa.String(80), nullable=False) + default_image_id = sa.Column(sa.String(36)) + cluster_configs = sa.Column(JsonDictType()) + node_groups = relationship('NodeGroup', cascade="all,delete", + backref='cluster') + # todo replace String type with sa.Enum(*CLUSTER_STATUSES) + status = sa.Column(sa.String(80)) + status_description = sa.Column(sa.String(200)) + # todo instances' credentials should be stored in cluster + base_cluster_template_id = sa.Column(sa.String(36), + sa.ForeignKey('ClusterTemplate.id')) + base_cluster_template = relationship('ClusterTemplate', + backref="clusters") + + def __init__(self, name, tenant_id, plugin_name, hadoop_version, + status=None, status_description=None, default_image_id=None, + cluster_configs=None, base_cluster_template_id=None, + extra=None): + self.name = name + self.tenant_id = tenant_id + self.plugin_name = plugin_name + self.hadoop_version = hadoop_version + self.status = status + self.status_description = status_description + self.default_image_id = default_image_id + self.cluster_configs = cluster_configs or {} + self.base_cluster_template_id = base_cluster_template_id + self.extra = extra or {} + + def to_dict(self): + d = super(Cluster, self).to_dict() + d['node_groups'] = [ng.dict for ng in self.node_groups] + return d + + +class NodeGroup(mb.SavannaBase, mb.IdMixin, mb.ExtraMixin): + """Specifies group of nodes within a cluster.""" + + __filter_cols__ = ['cluster_id'] + __table_args__ = ( + sa.UniqueConstraint('name', 'cluster_id'), + ) + + cluster_id = sa.Column(sa.String(36), sa.ForeignKey('Cluster.id')) + name = sa.Column(sa.String(80), nullable=False) + flavor_id = sa.Column(sa.String(36), nullable=False) + image_id = sa.Column(sa.String(36), nullable=False) + node_processes = sa.Column(JsonListType()) + node_configs = sa.Column(JsonDictType()) + anti_affinity_group = sa.Column(sa.String(36)) + count = sa.Column(sa.Integer, nullable=False) + instances = relationship('Instance', cascade="all,delete", + backref='node_group') + base_node_group_template_id = sa.Column(sa.String(36), + sa.ForeignKey( + 'NodeGroupTemplate.id')) + base_node_group_template = relationship('NodeGroupTemplate', + backref="node_groups") + + def __init__(self, name, flavor_id, image_id, node_processes, count, + node_configs=None, anti_affinity_group=None, extra=None, + base_node_group_template_id=None): + self.name = name + self.flavor_id = flavor_id + self.image_id = image_id + self.node_processes = node_processes + self.count = count + self.node_configs = node_configs or {} + self.anti_affinity_group = anti_affinity_group + self.extra = extra or {} + self.base_node_group_template_id = base_node_group_template_id + + +class Instance(mb.SavannaBase, mb.ExtraMixin): + """An OpenStack instance created for the cluster.""" + + __filter_cols__ = ['node_group_id'] + __table_args__ = ( + sa.UniqueConstraint('instance_id', 'node_group_id'), + ) + + node_group_id = sa.Column(sa.String(36), sa.ForeignKey('NodeGroup.id')) + instance_id = sa.Column(sa.String(36), primary_key=True) + management_ip = sa.Column(sa.String(15), nullable=False) + + def info(self): + """Returns info from nova about instance.""" + return novaclient().servers.get(self.instance_id) + + def __init__(self, node_group_id, instance_id, management_ip, extra=None): + self.node_group_id = node_group_id + self.instance_id = instance_id + self.management_ip = management_ip + self.extra = extra or {} + + +class ClusterTemplate(mb.SavannaBase, mb.IdMixin, mb.TenantMixin, + mb.PluginSpecificMixin): + """Template for Cluster.""" + + __table_args__ = ( + sa.UniqueConstraint('name', 'tenant_id'), + ) + + name = sa.Column(sa.String(80), nullable=False) + description = sa.Column(sa.String(200)) + cluster_configs = sa.Column(JsonDictType()) + + # todo add node_groups_suggestion helper + + def __init__(self, name, tenant_id, plugin_name, hadoop_version, + cluster_configs=None, description=None): + self.name = name + self.tenant_id = tenant_id + self.plugin_name = plugin_name + self.hadoop_version = hadoop_version + self.cluster_configs = cluster_configs or {} + self.description = description + + def add_node_group_template(self, node_group_template_id, name, count): + relation = TemplatesRelation(self.id, node_group_template_id, name, + count) + self.templates_relations.append(relation) + return relation + + def to_dict(self): + d = super(ClusterTemplate, self).to_dict() + d['node_group_templates'] = [tr.dict for tr in + self.templates_relations] + return d + + +class NodeGroupTemplate(mb.SavannaBase, mb.IdMixin, mb.TenantMixin, + mb.PluginSpecificMixin): + """Template for NodeGroup.""" + + __table_args__ = ( + sa.UniqueConstraint('name', 'tenant_id'), + ) + + name = sa.Column(sa.String(80), nullable=False) + description = sa.Column(sa.String(200)) + flavor_id = sa.Column(sa.String(36), nullable=False) + node_processes = sa.Column(JsonListType()) + node_configs = sa.Column(JsonDictType()) + + def __init__(self, name, tenant_id, flavor_id, plugin_name, + hadoop_version, node_processes, node_configs=None, + description=None): + self.name = name + self.tenant_id = tenant_id + self.flavor_id = flavor_id + self.plugin_name = plugin_name + self.hadoop_version = hadoop_version + self.node_processes = node_processes + self.node_configs = node_configs or {} + self.description = description + + +class TemplatesRelation(mb.SavannaBase): + """NodeGroupTemplate - ClusterTemplate relationship.""" + + __filter_cols__ = ['cluster_template_id', 'created', 'updated'] + + cluster_template_id = sa.Column(sa.String(36), + sa.ForeignKey('ClusterTemplate.id'), + primary_key=True) + node_group_template_id = sa.Column(sa.String(36), + sa.ForeignKey('NodeGroupTemplate.id'), + primary_key=True) + cluster_template = relationship(ClusterTemplate, + backref='templates_relations') + node_group_template = relationship(NodeGroupTemplate, + backref='templates_relations') + node_group_name = sa.Column(sa.String(80), nullable=False) + count = sa.Column(sa.Integer, nullable=False) + + def __init__(self, cluster_template_id, node_group_template_id, + node_group_name, count): + self.cluster_template_id = cluster_template_id + self.node_group_template_id = node_group_template_id + self.node_group_name = node_group_name + self.count = count + + +ClusterTemplate.node_group_templates = association_proxy("templates_relations", + "node_group_template") +NodeGroupTemplate.cluster_templates = association_proxy("templates_relations", + "cluster_template") diff --git a/savanna/db/storage.py b/savanna/db/storage.py new file mode 100644 index 00000000..2e0fb33a --- /dev/null +++ b/savanna/db/storage.py @@ -0,0 +1,104 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from savanna.context import ctx +from savanna.context import model_query +import savanna.db.models as m + + +## Cluster ops +# todo check tenant_id and etc. + +def get_clusters(**args): + return model_query(m.Cluster).filter_by(**args).all() + + +def get_cluster(**args): + return model_query(m.Cluster).filter_by(**args).first() + + +def create_cluster(values): + session = ctx().session + with session.begin(): + values['tenant_id'] = ctx().tenant_id + ngs_vals = values.pop('node_groups', []) + cluster = m.Cluster(**values) + for ng in ngs_vals: + node_group = m.NodeGroup(**ng) + cluster.node_groups.append(node_group) + session.add(node_group) + session.add(cluster) + + return cluster + + +def terminate_cluster(cluster): + with ctx().session.begin(): + ctx().session.delete() + + +## ClusterTemplate ops + +def get_cluster_templates(**args): + return model_query(m.ClusterTemplate).filter_by(**args).all() + + +def get_cluster_template(**args): + return model_query(m.ClusterTemplate).filter_by(**args).first() + + +def create_cluster_template(values): + session = ctx().session + with session.begin(): + values['tenant_id'] = ctx().tenant_id + ngts_vals = values.pop('node_group_templates', []) + cluster_template = m.ClusterTemplate(**values) + for ngt in ngts_vals: + relation = cluster_template.add_node_group_template( + ngt['node_group_template_id'], ngt['node_group_name'], + ngt['count']) + session.add(relation) + session.add(cluster_template) + + return cluster_template + + +def terminate_cluster_template(**args): + with ctx().session.begin(): + ctx().session.delete(get_cluster_template(**args)) + + +## NodeGroupTemplate ops + +def get_node_group_templates(**args): + return model_query(m.NodeGroupTemplate).filter_by(**args).all() + + +def get_node_group_template(**args): + return model_query(m.NodeGroupTemplate).filter_by(**args).first() + + +def create_node_group_template(values): + session = ctx().session + with session.begin(): + values['tenant_id'] = ctx().tenant_id + node_group_template = m.NodeGroupTemplate(**values) + session.add(node_group_template) + return node_group_template + + +def terminate_node_group_template(**args): + with ctx().session.begin(): + ctx().session.delete(get_node_group_template(**args)) diff --git a/savanna/exceptions.py b/savanna/exceptions.py index a86aa534..265fac93 100644 --- a/savanna/exceptions.py +++ b/savanna/exceptions.py @@ -37,102 +37,3 @@ class NotFoundException(SavannaException): def __init__(self, value): self.code = "NOT_FOUND" self.value = value - - -## Cluster operations exceptions - -class NotEnoughResourcesException(SavannaException): - def __init__(self, list): - self.message = "Nova available instances=%s, VCPUs=%s, RAM=%s. " \ - "Requested instances=%s, VCPUs=%s, RAM=%s" % tuple(list) - self.code = "NOT_ENOUGH_RESOURCES" - - -class ClusterNameExistedException(SavannaException): - def __init__(self, value): - self.message = "Cluster with name '%s' already exists" % value - self.code = "CLUSTER_NAME_ALREADY_EXISTS" - - -class ImageNotFoundException(SavannaException): - def __init__(self, value): - self.message = "Cannot find image with id '%s'" % value - self.code = "IMAGE_NOT_FOUND" - - -class NotSingleNameNodeException(SavannaException): - def __init__(self, nn_count): - self.message = "Hadoop cluster should contain only 1 NameNode. " \ - "Actual NN count is %s" % nn_count - self.code = "NOT_SINGLE_NAME_NODE" - - -class NotSingleJobTrackerException(SavannaException): - def __init__(self, jt_count): - self.message = "Hadoop cluster should contain only 1 JobTracker. " \ - "Actual JT count is %s" % jt_count - self.code = "NOT_SINGLE_JOB_TRACKER" - - -class ClusterNotFoundException(NotFoundException): - def __init__(self, value): - self.value = value - self.message = "Cluster '%s' not found" % self.value - self.code = "CLUSTER_NOT_FOUND" - - -## NodeTemplates operations exceptions - -class NodeTemplateNotFoundException(NotFoundException): - def __init__(self, value): - self.value = value - self.message = "NodeTemplate '%s' not found" % self.value - self.code = "NODE_TEMPLATE_NOT_FOUND" - - -class NodeTemplateExistedException(SavannaException): - def __init__(self, value): - self.message = "NodeTemplate with name '%s' already exists" % value - self.code = "NODE_TEMPLATE_ALREADY_EXISTS" - - -class FlavorNotFoundException(SavannaException): - def __init__(self, value): - self.message = "Cannot find flavor with name '%s'" % value - self.code = "FLAVOR_NOT_FOUND" - - -class DiscrepancyNodeProcessException(SavannaException): - def __init__(self, value): - self.message = "Discrepancies in Node Processes. Required: %s" % value - self.code = "NODE_PROCESS_DISCREPANCY" - - -class RequiredParamMissedException(SavannaException): - def __init__(self, process, param): - self.message = "Required parameter '%s' of process '%s' should be " \ - "specified" % (param, process) - self.code = "REQUIRED_PARAM_MISSED" - - -class AssociatedNodeTemplateTerminationException(SavannaException): - def __init__(self, value): - self.message = ("The are active nodes created using NodeTemplate '%s'" - " you trying to terminate") % value - self.code = "ASSOCIATED_NODE_TEMPLATE_TERMINATION" - - -class ParamNotAllowedException(SavannaException): - def __init__(self, param, process): - self.message = "Parameter '%s' of process '%s' is not " \ - "allowed to change" % (param, process) - self.code = "PARAM_IS_NOT_ALLOWED" - - -## NodeTypes operations exceptions - -class NodeTypeNotFoundException(NotFoundException): - def __init__(self, value): - self.value = value - self.message = "NodeType '%s' not found" % self.value - self.code = "NODE_TYPE_NOT_FOUND" diff --git a/savanna/main.py b/savanna/main.py index 13c25998..824305f7 100644 --- a/savanna/main.py +++ b/savanna/main.py @@ -17,12 +17,14 @@ from eventlet import monkey_patch from flask import Flask from keystoneclient.middleware.auth_token import filter_factory as auth_token from oslo.config import cfg +from savanna.context import ctx +from savanna.plugins.base import setup_plugins from werkzeug.exceptions import default_exceptions from werkzeug.exceptions import HTTPException -from savanna.api import v02 as api_v02 +from savanna.api import v10 as api_v10 +from savanna.db import api as db_api from savanna.middleware.auth_valid import filter_factory as auth_valid -from savanna.storage.db import setup_storage from savanna.utils.api import render from savanna.utils.scheduler import setup_scheduler @@ -71,14 +73,22 @@ def make_app(): def version_list(): return render({ "versions": [ - {"id": "v0.2", "status": "CURRENT"} + {"id": "v1.0", "status": "CURRENT"} ] }) - app.register_blueprint(api_v02.rest, url_prefix='/v0.2') + @app.teardown_request + def teardown_request(_ex=None): + # todo how it'll work in case of exception? + session = ctx().session + if session.transaction: + session.transaction.commit() - setup_storage(app) + app.register_blueprint(api_v10.rest, url_prefix='/v1.0') + + db_api.configure_db() setup_scheduler(app) + setup_plugins() def make_json_error(ex): status_code = (ex.code diff --git a/savanna/openstack/common/db/__init__.py b/savanna/openstack/common/db/__init__.py new file mode 100644 index 00000000..1b9b60de --- /dev/null +++ b/savanna/openstack/common/db/__init__.py @@ -0,0 +1,16 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 Cloudscaling Group, Inc +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. diff --git a/savanna/openstack/common/db/api.py b/savanna/openstack/common/db/api.py new file mode 100644 index 00000000..f88f9186 --- /dev/null +++ b/savanna/openstack/common/db/api.py @@ -0,0 +1,106 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2013 Rackspace Hosting +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Multiple DB API backend support. + +Supported configuration options: + +The following two parameters are in the 'database' group: +`backend`: DB backend name or full module path to DB backend module. +`use_tpool`: Enable thread pooling of DB API calls. + +A DB backend module should implement a method named 'get_backend' which +takes no arguments. The method can return any object that implements DB +API methods. + +*NOTE*: There are bugs in eventlet when using tpool combined with +threading locks. The python logging module happens to use such locks. To +work around this issue, be sure to specify thread=False with +eventlet.monkey_patch(). + +A bug for eventlet has been filed here: + +https://bitbucket.org/eventlet/eventlet/issue/137/ +""" +import functools + +from oslo.config import cfg + +from savanna.openstack.common import importutils +from savanna.openstack.common import lockutils + + +db_opts = [ + cfg.StrOpt('backend', + default='sqlalchemy', + deprecated_name='db_backend', + deprecated_group='DEFAULT', + help='The backend to use for db'), + cfg.BoolOpt('use_tpool', + default=False, + deprecated_name='dbapi_use_tpool', + deprecated_group='DEFAULT', + help='Enable the experimental use of thread pooling for ' + 'all DB API calls') +] + +CONF = cfg.CONF +CONF.register_opts(db_opts, 'database') + + +class DBAPI(object): + def __init__(self, backend_mapping=None): + if backend_mapping is None: + backend_mapping = {} + self.__backend = None + self.__backend_mapping = backend_mapping + + @lockutils.synchronized('dbapi_backend', 'savanna-') + def __get_backend(self): + """Get the actual backend. May be a module or an instance of + a class. Doesn't matter to us. We do this synchronized as it's + possible multiple greenthreads started very quickly trying to do + DB calls and eventlet can switch threads before self.__backend gets + assigned. + """ + if self.__backend: + # Another thread assigned it + return self.__backend + backend_name = CONF.database.backend + self.__use_tpool = CONF.database.use_tpool + if self.__use_tpool: + from eventlet import tpool + self.__tpool = tpool + # Import the untranslated name if we don't have a + # mapping. + backend_path = self.__backend_mapping.get(backend_name, + backend_name) + backend_mod = importutils.import_module(backend_path) + self.__backend = backend_mod.get_backend() + return self.__backend + + def __getattr__(self, key): + backend = self.__backend or self.__get_backend() + attr = getattr(backend, key) + if not self.__use_tpool or not hasattr(attr, '__call__'): + return attr + + def tpool_wrapper(*args, **kwargs): + return self.__tpool.execute(attr, *args, **kwargs) + + functools.update_wrapper(tpool_wrapper, attr) + return tpool_wrapper diff --git a/savanna/openstack/common/db/exception.py b/savanna/openstack/common/db/exception.py new file mode 100644 index 00000000..65bed21c --- /dev/null +++ b/savanna/openstack/common/db/exception.py @@ -0,0 +1,45 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""DB related custom exceptions.""" + +from savanna.openstack.common.gettextutils import _ + + +class DBError(Exception): + """Wraps an implementation specific exception.""" + def __init__(self, inner_exception=None): + self.inner_exception = inner_exception + super(DBError, self).__init__(str(inner_exception)) + + +class DBDuplicateEntry(DBError): + """Wraps an implementation specific exception.""" + def __init__(self, columns=[], inner_exception=None): + self.columns = columns + super(DBDuplicateEntry, self).__init__(inner_exception) + + +class DBDeadlock(DBError): + def __init__(self, inner_exception=None): + super(DBDeadlock, self).__init__(inner_exception) + + +class DBInvalidUnicodeParameter(Exception): + message = _("Invalid Parameter: " + "Unicode is not supported by the current database.") diff --git a/savanna/openstack/common/db/sqlalchemy/__init__.py b/savanna/openstack/common/db/sqlalchemy/__init__.py new file mode 100644 index 00000000..1b9b60de --- /dev/null +++ b/savanna/openstack/common/db/sqlalchemy/__init__.py @@ -0,0 +1,16 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 Cloudscaling Group, Inc +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. diff --git a/savanna/openstack/common/db/sqlalchemy/models.py b/savanna/openstack/common/db/sqlalchemy/models.py new file mode 100644 index 00000000..dce7a28b --- /dev/null +++ b/savanna/openstack/common/db/sqlalchemy/models.py @@ -0,0 +1,105 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2011 X.commerce, a business unit of eBay Inc. +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 Piston Cloud Computing, Inc. +# Copyright 2012 Cloudscaling Group, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +SQLAlchemy models. +""" + +from sqlalchemy import Column, Integer +from sqlalchemy import DateTime +from sqlalchemy.orm import object_mapper + +from savanna.openstack.common.db.sqlalchemy.session import get_session +from savanna.openstack.common import timeutils + + +class ModelBase(object): + """Base class for models.""" + __table_initialized__ = False + + def save(self, session=None): + """Save this object.""" + if not session: + session = get_session() + # NOTE(boris-42): This part of code should be look like: + # sesssion.add(self) + # session.flush() + # But there is a bug in sqlalchemy and eventlet that + # raises NoneType exception if there is no running + # transaction and rollback is called. As long as + # sqlalchemy has this bug we have to create transaction + # explicity. + with session.begin(subtransactions=True): + session.add(self) + session.flush() + + def __setitem__(self, key, value): + setattr(self, key, value) + + def __getitem__(self, key): + return getattr(self, key) + + def get(self, key, default=None): + return getattr(self, key, default) + + def __iter__(self): + columns = dict(object_mapper(self).columns).keys() + # NOTE(russellb): Allow models to specify other keys that can be looked + # up, beyond the actual db columns. An example would be the 'name' + # property for an Instance. + if hasattr(self, '_extra_keys'): + columns.extend(self._extra_keys()) + self._i = iter(columns) + return self + + def next(self): + n = self._i.next() + return n, getattr(self, n) + + def update(self, values): + """Make the model object behave like a dict.""" + for k, v in values.iteritems(): + setattr(self, k, v) + + def iteritems(self): + """Make the model object behave like a dict. + + Includes attributes from joins.""" + local = dict(self) + joined = dict([(k, v) for k, v in self.__dict__.iteritems() + if not k[0] == '_']) + local.update(joined) + return local.iteritems() + + +class TimestampMixin(object): + created_at = Column(DateTime, default=timeutils.utcnow) + updated_at = Column(DateTime, onupdate=timeutils.utcnow) + + +class SoftDeleteMixin(object): + deleted_at = Column(DateTime) + deleted = Column(Integer, default=0) + + def soft_delete(self, session=None): + """Mark this object as deleted.""" + self.deleted = self.id + self.deleted_at = timeutils.utcnow() + self.save(session=session) diff --git a/savanna/openstack/common/db/sqlalchemy/session.py b/savanna/openstack/common/db/sqlalchemy/session.py new file mode 100644 index 00000000..19606766 --- /dev/null +++ b/savanna/openstack/common/db/sqlalchemy/session.py @@ -0,0 +1,698 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Session Handling for SQLAlchemy backend. + +Initializing: + +* Call set_defaults with the minimal of the following kwargs: + sql_connection, sqlite_db + + Example: + + session.set_defaults( + sql_connection="sqlite:///var/lib/savanna/sqlite.db", + sqlite_db="/var/lib/savanna/sqlite.db") + +Recommended ways to use sessions within this framework: + +* Don't use them explicitly; this is like running with AUTOCOMMIT=1. + model_query() will implicitly use a session when called without one + supplied. This is the ideal situation because it will allow queries + to be automatically retried if the database connection is interrupted. + + Note: Automatic retry will be enabled in a future patch. + + It is generally fine to issue several queries in a row like this. Even though + they may be run in separate transactions and/or separate sessions, each one + will see the data from the prior calls. If needed, undo- or rollback-like + functionality should be handled at a logical level. For an example, look at + the code around quotas and reservation_rollback(). + + Examples: + + def get_foo(context, foo): + return model_query(context, models.Foo).\ + filter_by(foo=foo).\ + first() + + def update_foo(context, id, newfoo): + model_query(context, models.Foo).\ + filter_by(id=id).\ + update({'foo': newfoo}) + + def create_foo(context, values): + foo_ref = models.Foo() + foo_ref.update(values) + foo_ref.save() + return foo_ref + + +* Within the scope of a single method, keeping all the reads and writes within + the context managed by a single session. In this way, the session's __exit__ + handler will take care of calling flush() and commit() for you. + If using this approach, you should not explicitly call flush() or commit(). + Any error within the context of the session will cause the session to emit + a ROLLBACK. If the connection is dropped before this is possible, the + database will implicitly rollback the transaction. + + Note: statements in the session scope will not be automatically retried. + + If you create models within the session, they need to be added, but you + do not need to call model.save() + + def create_many_foo(context, foos): + session = get_session() + with session.begin(): + for foo in foos: + foo_ref = models.Foo() + foo_ref.update(foo) + session.add(foo_ref) + + def update_bar(context, foo_id, newbar): + session = get_session() + with session.begin(): + foo_ref = model_query(context, models.Foo, session).\ + filter_by(id=foo_id).\ + first() + model_query(context, models.Bar, session).\ + filter_by(id=foo_ref['bar_id']).\ + update({'bar': newbar}) + + Note: update_bar is a trivially simple example of using "with session.begin". + Whereas create_many_foo is a good example of when a transaction is needed, + it is always best to use as few queries as possible. The two queries in + update_bar can be better expressed using a single query which avoids + the need for an explicit transaction. It can be expressed like so: + + def update_bar(context, foo_id, newbar): + subq = model_query(context, models.Foo.id).\ + filter_by(id=foo_id).\ + limit(1).\ + subquery() + model_query(context, models.Bar).\ + filter_by(id=subq.as_scalar()).\ + update({'bar': newbar}) + + For reference, this emits approximagely the following SQL statement: + + UPDATE bar SET bar = ${newbar} + WHERE id=(SELECT bar_id FROM foo WHERE id = ${foo_id} LIMIT 1); + +* Passing an active session between methods. Sessions should only be passed + to private methods. The private method must use a subtransaction; otherwise + SQLAlchemy will throw an error when you call session.begin() on an existing + transaction. Public methods should not accept a session parameter and should + not be involved in sessions within the caller's scope. + + Note that this incurs more overhead in SQLAlchemy than the above means + due to nesting transactions, and it is not possible to implicitly retry + failed database operations when using this approach. + + This also makes code somewhat more difficult to read and debug, because a + single database transaction spans more than one method. Error handling + becomes less clear in this situation. When this is needed for code clarity, + it should be clearly documented. + + def myfunc(foo): + session = get_session() + with session.begin(): + # do some database things + bar = _private_func(foo, session) + return bar + + def _private_func(foo, session=None): + if not session: + session = get_session() + with session.begin(subtransaction=True): + # do some other database things + return bar + + +There are some things which it is best to avoid: + +* Don't keep a transaction open any longer than necessary. + + This means that your "with session.begin()" block should be as short + as possible, while still containing all the related calls for that + transaction. + +* Avoid "with_lockmode('UPDATE')" when possible. + + In MySQL/InnoDB, when a "SELECT ... FOR UPDATE" query does not match + any rows, it will take a gap-lock. This is a form of write-lock on the + "gap" where no rows exist, and prevents any other writes to that space. + This can effectively prevent any INSERT into a table by locking the gap + at the end of the index. Similar problems will occur if the SELECT FOR UPDATE + has an overly broad WHERE clause, or doesn't properly use an index. + + One idea proposed at ODS Fall '12 was to use a normal SELECT to test the + number of rows matching a query, and if only one row is returned, + then issue the SELECT FOR UPDATE. + + The better long-term solution is to use INSERT .. ON DUPLICATE KEY UPDATE. + However, this can not be done until the "deleted" columns are removed and + proper UNIQUE constraints are added to the tables. + + +Enabling soft deletes: + +* To use/enable soft-deletes, the SoftDeleteMixin must be added + to your model class. For example: + + class NovaBase(models.SoftDeleteMixin, models.ModelBase): + pass + + +Efficient use of soft deletes: + +* There are two possible ways to mark a record as deleted: + model.soft_delete() and query.soft_delete(). + + model.soft_delete() method works with single already fetched entry. + query.soft_delete() makes only one db request for all entries that correspond + to query. + +* In almost all cases you should use query.soft_delete(). Some examples: + + def soft_delete_bar(): + count = model_query(BarModel).find(some_condition).soft_delete() + if count == 0: + raise Exception("0 entries were soft deleted") + + def complex_soft_delete_with_synchronization_bar(session=None): + if session is None: + session = get_session() + with session.begin(subtransactions=True): + count = model_query(BarModel).\ + find(some_condition).\ + soft_delete(synchronize_session=True) + # Here synchronize_session is required, because we + # don't know what is going on in outer session. + if count == 0: + raise Exception("0 entries were soft deleted") + +* There is only one situation where model.soft_delete() is appropriate: when + you fetch a single record, work with it, and mark it as deleted in the same + transaction. + + def soft_delete_bar_model(): + session = get_session() + with session.begin(): + bar_ref = model_query(BarModel).find(some_condition).first() + # Work with bar_ref + bar_ref.soft_delete(session=session) + + However, if you need to work with all entries that correspond to query and + then soft delete them you should use query.soft_delete() method: + + def soft_delete_multi_models(): + session = get_session() + with session.begin(): + query = model_query(BarModel, session=session).\ + find(some_condition) + model_refs = query.all() + # Work with model_refs + query.soft_delete(synchronize_session=False) + # synchronize_session=False should be set if there is no outer + # session and these entries are not used after this. + + When working with many rows, it is very important to use query.soft_delete, + which issues a single query. Using model.soft_delete(), as in the following + example, is very inefficient. + + for bar_ref in bar_refs: + bar_ref.soft_delete(session=session) + # This will produce count(bar_refs) db requests. +""" + +import os.path +import re +import time + +from eventlet import greenthread +from oslo.config import cfg +import six +from sqlalchemy import exc as sqla_exc +import sqlalchemy.interfaces +from sqlalchemy.interfaces import PoolListener +import sqlalchemy.orm +from sqlalchemy.pool import NullPool, StaticPool +from sqlalchemy.sql.expression import literal_column + +from savanna.openstack.common.db import exception +from savanna.openstack.common import log as logging +from savanna.openstack.common.gettextutils import _ +from savanna.openstack.common import timeutils + +DEFAULT = 'DEFAULT' + +sqlite_db_opts = [ + cfg.StrOpt('sqlite_db', + default='savanna.sqlite', + help='the filename to use with sqlite'), + cfg.BoolOpt('sqlite_synchronous', + default=True, + help='If true, use synchronous mode for sqlite'), +] + +database_opts = [ + cfg.StrOpt('connection', + default='sqlite:///' + + os.path.abspath(os.path.join(os.path.dirname(__file__), + '../', '$sqlite_db')), + help='The SQLAlchemy connection string used to connect to the ' + 'database', + deprecated_name='sql_connection', + deprecated_group=DEFAULT, + secret=True), + cfg.IntOpt('idle_timeout', + default=3600, + deprecated_name='sql_idle_timeout', + deprecated_group=DEFAULT, + help='timeout before idle sql connections are reaped'), + cfg.IntOpt('min_pool_size', + default=1, + deprecated_name='sql_min_pool_size', + deprecated_group=DEFAULT, + help='Minimum number of SQL connections to keep open in a ' + 'pool'), + cfg.IntOpt('max_pool_size', + default=5, + deprecated_name='sql_max_pool_size', + deprecated_group=DEFAULT, + help='Maximum number of SQL connections to keep open in a ' + 'pool'), + cfg.IntOpt('max_retries', + default=10, + deprecated_name='sql_max_retries', + deprecated_group=DEFAULT, + help='maximum db connection retries during startup. ' + '(setting -1 implies an infinite retry count)'), + cfg.IntOpt('retry_interval', + default=10, + deprecated_name='sql_retry_interval', + deprecated_group=DEFAULT, + help='interval between retries of opening a sql connection'), + cfg.IntOpt('max_overflow', + default=None, + deprecated_name='sql_max_overflow', + deprecated_group=DEFAULT, + help='If set, use this value for max_overflow with sqlalchemy'), + cfg.IntOpt('connection_debug', + default=0, + deprecated_name='sql_connection_debug', + deprecated_group=DEFAULT, + help='Verbosity of SQL debugging information. 0=None, ' + '100=Everything'), + cfg.BoolOpt('connection_trace', + default=False, + deprecated_name='sql_connection_trace', + deprecated_group=DEFAULT, + help='Add python stack traces to SQL as comment strings'), +] + +CONF = cfg.CONF +CONF.register_opts(sqlite_db_opts) +CONF.register_opts(database_opts, 'database') +LOG = logging.getLogger(__name__) + +_ENGINE = None +_MAKER = None + + +def set_defaults(sql_connection, sqlite_db): + """Set defaults for configuration variables.""" + cfg.set_defaults(database_opts, + connection=sql_connection) + cfg.set_defaults(sqlite_db_opts, + sqlite_db=sqlite_db) + + +def cleanup(): + global _ENGINE, _MAKER + + if _MAKER: + _MAKER.close_all() + _MAKER = None + if _ENGINE: + _ENGINE.dispose() + _ENGINE = None + + +class SqliteForeignKeysListener(PoolListener): + """ + Ensures that the foreign key constraints are enforced in SQLite. + + The foreign key constraints are disabled by default in SQLite, + so the foreign key constraints will be enabled here for every + database connection + """ + def connect(self, dbapi_con, con_record): + dbapi_con.execute('pragma foreign_keys=ON') + + +def get_session(autocommit=True, expire_on_commit=False, + sqlite_fk=False): + """Return a SQLAlchemy session.""" + global _MAKER + + if _MAKER is None: + engine = get_engine(sqlite_fk=sqlite_fk) + _MAKER = get_maker(engine, autocommit, expire_on_commit) + + session = _MAKER() + return session + + +# note(boris-42): In current versions of DB backends unique constraint +# violation messages follow the structure: +# +# sqlite: +# 1 column - (IntegrityError) column c1 is not unique +# N columns - (IntegrityError) column c1, c2, ..., N are not unique +# +# postgres: +# 1 column - (IntegrityError) duplicate key value violates unique +# constraint "users_c1_key" +# N columns - (IntegrityError) duplicate key value violates unique +# constraint "name_of_our_constraint" +# +# mysql: +# 1 column - (IntegrityError) (1062, "Duplicate entry 'value_of_c1' for key +# 'c1'") +# N columns - (IntegrityError) (1062, "Duplicate entry 'values joined +# with -' for key 'name_of_our_constraint'") +_DUP_KEY_RE_DB = { + "sqlite": re.compile(r"^.*columns?([^)]+)(is|are)\s+not\s+unique$"), + "postgresql": re.compile(r"^.*duplicate\s+key.*\"([^\"]+)\"\s*\n.*$"), + "mysql": re.compile(r"^.*\(1062,.*'([^\']+)'\"\)$") +} + + +def _raise_if_duplicate_entry_error(integrity_error, engine_name): + """ + In this function will be raised DBDuplicateEntry exception if integrity + error wrap unique constraint violation. + """ + + def get_columns_from_uniq_cons_or_name(columns): + # note(vsergeyev): UniqueConstraint name convention: "uniq_t$c1$c2" + # where `t` it is table name and columns `c1`, `c2` + # are in UniqueConstraint. + uniqbase = "uniq_" + if not columns.startswith(uniqbase): + if engine_name == "postgresql": + return [columns[columns.index("_") + 1:columns.rindex("_")]] + return [columns] + return columns[len(uniqbase):].split("$")[1:] + + if engine_name not in ["mysql", "sqlite", "postgresql"]: + return + + m = _DUP_KEY_RE_DB[engine_name].match(integrity_error.message) + if not m: + return + columns = m.group(1) + + if engine_name == "sqlite": + columns = columns.strip().split(", ") + else: + columns = get_columns_from_uniq_cons_or_name(columns) + raise exception.DBDuplicateEntry(columns, integrity_error) + + +# NOTE(comstud): In current versions of DB backends, Deadlock violation +# messages follow the structure: +# +# mysql: +# (OperationalError) (1213, 'Deadlock found when trying to get lock; try ' +# 'restarting transaction') +_DEADLOCK_RE_DB = { + "mysql": re.compile(r"^.*\(1213, 'Deadlock.*") +} + + +def _raise_if_deadlock_error(operational_error, engine_name): + """ + Raise DBDeadlock exception if OperationalError contains a Deadlock + condition. + """ + re = _DEADLOCK_RE_DB.get(engine_name) + if re is None: + return + m = re.match(operational_error.message) + if not m: + return + raise exception.DBDeadlock(operational_error) + + +def _wrap_db_error(f): + def _wrap(*args, **kwargs): + try: + return f(*args, **kwargs) + except UnicodeEncodeError: + raise exception.DBInvalidUnicodeParameter() + # note(boris-42): We should catch unique constraint violation and + # wrap it by our own DBDuplicateEntry exception. Unique constraint + # violation is wrapped by IntegrityError. + except sqla_exc.OperationalError as e: + _raise_if_deadlock_error(e, get_engine().name) + # NOTE(comstud): A lot of code is checking for OperationalError + # so let's not wrap it for now. + raise + except sqla_exc.IntegrityError as e: + # note(boris-42): SqlAlchemy doesn't unify errors from different + # DBs so we must do this. Also in some tables (for example + # instance_types) there are more than one unique constraint. This + # means we should get names of columns, which values violate + # unique constraint, from error message. + _raise_if_duplicate_entry_error(e, get_engine().name) + raise exception.DBError(e) + except Exception as e: + LOG.exception(_('DB exception wrapped.')) + raise exception.DBError(e) + _wrap.func_name = f.func_name + return _wrap + + +def get_engine(sqlite_fk=False): + """Return a SQLAlchemy engine.""" + global _ENGINE + if _ENGINE is None: + _ENGINE = create_engine(CONF.database.connection, + sqlite_fk=sqlite_fk) + return _ENGINE + + +def _synchronous_switch_listener(dbapi_conn, connection_rec): + """Switch sqlite connections to non-synchronous mode.""" + dbapi_conn.execute("PRAGMA synchronous = OFF") + + +def _add_regexp_listener(dbapi_con, con_record): + """Add REGEXP function to sqlite connections.""" + + def regexp(expr, item): + reg = re.compile(expr) + return reg.search(six.text_type(item)) is not None + dbapi_con.create_function('regexp', 2, regexp) + + +def _greenthread_yield(dbapi_con, con_record): + """ + Ensure other greenthreads get a chance to execute by forcing a context + switch. With common database backends (eg MySQLdb and sqlite), there is + no implicit yield caused by network I/O since they are implemented by + C libraries that eventlet cannot monkey patch. + """ + greenthread.sleep(0) + + +def _ping_listener(dbapi_conn, connection_rec, connection_proxy): + """ + Ensures that MySQL connections checked out of the + pool are alive. + + Borrowed from: + http://groups.google.com/group/sqlalchemy/msg/a4ce563d802c929f + """ + try: + dbapi_conn.cursor().execute('select 1') + except dbapi_conn.OperationalError as ex: + if ex.args[0] in (2006, 2013, 2014, 2045, 2055): + LOG.warn(_('Got mysql server has gone away: %s'), ex) + raise sqla_exc.DisconnectionError("Database server went away") + else: + raise + + +def _is_db_connection_error(args): + """Return True if error in connecting to db.""" + # NOTE(adam_g): This is currently MySQL specific and needs to be extended + # to support Postgres and others. + conn_err_codes = ('2002', '2003', '2006') + for err_code in conn_err_codes: + if args.find(err_code) != -1: + return True + return False + + +def create_engine(sql_connection, sqlite_fk=False): + """Return a new SQLAlchemy engine.""" + connection_dict = sqlalchemy.engine.url.make_url(sql_connection) + + engine_args = { + "pool_recycle": CONF.database.idle_timeout, + "echo": False, + 'convert_unicode': True, + } + + # Map our SQL debug level to SQLAlchemy's options + if CONF.database.connection_debug >= 100: + engine_args['echo'] = 'debug' + elif CONF.database.connection_debug >= 50: + engine_args['echo'] = True + + if "sqlite" in connection_dict.drivername: + if sqlite_fk: + engine_args["listeners"] = [SqliteForeignKeysListener()] + engine_args["poolclass"] = NullPool + + if CONF.database.connection == "sqlite://": + engine_args["poolclass"] = StaticPool + engine_args["connect_args"] = {'check_same_thread': False} + else: + engine_args['pool_size'] = CONF.database.max_pool_size + if CONF.database.max_overflow is not None: + engine_args['max_overflow'] = CONF.database.max_overflow + + engine = sqlalchemy.create_engine(sql_connection, **engine_args) + + sqlalchemy.event.listen(engine, 'checkin', _greenthread_yield) + + if 'mysql' in connection_dict.drivername: + sqlalchemy.event.listen(engine, 'checkout', _ping_listener) + elif 'sqlite' in connection_dict.drivername: + if not CONF.sqlite_synchronous: + sqlalchemy.event.listen(engine, 'connect', + _synchronous_switch_listener) + sqlalchemy.event.listen(engine, 'connect', _add_regexp_listener) + + if (CONF.database.connection_trace and + engine.dialect.dbapi.__name__ == 'MySQLdb'): + _patch_mysqldb_with_stacktrace_comments() + + try: + engine.connect() + except sqla_exc.OperationalError as e: + if not _is_db_connection_error(e.args[0]): + raise + + remaining = CONF.database.max_retries + if remaining == -1: + remaining = 'infinite' + while True: + msg = _('SQL connection failed. %s attempts left.') + LOG.warn(msg % remaining) + if remaining != 'infinite': + remaining -= 1 + time.sleep(CONF.database.retry_interval) + try: + engine.connect() + break + except sqla_exc.OperationalError as e: + if (remaining != 'infinite' and remaining == 0) or \ + not _is_db_connection_error(e.args[0]): + raise + return engine + + +class Query(sqlalchemy.orm.query.Query): + """Subclass of sqlalchemy.query with soft_delete() method.""" + def soft_delete(self, synchronize_session='evaluate'): + return self.update({'deleted': literal_column('id'), + 'updated_at': literal_column('updated_at'), + 'deleted_at': timeutils.utcnow()}, + synchronize_session=synchronize_session) + + +class Session(sqlalchemy.orm.session.Session): + """Custom Session class to avoid SqlAlchemy Session monkey patching.""" + @_wrap_db_error + def query(self, *args, **kwargs): + return super(Session, self).query(*args, **kwargs) + + @_wrap_db_error + def flush(self, *args, **kwargs): + return super(Session, self).flush(*args, **kwargs) + + @_wrap_db_error + def execute(self, *args, **kwargs): + return super(Session, self).execute(*args, **kwargs) + + +def get_maker(engine, autocommit=True, expire_on_commit=False): + """Return a SQLAlchemy sessionmaker using the given engine.""" + return sqlalchemy.orm.sessionmaker(bind=engine, + class_=Session, + autocommit=autocommit, + expire_on_commit=expire_on_commit, + query_cls=Query) + + +def _patch_mysqldb_with_stacktrace_comments(): + """Adds current stack trace as a comment in queries by patching + MySQLdb.cursors.BaseCursor._do_query. + """ + import MySQLdb.cursors + import traceback + + old_mysql_do_query = MySQLdb.cursors.BaseCursor._do_query + + def _do_query(self, q): + stack = '' + for file, line, method, function in traceback.extract_stack(): + # exclude various common things from trace + if file.endswith('session.py') and method == '_do_query': + continue + if file.endswith('api.py') and method == 'wrapper': + continue + if file.endswith('utils.py') and method == '_inner': + continue + if file.endswith('exception.py') and method == '_wrap': + continue + # db/api is just a wrapper around db/sqlalchemy/api + if file.endswith('db/api.py'): + continue + # only trace inside savanna + index = file.rfind('savanna') + if index == -1: + continue + stack += "File:%s:%s Method:%s() Line:%s | " \ + % (file[index:], line, method, function) + + # strip trailing " | " from stack + if stack: + stack = stack[:-3] + qq = "%s /* %s */" % (q, stack) + else: + qq = q + old_mysql_do_query(self, qq) + + setattr(MySQLdb.cursors.BaseCursor, '_do_query', _do_query) diff --git a/savanna/openstack/common/db/sqlalchemy/utils.py b/savanna/openstack/common/db/sqlalchemy/utils.py new file mode 100644 index 00000000..6e4a245c --- /dev/null +++ b/savanna/openstack/common/db/sqlalchemy/utils.py @@ -0,0 +1,132 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2010-2011 OpenStack Foundation. +# Copyright 2012 Justin Santa Barbara +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Implementation of paginate query.""" + +import sqlalchemy + +from savanna.openstack.common.gettextutils import _ +from savanna.openstack.common import log as logging + + +LOG = logging.getLogger(__name__) + + +class InvalidSortKey(Exception): + message = _("Sort key supplied was not valid.") + + +# copy from glance/db/sqlalchemy/api.py +def paginate_query(query, model, limit, sort_keys, marker=None, + sort_dir=None, sort_dirs=None): + """Returns a query with sorting / pagination criteria added. + + Pagination works by requiring a unique sort_key, specified by sort_keys. + (If sort_keys is not unique, then we risk looping through values.) + We use the last row in the previous page as the 'marker' for pagination. + So we must return values that follow the passed marker in the order. + With a single-valued sort_key, this would be easy: sort_key > X. + With a compound-values sort_key, (k1, k2, k3) we must do this to repeat + the lexicographical ordering: + (k1 > X1) or (k1 == X1 && k2 > X2) or (k1 == X1 && k2 == X2 && k3 > X3) + + We also have to cope with different sort_directions. + + Typically, the id of the last row is used as the client-facing pagination + marker, then the actual marker object must be fetched from the db and + passed in to us as marker. + + :param query: the query object to which we should add paging/sorting + :param model: the ORM model class + :param limit: maximum number of items to return + :param sort_keys: array of attributes by which results should be sorted + :param marker: the last item of the previous page; we returns the next + results after this value. + :param sort_dir: direction in which results should be sorted (asc, desc) + :param sort_dirs: per-column array of sort_dirs, corresponding to sort_keys + + :rtype: sqlalchemy.orm.query.Query + :return: The query with sorting/pagination added. + """ + + if 'id' not in sort_keys: + # TODO(justinsb): If this ever gives a false-positive, check + # the actual primary key, rather than assuming its id + LOG.warn(_('Id not in sort_keys; is sort_keys unique?')) + + assert(not (sort_dir and sort_dirs)) + + # Default the sort direction to ascending + if sort_dirs is None and sort_dir is None: + sort_dir = 'asc' + + # Ensure a per-column sort direction + if sort_dirs is None: + sort_dirs = [sort_dir for _sort_key in sort_keys] + + assert(len(sort_dirs) == len(sort_keys)) + + # Add sorting + for current_sort_key, current_sort_dir in zip(sort_keys, sort_dirs): + sort_dir_func = { + 'asc': sqlalchemy.asc, + 'desc': sqlalchemy.desc, + }[current_sort_dir] + + try: + sort_key_attr = getattr(model, current_sort_key) + except AttributeError: + raise InvalidSortKey() + query = query.order_by(sort_dir_func(sort_key_attr)) + + # Add pagination + if marker is not None: + marker_values = [] + for sort_key in sort_keys: + v = getattr(marker, sort_key) + marker_values.append(v) + + # Build up an array of sort criteria as in the docstring + criteria_list = [] + for i in range(0, len(sort_keys)): + crit_attrs = [] + for j in range(0, i): + model_attr = getattr(model, sort_keys[j]) + crit_attrs.append((model_attr == marker_values[j])) + + model_attr = getattr(model, sort_keys[i]) + if sort_dirs[i] == 'desc': + crit_attrs.append((model_attr < marker_values[i])) + elif sort_dirs[i] == 'asc': + crit_attrs.append((model_attr > marker_values[i])) + else: + raise ValueError(_("Unknown sort direction, " + "must be 'desc' or 'asc'")) + + criteria = sqlalchemy.sql.and_(*crit_attrs) + criteria_list.append(criteria) + + f = sqlalchemy.sql.or_(*criteria_list) + query = query.filter(f) + + if limit is not None: + query = query.limit(limit) + + return query diff --git a/savanna/openstack/common/fileutils.py b/savanna/openstack/common/fileutils.py new file mode 100644 index 00000000..b988ad03 --- /dev/null +++ b/savanna/openstack/common/fileutils.py @@ -0,0 +1,35 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import errno +import os + + +def ensure_tree(path): + """Create a directory (and any ancestor directories required) + + :param path: Directory to create + """ + try: + os.makedirs(path) + except OSError as exc: + if exc.errno == errno.EEXIST: + if not os.path.isdir(path): + raise + else: + raise diff --git a/savanna/openstack/common/jsonutils.py b/savanna/openstack/common/jsonutils.py index 524aabcb..9af7a985 100644 --- a/savanna/openstack/common/jsonutils.py +++ b/savanna/openstack/common/jsonutils.py @@ -41,6 +41,8 @@ import json import types import xmlrpclib +import six + from savanna.openstack.common import timeutils @@ -93,7 +95,7 @@ def to_primitive(value, convert_instances=False, convert_datetime=True, # value of itertools.count doesn't get caught by nasty_type_tests # and results in infinite loop when list(value) is called. if type(value) == itertools.count: - return unicode(value) + return six.text_type(value) # FIXME(vish): Workaround for LP bug 852095. Without this workaround, # tests that raise an exception in a mocked method that @@ -137,12 +139,12 @@ def to_primitive(value, convert_instances=False, convert_datetime=True, return recursive(value.__dict__, level=level + 1) else: if any(test(value) for test in _nasty_type_tests): - return unicode(value) + return six.text_type(value) return value except TypeError: # Class objects are tricky since they may define something like # __iter__ defined but it isn't callable as list(). - return unicode(value) + return six.text_type(value) def dumps(value, default=to_primitive, **kwargs): diff --git a/savanna/openstack/common/local.py b/savanna/openstack/common/local.py index f1bfc824..5e88e20e 100644 --- a/savanna/openstack/common/local.py +++ b/savanna/openstack/common/local.py @@ -15,7 +15,7 @@ # License for the specific language governing permissions and limitations # under the License. -"""Greenthread local storage of variables using weak references""" +"""Greenthread local db of variables using weak references""" import weakref @@ -41,7 +41,7 @@ class WeakLocal(corolocal.local): store = WeakLocal() # A "weak" store uses weak references and allows an object to fall out of scope -# when it falls out of scope in the code that uses the thread local storage. A +# when it falls out of scope in the code that uses the thread local db. A # "strong" store will hold a reference to the object so that it never falls out # of scope. weak_store = WeakLocal() diff --git a/savanna/openstack/common/lockutils.py b/savanna/openstack/common/lockutils.py new file mode 100644 index 00000000..f43ac228 --- /dev/null +++ b/savanna/openstack/common/lockutils.py @@ -0,0 +1,278 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import errno +import functools +import os +import shutil +import tempfile +import time +import weakref + +from eventlet import semaphore +from oslo.config import cfg + +from savanna.openstack.common import fileutils +from savanna.openstack.common.gettextutils import _ +from savanna.openstack.common import local +from savanna.openstack.common import log as logging + + +LOG = logging.getLogger(__name__) + + +util_opts = [ + cfg.BoolOpt('disable_process_locking', default=False, + help='Whether to disable inter-process locks'), + cfg.StrOpt('lock_path', + help=('Directory to use for lock files. Default to a ' + 'temp directory')) +] + + +CONF = cfg.CONF +CONF.register_opts(util_opts) + + +def set_defaults(lock_path): + cfg.set_defaults(util_opts, lock_path=lock_path) + + +class _InterProcessLock(object): + """Lock implementation which allows multiple locks, working around + issues like bugs.debian.org/cgi-bin/bugreport.cgi?bug=632857 and does + not require any cleanup. Since the lock is always held on a file + descriptor rather than outside of the process, the lock gets dropped + automatically if the process crashes, even if __exit__ is not executed. + + There are no guarantees regarding usage by multiple green threads in a + single process here. This lock works only between processes. Exclusive + access between local threads should be achieved using the semaphores + in the @synchronized decorator. + + Note these locks are released when the descriptor is closed, so it's not + safe to close the file descriptor while another green thread holds the + lock. Just opening and closing the lock file can break synchronisation, + so lock files must be accessed only using this abstraction. + """ + + def __init__(self, name): + self.lockfile = None + self.fname = name + + def __enter__(self): + self.lockfile = open(self.fname, 'w') + + while True: + try: + # Using non-blocking locks since green threads are not + # patched to deal with blocking locking calls. + # Also upon reading the MSDN docs for locking(), it seems + # to have a laughable 10 attempts "blocking" mechanism. + self.trylock() + return self + except IOError as e: + if e.errno in (errno.EACCES, errno.EAGAIN): + # external locks synchronise things like iptables + # updates - give it some time to prevent busy spinning + time.sleep(0.01) + else: + raise + + def __exit__(self, exc_type, exc_val, exc_tb): + try: + self.unlock() + self.lockfile.close() + except IOError: + LOG.exception(_("Could not release the acquired lock `%s`"), + self.fname) + + def trylock(self): + raise NotImplementedError() + + def unlock(self): + raise NotImplementedError() + + +class _WindowsLock(_InterProcessLock): + def trylock(self): + msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_NBLCK, 1) + + def unlock(self): + msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_UNLCK, 1) + + +class _PosixLock(_InterProcessLock): + def trylock(self): + fcntl.lockf(self.lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB) + + def unlock(self): + fcntl.lockf(self.lockfile, fcntl.LOCK_UN) + + +if os.name == 'nt': + import msvcrt + InterProcessLock = _WindowsLock +else: + import fcntl + InterProcessLock = _PosixLock + +_semaphores = weakref.WeakValueDictionary() + + +def synchronized(name, lock_file_prefix, external=False, lock_path=None): + """Synchronization decorator. + + Decorating a method like so:: + + @synchronized('mylock') + def foo(self, *args): + ... + + ensures that only one thread will execute the foo method at a time. + + Different methods can share the same lock:: + + @synchronized('mylock') + def foo(self, *args): + ... + + @synchronized('mylock') + def bar(self, *args): + ... + + This way only one of either foo or bar can be executing at a time. + + The lock_file_prefix argument is used to provide lock files on disk with a + meaningful prefix. The prefix should end with a hyphen ('-') if specified. + + The external keyword argument denotes whether this lock should work across + multiple processes. This means that if two different workers both run a + a method decorated with @synchronized('mylock', external=True), only one + of them will execute at a time. + + The lock_path keyword argument is used to specify a special location for + external lock files to live. If nothing is set, then CONF.lock_path is + used as a default. + """ + + def wrap(f): + @functools.wraps(f) + def inner(*args, **kwargs): + # NOTE(soren): If we ever go natively threaded, this will be racy. + # See http://stackoverflow.com/questions/5390569/dyn + # amically-allocating-and-destroying-mutexes + sem = _semaphores.get(name, semaphore.Semaphore()) + if name not in _semaphores: + # this check is not racy - we're already holding ref locally + # so GC won't remove the item and there was no IO switch + # (only valid in greenthreads) + _semaphores[name] = sem + + with sem: + LOG.debug(_('Got semaphore "%(lock)s" for method ' + '"%(method)s"...'), {'lock': name, + 'method': f.__name__}) + + # NOTE(mikal): I know this looks odd + if not hasattr(local.strong_store, 'locks_held'): + local.strong_store.locks_held = [] + local.strong_store.locks_held.append(name) + + try: + if external and not CONF.disable_process_locking: + LOG.debug(_('Attempting to grab file lock "%(lock)s" ' + 'for method "%(method)s"...'), + {'lock': name, 'method': f.__name__}) + cleanup_dir = False + + # We need a copy of lock_path because it is non-local + local_lock_path = lock_path + if not local_lock_path: + local_lock_path = CONF.lock_path + + if not local_lock_path: + cleanup_dir = True + local_lock_path = tempfile.mkdtemp() + + if not os.path.exists(local_lock_path): + fileutils.ensure_tree(local_lock_path) + + # NOTE(mikal): the lock name cannot contain directory + # separators + safe_name = name.replace(os.sep, '_') + lock_file_name = '%s%s' % (lock_file_prefix, safe_name) + lock_file_path = os.path.join(local_lock_path, + lock_file_name) + + try: + lock = InterProcessLock(lock_file_path) + with lock: + LOG.debug(_('Got file lock "%(lock)s" at ' + '%(path)s for method ' + '"%(method)s"...'), + {'lock': name, + 'path': lock_file_path, + 'method': f.__name__}) + retval = f(*args, **kwargs) + finally: + LOG.debug(_('Released file lock "%(lock)s" at ' + '%(path)s for method "%(method)s"...'), + {'lock': name, + 'path': lock_file_path, + 'method': f.__name__}) + # NOTE(vish): This removes the tempdir if we needed + # to create one. This is used to + # cleanup the locks left behind by unit + # tests. + if cleanup_dir: + shutil.rmtree(local_lock_path) + else: + retval = f(*args, **kwargs) + + finally: + local.strong_store.locks_held.remove(name) + + return retval + return inner + return wrap + + +def synchronized_with_prefix(lock_file_prefix): + """Partial object generator for the synchronization decorator. + + Redefine @synchronized in each project like so:: + + (in nova/utils.py) + from nova.openstack.common import lockutils + + synchronized = lockutils.synchronized_with_prefix('nova-') + + + (in nova/foo.py) + from nova import utils + + @utils.synchronized('mylock') + def bar(self, *args): + ... + + The lock_file_prefix argument is used to provide lock files on disk with a + meaningful prefix. The prefix should end with a hyphen ('-') if specified. + """ + + return functools.partial(synchronized, lock_file_prefix=lock_file_prefix) diff --git a/savanna/openstack/common/log.py b/savanna/openstack/common/log.py index 06e13682..14bb4c8c 100644 --- a/savanna/openstack/common/log.py +++ b/savanna/openstack/common/log.py @@ -43,12 +43,11 @@ import traceback from oslo.config import cfg from savanna.openstack.common.gettextutils import _ +from savanna.openstack.common import importutils from savanna.openstack.common import jsonutils from savanna.openstack.common import local -from savanna.openstack.common import notifier -_DEFAULT_LOG_FORMAT = "%(asctime)s %(levelname)8s [%(name)s] %(message)s" _DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" common_cli_opts = [ @@ -73,11 +72,13 @@ logging_cli_opts = [ 'documentation for details on logging configuration ' 'files.'), cfg.StrOpt('log-format', - default=_DEFAULT_LOG_FORMAT, + default=None, metavar='FORMAT', help='A logging.Formatter log message format string which may ' 'use any of the available logging.LogRecord attributes. ' - 'Default: %(default)s'), + 'This option is deprecated. Please use ' + 'logging_context_format_string and ' + 'logging_default_format_string instead.'), cfg.StrOpt('log-date-format', default=_DEFAULT_LOG_DATE_FORMAT, metavar='DATE_FORMAT', @@ -207,7 +208,27 @@ def _get_log_file_path(binary=None): return '%s.log' % (os.path.join(logdir, binary),) -class ContextAdapter(logging.LoggerAdapter): +class BaseLoggerAdapter(logging.LoggerAdapter): + + def audit(self, msg, *args, **kwargs): + self.log(logging.AUDIT, msg, *args, **kwargs) + + +class LazyAdapter(BaseLoggerAdapter): + def __init__(self, name='unknown', version='unknown'): + self._logger = None + self.extra = {} + self.name = name + self.version = version + + @property + def logger(self): + if not self._logger: + self._logger = getLogger(self.name, self.version) + return self._logger + + +class ContextAdapter(BaseLoggerAdapter): warn = logging.LoggerAdapter.warning def __init__(self, logger, project_name, version_string): @@ -215,8 +236,9 @@ class ContextAdapter(logging.LoggerAdapter): self.project = project_name self.version = version_string - def audit(self, msg, *args, **kwargs): - self.log(logging.AUDIT, msg, *args, **kwargs) + @property + def handlers(self): + return self.logger.handlers def deprecated(self, msg, *args, **kwargs): stdmsg = _("Deprecated: %s") % msg @@ -300,17 +322,6 @@ class JSONFormatter(logging.Formatter): return jsonutils.dumps(message) -class PublishErrorsHandler(logging.Handler): - def emit(self, record): - if ('savanna.openstack.common.notifier.log_notifier' in - CONF.notification_driver): - return - notifier.api.notify(None, 'error.publisher', - 'error_notification', - notifier.api.ERROR, - dict(error=record.msg)) - - def _create_logging_excepthook(product_name): def logging_excepthook(type, value, tb): extra = {} @@ -406,15 +417,22 @@ def _setup_logging_from_conf(): log_root.addHandler(streamlog) if CONF.publish_errors: - log_root.addHandler(PublishErrorsHandler(logging.ERROR)) + handler = importutils.import_object( + "savanna.openstack.common.log_handler.PublishErrorsHandler", + logging.ERROR) + log_root.addHandler(handler) + datefmt = CONF.log_date_format for handler in log_root.handlers: - datefmt = CONF.log_date_format + # NOTE(alaski): CONF.log_format overrides everything currently. This + # should be deprecated in favor of context aware formatting. if CONF.log_format: handler.setFormatter(logging.Formatter(fmt=CONF.log_format, datefmt=datefmt)) + log_root.info('Deprecated: log_format is now deprecated and will ' + 'be removed in the next release') else: - handler.setFormatter(LegacyFormatter(datefmt=datefmt)) + handler.setFormatter(ContextFormatter(datefmt=datefmt)) if CONF.debug: log_root.setLevel(logging.DEBUG) @@ -440,6 +458,15 @@ def getLogger(name='unknown', version='unknown'): return _loggers[name] +def getLazyLogger(name='unknown', version='unknown'): + """ + create a pass-through logger that does not create the real logger + until it is really needed and delegates all calls to the real logger + once it is created + """ + return LazyAdapter(name, version) + + class WritableLogger(object): """A thin wrapper that responds to `write` and logs.""" @@ -451,7 +478,7 @@ class WritableLogger(object): self.logger.log(self.level, msg) -class LegacyFormatter(logging.Formatter): +class ContextFormatter(logging.Formatter): """A context.RequestContext aware formatter configured through flags. The flags used to set format strings are: logging_context_format_string diff --git a/savanna/openstack/common/loopingcall.py b/savanna/openstack/common/loopingcall.py new file mode 100644 index 00000000..30d592f7 --- /dev/null +++ b/savanna/openstack/common/loopingcall.py @@ -0,0 +1,147 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 Justin Santa Barbara +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys + +from eventlet import event +from eventlet import greenthread + +from savanna.openstack.common.gettextutils import _ +from savanna.openstack.common import log as logging +from savanna.openstack.common import timeutils + +LOG = logging.getLogger(__name__) + + +class LoopingCallDone(Exception): + """Exception to break out and stop a LoopingCall. + + The poll-function passed to LoopingCall can raise this exception to + break out of the loop normally. This is somewhat analogous to + StopIteration. + + An optional return-value can be included as the argument to the exception; + this return-value will be returned by LoopingCall.wait() + + """ + + def __init__(self, retvalue=True): + """:param retvalue: Value that LoopingCall.wait() should return.""" + self.retvalue = retvalue + + +class LoopingCallBase(object): + def __init__(self, f=None, *args, **kw): + self.args = args + self.kw = kw + self.f = f + self._running = False + self.done = None + + def stop(self): + self._running = False + + def wait(self): + return self.done.wait() + + +class FixedIntervalLoopingCall(LoopingCallBase): + """A fixed interval looping call.""" + + def start(self, interval, initial_delay=None): + self._running = True + done = event.Event() + + def _inner(): + if initial_delay: + greenthread.sleep(initial_delay) + + try: + while self._running: + start = timeutils.utcnow() + self.f(*self.args, **self.kw) + end = timeutils.utcnow() + if not self._running: + break + delay = interval - timeutils.delta_seconds(start, end) + if delay <= 0: + LOG.warn(_('task run outlasted interval by %s sec') % + -delay) + greenthread.sleep(delay if delay > 0 else 0) + except LoopingCallDone as e: + self.stop() + done.send(e.retvalue) + except Exception: + LOG.exception(_('in fixed duration looping call')) + done.send_exception(*sys.exc_info()) + return + else: + done.send(True) + + self.done = done + + greenthread.spawn_n(_inner) + return self.done + + +# TODO(mikal): this class name is deprecated in Havana and should be removed +# in the I release +LoopingCall = FixedIntervalLoopingCall + + +class DynamicLoopingCall(LoopingCallBase): + """A looping call which sleeps until the next known event. + + The function called should return how long to sleep for before being + called again. + """ + + def start(self, initial_delay=None, periodic_interval_max=None): + self._running = True + done = event.Event() + + def _inner(): + if initial_delay: + greenthread.sleep(initial_delay) + + try: + while self._running: + idle = self.f(*self.args, **self.kw) + if not self._running: + break + + if periodic_interval_max is not None: + idle = min(idle, periodic_interval_max) + LOG.debug(_('Dynamic looping call sleeping for %.02f ' + 'seconds'), idle) + greenthread.sleep(idle) + except LoopingCallDone as e: + self.stop() + done.send(e.retvalue) + except Exception: + LOG.exception(_('in dynamic looping call')) + done.send_exception(*sys.exc_info()) + return + else: + done.send(True) + + self.done = done + + greenthread.spawn(_inner) + return self.done diff --git a/savanna/openstack/common/threadgroup.py b/savanna/openstack/common/threadgroup.py new file mode 100644 index 00000000..7d1c6b4a --- /dev/null +++ b/savanna/openstack/common/threadgroup.py @@ -0,0 +1,121 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from eventlet import greenlet +from eventlet import greenpool +from eventlet import greenthread + +from savanna.openstack.common import log as logging +from savanna.openstack.common import loopingcall + + +LOG = logging.getLogger(__name__) + + +def _thread_done(gt, *args, **kwargs): + """ Callback function to be passed to GreenThread.link() when we spawn() + Calls the :class:`ThreadGroup` to notify if. + + """ + kwargs['group'].thread_done(kwargs['thread']) + + +class Thread(object): + """ Wrapper around a greenthread, that holds a reference to the + :class:`ThreadGroup`. The Thread will notify the :class:`ThreadGroup` when + it has done so it can be removed from the threads list. + """ + def __init__(self, thread, group): + self.thread = thread + self.thread.link(_thread_done, group=group, thread=self) + + def stop(self): + self.thread.kill() + + def wait(self): + return self.thread.wait() + + +class ThreadGroup(object): + """ The point of the ThreadGroup classis to: + + * keep track of timers and greenthreads (making it easier to stop them + when need be). + * provide an easy API to add timers. + """ + def __init__(self, thread_pool_size=10): + self.pool = greenpool.GreenPool(thread_pool_size) + self.threads = [] + self.timers = [] + + def add_dynamic_timer(self, callback, initial_delay=None, + periodic_interval_max=None, *args, **kwargs): + timer = loopingcall.DynamicLoopingCall(callback, *args, **kwargs) + timer.start(initial_delay=initial_delay, + periodic_interval_max=periodic_interval_max) + self.timers.append(timer) + + def add_timer(self, interval, callback, initial_delay=None, + *args, **kwargs): + pulse = loopingcall.FixedIntervalLoopingCall(callback, *args, **kwargs) + pulse.start(interval=interval, + initial_delay=initial_delay) + self.timers.append(pulse) + + def add_thread(self, callback, *args, **kwargs): + gt = self.pool.spawn(callback, *args, **kwargs) + th = Thread(gt, self) + self.threads.append(th) + + def thread_done(self, thread): + self.threads.remove(thread) + + def stop(self): + current = greenthread.getcurrent() + for x in self.threads: + if x is current: + # don't kill the current thread. + continue + try: + x.stop() + except Exception as ex: + LOG.exception(ex) + + for x in self.timers: + try: + x.stop() + except Exception as ex: + LOG.exception(ex) + self.timers = [] + + def wait(self): + for x in self.timers: + try: + x.wait() + except greenlet.GreenletExit: + pass + except Exception as ex: + LOG.exception(ex) + current = greenthread.getcurrent() + for x in self.threads: + if x is current: + continue + try: + x.wait() + except greenlet.GreenletExit: + pass + except Exception as ex: + LOG.exception(ex) diff --git a/savanna/plugins/__init__.py b/savanna/plugins/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/savanna/plugins/base.py b/savanna/plugins/base.py new file mode 100644 index 00000000..4e776243 --- /dev/null +++ b/savanna/plugins/base.py @@ -0,0 +1,167 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABCMeta +from abc import abstractmethod +import inspect +from oslo.config import cfg +from savanna.config import parse_configs +from savanna.openstack.common import importutils +from savanna.openstack.common import log as logging +from savanna.utils.resources import BaseResource + +LOG = logging.getLogger(__name__) + +opts = [ + cfg.ListOpt('plugins', + default=[], + help='List of plugins to be loaded'), +] + +CONF = cfg.CONF +CONF.register_opts(opts) + + +class PluginInterface(BaseResource): + __metaclass__ = ABCMeta + + name = 'plugin_interface' + + def get_plugin_opts(self): + """Plugin can expose some options that should be specified in conf file + + For example: + + def get_plugin_opts(self): + return [ + cfg.StrOpt('mandatory-conf', required=True), + cfg.StrOpt('optional_conf', default="42"), + ] + """ + return [] + + def setup(self, conf): + """Plugin initialization + + :param conf: plugin-specific configurations + """ + pass + + @abstractmethod + def get_title(self): + """Plugin title + + For example: + + "Vanilla Provisioning" + """ + pass + + def get_description(self): + """Optional description of the plugin + + This information is targeted to be displayed in UI. + """ + pass + + def to_dict(self): + return { + 'name': self.name, + 'title': self.get_title(), + 'description': self.get_description(), + } + + +class PluginManager(object): + def __init__(self): + self.plugins = {} + self._load_all_plugins() + + def _load_all_plugins(self): + LOG.debug("List of requested plugins: %s" % CONF.plugins) + + if len(CONF.plugins) > len(set(CONF.plugins)): + raise RuntimeError("plugins config contains non-unique entries") + + # register required 'plugin_factory' property for each plugin + for plugin in CONF.plugins: + opts = [ + cfg.StrOpt('plugin_class', required=True), + ] + CONF.register_opts(opts, group='plugin:%s' % plugin) + + parse_configs() + + # register plugin-specific configs + for plugin_name in CONF.plugins: + self.plugins[plugin_name] = self._get_plugin_instance(plugin_name) + + parse_configs() + + titles = [] + for plugin_name in CONF.plugins: + plugin = self.plugins[plugin_name] + plugin.setup(CONF['plugin:%s' % plugin_name]) + + title = plugin.get_title() + if title in titles: + # replace with specific error + raise RuntimeError( + "Title of plugin '%s' isn't unique" % plugin_name) + titles.append(title) + + LOG.info("Plugin '%s' defined and loaded" % plugin_name) + + def _get_plugin_instance(self, plugin_name): + plugin_path = CONF['plugin:%s' % plugin_name].plugin_class + module_path, klass = [s.strip() for s in plugin_path.split(':')] + if not module_path or not klass: + # todo replace with specific error + raise RuntimeError("Incorrect plugin_class: '%s'" % + plugin_path) + module = importutils.try_import(module_path) + if not hasattr(module, klass): + # todo replace with specific error + raise RuntimeError("Class not found: '%s'" % plugin_path) + + plugin_class = getattr(module, klass) + if not inspect.isclass(plugin_class): + # todo replace with specific error + raise RuntimeError("'%s' isn't a class" % plugin_path) + + plugin = plugin_class() + plugin.name = plugin_name + + CONF.register_opts(plugin.get_plugin_opts(), + group='plugin:%s' % plugin_name) + + return plugin + + def get_plugins(self, base): + return [ + self.plugins[plugin] for plugin in self.plugins + if not base or issubclass(self.plugins[plugin].__class__, base) + ] + + def get_plugin(self, plugin_name): + return self.plugins.get(plugin_name) + + +PLUGINS = None + + +def setup_plugins(): + global PLUGINS + PLUGINS = PluginManager() diff --git a/savanna/plugins/provisioning.py b/savanna/plugins/provisioning.py new file mode 100644 index 00000000..2ea97ebe --- /dev/null +++ b/savanna/plugins/provisioning.py @@ -0,0 +1,118 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import abstractmethod +import functools +from savanna.plugins.base import PluginInterface +import savanna.utils.openstack.nova as nova +from savanna.utils.resources import BaseResource + + +class ProvisioningPluginContext(object): + def __init__(self, headers): + self.headers = headers + self.nova = self._autoheaders(nova.novaclient) + + def _autoheaders(self, func): + return functools.partial(func, headers=self.headers) + + +class ProvisioningPluginBase(PluginInterface): + @abstractmethod + def get_versions(self): + pass + + @abstractmethod + def get_configs(self, ctx, hadoop_version): + pass + + @abstractmethod + def get_node_processes(self, ctx, hadoop_version): + pass + + def validate(self, ctx, cluster): + pass + + def update_infra(self, ctx, cluster): + pass + + @abstractmethod + def configure_cluster(self, ctx, cluster): + pass + + @abstractmethod + def start_cluster(self, ctx, cluster): + pass + + def convert(self, ctx, cluster, input_file): + pass + + def on_terminate_cluster(self, ctx, cluster): + pass + + def to_dict(self): + res = super(ProvisioningPluginBase, self).to_dict() + res['versions'] = self.get_versions() + return res + + +class Config(BaseResource): + """Describes a single config parameter. + + For example: + + "some_conf", "jot_tracker", is_optional=True + """ + + def __init__(self, name, applicable_target, config_type="str", + config_values=None, default_value=None, is_optional=False, + description=None): + self.name = name + self.applicable_target = applicable_target + self.config_type = config_type + self.config_values = config_values + self.default_value = default_value + self.is_optional = is_optional + self.description = description + + def to_dict(self): + res = super(Config, self).to_dict() + # todo all custom fields from res + return res + + def __repr__(self): + return '' % (self.name, self.applicable_target) + + +class UserInput(object): + """Value provided by the Savanna user for a specific config entry.""" + + def __init__(self, config, value): + self.config = config + self.value = value + + def __repr__(self): + return '' % (self.config.name, self.value) + + +class ValidationError(object): + """Describes what is wrong with one of the values provided by user.""" + + def __init__(self, config, message): + self.config = config + self.message = message + + def __repr__(self): + return "" % self.config.name diff --git a/savanna/plugins/vanilla/__init__.py b/savanna/plugins/vanilla/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/savanna/plugins/vanilla/plugin.py b/savanna/plugins/vanilla/plugin.py new file mode 100644 index 00000000..1d99ab34 --- /dev/null +++ b/savanna/plugins/vanilla/plugin.py @@ -0,0 +1,59 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from savanna.plugins.provisioning import Config +from savanna.plugins.provisioning import ProvisioningPluginBase + + +class VanillaProvider(ProvisioningPluginBase): + def get_plugin_opts(self): + return [] + + def setup(self, conf): + self.conf = conf + + def get_title(self): + return "Vanilla Apache Hadoop" + + def get_description(self): + return ( + "This plugin provides an ability to launch vanilla Apache Hadoop " + "cluster without any management consoles.") + + def get_versions(self): + return ['Hadoop 1.1.1'] + + def get_configs(self, ctx, hadoop_version): + return [ + Config('heap_size', 'tasktracker', default_value='1024M') + ] + + def get_node_processes(self, ctx, hadoop_version): + return [ + 'jobtracker', 'tasktracker', + 'namenode', 'datanode', + ] + + def validate(self, ctx, cluster): + pass + + def update_infra(self, ctx, cluster): + pass + + def configure_cluster(self, ctx, cluster): + pass + + def start_cluster(self, ctx, cluster): + pass diff --git a/savanna/resources/core-default.xml b/savanna/resources/core-default.xml deleted file mode 100644 index 7e9e14d4..00000000 --- a/savanna/resources/core-default.xml +++ /dev/null @@ -1,580 +0,0 @@ - - - - - - - - - - - - - hadoop.tmp.dir - /tmp/hadoop-${user.name} - A base for other temporary directories. - - - - hadoop.native.lib - true - Should native hadoop libraries, if present, be used. - - - - hadoop.http.filter.initializers - - A comma separated list of class names. Each class in the list - must extend org.apache.hadoop.http.FilterInitializer. The corresponding - Filter will be initialized. Then, the Filter will be applied to all user - facing jsp and servlet web pages. The ordering of the list defines the - ordering of the filters. - - - - hadoop.security.group.mapping - org.apache.hadoop.security.ShellBasedUnixGroupsMapping - Class for user to group mapping (get groups for a given user) - - - - - hadoop.security.authorization - false - Is service-level authorization enabled? - - - - hadoop.security.authentication - simple - Possible values are simple (no authentication), and kerberos - - - - - hadoop.security.token.service.use_ip - true - Controls whether tokens always use IP addresses. DNS changes - will not be detected if this option is enabled. Existing client connections - that break will always reconnect to the IP of the original host. New clients - will connect to the host's new IP but fail to locate a token. Disabling - this option will allow existing and new clients to detect an IP change and - continue to locate the new host's token. - - - - - hadoop.security.use-weak-http-crypto - false - If enabled, use KSSL to authenticate HTTP connections to the - NameNode. Due to a bug in JDK6, using KSSL requires one to configure - Kerberos tickets to use encryption types that are known to be - cryptographically weak. If disabled, SPNEGO will be used for HTTP - authentication, which supports stronger encryption types. - - - - - - - - - hadoop.logfile.size - 10000000 - The max size of each log file - - - - hadoop.logfile.count - 10 - The max number of log files - - - - - io.file.buffer.size - 4096 - The size of buffer for use in sequence files. - The size of this buffer should probably be a multiple of hardware - page size (4096 on Intel x86), and it determines how much data is - buffered during read and write operations. - - - - io.bytes.per.checksum - 512 - The number of bytes per checksum. Must not be larger than - io.file.buffer.size. - - - - io.skip.checksum.errors - false - If true, when a checksum error is encountered while - reading a sequence file, entries are skipped, instead of throwing an - exception. - - - - io.compression.codecs - org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.SnappyCodec - A list of the compression codec classes that can be used - for compression/decompression. - - - - io.serializations - org.apache.hadoop.io.serializer.WritableSerialization - A list of serialization classes that can be used for - obtaining serializers and deserializers. - - - - - - fs.default.name - file:/// - The name of the default file system. A URI whose - scheme and authority determine the FileSystem implementation. The - uri's scheme determines the config property (fs.SCHEME.impl) naming - the FileSystem implementation class. The uri's authority is used to - determine the host, port, etc. for a filesystem. - - - - fs.trash.interval - 0 - Number of minutes between trash checkpoints. - If zero, the trash feature is disabled. - - - - - fs.file.impl - org.apache.hadoop.fs.LocalFileSystem - The FileSystem for file: uris. - - - - fs.hdfs.impl - org.apache.hadoop.hdfs.DistributedFileSystem - The FileSystem for hdfs: uris. - - - - fs.s3.impl - org.apache.hadoop.fs.s3.S3FileSystem - The FileSystem for s3: uris. - - - - fs.s3n.impl - org.apache.hadoop.fs.s3native.NativeS3FileSystem - The FileSystem for s3n: (Native S3) uris. - - - - fs.kfs.impl - org.apache.hadoop.fs.kfs.KosmosFileSystem - The FileSystem for kfs: uris. - - - - fs.hftp.impl - org.apache.hadoop.hdfs.HftpFileSystem - - - - fs.hsftp.impl - org.apache.hadoop.hdfs.HsftpFileSystem - - - - fs.webhdfs.impl - org.apache.hadoop.hdfs.web.WebHdfsFileSystem - - - - fs.ftp.impl - org.apache.hadoop.fs.ftp.FTPFileSystem - The FileSystem for ftp: uris. - - - - fs.ramfs.impl - org.apache.hadoop.fs.InMemoryFileSystem - The FileSystem for ramfs: uris. - - - - fs.har.impl - org.apache.hadoop.fs.HarFileSystem - The filesystem for Hadoop archives. - - - - fs.har.impl.disable.cache - true - Don't cache 'har' filesystem instances. - - - - fs.checkpoint.dir - ${hadoop.tmp.dir}/dfs/namesecondary - Determines where on the local filesystem the DFS secondary - name node should store the temporary images to merge. - If this is a comma-delimited list of directories then the image is - replicated in all of the directories for redundancy. - - - - - fs.checkpoint.edits.dir - ${fs.checkpoint.dir} - Determines where on the local filesystem the DFS secondary - name node should store the temporary edits to merge. - If this is a comma-delimited list of directoires then teh edits is - replicated in all of the directoires for redundancy. - Default value is same as fs.checkpoint.dir - - - - - fs.checkpoint.period - 3600 - The number of seconds between two periodic checkpoints. - - - - - fs.checkpoint.size - 67108864 - The size of the current edit log (in bytes) that triggers - a periodic checkpoint even if the fs.checkpoint.period hasn't expired. - - - - - - - fs.s3.block.size - 67108864 - Block size to use when writing files to S3. - - - - fs.s3.buffer.dir - ${hadoop.tmp.dir}/s3 - Determines where on the local filesystem the S3 filesystem - should store files before sending them to S3 - (or after retrieving them from S3). - - - - - fs.s3.maxRetries - 4 - The maximum number of retries for reading or writing files to S3, - before we signal failure to the application. - - - - - fs.s3.sleepTimeSeconds - 10 - The number of seconds to sleep between each S3 retry. - - - - - - local.cache.size - 10737418240 - The limit on the size of cache you want to keep, set by default - to 10GB. This will act as a soft limit on the cache directory for out of band data. - - - - - io.seqfile.compress.blocksize - 1000000 - The minimum block size for compression in block compressed - SequenceFiles. - - - - - io.seqfile.lazydecompress - true - Should values of block-compressed SequenceFiles be decompressed - only when necessary. - - - - - io.seqfile.sorter.recordlimit - 1000000 - The limit on number of records to be kept in memory in a spill - in SequenceFiles.Sorter - - - - - io.mapfile.bloom.size - 1048576 - The size of BloomFilter-s used in BloomMapFile. Each time this many - keys is appended the next BloomFilter will be created (inside a DynamicBloomFilter). - Larger values minimize the number of filters, which slightly increases the performance, - but may waste too much space if the total number of keys is usually much smaller - than this number. - - - - - io.mapfile.bloom.error.rate - 0.005 - The rate of false positives in BloomFilter-s used in BloomMapFile. - As this value decreases, the size of BloomFilter-s increases exponentially. This - value is the probability of encountering false positives (default is 0.5%). - - - - - hadoop.util.hash.type - murmur - The default implementation of Hash. Currently this can take one of the - two values: 'murmur' to select MurmurHash and 'jenkins' to select JenkinsHash. - - - - - - - - ipc.client.idlethreshold - 4000 - Defines the threshold number of connections after which - connections will be inspected for idleness. - - - - - ipc.client.kill.max - 10 - Defines the maximum number of clients to disconnect in one go. - - - - - ipc.client.connection.maxidletime - 10000 - The maximum time in msec after which a client will bring down the - connection to the server. - - - - - ipc.client.connect.max.retries - 10 - Indicates the number of retries a client will make to establish - a server connection. - - - - - ipc.server.listen.queue.size - 128 - Indicates the length of the listen queue for servers accepting - client connections. - - - - - ipc.server.tcpnodelay - false - Turn on/off Nagle's algorithm for the TCP socket connection on - the server. Setting to true disables the algorithm and may decrease latency - with a cost of more/smaller packets. - - - - - ipc.client.tcpnodelay - false - Turn on/off Nagle's algorithm for the TCP socket connection on - the client. Setting to true disables the algorithm and may decrease latency - with a cost of more/smaller packets. - - - - - - - - webinterface.private.actions - false - If set to true, the web interfaces of JT and NN may contain - actions, such as kill job, delete file, etc., that should - not be exposed to public. Enable this option if the interfaces - are only reachable by those who have the right authorization. - - - - - - - hadoop.rpc.socket.factory.class.default - org.apache.hadoop.net.StandardSocketFactory - Default SocketFactory to use. This parameter is expected to be - formatted as "package.FactoryClassName". - - - - - hadoop.rpc.socket.factory.class.ClientProtocol - - SocketFactory to use to connect to a DFS. If null or empty, use - hadoop.rpc.socket.class.default. This socket factory is also used by - DFSClient to create sockets to DataNodes. - - - - - - - hadoop.socks.server - - Address (host:port) of the SOCKS server to be used by the - SocksSocketFactory. - - - - - - - topology.node.switch.mapping.impl - org.apache.hadoop.net.ScriptBasedMapping - The default implementation of the DNSToSwitchMapping. It - invokes a script specified in topology.script.file.name to resolve - node names. If the value for topology.script.file.name is not set, the - default value of DEFAULT_RACK is returned for all node names. - - - - - topology.script.file.name - - The script name that should be invoked to resolve DNS names to - NetworkTopology names. Example: the script would take host.foo.bar as an - argument, and return /rack1 as the output. - - - - - topology.script.number.args - 100 - The max number of args that the script configured with - topology.script.file.name should be run with. Each arg is an - IP address. - - - - - hadoop.security.uid.cache.secs - 14400 - NativeIO maintains a cache from UID to UserName. This is - the timeout for an entry in that cache. - - - - - - hadoop.http.authentication.type - simple - - Defines authentication used for Oozie HTTP endpoint. - Supported values are: simple | kerberos | #AUTHENTICATION_HANDLER_CLASSNAME# - - - - - hadoop.http.authentication.token.validity - 36000 - - Indicates how long (in seconds) an authentication token is valid before it has - to be renewed. - - - - - hadoop.http.authentication.signature.secret.file - ${user.home}/hadoop-http-auth-signature-secret - - The signature secret for signing the authentication tokens. - If not set a random secret is generated at startup time. - The same secret should be used for JT/NN/DN/TT configurations. - - - - - hadoop.http.authentication.cookie.domain - - - The domain to use for the HTTP cookie that stores the authentication token. - In order to authentiation to work correctly across all Hadoop nodes web-consoles - the domain must be correctly set. - IMPORTANT: when using IP addresses, browsers ignore cookies with domain settings. - For this setting to work properly all nodes in the cluster must be configured - to generate URLs with hostname.domain names on it. - - - - - hadoop.http.authentication.simple.anonymous.allowed - true - - Indicates if anonymous requests are allowed when using 'simple' authentication. - - - - - hadoop.http.authentication.kerberos.principal - HTTP/localhost@LOCALHOST - - Indicates the Kerberos principal to be used for HTTP endpoint. - The principal MUST start with 'HTTP/' as per Kerberos HTTP SPNEGO specification. - - - - - hadoop.http.authentication.kerberos.keytab - ${user.home}/hadoop.keytab - - Location of the keytab file with the credentials for the principal. - Referring to the same keytab file Oozie uses for its Kerberos credentials for Hadoop. - - - - - hadoop.relaxed.worker.version.check - false - - By default datanodes refuse to connect to namenodes if their build - revision (svn revision) do not match, and tasktrackers refuse to - connect to jobtrackers if their build version (version, revision, - user, and source checksum) do not match. This option changes the - behavior of hadoop workers to only check for a version match (eg - "1.0.2") but ignore the other build fields (revision, user, and - source checksum). - - - - diff --git a/savanna/resources/hdfs-default.xml b/savanna/resources/hdfs-default.xml deleted file mode 100644 index b24c79c7..00000000 --- a/savanna/resources/hdfs-default.xml +++ /dev/null @@ -1,547 +0,0 @@ - - - - - - - - - - - dfs.namenode.logging.level - info - The logging level for dfs namenode. Other values are "dir"(trac -e namespace mutations), "block"(trace block under/over replications and block -creations/deletions), or "all". - - - - dfs.secondary.http.address - 0.0.0.0:50090 - - The secondary namenode http server address and port. - If the port is 0 then the server will start on a free port. - - - - - dfs.datanode.address - 0.0.0.0:50010 - - The datanode server address and port for data transfer. - If the port is 0 then the server will start on a free port. - - - - - dfs.datanode.http.address - 0.0.0.0:50075 - - The datanode http server address and port. - If the port is 0 then the server will start on a free port. - - - - - dfs.datanode.ipc.address - 0.0.0.0:50020 - - The datanode ipc server address and port. - If the port is 0 then the server will start on a free port. - - - - - dfs.datanode.handler.count - 3 - The number of server threads for the datanode. - - - - dfs.http.address - 0.0.0.0:50070 - - The address and the base port where the dfs namenode web ui will listen on. - If the port is 0 then the server will start on a free port. - - - - - dfs.https.enable - false - Decide if HTTPS(SSL) is supported on HDFS - - - - - dfs.https.need.client.auth - false - Whether SSL client certificate authentication is required - - - - - dfs.https.server.keystore.resource - ssl-server.xml - Resource file from which ssl server keystore - information will be extracted - - - - - dfs.https.client.keystore.resource - ssl-client.xml - Resource file from which ssl client keystore - information will be extracted - - - - - dfs.datanode.https.address - 0.0.0.0:50475 - - - - dfs.https.address - 0.0.0.0:50470 - - - - dfs.datanode.dns.interface - default - The name of the Network Interface from which a data node should - report its IP address. - - - - - dfs.datanode.dns.nameserver - default - The host name or IP address of the name server (DNS) - which a DataNode should use to determine the host name used by the - NameNode for communication and display purposes. - - - - - - - dfs.replication.considerLoad - true - Decide if chooseTarget considers the target's load or not - - - - dfs.default.chunk.view.size - 32768 - The number of bytes to view for a file on the browser. - - - - - dfs.datanode.du.reserved - 0 - Reserved space in bytes per volume. Always leave this much space free for non dfs use. - - - - - dfs.name.dir - ${hadoop.tmp.dir}/dfs/name - Determines where on the local filesystem the DFS name node - should store the name table(fsimage). If this is a comma-delimited list - of directories then the name table is replicated in all of the - directories, for redundancy. - - - - dfs.name.edits.dir - ${dfs.name.dir} - Determines where on the local filesystem the DFS name node - should store the transaction (edits) file. If this is a comma-delimited list - of directories then the transaction file is replicated in all of the - directories, for redundancy. Default value is same as dfs.name.dir - - - - dfs.web.ugi - webuser,webgroup - The user account used by the web interface. - Syntax: USERNAME,GROUP1,GROUP2, ... - - - - - dfs.permissions - true - - If "true", enable permission checking in HDFS. - If "false", permission checking is turned off, - but all other behavior is unchanged. - Switching from one parameter value to the other does not change the mode, - owner or group of files or directories. - - - - - dfs.permissions.supergroup - supergroup - The name of the group of super-users. - - - - dfs.block.access.token.enable - false - - If "true", access tokens are used as capabilities for accessing datanodes. - If "false", no access tokens are checked on accessing datanodes. - - - - - dfs.block.access.key.update.interval - 600 - - Interval in minutes at which namenode updates its access keys. - - - - - dfs.block.access.token.lifetime - 600 - The lifetime of access tokens in minutes. - - - - - dfs.data.dir - ${hadoop.tmp.dir}/dfs/data - Determines where on the local filesystem an DFS data node - should store its blocks. If this is a comma-delimited - list of directories, then data will be stored in all named - directories, typically on different devices. - Directories that do not exist are ignored. - - - - - dfs.datanode.data.dir.perm - 755 - Permissions for the directories on on the local filesystem where - the DFS data node store its blocks. The permissions can either be octal or - symbolic. - - - - dfs.replication - 3 - Default block replication. - The actual number of replications can be specified when the file is created. - The default is used if replication is not specified in create time. - - - - - dfs.replication.max - 512 - Maximal block replication. - - - - - dfs.replication.min - 1 - Minimal block replication. - - - - - dfs.block.size - 67108864 - The default block size for new files. - - - - dfs.df.interval - 60000 - Disk usage statistics refresh interval in msec. - - - - dfs.client.block.write.retries - 3 - The number of retries for writing blocks to the data nodes, - before we signal failure to the application. - - - - - dfs.blockreport.intervalMsec - 3600000 - Determines block reporting interval in milliseconds. - - - - dfs.blockreport.initialDelay 0 - Delay for first block report in seconds. - - - - dfs.heartbeat.interval - 3 - Determines datanode heartbeat interval in seconds. - - - - dfs.namenode.handler.count - 10 - The number of server threads for the namenode. - - - - dfs.safemode.threshold.pct - 0.999f - - Specifies the percentage of blocks that should satisfy - the minimal replication requirement defined by dfs.replication.min. - Values less than or equal to 0 mean not to wait for any particular - percentage of blocks before exiting safemode. - Values greater than 1 will make safe mode permanent. - - - - - dfs.namenode.safemode.min.datanodes - 0 - - Specifies the number of datanodes that must be considered alive - before the name node exits safemode. - Values less than or equal to 0 mean not to take the number of live - datanodes into account when deciding whether to remain in safe mode - during startup. - Values greater than the number of datanodes in the cluster - will make safe mode permanent. - - - - - dfs.safemode.extension - 30000 - - Determines extension of safe mode in milliseconds - after the threshold level is reached. - - - - - dfs.balance.bandwidthPerSec - 1048576 - - Specifies the maximum amount of bandwidth that each datanode - can utilize for the balancing purpose in term of - the number of bytes per second. - - - - - dfs.hosts - - Names a file that contains a list of hosts that are - permitted to connect to the namenode. The full pathname of the file - must be specified. If the value is empty, all hosts are - permitted. - - - - dfs.hosts.exclude - - Names a file that contains a list of hosts that are - not permitted to connect to the namenode. The full pathname of the - file must be specified. If the value is empty, no hosts are - excluded. - - - - dfs.max.objects - 0 - The maximum number of files, directories and blocks - dfs supports. A value of zero indicates no limit to the number - of objects that dfs supports. - - - - - dfs.namenode.decommission.interval - 30 - Namenode periodicity in seconds to check if decommission is - complete. - - - - dfs.namenode.decommission.nodes.per.interval - 5 - The number of nodes namenode checks if decommission is complete - in each dfs.namenode.decommission.interval. - - - - dfs.replication.interval - 3 - The periodicity in seconds with which the namenode computes - repliaction work for datanodes. - - - - dfs.access.time.precision - 3600000 - The access time for HDFS file is precise upto this value. - The default value is 1 hour. Setting a value of 0 disables - access times for HDFS. - - - - - dfs.support.append - - This option is no longer supported. HBase no longer requires that - this option be enabled as sync is now enabled by default. See - HADOOP-8230 for additional information. - - - - - dfs.namenode.delegation.key.update-interval - 86400000 - The update interval for master key for delegation tokens - in the namenode in milliseconds. - - - - - dfs.namenode.delegation.token.max-lifetime - 604800000 - The maximum lifetime in milliseconds for which a delegation - token is valid. - - - - - dfs.namenode.delegation.token.renew-interval - 86400000 - The renewal interval for delegation token in milliseconds. - - - - - dfs.datanode.failed.volumes.tolerated - 0 - The number of volumes that are allowed to - fail before a datanode stops offering service. By default - any volume failure will cause a datanode to shutdown. - - - - - dfs.datanode.max.xcievers - 4096 - Specifies the maximum number of threads to use for transferring data - in and out of the DN. - - - - - dfs.client.use.datanode.hostname - false - Whether clients should use datanode hostnames when - connecting to datanodes. - - - - - dfs.datanode.use.datanode.hostname - false - Whether datanodes should use datanode hostnames when - connecting to other datanodes for data transfer. - - - - - dfs.client.local.interfaces - - A comma separated list of network interface names to use - for data transfer between the client and datanodes. When creating - a connection to read from or write to a datanode, the client - chooses one of the specified interfaces at random and binds its - socket to the IP of that interface. Individual names may be - specified as either an interface name (eg "eth0"), a subinterface - name (eg "eth0:0"), or an IP address (which may be specified using - CIDR notation to match a range of IPs). - - - - - dfs.namenode.kerberos.internal.spnego.principal - ${dfs.web.authentication.kerberos.principal} - - - - dfs.secondary.namenode.kerberos.internal.spnego.principal - ${dfs.web.authentication.kerberos.principal} - - - - dfs.namenode.invalidate.work.pct.per.iteration - 0.32f - - *Note*: Advanced property. Change with caution. - This determines the percentage amount of block - invalidations (deletes) to do over a single DN heartbeat - deletion command. The final deletion count is determined by applying this - percentage to the number of live nodes in the system. - The resultant number is the number of blocks from the deletion list - chosen for proper invalidation over a single heartbeat of a single DN. - Value should be a positive, non-zero percentage in float notation (X.Yf), - with 1.0f meaning 100%. - - - - - dfs.namenode.replication.work.multiplier.per.iteration - 2 - - *Note*: Advanced property. Change with caution. - This determines the total amount of block transfers to begin in - parallel at a DN, for replication, when such a command list is being - sent over a DN heartbeat by the NN. The actual number is obtained by - multiplying this multiplier with the total number of live nodes in the - cluster. The result number is the number of blocks to begin transfers - immediately for, per DN heartbeat. This number can be any positive, - non-zero integer. - - - - - dfs.namenode.check.stale.datanode - false - - Indicate whether or not to check "stale" datanodes whose - heartbeat messages have not been received by the namenode - for more than a specified time interval. If this configuration - parameter is set as true, the stale datanodes will be moved to - the end of the target node list for reading. The writing will - also try to avoid stale nodes. - - - - - dfs.namenode.stale.datanode.interval - 30000 - - Default time interval for marking a datanode as "stale", i.e., if - the namenode has not received heartbeat msg from a datanode for - more than this time interval, the datanode will be marked and treated - as "stale" by default. - - - - diff --git a/savanna/resources/mapred-default.xml b/savanna/resources/mapred-default.xml deleted file mode 100644 index c06e20fa..00000000 --- a/savanna/resources/mapred-default.xml +++ /dev/null @@ -1,1282 +0,0 @@ - - - - - - - - - - - hadoop.job.history.location - - If job tracker is static the history files are stored - in this single well known place. If No value is set here, by default, - it is in the local file system at ${hadoop.log.dir}/history. - - - - - hadoop.job.history.user.location - - User can specify a location to store the history files of - a particular job. If nothing is specified, the logs are stored in - output directory. The files are stored in "_logs/history/" in the directory. - User can stop logging by giving the value "none". - - - - - mapred.job.tracker.history.completed.location - - The completed job history files are stored at this single well - known location. If nothing is specified, the files are stored at - ${hadoop.job.history.location}/done. - - - - - - - io.sort.factor - 10 - The number of streams to merge at once while sorting - files. This determines the number of open file handles. - - - - io.sort.mb - 100 - The total amount of buffer memory to use while sorting - files, in megabytes. By default, gives each merge stream 1MB, which - should minimize seeks. - - - - io.sort.record.percent - 0.05 - The percentage of io.sort.mb dedicated to tracking record - boundaries. Let this value be r, io.sort.mb be x. The maximum number - of records collected before the collection thread must block is equal - to (r * x) / 4 - - - - io.sort.spill.percent - 0.80 - The soft limit in either the buffer or record collection - buffers. Once reached, a thread will begin to spill the contents to disk - in the background. Note that this does not imply any chunking of data to - the spill. A value less than 0.5 is not recommended. - - - - io.map.index.skip - 0 - Number of index entries to skip between each entry. - Zero by default. Setting this to values larger than zero can - facilitate opening large map files using less memory. - - - - mapred.job.tracker - local - The host and port that the MapReduce job tracker runs - at. If "local", then jobs are run in-process as a single map - and reduce task. - - - - - mapred.job.tracker.http.address - 0.0.0.0:50030 - - The job tracker http server address and port the server will listen on. - If the port is 0 then the server will start on a free port. - - - - - mapred.job.tracker.handler.count - 10 - - The number of server threads for the JobTracker. This should be roughly - 4% of the number of tasktracker nodes. - - - - - mapred.task.tracker.report.address - 127.0.0.1:0 - The interface and port that task tracker server listens on. - Since it is only connected to by the tasks, it uses the local interface. - EXPERT ONLY. Should only be changed if your host does not have the loopback - interface. - - - - mapred.local.dir - ${hadoop.tmp.dir}/mapred/local - The local directory where MapReduce stores intermediate - data files. May be a comma-separated list of - directories on different devices in order to spread disk i/o. - Directories that do not exist are ignored. - - - - - mapred.system.dir - ${hadoop.tmp.dir}/mapred/system - The directory where MapReduce stores control files. - - - - - mapreduce.jobtracker.staging.root.dir - ${hadoop.tmp.dir}/mapred/staging - The root of the staging area for users' job files - In practice, this should be the directory where users' home - directories are located (usually /user) - - - - - mapred.temp.dir - ${hadoop.tmp.dir}/mapred/temp - A shared directory for temporary files. - - - - - mapred.local.dir.minspacestart - 0 - If the space in mapred.local.dir drops under this, - do not ask for more tasks. - Value in bytes. - - - - - mapred.local.dir.minspacekill - 0 - If the space in mapred.local.dir drops under this, - do not ask more tasks until all the current ones have finished and - cleaned up. Also, to save the rest of the tasks we have running, - kill one of them, to clean up some space. Start with the reduce tasks, - then go with the ones that have finished the least. - Value in bytes. - - - - - mapred.tasktracker.expiry.interval - 600000 - Expert: The time-interval, in miliseconds, after which - a tasktracker is declared 'lost' if it doesn't send heartbeats. - - - - - - - mapred.tasktracker.resourcecalculatorplugin - - - Name of the class whose instance will be used to query resource information - on the tasktracker. - - The class must be an instance of - org.apache.hadoop.util.ResourceCalculatorPlugin. If the value is null, the - tasktracker attempts to use a class appropriate to the platform. - Currently, the only platform supported is Linux. - - - - - mapred.tasktracker.taskmemorymanager.monitoring-interval - 5000 - The interval, in milliseconds, for which the tasktracker waits - between two cycles of monitoring its tasks' memory usage. Used only if - tasks' memory management is enabled via mapred.tasktracker.tasks.maxmemory. - - - - - mapred.tasktracker.tasks.sleeptime-before-sigkill - 5000 - The time, in milliseconds, the tasktracker waits for sending a - SIGKILL to a process, after it has been sent a SIGTERM. - - - - mapred.map.tasks - 2 - The default number of map tasks per job. - Ignored when mapred.job.tracker is "local". - - - - - mapred.reduce.tasks - 1 - The default number of reduce tasks per job. Typically set to 99% - of the cluster's reduce capacity, so that if a node fails the reduces can - still be executed in a single wave. - Ignored when mapred.job.tracker is "local". - - - - - mapreduce.tasktracker.outofband.heartbeat - false - Expert: Set this to true to let the tasktracker send an - out-of-band heartbeat on task-completion for better latency. - - - - - mapreduce.tasktracker.outofband.heartbeat.damper - 1000000 - When out-of-band heartbeats are enabled, provides - damping to avoid overwhelming the JobTracker if too many out-of-band - heartbeats would occur. The damping is calculated such that the - heartbeat interval is divided by (T*D + 1) where T is the number - of completed tasks and D is the damper value. - - Setting this to a high value like the default provides no damping -- - as soon as any task finishes, a heartbeat will be sent. Setting this - parameter to 0 is equivalent to disabling the out-of-band heartbeat feature. - A value of 1 would indicate that, after one task has completed, the - time to wait before the next heartbeat would be 1/2 the usual time. - After two tasks have finished, it would be 1/3 the usual time, etc. - - - - - mapred.jobtracker.restart.recover - false - "true" to enable (job) recovery upon restart, - "false" to start afresh - - - - - mapred.jobtracker.job.history.block.size - 3145728 - The block size of the job history file. Since the job recovery - uses job history, its important to dump job history to disk as - soon as possible. Note that this is an expert level parameter. - The default value is set to 3 MB. - - - - - mapreduce.job.split.metainfo.maxsize - 10000000 - The maximum permissible size of the split metainfo file. - The JobTracker won't attempt to read split metainfo files bigger than - the configured value. - No limits if set to -1. - - - - - mapred.jobtracker.taskScheduler - org.apache.hadoop.mapred.JobQueueTaskScheduler - The class responsible for scheduling the tasks. - - - - mapred.jobtracker.taskScheduler.maxRunningTasksPerJob - - The maximum number of running tasks for a job before - it gets preempted. No limits if undefined. - - - - - mapred.map.max.attempts - 4 - Expert: The maximum number of attempts per map task. - In other words, framework will try to execute a map task these many number - of times before giving up on it. - - - - - mapred.reduce.max.attempts - 4 - Expert: The maximum number of attempts per reduce task. - In other words, framework will try to execute a reduce task these many number - of times before giving up on it. - - - - - mapred.reduce.parallel.copies - 5 - The default number of parallel transfers run by reduce - during the copy(shuffle) phase. - - - - - mapreduce.reduce.shuffle.maxfetchfailures - 10 - The maximum number of times a reducer tries to - fetch a map output before it reports it. - - - - mapreduce.reduce.shuffle.connect.timeout - 180000 - Expert: The maximum amount of time (in milli seconds) a reduce - task spends in trying to connect to a tasktracker for getting map output. - - - - - mapreduce.reduce.shuffle.read.timeout - 180000 - Expert: The maximum amount of time (in milli seconds) a reduce - task waits for map output data to be available for reading after obtaining - connection. - - - - - mapred.task.timeout - 600000 - The number of milliseconds before a task will be - terminated if it neither reads an input, writes an output, nor - updates its status string. - - - - - mapred.tasktracker.map.tasks.maximum - 2 - The maximum number of map tasks that will be run - simultaneously by a task tracker. - - - - - mapred.tasktracker.reduce.tasks.maximum - 2 - The maximum number of reduce tasks that will be run - simultaneously by a task tracker. - - - - - mapred.jobtracker.completeuserjobs.maximum - 100 - The maximum number of complete jobs per user to keep around - before delegating them to the job history. - - - - mapreduce.reduce.input.limit - -1 - The limit on the input size of the reduce. If the estimated - input size of the reduce is greater than this value, job is failed. A - value of -1 means that there is no limit set. - - - - mapred.job.tracker.retiredjobs.cache.size - 1000 - The number of retired job status to keep in the cache. - - - - - mapred.job.tracker.jobhistory.lru.cache.size - 5 - The number of job history files loaded in memory. The jobs are - loaded when they are first accessed. The cache is cleared based on LRU. - - - - - - - mapred.child.java.opts - -Xmx200m - Java opts for the task tracker child processes. - The following symbol, if present, will be interpolated: @taskid@ is replaced - by current TaskID. Any other occurrences of '@' will go unchanged. - For example, to enable verbose gc logging to a file named for the taskid in - /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of: - -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc - - The configuration variable mapred.child.ulimit can be used to control the - maximum virtual memory of the child processes. - - - - - mapred.child.env - - User added environment variables for the task tracker child - processes. Example : - 1) A=foo This will set the env variable A to foo - 2) B=$B:c This is inherit tasktracker's B env variable. - - - - - mapred.child.ulimit - - The maximum virtual memory, in KB, of a process launched by the - Map-Reduce framework. This can be used to control both the Mapper/Reducer - tasks and applications using Hadoop Pipes, Hadoop Streaming etc. - By default it is left unspecified to let cluster admins control it via - limits.conf and other such relevant mechanisms. - - Note: mapred.child.ulimit must be greater than or equal to the -Xmx passed to - JavaVM, else the VM might not start. - - - - - mapred.cluster.map.memory.mb - -1 - The size, in terms of virtual memory, of a single map slot - in the Map-Reduce framework, used by the scheduler. - A job can ask for multiple slots for a single map task via - mapred.job.map.memory.mb, upto the limit specified by - mapred.cluster.max.map.memory.mb, if the scheduler supports the feature. - The value of -1 indicates that this feature is turned off. - - - - - mapred.cluster.reduce.memory.mb - -1 - The size, in terms of virtual memory, of a single reduce slot - in the Map-Reduce framework, used by the scheduler. - A job can ask for multiple slots for a single reduce task via - mapred.job.reduce.memory.mb, upto the limit specified by - mapred.cluster.max.reduce.memory.mb, if the scheduler supports the feature. - The value of -1 indicates that this feature is turned off. - - - - - mapred.cluster.max.map.memory.mb - -1 - The maximum size, in terms of virtual memory, of a single map - task launched by the Map-Reduce framework, used by the scheduler. - A job can ask for multiple slots for a single map task via - mapred.job.map.memory.mb, upto the limit specified by - mapred.cluster.max.map.memory.mb, if the scheduler supports the feature. - The value of -1 indicates that this feature is turned off. - - - - - mapred.cluster.max.reduce.memory.mb - -1 - The maximum size, in terms of virtual memory, of a single reduce - task launched by the Map-Reduce framework, used by the scheduler. - A job can ask for multiple slots for a single reduce task via - mapred.job.reduce.memory.mb, upto the limit specified by - mapred.cluster.max.reduce.memory.mb, if the scheduler supports the feature. - The value of -1 indicates that this feature is turned off. - - - - - mapred.job.map.memory.mb - -1 - The size, in terms of virtual memory, of a single map task - for the job. - A job can ask for multiple slots for a single map task, rounded up to the - next multiple of mapred.cluster.map.memory.mb and upto the limit - specified by mapred.cluster.max.map.memory.mb, if the scheduler supports - the feature. - The value of -1 indicates that this feature is turned off iff - mapred.cluster.map.memory.mb is also turned off (-1). - - - - - mapred.job.reduce.memory.mb - -1 - The size, in terms of virtual memory, of a single reduce task - for the job. - A job can ask for multiple slots for a single map task, rounded up to the - next multiple of mapred.cluster.reduce.memory.mb and upto the limit - specified by mapred.cluster.max.reduce.memory.mb, if the scheduler supports - the feature. - The value of -1 indicates that this feature is turned off iff - mapred.cluster.reduce.memory.mb is also turned off (-1). - - - - - mapred.child.tmp - ./tmp - To set the value of tmp directory for map and reduce tasks. - If the value is an absolute path, it is directly assigned. Otherwise, it is - prepended with task's working directory. The java tasks are executed with - option -Djava.io.tmpdir='the absolute path of the tmp dir'. Pipes and - streaming are set with environment variable, - TMPDIR='the absolute path of the tmp dir' - - - - - mapred.inmem.merge.threshold - 1000 - The threshold, in terms of the number of files - for the in-memory merge process. When we accumulate threshold number of files - we initiate the in-memory merge and spill to disk. A value of 0 or less than - 0 indicates we want to DON'T have any threshold and instead depend only on - the ramfs's memory consumption to trigger the merge. - - - - - mapred.job.shuffle.merge.percent - 0.66 - The usage threshold at which an in-memory merge will be - initiated, expressed as a percentage of the total memory allocated to - storing in-memory map outputs, as defined by - mapred.job.shuffle.input.buffer.percent. - - - - - mapred.job.shuffle.input.buffer.percent - 0.70 - The percentage of memory to be allocated from the maximum heap - size to storing map outputs during the shuffle. - - - - - mapred.job.reduce.input.buffer.percent - 0.0 - The percentage of memory- relative to the maximum heap size- to - retain map outputs during the reduce. When the shuffle is concluded, any - remaining map outputs in memory must consume less than this threshold before - the reduce can begin. - - - - - mapred.map.tasks.speculative.execution - true - If true, then multiple instances of some map tasks - may be executed in parallel. - - - - mapred.reduce.tasks.speculative.execution - true - If true, then multiple instances of some reduce tasks - may be executed in parallel. - - - - mapred.job.reuse.jvm.num.tasks - 1 - How many tasks to run per jvm. If set to -1, there is - no limit. - - - - - mapred.min.split.size - 0 - The minimum size chunk that map input should be split - into. Note that some file formats may have minimum split sizes that - take priority over this setting. - - - - mapred.jobtracker.maxtasks.per.job - -1 - The maximum number of tasks for a single job. - A value of -1 indicates that there is no maximum. - - - - mapred.submit.replication - 10 - The replication level for submitted job files. This - should be around the square root of the number of nodes. - - - - - - mapred.tasktracker.dns.interface - default - The name of the Network Interface from which a task - tracker should report its IP address. - - - - - mapred.tasktracker.dns.nameserver - default - The host name or IP address of the name server (DNS) - which a TaskTracker should use to determine the host name used by - the JobTracker for communication and display purposes. - - - - - tasktracker.http.threads - 40 - The number of worker threads that for the http server. This is - used for map output fetching - - - - - mapred.task.tracker.http.address - 0.0.0.0:50060 - - The task tracker http server address and port. - If the port is 0 then the server will start on a free port. - - - - - keep.failed.task.files - false - Should the files for failed tasks be kept. This should only be - used on jobs that are failing, because the storage is never - reclaimed. It also prevents the map outputs from being erased - from the reduce directory as they are consumed. - - - - - - - mapred.output.compress - false - Should the job outputs be compressed? - - - - - mapred.output.compression.type - RECORD - If the job outputs are to compressed as SequenceFiles, how should - they be compressed? Should be one of NONE, RECORD or BLOCK. - - - - - mapred.output.compression.codec - org.apache.hadoop.io.compress.DefaultCodec - If the job outputs are compressed, how should they be compressed? - - - - - mapred.compress.map.output - false - Should the outputs of the maps be compressed before being - sent across the network. Uses SequenceFile compression. - - - - - mapred.map.output.compression.codec - org.apache.hadoop.io.compress.DefaultCodec - If the map outputs are compressed, how should they be - compressed? - - - - - map.sort.class - org.apache.hadoop.util.QuickSort - The default sort class for sorting keys. - - - - - mapred.userlog.limit.kb - 0 - The maximum size of user-logs of each task in KB. 0 disables the cap. - - - - - mapred.userlog.retain.hours - 24 - The maximum time, in hours, for which the user-logs are to be - retained after the job completion. - - - - - mapred.user.jobconf.limit - 5242880 - The maximum allowed size of the user jobconf. The - default is set to 5 MB - - - - mapred.hosts - - Names a file that contains the list of nodes that may - connect to the jobtracker. If the value is empty, all hosts are - permitted. - - - - mapred.hosts.exclude - - Names a file that contains the list of hosts that - should be excluded by the jobtracker. If the value is empty, no - hosts are excluded. - - - - mapred.heartbeats.in.second - 100 - Expert: Approximate number of heart-beats that could arrive - at JobTracker in a second. Assuming each RPC can be processed - in 10msec, the default value is made 100 RPCs in a second. - - - - - mapred.max.tracker.blacklists - 4 - The number of blacklists for a tasktracker by various jobs - after which the tasktracker will be marked as potentially - faulty and is a candidate for graylisting across all jobs. - (Unlike blacklisting, this is advisory; the tracker remains - active. However, it is reported as graylisted in the web UI, - with the expectation that chronically graylisted trackers - will be manually decommissioned.) This value is tied to - mapred.jobtracker.blacklist.fault-timeout-window; faults - older than the window width are forgiven, so the tracker - will recover from transient problems. It will also become - healthy after a restart. - - - - - mapred.jobtracker.blacklist.fault-timeout-window - 180 - The timeout (in minutes) after which per-job tasktracker - faults are forgiven. The window is logically a circular - buffer of time-interval buckets whose width is defined by - mapred.jobtracker.blacklist.fault-bucket-width; when the - "now" pointer moves across a bucket boundary, the previous - contents (faults) of the new bucket are cleared. In other - words, the timeout's granularity is determined by the bucket - width. - - - - - mapred.jobtracker.blacklist.fault-bucket-width - 15 - The width (in minutes) of each bucket in the tasktracker - fault timeout window. Each bucket is reused in a circular - manner after a full timeout-window interval (defined by - mapred.jobtracker.blacklist.fault-timeout-window). - - - - - mapred.max.tracker.failures - 4 - The number of task-failures on a tasktracker of a given job - after which new tasks of that job aren't assigned to it. - - - - - jobclient.output.filter - FAILED - The filter for controlling the output of the task's userlogs sent - to the console of the JobClient. - The permissible options are: NONE, KILLED, FAILED, SUCCEEDED and - ALL. - - - - - mapred.job.tracker.persist.jobstatus.active - false - Indicates if persistency of job status information is - active or not. - - - - - mapred.job.tracker.persist.jobstatus.hours - 0 - The number of hours job status information is persisted in DFS. - The job status information will be available after it drops of the memory - queue and between jobtracker restarts. With a zero value the job status - information is not persisted at all in DFS. - - - - - mapred.job.tracker.persist.jobstatus.dir - /jobtracker/jobsInfo - The directory where the job status information is persisted - in a file system to be available after it drops of the memory queue and - between jobtracker restarts. - - - - - mapreduce.job.complete.cancel.delegation.tokens - true - if false - do not unregister/cancel delegation tokens - from renewal, because same tokens may be used by spawned jobs - - - - - mapred.task.profile - false - To set whether the system should collect profiler - information for some of the tasks in this job? The information is stored - in the user log directory. The value is "true" if task profiling - is enabled. - - - - mapred.task.profile.maps - 0-2 - To set the ranges of map tasks to profile. - mapred.task.profile has to be set to true for the value to be accounted. - - - - - mapred.task.profile.reduces - 0-2 - To set the ranges of reduce tasks to profile. - mapred.task.profile has to be set to true for the value to be accounted. - - - - - mapred.line.input.format.linespermap - 1 - Number of lines per split in NLineInputFormat. - - - - - mapred.skip.attempts.to.start.skipping - 2 - The number of Task attempts AFTER which skip mode - will be kicked off. When skip mode is kicked off, the - tasks reports the range of records which it will process - next, to the TaskTracker. So that on failures, TT knows which - ones are possibly the bad records. On further executions, - those are skipped. - - - - - mapred.skip.map.auto.incr.proc.count - true - The flag which if set to true, - SkipBadRecords.COUNTER_MAP_PROCESSED_RECORDS is incremented - by MapRunner after invoking the map function. This value must be set to - false for applications which process the records asynchronously - or buffer the input records. For example streaming. - In such cases applications should increment this counter on their own. - - - - - mapred.skip.reduce.auto.incr.proc.count - true - The flag which if set to true, - SkipBadRecords.COUNTER_REDUCE_PROCESSED_GROUPS is incremented - by framework after invoking the reduce function. This value must be set to - false for applications which process the records asynchronously - or buffer the input records. For example streaming. - In such cases applications should increment this counter on their own. - - - - - mapred.skip.out.dir - - If no value is specified here, the skipped records are - written to the output directory at _logs/skip. - User can stop writing skipped records by giving the value "none". - - - - - mapred.skip.map.max.skip.records - 0 - The number of acceptable skip records surrounding the bad - record PER bad record in mapper. The number includes the bad record as well. - To turn the feature of detection/skipping of bad records off, set the - value to 0. - The framework tries to narrow down the skipped range by retrying - until this threshold is met OR all attempts get exhausted for this task. - Set the value to Long.MAX_VALUE to indicate that framework need not try to - narrow down. Whatever records(depends on application) get skipped are - acceptable. - - - - - mapred.skip.reduce.max.skip.groups - 0 - The number of acceptable skip groups surrounding the bad - group PER bad group in reducer. The number includes the bad group as well. - To turn the feature of detection/skipping of bad groups off, set the - value to 0. - The framework tries to narrow down the skipped range by retrying - until this threshold is met OR all attempts get exhausted for this task. - Set the value to Long.MAX_VALUE to indicate that framework need not try to - narrow down. Whatever groups(depends on application) get skipped are - acceptable. - - - - - mapreduce.ifile.readahead - true - Configuration key to enable/disable IFile readahead. - - - - - mapreduce.ifile.readahead.bytes - 4194304 - Configuration key to set the IFile readahead length in bytes. - - - - - - - - - job.end.retry.attempts - 0 - Indicates how many times hadoop should attempt to contact the - notification URL - - - - job.end.retry.interval - 30000 - Indicates time in milliseconds between notification URL retry - calls - - - - - hadoop.rpc.socket.factory.class.JobSubmissionProtocol - - SocketFactory to use to connect to a Map/Reduce master - (JobTracker). If null or empty, then use hadoop.rpc.socket.class.default. - - - - - mapred.task.cache.levels - 2 - This is the max level of the task cache. For example, if - the level is 2, the tasks cached are at the host level and at the rack - level. - - - - - mapred.queue.names - default - Comma separated list of queues configured for this jobtracker. - Jobs are added to queues and schedulers can configure different - scheduling properties for the various queues. To configure a property - for a queue, the name of the queue must match the name specified in this - value. Queue properties that are common to all schedulers are configured - here with the naming convention, mapred.queue.$QUEUE-NAME.$PROPERTY-NAME, - for e.g. mapred.queue.default.submit-job-acl. - The number of queues configured in this parameter could depend on the - type of scheduler being used, as specified in - mapred.jobtracker.taskScheduler. For example, the JobQueueTaskScheduler - supports only a single queue, which is the default configured here. - Before adding more queues, ensure that the scheduler you've configured - supports multiple queues. - - - - - mapred.acls.enabled - false - Specifies whether ACLs should be checked - for authorization of users for doing various queue and job level operations. - ACLs are disabled by default. If enabled, access control checks are made by - JobTracker and TaskTracker when requests are made by users for queue - operations like submit job to a queue and kill a job in the queue and job - operations like viewing the job-details (See mapreduce.job.acl-view-job) - or for modifying the job (See mapreduce.job.acl-modify-job) using - Map/Reduce APIs, RPCs or via the console and web user interfaces. - - - - - mapred.queue.default.state - RUNNING - - This values defines the state , default queue is in. - the values can be either "STOPPED" or "RUNNING" - This value can be changed at runtime. - - - - - mapred.job.queue.name - default - Queue to which a job is submitted. This must match one of the - queues defined in mapred.queue.names for the system. Also, the ACL setup - for the queue must allow the current user to submit a job to the queue. - Before specifying a queue, ensure that the system is configured with - the queue, and access is allowed for submitting jobs to the queue. - - - - - mapreduce.job.acl-modify-job - - Job specific access-control list for 'modifying' the job. It - is only used if authorization is enabled in Map/Reduce by setting the - configuration property mapred.acls.enabled to true. - This specifies the list of users and/or groups who can do modification - operations on the job. For specifying a list of users and groups the - format to use is "user1,user2 group1,group". If set to '*', it allows all - users/groups to modify this job. If set to ' '(i.e. space), it allows - none. This configuration is used to guard all the modifications with respect - to this job and takes care of all the following operations: - o killing this job - o killing a task of this job, failing a task of this job - o setting the priority of this job - Each of these operations are also protected by the per-queue level ACL - "acl-administer-jobs" configured via mapred-queues.xml. So a caller should - have the authorization to satisfy either the queue-level ACL or the - job-level ACL. - - Irrespective of this ACL configuration, job-owner, the user who started the - cluster, cluster administrators configured via - mapreduce.cluster.administrators and queue administrators of the queue to - which this job is submitted to configured via - mapred.queue.queue-name.acl-administer-jobs in mapred-queue-acls.xml can - do all the modification operations on a job. - - By default, nobody else besides job-owner, the user who started the cluster, - cluster administrators and queue administrators can perform modification - operations on a job. - - - - - mapreduce.job.acl-view-job - - Job specific access-control list for 'viewing' the job. It is - only used if authorization is enabled in Map/Reduce by setting the - configuration property mapred.acls.enabled to true. - This specifies the list of users and/or groups who can view private details - about the job. For specifying a list of users and groups the - format to use is "user1,user2 group1,group". If set to '*', it allows all - users/groups to modify this job. If set to ' '(i.e. space), it allows - none. This configuration is used to guard some of the job-views and at - present only protects APIs that can return possibly sensitive information - of the job-owner like - o job-level counters - o task-level counters - o tasks' diagnostic information - o task-logs displayed on the TaskTracker web-UI and - o job.xml showed by the JobTracker's web-UI - Every other piece of information of jobs is still accessible by any other - user, for e.g., JobStatus, JobProfile, list of jobs in the queue, etc. - - Irrespective of this ACL configuration, job-owner, the user who started the - cluster, cluster administrators configured via - mapreduce.cluster.administrators and queue administrators of the queue to - which this job is submitted to configured via - mapred.queue.queue-name.acl-administer-jobs in mapred-queue-acls.xml can do - all the view operations on a job. - - By default, nobody else besides job-owner, the user who started the - cluster, cluster administrators and queue administrators can perform - view operations on a job. - - - - - mapred.tasktracker.indexcache.mb - 10 - The maximum memory that a task tracker allows for the - index cache that is used when serving map outputs to reducers. - - - - - mapred.combine.recordsBeforeProgress - 10000 - The number of records to process during combine output collection - before sending a progress notification to the TaskTracker. - - - - - mapred.merge.recordsBeforeProgress - 10000 - The number of records to process during merge before - sending a progress notification to the TaskTracker. - - - - - mapred.reduce.slowstart.completed.maps - 0.05 - Fraction of the number of maps in the job which should be - complete before reduces are scheduled for the job. - - - - - mapred.task.tracker.task-controller - org.apache.hadoop.mapred.DefaultTaskController - TaskController which is used to launch and manage task execution - - - - - mapreduce.tasktracker.group - - Expert: Group to which TaskTracker belongs. If - LinuxTaskController is configured via mapreduce.tasktracker.taskcontroller, - the group owner of the task-controller binary should be same as this group. - - - - - mapred.disk.healthChecker.interval - 60000 - How often the TaskTracker checks the health of its - local directories. Configuring this to a value smaller than the - heartbeat interval is equivalent to setting this to heartbeat - interval value. - - - - - - - mapred.healthChecker.script.path - - Absolute path to the script which is - periodicallyrun by the node health monitoring service to determine if - the node is healthy or not. If the value of this key is empty or the - file does not exist in the location configured here, the node health - monitoring service is not started. - - - - mapred.healthChecker.interval - 60000 - Frequency of the node health script to be run, - in milliseconds - - - - mapred.healthChecker.script.timeout - 600000 - Time after node health script should be killed if - unresponsive and considered that the script has failed. - - - - mapred.healthChecker.script.args - - List of arguments which are to be passed to - node health script when it is being launched comma seperated. - - - - - - - mapreduce.job.counters.max - 120 - Limit on the number of counters allowed per job. - - - - - mapreduce.job.counters.groups.max - 50 - Limit on the number of counter groups allowed per job. - - - - - mapreduce.job.counters.counter.name.max - 64 - Limit on the length of counter names in jobs. Names - exceeding this limit will be truncated. - - - - - mapreduce.job.counters.group.name.max - 128 - Limit on the length of counter group names in jobs. Names - exceeding this limit will be truncated. - - - - diff --git a/savanna/resources/setup-general.sh.template b/savanna/resources/setup-general.sh.template deleted file mode 100644 index accce397..00000000 --- a/savanna/resources/setup-general.sh.template +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -echo "----- Setting up Hadoop enviroment config" - -{% for envconf in env_configs -%} - echo "{{envconf}}" >> /tmp/hadoop-env.sh -{% endfor %} - -cat /etc/hadoop/hadoop-env.sh >> /tmp/hadoop-env.sh -mv /tmp/hadoop-env.sh /etc/hadoop/hadoop-env.sh - - -echo "----- Creating directories permissions" - -#TODO(aignatov): Need to put here /mnt via args in the future when HDFS placement feature will be ready -chown -R hadoop:hadoop /mnt -chmod -R 755 /mnt - -{% block master %} -{% endblock %} diff --git a/savanna/resources/setup-master.sh.template b/savanna/resources/setup-master.sh.template deleted file mode 100644 index 29f6c7c5..00000000 --- a/savanna/resources/setup-master.sh.template +++ /dev/null @@ -1,21 +0,0 @@ -{% extends "setup-general.sh.template" %} - -{% block master %} -echo "----- Populating slaves file" - -echo -e ' -{%- for slave in slaves -%} -{{slave}}\n -{%- endfor -%} -' | tee /etc/hadoop/slaves - - -echo "----- Populating master file" - -echo {{master_hostname}} | tee /etc/hadoop/masters - - -echo "----- Formatting Hadoop NameNode" - -su -c 'hadoop namenode -format' hadoop -{% endblock %} diff --git a/savanna/service/api.py b/savanna/service/api.py index 4947c221..9ad4d375 100644 --- a/savanna/service/api.py +++ b/savanna/service/api.py @@ -13,257 +13,57 @@ # See the License for the specific language governing permissions and # limitations under the License. -import eventlet -from oslo.config import cfg - -from savanna import exceptions as ex +import savanna.db.storage as s from savanna.openstack.common import log as logging -from savanna.service import cluster_ops -import savanna.storage.storage as storage +import savanna.plugins.base as plugin_base +from savanna.plugins.provisioning import ProvisioningPluginBase LOG = logging.getLogger(__name__) -CONF = cfg.CONF -CONF.import_opt('allow_cluster_ops', 'savanna.config') +## Cluster ops -## Node Template ops: +get_clusters = s.get_clusters +get_cluster = s.get_cluster -def get_node_template(**args): - return _node_template(storage.get_node_template(**args)) +def create_cluster(values): + # todo initiate cluster creation here :) + return s.create_cluster(values) -def get_node_templates(**args): - return [_node_template(tmpl) for tmpl - in storage.get_node_templates(**args)] +def terminate_cluster(**args): + # todo initiate cluster termination here :) + cluster = get_cluster(**args) + s.terminate_cluster(cluster) -def is_node_template_associated(**args): - return storage.is_node_template_associated(**args) +## ClusterTemplate ops -def create_node_template(values, headers): - """Creates new node template from values dict. +get_cluster_templates = s.get_cluster_templates +get_cluster_template = s.get_cluster_template +create_cluster_template = s.create_cluster_template +terminate_cluster_template = s.terminate_cluster_template - :param values: dict - :return: created node template resource - """ - values = values.pop('node_template') - name = values.pop('name') - node_type_id = storage.get_node_type(name=values.pop('node_type')).id - flavor_id = values.pop('flavor_id') +## NodeGroupTemplate ops - nt = storage.create_node_template(name, node_type_id, flavor_id, values) +get_node_group_templates = s.get_node_group_templates +get_node_group_template = s.get_node_group_template +create_node_group_template = s.create_node_group_template +terminate_node_group_template = s.terminate_node_group_template - return get_node_template(id=nt.id) +## Plugins ops -def terminate_node_template(**args): - return storage.terminate_node_template(**args) +def get_plugins(): + return plugin_base.PLUGINS.get_plugins(base=ProvisioningPluginBase) -## Cluster ops: - -def get_cluster(**args): - return _cluster(storage.get_cluster(**args)) - - -def get_clusters(**args): - return [_cluster(cluster) for cluster in - storage.get_clusters(**args)] - - -def create_cluster(values, headers): - values = values.pop('cluster') - - name = values.pop('name') - base_image_id = values.pop('base_image_id') - tenant_id = headers['X-Tenant-Id'] - templates = values.pop('node_templates') - - # todo(slukjanov): check that we can create objects in the specified tenant - - cluster = storage.create_cluster(name, base_image_id, tenant_id, templates) - - eventlet.spawn(_cluster_creation_job, headers, cluster.id) - - return get_cluster(id=cluster.id) - - -def _cluster_creation_job(headers, cluster_id): - cluster = storage.get_cluster(id=cluster_id) - LOG.debug("Starting cluster '%s' creation: %s", cluster_id, - _cluster(cluster).dict) - - if CONF.allow_cluster_ops: - launched = cluster_ops.launch_cluster(headers, cluster) - else: - LOG.info("Cluster ops are disabled, use --allow-cluster-ops flag") - launched = True - - if launched: - storage.update_cluster_status('Active', id=cluster.id) - - -def terminate_cluster(headers, **args): - cluster = storage.update_cluster_status('Stopping', **args) - - eventlet.spawn(_cluster_termination_job, headers, cluster.id) - - -def _cluster_termination_job(headers, cluster_id): - cluster = storage.get_cluster(id=cluster_id) - LOG.debug("Stopping cluster '%s' creation: %s", cluster_id, - _cluster(cluster).dict) - - if CONF.allow_cluster_ops: - cluster_ops.stop_cluster(headers, cluster) - else: - LOG.info("Cluster ops are disabled, use --allow-cluster-ops flag") - - storage.terminate_cluster(id=cluster.id) - - -## Node Type ops: - -def get_node_type(**args): - return _node_type(storage.get_node_type(**args)) - - -def get_node_types(**args): - return [_node_type(t) for t in storage.get_node_types(**args)] - - -def get_node_type_required_params(**args): - result = {} - for process in storage.get_node_type(**args).processes: - result[process.name] = [] - for prop in process.node_process_properties: - if prop.required and not prop.default: - result[process.name] += [prop.name] - - return result - - -def get_node_type_all_params(**args): - result = {} - for process in storage.get_node_type(**args).processes: - result[process.name] = [prop.name - for prop in process.node_process_properties] - return result - - -## Utils and DB object to Resource converters - -def _clean_nones(obj): - if not isinstance(obj, dict) and not isinstance(obj, list): - return obj - - if isinstance(obj, dict): - remove = [] - for key, value in obj.iteritems(): - if value is None: - remove.append(key) - for key in remove: - obj.pop(key) - for value in obj.values(): - _clean_nones(value) - elif isinstance(obj, list): - new_list = [] - for elem in obj: - elem = _clean_nones(elem) - if elem is not None: - new_list.append(elem) - return new_list - - return obj - - -class Resource(object): - def __init__(self, _name, _info): - self._name = _name - self._info = _clean_nones(_info) - - def __getattr__(self, k): - if k not in self.__dict__: - return self._info.get(k) - return self.__dict__[k] - - def __repr__(self): - return '<%s %s>' % (self._name, self._info) - - def __eq__(self, other): - return self._name == other._name and self._info == other._info - - @property - def dict(self): - return self._info - - @property - def wrapped_dict(self): - return {self._name: self._info} - - -def _node_template(nt): - if not nt: - raise ex.NodeTemplateNotFoundException(nt) - - d = { - 'id': nt.id, - 'name': nt.name, - 'node_type': { - 'name': nt.node_type.name, - 'processes': [p.name for p in nt.node_type.processes]}, - 'flavor_id': nt.flavor_id - } - - for conf in nt.node_template_configs: - c_section = conf.node_process_property.node_process.name - c_name = conf.node_process_property.name - c_value = conf.value - if c_section not in d: - d[c_section] = dict() - d[c_section][c_name] = c_value - - return Resource('node_template', d) - - -def _cluster(cluster): - if not cluster: - raise ex.ClusterNotFoundException(cluster) - - d = { - 'id': cluster.id, - 'name': cluster.name, - 'base_image_id': cluster.base_image_id, - 'status': cluster.status, - 'service_urls': {}, - 'node_templates': {}, - 'nodes': [{'vm_id': n.vm_id, - 'node_template': { - 'id': n.node_template.id, - 'name': n.node_template.name - }} - for n in cluster.nodes] - } - for ntc in cluster.node_counts: - d['node_templates'][ntc.node_template.name] = ntc.count - - for service in cluster.service_urls: - d['service_urls'][service.name] = service.url - - return Resource('cluster', d) - - -def _node_type(nt): - if not nt: - raise ex.NodeTypeNotFoundException(nt) - - d = { - 'id': nt.id, - 'name': nt.name, - 'processes': [p.name for p in nt.processes] - } - - return Resource('node_type', d) +def get_plugin(plugin_name, version=None): + plugin = plugin_base.PLUGINS.get_plugin(plugin_name) + res = plugin.as_resource() + if version: + res._info['configs'] = [c.dict for c in plugin.get_configs(version)] + res._info['node_processes'] = plugin.get_node_processes(version) + return res diff --git a/savanna/service/cluster_ops.py b/savanna/service/cluster_ops.py deleted file mode 100644 index 5b96881d..00000000 --- a/savanna/service/cluster_ops.py +++ /dev/null @@ -1,510 +0,0 @@ -# Copyright (c) 2013 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import eventlet -from jinja2 import Environment -from jinja2 import PackageLoader -from oslo.config import cfg -import paramiko -from pkg_resources import resource_filename -import xml.dom.minidom as xml - -from savanna.openstack.common import log as logging -from savanna.storage.db import DB -from savanna.storage.models import Node, ServiceUrl -from savanna.storage.storage import update_cluster_status -from savanna.utils.openstack.nova import novaclient -from savanna.utils.patches import patch_minidom_writexml - - -LOG = logging.getLogger(__name__) - -CONF = cfg.CONF - -cluster_node_opts = [ - cfg.StrOpt('username', - default='root', - help='An existing user on Hadoop image'), - cfg.StrOpt('password', - default='swordfish', - help='User\'s password'), - cfg.BoolOpt('use_floating_ips', - default=True, - help='When set to false, Savanna uses only internal IP of VMs.' - ' When set to true, Savanna expects OpenStack to auto-' - 'assign floating IPs to cluster nodes. Internal IPs will ' - 'be used for inter-cluster communication, while floating ' - 'ones will be used by Savanna to configure nodes. Also ' - 'floating IPs will be exposed in service URLs') -] - -CONF.register_opts(cluster_node_opts, 'cluster_node') - - -def _find_by_id(lst, id): - for entity in lst: - if entity.id == id: - return entity - - return None - - -def _find_by_name(lst, name): - for entity in lst: - if entity.name == name: - return entity - - return None - - -def _check_finding(entity, attr, value): - if entity is None: - raise RuntimeError("Unable to find entity with %s " - "\'%s\'" % (attr, value)) - - -def _ensure_zero(ret): - if ret != 0: - raise RuntimeError('Command returned non-zero status code - %i' % ret) - - -def _setup_ssh_connection(host, ssh): - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh.connect( - host, - username=CONF.cluster_node.username, - password=CONF.cluster_node.password - ) - - -def _open_channel_and_execute(ssh, cmd): - chan = ssh.get_transport().open_session() - chan.exec_command(cmd) - return chan.recv_exit_status() - - -def _execute_command_on_node(host, cmd): - ssh = paramiko.SSHClient() - try: - _setup_ssh_connection(host, ssh) - return _open_channel_and_execute(ssh, cmd) - finally: - ssh.close() - - -def launch_cluster(headers, cluster): - nova = novaclient(headers) - - clmap = dict() - clmap['id'] = cluster.id - clmap['name'] = cluster.name - clmap['image'] = _find_by_id(nova.images.list(), - cluster.base_image_id) - _check_finding(clmap['image'], 'id', cluster.base_image_id) - - clmap['nodes'] = [] - num = 1 - - for nc in cluster.node_counts: - configs = dict() - for cf in nc.node_template.node_template_configs: - proc_name = cf.node_process_property.node_process.name - if proc_name not in configs: - configs[proc_name] = dict() - - name = cf.node_process_property.name - configs[proc_name][name] = cf.value - - ntype = nc.node_template.node_type.name - templ_id = nc.node_template.id - flv_id = nc.node_template.flavor_id - flv = _find_by_name(nova.flavors.list(), flv_id) - _check_finding(flv, 'id', flv_id) - - for _ in xrange(0, nc.count): - node = dict() - node['id'] = None - if ntype == 'JT+NN': - node['name'] = '%s-master' % cluster.name - else: - node['name'] = '%s-%i' % (cluster.name, num) - num += 1 - node['type'] = ntype - node['templ_id'] = templ_id - node['flavor'] = flv - node['configs'] = configs - node['is_up'] = False - clmap['nodes'].append(node) - - try: - for node in clmap['nodes']: - LOG.debug("Starting node for cluster '%s', node: %s, image: %s", - cluster.name, node, clmap['image']) - _launch_node(nova, node, clmap['image']) - except Exception, e: - _rollback_cluster_creation(cluster, clmap, nova, e) - return False - - all_set = False - - LOG.debug("All nodes for cluster '%s' have been started, " - "waiting for them to come up", cluster.name) - - while not all_set: - all_set = True - - for node in clmap['nodes']: - _check_if_up(nova, node) - - if not node['is_up']: - all_set = False - - eventlet.sleep(1) - - LOG.debug("All nodes of cluster '%s' are up: %s", - cluster.name, all_set) - - _pre_cluster_setup(clmap) - for node in clmap['nodes']: - _setup_node(node, clmap) - _register_node(node, cluster) - - LOG.debug("All nodes of cluster '%s' are configured and registered, " - "starting the cluster...", cluster.name) - - _start_cluster(cluster, clmap) - - return True - - -def _launch_node(nova, node, image): - srv = nova.servers.create(node['name'], image, node['flavor']) - node['id'] = srv.id - - -def _rollback_cluster_creation(cluster, clmap, nova, error): - update_cluster_status("Error", id=cluster.id) - - LOG.warn("Can't launch all vms for cluster '%s': %s", cluster.id, error) - for node in clmap['nodes']: - if node['id']: - _stop_node_silently(nova, cluster, node['id']) - - LOG.info("All vms of cluster '%s' has been stopped", cluster.id) - - -def _stop_node_silently(nova, cluster, vm_id): - LOG.debug("Stopping vm '%s' of cluster '%s'", vm_id, cluster.id) - try: - nova.servers.delete(vm_id) - except Exception, e: - LOG.error("Can't silently remove node '%s': %s", vm_id, e) - - -def _check_if_up(nova, node): - if node['is_up']: - # all set - return - - if 'ip' not in node: - srv = _find_by_id(nova.servers.list(), node['id']) - nets = srv.networks - - if len(nets) == 0: - # VM's networking is not configured yet - return - - ips = nets.values()[0] - - if CONF.cluster_node.use_floating_ips: - if len(ips) < 2: - # floating IP is not assigned yet - return - - # we assume that floating IP comes last in the list - node['ip'] = ips[-1] - node['internal_ip'] = ips[0] - else: - if len(ips) < 1: - # private IP is not assigned yet - return - node['ip'] = ips[0] - node['internal_ip'] = ips[0] - - try: - ret = _execute_command_on_node(node['ip'], 'ls -l /') - _ensure_zero(ret) - except Exception: - # ssh is not up yet - return - - node['is_up'] = True - - -env = Environment(loader=PackageLoader('savanna', 'resources')) - - -def _render_template(template_name, **kwargs): - templ = env.get_template('%s.template' % template_name) - return templ.render(**kwargs) - - -ENV_CONFS = { - 'job_tracker': { - 'heap_size': 'HADOOP_JOBTRACKER_OPTS=\\"-Xmx%sm\\"' - }, - 'name_node': { - 'heap_size': 'HADOOP_NAMENODE_OPTS=\\"-Xmx%sm\\"' - }, - 'task_tracker': { - 'heap_size': 'HADOOP_TASKTRACKER_OPTS=\\"-Xmx%sm\\"' - }, - 'data_node': { - 'heap_size': 'HADOOP_DATANODE_OPTS=\\"-Xmx%sm\\"' - } -} - - -def _load_xml_default_configs(file_name): - doc = xml.parse(resource_filename("savanna", 'resources/%s' % file_name)) - properties = doc.getElementsByTagName("name") - return [prop.childNodes[0].data for prop in properties] - -CORE_CONF = _load_xml_default_configs('core-default.xml') -MAPRED_CONF = _load_xml_default_configs('mapred-default.xml') -HDFS_CONF = _load_xml_default_configs('hdfs-default.xml') - - -def _generate_xml_configs(node, clmap): - # inserting common configs depends on provisioned VMs and HDFS placement - cfg = { - 'fs.default.name': 'hdfs://%s:8020' % clmap['master_hostname'], - 'mapred.job.tracker': '%s:8021' % clmap['master_hostname'], - 'dfs.name.dir': '/mnt/lib/hadoop/hdfs/namenode', - 'dfs.data.dir': '/mnt/lib/hadoop/hdfs/datanode', - 'mapred.system.dir': '/mnt/mapred/mapredsystem', - 'mapred.local.dir': '/mnt/lib/hadoop/mapred' - } - - # inserting user-defined configs from NodeTemplates - for key, value in _extract_xml_confs(node['configs']): - cfg[key] = value - - # invoking applied configs to appropriate xml files - xml_configs = { - 'core-site': _create_xml(cfg, CORE_CONF), - 'mapred-site': _create_xml(cfg, MAPRED_CONF), - 'hdfs-site': _create_xml(cfg, HDFS_CONF) - } - - return xml_configs - - -# Patches minidom's writexml to avoid excess whitespaces in generated xml -# configuration files that brakes Hadoop. -patch_minidom_writexml() - - -def _create_xml(configs, global_conf): - doc = xml.Document() - - pi = doc.createProcessingInstruction('xml-stylesheet', - 'type="text/xsl" ' - 'href="configuration.xsl"') - doc.insertBefore(pi, doc.firstChild) - - # Create the base element - configuration = doc.createElement("configuration") - doc.appendChild(configuration) - - for prop_name, prop_value in configs.items(): - if prop_name in global_conf: - # Create the element - property = doc.createElement("property") - configuration.appendChild(property) - - # Create a element in - name = doc.createElement("name") - property.appendChild(name) - - # Give the element some hadoop config name - name_text = doc.createTextNode(prop_name) - name.appendChild(name_text) - - # Create a element in - value = doc.createElement("value") - property.appendChild(value) - - # Give the element some hadoop config value - value_text = doc.createTextNode(prop_value) - value.appendChild(value_text) - - # Return newly created XML - return doc.toprettyxml(indent=" ") - - -def _keys_exist(map, key1, key2): - return key1 in map and key2 in map[key1] - - -def _extract_environment_confs(node_configs): - """Returns list of Hadoop parameters which should be passed via environment - """ - - lst = [] - - for process, proc_confs in ENV_CONFS.items(): - for param_name, param_format_str in proc_confs.items(): - if (_keys_exist(node_configs, process, param_name) and - not node_configs[process][param_name] is None): - lst.append(param_format_str % - node_configs[process][param_name]) - - return lst - - -def _extract_xml_confs(node_configs): - """Returns list of Hadoop parameters which should be passed into general - configs like core-site.xml - """ - - # For now we assume that all parameters outside of ENV_CONFS - # are passed to xml files - - lst = [] - - for process, proc_confs in node_configs.items(): - for param_name, param_value in proc_confs.items(): - if (not _keys_exist(ENV_CONFS, process, param_name) and - not param_value is None): - lst.append((param_name, param_value)) - - return lst - - -def _analyze_templates(clmap): - clmap['master_ip'] = None - clmap['slaves'] = [] - for node in clmap['nodes']: - if node['type'] == 'JT+NN': - clmap['master_ip'] = node['ip'] - clmap['master_hostname'] = node['name'] - node['is_master'] = True - elif node['type'] == 'TT+DN': - clmap['slaves'].append(node['name']) - node['is_master'] = False - - if clmap['master_ip'] is None: - raise RuntimeError("No master node is defined in the cluster") - - -def _generate_hosts(clmap): - hosts = "127.0.0.1 localhost\n" - for node in clmap['nodes']: - hosts += "%s %s\n" % (node['internal_ip'], node['name']) - - clmap['hosts'] = hosts - - -def _pre_cluster_setup(clmap): - _analyze_templates(clmap) - _generate_hosts(clmap) - - for node in clmap['nodes']: - if node['is_master']: - script_file = 'setup-master.sh' - else: - script_file = 'setup-general.sh' - - templ_args = { - 'slaves': clmap['slaves'], - 'master_hostname': clmap['master_hostname'], - 'env_configs': _extract_environment_confs(node['configs']) - } - - node['setup_script'] = _render_template(script_file, **templ_args) - node['xml'] = _generate_xml_configs(node, clmap) - - -def _setup_node(node, clmap): - ssh = paramiko.SSHClient() - try: - _setup_ssh_connection(node['ip'], ssh) - sftp = ssh.open_sftp() - - fl = sftp.file('/etc/hosts', 'w') - fl.write(clmap['hosts']) - fl.close() - - fl = sftp.file('/etc/hadoop/core-site.xml', 'w') - fl.write(node['xml']['core-site']) - fl.close() - - fl = sftp.file('/etc/hadoop/hdfs-site.xml', 'w') - fl.write(node['xml']['hdfs-site']) - fl.close() - - fl = sftp.file('/etc/hadoop/mapred-site.xml', 'w') - fl.write(node['xml']['mapred-site']) - fl.close() - - fl = sftp.file('/tmp/savanna-hadoop-init.sh', 'w') - fl.write(node['setup_script']) - fl.close() - - sftp.chmod('/tmp/savanna-hadoop-init.sh', 0500) - - ret = _open_channel_and_execute(ssh, - '/tmp/savanna-hadoop-init.sh ' - '>> /tmp/savanna-hadoop-init.log 2>&1') - _ensure_zero(ret) - finally: - ssh.close() - - -def _register_node(node, cluster): - node_obj = Node(node['id'], cluster.id, node['templ_id']) - DB.session.add(node_obj) - - if node['is_master']: - srv_url_jt = ServiceUrl(cluster.id, 'jobtracker', 'http://%s:50030' - % node['ip']) - srv_url_nn = ServiceUrl(cluster.id, 'namenode', 'http://%s:50070' - % node['ip']) - - DB.session.add(srv_url_jt) - DB.session.add(srv_url_nn) - - DB.session.commit() - - -def _start_cluster(cluster, clmap): - ret = _execute_command_on_node( - clmap['master_ip'], - 'su -c start-all.sh hadoop >> /tmp/savanna-hadoop-start-all.log') - _ensure_zero(ret) - - LOG.info("Cluster '%s' successfully started!", cluster.name) - - -def stop_cluster(headers, cluster): - nova = novaclient(headers) - - for node in cluster.nodes: - try: - nova.servers.delete(node.vm_id) - LOG.debug("vm '%s' has been stopped", node.vm_id) - except Exception, e: - LOG.info("Can't stop vm '%s': %s", node.vm_id, e) diff --git a/savanna/service/validation.py b/savanna/service/validation.py deleted file mode 100644 index dc1e2114..00000000 --- a/savanna/service/validation.py +++ /dev/null @@ -1,296 +0,0 @@ -# Copyright (c) 2013 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from flask import request -import functools -import jsonschema -from oslo.config import cfg - -from savanna import exceptions as ex -import savanna.openstack.common.exception as os_ex -from savanna.openstack.common import log as logging -from savanna.service import api -import savanna.utils.api as api_u -from savanna.utils.openstack import nova - - -LOG = logging.getLogger(__name__) - -CONF = cfg.CONF -CONF.import_opt('allow_cluster_ops', 'savanna.config') - -# Base validation schema of cluster creation operation -CLUSTER_CREATE_SCHEMA = { - "title": "Cluster creation schema", - "type": "object", - "properties": { - "cluster": { - "type": "object", - "properties": { - "name": {"type": "string", - "minLength": 1, - "maxLength": 50, - "pattern": r"^(([a-zA-Z]|[a-zA-Z][a-zA-Z0-9\-]" - r"*[a-zA-Z0-9])\.)*([A-Za-z]|[A-Za-z]" - r"[A-Za-z0-9\-]*[A-Za-z0-9])$"}, - "base_image_id": {"type": "string", - "minLength": 1, - "maxLength": 240}, - "node_templates": { - "type": "object" - } - }, - "required": ["name", "base_image_id", "node_templates"] - } - }, - "required": ["cluster"] -} - -# Base validation schema of node template creation operation -TEMPLATE_CREATE_SCHEMA = { - "title": "Node Template creation schema", - "type": "object", - "properties": { - "node_template": { - "type": "object", - "properties": { - "name": {"type": "string", - "minLength": 1, - "maxLength": 240, - "pattern": r"^(([a-zA-Z]|[a-zA-Z][a-zA-Z0-9\-_]" - r"*[a-zA-Z0-9])\.)*([A-Za-z]|[A-Za-z]" - r"[A-Za-z0-9\-_]*[A-Za-z0-9])$"}, - "node_type": {"type": "string", - "minLength": 1, - "maxLength": 240}, - "flavor_id": {"type": "string", - "minLength": 1, - "maxLength": 240}, - "task_tracker": { - "type": "object" - }, - "job_tracker": { - "type": "object" - }, - "name_node": { - "type": "object" - }, - "data_node": { - "type": "object" - } - }, - "required": ["name", "node_type", "flavor_id"] - } - }, - "required": ["node_template"] -} - - -def validate(validate_func): - def decorator(func): - @functools.wraps(func) - def handler(*args, **kwargs): - try: - validate_func(api_u.request_data(), **kwargs) - except jsonschema.ValidationError, e: - e.code = "VALIDATION_ERROR" - return api_u.bad_request(e) - except ex.SavannaException, e: - return api_u.bad_request(e) - except os_ex.MalformedRequestBody, e: - e.code = "MALFORMED_REQUEST_BODY" - return api_u.bad_request(e) - except Exception, e: - return api_u.internal_error( - 500, "Error occurred during validation", e) - - return func(*args, **kwargs) - - return handler - - return decorator - - -def exists_by_id(service_func, id_prop, tenant_specific=False): - def decorator(func): - @functools.wraps(func) - def handler(*args, **kwargs): - try: - if tenant_specific: - tenant = request.headers['X-Tenant-Id'] - service_func(*args, id=kwargs[id_prop], tenant_id=tenant) - else: - service_func(*args, id=kwargs[id_prop]) - return func(*args, **kwargs) - except ex.NotFoundException, e: - e.__init__(kwargs[id_prop]) - return api_u.not_found(e) - except Exception, e: - return api_u.internal_error( - 500, "Unexpected error occurred", e) - - return handler - - return decorator - - -def validate_cluster_create(cluster_values): - jsonschema.validate(cluster_values, CLUSTER_CREATE_SCHEMA) - values = cluster_values['cluster'] - - # check that requested cluster name is unique - unique_names = [cluster.name for cluster in api.get_clusters()] - if values['name'] in unique_names: - raise ex.ClusterNameExistedException(values['name']) - - # check that requested templates are from already defined values - node_templates = values['node_templates'] - possible_node_templates = [nt.name for nt in api.get_node_templates()] - for nt in node_templates: - if nt not in possible_node_templates: - raise ex.NodeTemplateNotFoundException(nt) - # check node count is integer and non-zero value - jsonschema.validate(node_templates[nt], - {"type": "integer", "minimum": 1}) - - # check that requested cluster contains only 1 instance of NameNode - # and 1 instance of JobTracker - jt_count = 0 - nn_count = 0 - - for nt_name in node_templates: - processes = api.get_node_template(name=nt_name).dict['node_type'][ - 'processes'] - if "job_tracker" in processes: - jt_count += node_templates[nt_name] - if "name_node" in processes: - nn_count += node_templates[nt_name] - - if nn_count != 1: - raise ex.NotSingleNameNodeException(nn_count) - - if jt_count != 1: - raise ex.NotSingleJobTrackerException(jt_count) - - if CONF.allow_cluster_ops: - image_id = values['base_image_id'] - nova_images = nova.get_images(request.headers) - if image_id not in nova_images: - LOG.debug("Could not find %s image in %s", image_id, nova_images) - raise ex.ImageNotFoundException(values['base_image_id']) - - # check available Nova absolute limits - _check_limits(nova.get_limits(request.headers), - values['node_templates']) - else: - LOG.info("Cluster ops are disabled, use --allow-cluster-ops flag") - - -def validate_node_template_create(nt_values): - jsonschema.validate(nt_values, TEMPLATE_CREATE_SCHEMA) - values = nt_values['node_template'] - - # check that requested node_template name is unique - unique_names = [nt.name for nt in api.get_node_templates()] - if values['name'] in unique_names: - raise ex.NodeTemplateExistedException(values['name']) - - node_types = [nt.name for nt in api.get_node_types()] - - if values['node_type'] not in node_types: - raise ex.NodeTypeNotFoundException(values['node_type']) - - req_procs = [] - if "TT" in values['node_type']: - req_procs.append("task_tracker") - if "DN" in values['node_type']: - req_procs.append("data_node") - if "NN" in values['node_type']: - req_procs.append("name_node") - if "JT" in values['node_type']: - req_procs.append("job_tracker") - - LOG.debug("Required properties are: %s", req_procs) - - jsonschema.validate(values, {"required": req_procs}) - - processes = values.copy() - del processes['name'] - del processes['node_type'] - del processes['flavor_id'] - - LOG.debug("Incoming properties are: %s", processes) - - for proc in processes: - if proc not in req_procs: - raise ex.DiscrepancyNodeProcessException(req_procs) - - req_params = api.get_node_type_required_params(name=values['node_type']) - for process in req_params: - for param in req_params[process]: - if param not in values[process] or not values[process][param]: - raise ex.RequiredParamMissedException(process, param) - - all_params = api.get_node_type_all_params(name=values['node_type']) - for process in all_params: - for param in processes[process]: - if param not in all_params[process]: - raise ex.ParamNotAllowedException(param, process) - - if api.CONF.allow_cluster_ops: - flavor = values['flavor_id'] - nova_flavors = nova.get_flavors(request.headers) - if flavor not in nova_flavors: - LOG.debug("Could not find %s flavor in %s", flavor, nova_flavors) - raise ex.FlavorNotFoundException(flavor) - else: - LOG.info("Cluster ops are disabled, use --allow-cluster-ops flag") - - -def _check_limits(limits, node_templates): - all_vcpus = limits['maxTotalCores'] - limits['totalCoresUsed'] - all_ram = limits['maxTotalRAMSize'] - limits['totalRAMUsed'] - all_inst = limits['maxTotalInstances'] - limits['totalInstancesUsed'] - LOG.info("List of available VCPUs: %d, RAM: %d, Instances: %d", - all_vcpus, all_ram, all_inst) - - need_vcpus = 0 - need_ram = 0 - need_inst = 0 - for nt_name in node_templates: - nt_flavor_name = api.get_node_template(name=nt_name).dict['flavor_id'] - nt_flavor_count = node_templates[nt_name] - LOG.debug("User requested flavor: %s, count: %s", - nt_flavor_name, nt_flavor_count) - nova_flavor = nova.get_flavor(request.headers, name=nt_flavor_name) - LOG.debug("Nova has flavor %s with VCPUs=%d, RAM=%d", - nova_flavor.name, nova_flavor.vcpus, nova_flavor.ram) - - need_vcpus += nova_flavor.vcpus * nt_flavor_count - need_ram += nova_flavor.ram * nt_flavor_count - need_inst += nt_flavor_count - - LOG.info("User requested %d instances with total VCPUs=%d and RAM=%d", - need_inst, need_vcpus, need_ram) - - if need_inst > all_inst or need_vcpus > all_vcpus or need_ram > all_ram: - raise ex.NotEnoughResourcesException([all_inst, all_vcpus, all_ram, - need_inst, need_vcpus, need_ram]) - - -def validate_node_template_terminate(_, template_id): - if api.is_node_template_associated(id=template_id): - name = api.get_node_template(id=template_id).name - raise ex.AssociatedNodeTemplateTerminationException(name) diff --git a/savanna/storage/db.py b/savanna/storage/db.py deleted file mode 100644 index 21144bc9..00000000 --- a/savanna/storage/db.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (c) 2013 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from flask.ext.sqlalchemy import SQLAlchemy -from oslo.config import cfg - -DB = SQLAlchemy() - -opts = [ - cfg.StrOpt('database_uri', - default='sqlite:////tmp/savanna.db', - help='URL for sqlalchemy database'), - cfg.BoolOpt('echo', - default=False, - help='Sqlalchemy echo') -] - -CONF = cfg.CONF -CONF.register_opts(opts, group='sqlalchemy') - - -def setup_storage(app): - app.config['SQLALCHEMY_DATABASE_URI'] = CONF.sqlalchemy.database_uri - app.config['SQLALCHEMY_ECHO'] = CONF.sqlalchemy.echo - - DB.app = app - DB.init_app(app) - DB.create_all() diff --git a/savanna/storage/defaults.py b/savanna/storage/defaults.py deleted file mode 100644 index f1ba2c75..00000000 --- a/savanna/storage/defaults.py +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright (c) 2013 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from savanna.storage.storage import create_node_type, \ - create_node_template, create_node_process -from savanna.openstack.common import log as logging - -LOG = logging.getLogger(__name__) - - -def setup_defaults(reset_db=False, gen_templates=False): - nt_jt_nn = None - nt_jt = None - nt_nn = None - nt_tt_dn = None - - if reset_db: - # setup default processes - p_jt = create_node_process('job_tracker', - [('heap_size', True, None), - ('mapred.job.tracker.handler.count', - False, None)]) - p_nn = create_node_process('name_node', - [('heap_size', True, None), - ('dfs.namenode.handler.count', - False, None), - ('dfs.block.size', False, None), - ('dfs.replication', False, None)]) - p_tt = create_node_process('task_tracker', - [('heap_size', True, None), - ('mapred.child.java.opts', False, None), - ('mapred.map.tasks', False, None), - ('mapred.tasktracker.map.tasks.maximum', - False, None), - ('mapred.reduce.tasks', False, None), - ('mapred.tasktracker.reduce.tasks.maximum', - False, None)]) - - p_dn = create_node_process('data_node', - [('heap_size', True, None), - ('dfs.datanode.max.xcievers', False, None), - ('dfs.block.size', False, None), - ('dfs.replication', False, None), - ('dfs.datanode.handler.count', - False, None)]) - - for p in [p_jt, p_nn, p_tt, p_dn]: - LOG.info('New NodeProcess: \'%s\'', p.name) - - # setup default node types - nt_jt_nn = create_node_type('JT+NN', [p_jt, p_nn]) - nt_jt = create_node_type('JT', [p_jt]) - nt_nn = create_node_type('NN', [p_nn]) - nt_tt_dn = create_node_type('TT+DN', [p_tt, p_dn]) - - for nt in [nt_jt_nn, nt_jt, nt_nn, nt_tt_dn]: - LOG.info('New NodeType: \'%s\' %s', - nt.name, [p.name.__str__() for p in nt.processes]) - - if gen_templates: - _generate_templates(nt_jt_nn, nt_jt, nt_nn, nt_tt_dn) - - LOG.info('All defaults has been inserted') - - -def _generate_templates(nt_jt_nn, nt_jt, nt_nn, nt_tt_dn): - jt_nn_small = create_node_template('jt_nn.small', nt_jt_nn.id, 'm1.small', - { - 'job_tracker': { - 'heap_size': '896' - }, - 'name_node': { - 'heap_size': '896' - } - }) - jt_nn_medium = create_node_template('jt_nn.medium', nt_jt_nn.id, - 'm1.medium', - { - 'job_tracker': { - 'heap_size': '1792' - }, - 'name_node': { - 'heap_size': '1792' - } - }) - tt_dn_small = create_node_template('tt_dn.small', nt_tt_dn.id, 'm1.small', - { - 'task_tracker': { - 'heap_size': '896' - }, - 'data_node': { - 'heap_size': '896' - } - }) - tt_dn_medium = create_node_template('tt_dn.medium', nt_tt_dn.id, - 'm1.medium', - { - 'task_tracker': { - 'heap_size': '1792' - }, - 'data_node': { - 'heap_size': '1792' - } - }) - - for tmpl in [jt_nn_small, jt_nn_medium, tt_dn_small, tt_dn_medium]: - LOG.info('New NodeTemplate: \'%s\' %s', tmpl.name, tmpl.flavor_id) diff --git a/savanna/storage/models.py b/savanna/storage/models.py deleted file mode 100644 index 00cb04fa..00000000 --- a/savanna/storage/models.py +++ /dev/null @@ -1,232 +0,0 @@ -# Copyright (c) 2013 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import uuid - -from savanna.storage.db import DB - - -class NodeTemplate(DB.Model): - __tablename__ = 'NodeTemplate' - - id = DB.Column(DB.String(36), primary_key=True) - name = DB.Column(DB.String(80), unique=True, nullable=False) - node_type_id = DB.Column(DB.String(36), DB.ForeignKey('NodeType.id'), - nullable=False) - flavor_id = DB.Column(DB.String(36), nullable=False) - - node_template_configs = DB.relationship('NodeTemplateConfig', - cascade="all,delete", - backref='node_template') - cluster_node_counts = DB.relationship('ClusterNodeCount', - cascade="all,delete", - backref='node_template') - nodes = DB.relationship('Node', cascade="all,delete", - backref='node_template') - - def __init__(self, name, node_type_id, flavor_id): - self.id = uuid.uuid4().hex - self.name = name - self.node_type_id = node_type_id - self.flavor_id = flavor_id - - def __repr__(self): - return '' % (self.name, self.node_type_id) - - -class Cluster(DB.Model): - __tablename__ = 'Cluster' - - id = DB.Column(DB.String(36), primary_key=True) - name = DB.Column(DB.String(80), unique=True, nullable=False) - base_image_id = DB.Column(DB.String(36), nullable=False) - status = DB.Column(DB.String(80)) - tenant_id = DB.Column(DB.String(36), nullable=False) - - nodes = DB.relationship('Node', cascade="all,delete", backref='cluster') - service_urls = DB.relationship('ServiceUrl', cascade="all,delete", - backref='cluster') - node_counts = DB.relationship('ClusterNodeCount', cascade="all,delete", - backref='cluster') - - # node_templates: [(node_template_id, count), ...] - - def __init__(self, name, base_image_id, tenant_id, status=None): - self.id = uuid.uuid4().hex - self.name = name - self.base_image_id = base_image_id - if not status: - status = 'Starting' - self.status = status - self.tenant_id = tenant_id - - def __repr__(self): - return '' % (self.name, self.status) - - -NODE_TYPE_NODE_PROCESS = DB.Table('NodeType_NodeProcess', DB.metadata, - DB.Column('node_type_id', DB.String(36), - DB.ForeignKey('NodeType.id')), - DB.Column('node_process_id', DB.String(36), - DB.ForeignKey('NodeProcess.id'))) - - -class NodeType(DB.Model): - __tablename__ = 'NodeType' - - id = DB.Column(DB.String(36), primary_key=True) - name = DB.Column(DB.String(80), unique=True, nullable=False) - processes = DB.relationship('NodeProcess', - cascade="all,delete", - secondary=NODE_TYPE_NODE_PROCESS, - backref='node_types') - node_templates = DB.relationship('NodeTemplate', cascade="all,delete", - backref='node_type') - - def __init__(self, name): - self.id = uuid.uuid4().hex - self.name = name - - def __repr__(self): - return '' % self.name - - -class NodeProcess(DB.Model): - __tablename__ = 'NodeProcess' - - id = DB.Column(DB.String(36), primary_key=True) - name = DB.Column(DB.String(80), unique=True, nullable=False) - node_process_properties = DB.relationship('NodeProcessProperty', - cascade="all,delete", - backref='node_process') - - def __init__(self, name): - self.id = uuid.uuid4().hex - self.name = name - - def __repr__(self): - return '' % self.name - - -class NodeProcessProperty(DB.Model): - __tablename__ = 'NodeProcessProperty' - __table_args__ = ( - DB.UniqueConstraint('node_process_id', 'name'), - ) - - id = DB.Column(DB.String(36), primary_key=True) - node_process_id = DB.Column(DB.String(36), DB.ForeignKey('NodeProcess.id')) - name = DB.Column(DB.String(80), nullable=False) - required = DB.Column(DB.Boolean, nullable=False) - default = DB.Column(DB.String(36)) - node_template_configs = DB.relationship('NodeTemplateConfig', - cascade="all,delete", - backref='node_process_property') - - def __init__(self, node_process_id, name, required=True, default=None): - self.id = uuid.uuid4().hex - self.node_process_id = node_process_id - self.name = name - self.required = required - self.default = default - - def __repr__(self): - return '' % self.name - - -class NodeTemplateConfig(DB.Model): - __tablename__ = 'NodeTemplateConfig' - __table_args__ = ( - DB.UniqueConstraint('node_template_id', 'node_process_property_id'), - ) - - id = DB.Column(DB.String(36), primary_key=True) - node_template_id = DB.Column( - DB.String(36), - DB.ForeignKey('NodeTemplate.id')) - node_process_property_id = DB.Column( - DB.String(36), - DB.ForeignKey('NodeProcessProperty.id')) - value = DB.Column(DB.String(36)) - - def __init__(self, node_template_id, node_process_property_id, value): - self.id = uuid.uuid4().hex - self.node_template_id = node_template_id - self.node_process_property_id = node_process_property_id - self.value = value - - def __repr__(self): - return '' \ - % (self.node_template_id, self.node_process_property_id, - self.value) - - -class ClusterNodeCount(DB.Model): - __tablename__ = 'ClusterNodeCount' - __table_args__ = ( - DB.UniqueConstraint('cluster_id', 'node_template_id'), - ) - - id = DB.Column(DB.String(36), primary_key=True) - cluster_id = DB.Column(DB.String(36), DB.ForeignKey('Cluster.id')) - node_template_id = DB.Column(DB.String(36), - DB.ForeignKey('NodeTemplate.id')) - count = DB.Column(DB.Integer, nullable=False) - - def __init__(self, cluster_id, node_template_id, count): - self.id = uuid.uuid4().hex - self.cluster_id = cluster_id - self.node_template_id = node_template_id - self.count = count - - def __repr__(self): - return '' \ - % (self.node_template_id, self.count) - - -class Node(DB.Model): - __tablename__ = 'Node' - - # do we need own id? - vm_id = DB.Column(DB.String(36), primary_key=True) - cluster_id = DB.Column(DB.String(36), DB.ForeignKey('Cluster.id')) - node_template_id = DB.Column(DB.String(36), - DB.ForeignKey('NodeTemplate.id')) - - def __init__(self, vm_id, cluster_id, node_template_id): - self.vm_id = vm_id - self.cluster_id = cluster_id - self.node_template_id = node_template_id - - def __repr__(self): - return '' % self.node_template.name - - -class ServiceUrl(DB.Model): - __tablename__ = 'ServiceUrl' - - id = DB.Column(DB.String(36), primary_key=True) - cluster_id = DB.Column(DB.String(36), DB.ForeignKey('Cluster.id')) - name = DB.Column(DB.String(80)) - url = DB.Column(DB.String(80), nullable=False) - - def __init__(self, cluster_id, name, url): - self.id = uuid.uuid4().hex - self.cluster_id = cluster_id - self.name = name - self.url = url - - def __repr__(self): - return '' % (self.name, self.url) diff --git a/savanna/storage/storage.py b/savanna/storage/storage.py deleted file mode 100644 index 39d960e0..00000000 --- a/savanna/storage/storage.py +++ /dev/null @@ -1,163 +0,0 @@ -# Copyright (c) 2013 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from savanna.storage.db import DB - -from savanna.storage.models import NodeTemplate, NodeProcess, Cluster, \ - ClusterNodeCount, NodeTemplateConfig, NodeType, NodeProcessProperty - - -## Node Template ops: - -def get_node_template(**args): - return NodeTemplate.query.filter_by(**args).first() - - -def get_node_templates(**args): - return NodeTemplate.query.filter_by(**args).all() - - -def is_node_template_associated(**args): - nt = get_node_template(**args) - return nt and (len(nt.nodes) or len(nt.cluster_node_counts)) - - -def create_node_template(name, node_type_id, flavor_id, configs): - """Creates new node templates. - - :param name: template name - :param node_type_id: node type - :param flavor_id: flavor - :param configs: dict of process->property->value - :return: created node template - """ - node_template = NodeTemplate(name, node_type_id, flavor_id) - DB.session.add(node_template) - for process_name in configs: - process = NodeProcess.query.filter_by(name=process_name).first() - conf = configs.get(process_name) - for prop in process.node_process_properties: - val = conf.get(prop.name, None) - if not val and prop.required: - if not prop.default: - raise RuntimeError('Template \'%s\', value missed ' - 'for required param: %s %s' - % (name, process.name, prop.name)) - val = prop.default - DB.session.add(NodeTemplateConfig(node_template.id, prop.id, val)) - DB.session.commit() - - return node_template - - -def terminate_node_template(**args): - template = get_node_template(**args) - if template: - DB.session.delete(template) - DB.session.commit() - return True - else: - return False - - -## Cluster ops: - -def get_cluster(**args): - return Cluster.query.filter_by(**args).first() - - -def get_clusters(**args): - return Cluster.query.filter_by(**args).all() - - -def create_cluster(name, base_image_id, tenant_id, templates): - """Creates new cluster. - - :param name: cluster name - :param base_image_id: base image - :param tenant_id: tenant - :param templates: dict of template->count - :return: created cluster - """ - cluster = Cluster(name, base_image_id, tenant_id) - DB.session.add(cluster) - for template in templates: - count = templates.get(template) - template_id = get_node_template(name=template).id - cnc = ClusterNodeCount(cluster.id, template_id, int(count)) - DB.session.add(cnc) - DB.session.commit() - - return cluster - - -def terminate_cluster(**args): - cluster = get_cluster(**args) - DB.session.delete(cluster) - DB.session.commit() - - -def update_cluster_status(new_status, **args): - cluster = Cluster.query.filter_by(**args).first() - cluster.status = new_status - DB.session.add(cluster) - DB.session.commit() - - return cluster - - -## Node Process ops: - -def create_node_process(name, properties): - """Creates new node process and node process properties. - - :param name: process name - :param properties: array of triples (name, required, default) - :return: created node process - """ - process = NodeProcess(name) - DB.session.add(process) - DB.session.commit() - for p in properties: - prop = NodeProcessProperty(process.id, p[0], p[1], p[2]) - DB.session.add(prop) - DB.session.commit() - - return process - - -## Node Type ops: - -def get_node_type(**args): - return NodeType.query.filter_by(**args).first() - - -def get_node_types(**args): - return NodeType.query.filter_by(**args).all() - - -def create_node_type(name, processes): - """Creates new node type using specified list of processes - - :param name: - :param processes: - :return: - """ - node_type = NodeType(name) - node_type.processes = processes - DB.session.add(node_type) - DB.session.commit() - - return node_type diff --git a/savanna/tests/integration/README.rst b/savanna/tests/integration/README.rst deleted file mode 100644 index bfb4c368..00000000 --- a/savanna/tests/integration/README.rst +++ /dev/null @@ -1,17 +0,0 @@ -Integration tests for Savanna project -===================================== - -How to run ----------- - -Create config file for integration tests - `/savanna/tests/integration/config.py`. -You can take a look at the sample config file - `/savanna/tests/integration/config.py.sample`. -All values used in a sample config file are defaults, so, if their are applicable for your -environment than you can skip config file creation. - -To run integration tests you should use the corresponding tox env: `tox -e integration`. - -Contents --------- - -TBD diff --git a/savanna/tests/integration/config.py.sample b/savanna/tests/integration/config.py.sample deleted file mode 100644 index 69b20a17..00000000 --- a/savanna/tests/integration/config.py.sample +++ /dev/null @@ -1,24 +0,0 @@ -OS_USERNAME = 'admin' # username for nova -OS_PASSWORD = 'password' # password for nova -OS_TENANT_NAME = 'admin' -OS_AUTH_URL = 'http://192.168.1.1:35357/v2.0/' # URL for keystone - -SAVANNA_HOST = '192.168.1.1' # IP for Savanna API -SAVANNA_PORT = '8080' # port for Savanna API - -IMAGE_ID = '42' # ID for instance image -FLAVOR_ID = 'abc' - -IP_PREFIX = '172.' # prefix for IP address which is used for ssh connect to worker nodes - -NODE_USERNAME = 'username' # username for master node -NODE_PASSWORD = 'password' # password for master node - -CLUSTER_NAME_CRUD = 'cluster-name-crud' # cluster name for crud operations -CLUSTER_NAME_HADOOP = 'cluster-name-hadoop' # cluster name for hadoop testing - -TIMEOUT = 15 # cluster creation timeout (in minutes) - -HADOOP_VERSION = '1.1.1' -HADOOP_DIRECTORY = '/usr/share/hadoop' -HADOOP_LOG_DIRECTORY = '/mnt/log/hadoop/hadoop/userlogs' diff --git a/savanna/tests/integration/db.py b/savanna/tests/integration/db.py deleted file mode 100644 index f24f6b2e..00000000 --- a/savanna/tests/integration/db.py +++ /dev/null @@ -1,244 +0,0 @@ -# Copyright (c) 2013 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import eventlet -import json -from keystoneclient.v2_0 import Client as keystone_client -import requests -import savanna.tests.integration.parameters as param -import unittest - - -class ITestCase(unittest.TestCase): - - def setUp(self): - self.port = param.SAVANNA_PORT - self.host = param.SAVANNA_HOST - - self.maxDiff = None - - self.baseurl = 'http://' + self.host + ':' + self.port - - self.keystone = keystone_client( - username=param.OS_USERNAME, - password=param.OS_PASSWORD, - tenant_name=param.OS_TENANT_NAME, - auth_url=param.OS_AUTH_URL - ) - - self.tenant = self.keystone.tenant_id - self.token = self.keystone.auth_token - - self.flavor_id = param.FLAVOR_ID - self.image_id = param.IMAGE_ID - - self.url_nt = '/v0.2/%s/node-templates' % self.tenant - self.url_nt_with_slash = '/v0.2/%s/node-templates/' % self.tenant - self.url_cluster = '/v0.2/%s/clusters' % self.tenant - self.url_cl_with_slash = '/v0.2/%s/clusters/' % self.tenant - -#----------------------CRUD_comands-------------------------------------------- - - def post(self, url, body): - URL = self.baseurl + url - resp = requests.post(URL, data=body, headers={ - 'x-auth-token': self.token, 'Content-Type': 'application/json'}) - data = json.loads(resp.content) if resp.status_code == 202 \ - else resp.content - print('URL = %s\ndata = %s\nresponse = %s\ndata = %s\n' - % (URL, body, resp.status_code, data)) - return resp - - def put(self, url, body): - URL = self.baseurl + url - resp = requests.put(URL, data=body, headers={ - 'x-auth-token': self.token, 'Content-Type': 'application/json'}) - data = json.loads(resp.content) - print('URL = %s\ndata = %s\nresponse = %s\ndata = %s\n' - % (URL, body, resp.status_code, data)) - return resp - - def get(self, url, printing): - URL = self.baseurl + url - resp = requests.get(URL, headers={'x-auth-token': self.token}) - if printing: - print('URL = %s\nresponse = %s\n' % (URL, resp.status_code)) - if resp.status_code != 200: - data = json.loads(resp.content) - print('data= %s\n') % data - return resp - - def delete(self, url): - URL = self.baseurl + url - resp = requests.delete(URL, headers={'x-auth-token': self.token}) - print('URL = %s\nresponse = %s\n' % (URL, resp.status_code)) - if resp.status_code != 204: - data = json.loads(resp.content) - print('data= %s\n') % data - return resp - - def _post_object(self, url, body, code): - post = self.post(url, json.dumps(body)) - self.assertEquals(post.status_code, code) - data = json.loads(post.content) - return data - - def _get_object(self, url, obj_id, code, printing=False): - rv = self.get(url + obj_id, printing) - self.assertEquals(rv.status_code, code) - data = json.loads(rv.content) - return data - - def _del_object(self, url, obj_id, code): - rv = self.delete(url + obj_id) - self.assertEquals(rv.status_code, code) - if rv.status_code != 204: - data = json.loads(rv.content) - return data - else: - code = self.delete(url + obj_id).status_code - while code != 404: - eventlet.sleep(1) - code = self.delete(url + obj_id).status_code - -#----------------------other_commands------------------------------------------ - - def _get_body_nt(self, name, nt_type, hs1, hs2): - node = 'name' if nt_type in ['JT+NN', 'NN'] else 'data' - tracker = 'job' if nt_type in ['JT+NN', 'JT'] else 'task' - processes_name = nt_type - nt = { - u'name': u'%s.%s' % (name, param.FLAVOR_ID), - u'%s_node' % node: {u'heap_size': u'%d' % hs1}, - u'%s_tracker' % tracker: {u'heap_size': u'%d' % hs2}, - u'node_type': { - u'processes': [u'%s_tracker' % tracker, - u'%s_node' % node], - u'name': u'%s' % processes_name}, - u'flavor_id': u'%s' % self.flavor_id - } - if nt_type == 'NN': - del nt[u'%s_tracker' % tracker] - nt[u'node_type'][u'processes'] = [u'%s_node' % node] - elif nt_type == 'JT': - del nt[u'%s_node' % node] - nt[u'node_type'][u'processes'] = [u'%s_tracker' % tracker] - return nt - - def _get_body_cluster(self, name, master_name, worker_name, node_number): - return { - u'status': u'Starting', - u'service_urls': {}, - u'name': u'%s' % name, - u'base_image_id': u'%s' % self.image_id, - u'node_templates': - { - u'%s.%s' % (master_name, param.FLAVOR_ID): 1, - u'%s.%s' % (worker_name, param.FLAVOR_ID): node_number - }, - u'nodes': [] - } - - def change_field_nt(self, data, old_field, new_field): - val = data['node_template'][old_field] - del data['node_template'][old_field] - data['node_template'][new_field] = val - return data - - def make_nt(self, nt_name, node_type, jt_heap_size, nn_heap_size): - nt = dict( - node_template=dict( - name='%s.%s' % (nt_name, param.FLAVOR_ID), - node_type='JT+NN', - flavor_id=self.flavor_id, - job_tracker={ - 'heap_size': '%d' % jt_heap_size - }, - name_node={ - 'heap_size': '%d' % nn_heap_size - } - )) - if node_type == 'TT+DN': - nt['node_template']['node_type'] = 'TT+DN' - nt = self.change_field_nt(nt, 'job_tracker', 'task_tracker') - nt = self.change_field_nt(nt, 'name_node', 'data_node') - elif node_type == 'NN': - nt['node_template']['node_type'] = 'NN' - del nt['node_template']['job_tracker'] - elif node_type == 'JT': - nt['node_template']['node_type'] = 'JT' - del nt['node_template']['name_node'] - return nt - - def make_cluster_body(self, cluster_name, name_master_node, - name_worker_node, number_workers): - body = dict( - cluster=dict( - name=cluster_name, - base_image_id=self.image_id, - node_templates={ - '%s.%s' % (name_master_node, param.FLAVOR_ID): 1, - '%s.%s' % - (name_worker_node, param.FLAVOR_ID): number_workers - } - )) - return body - - def delete_node_template(self, data): - data = data['node_template'] - object_id = data.pop(u'id') - self._del_object(self.url_nt_with_slash, object_id, 204) - - def _crud_object(self, body, get_body, url): - data = self._post_object(url, body, 202) - get_url = None - object_id = None - try: - obj = 'node_template' if url == self.url_nt else 'cluster' - get_url = self.url_nt_with_slash if url == self.url_nt \ - else self.url_cl_with_slash - data = data['%s' % obj] - object_id = data.pop(u'id') - self.assertEquals(data, get_body) - get_data = self._get_object(get_url, object_id, 200) - get_data = get_data['%s' % obj] - del get_data[u'id'] - if obj == 'cluster': - self._await_cluster_active( - get_body, get_data, get_url, object_id) - except Exception as e: - self.fail('failure:' + str(e)) - finally: - self._del_object(get_url, object_id, 204) - return object_id - - def _await_cluster_active(self, get_body, get_data, get_url, object_id): - get_body[u'status'] = u'Active' - del get_body[u'service_urls'] - del get_body[u'nodes'] - i = 1 - while get_data[u'status'] != u'Active': - if i > int(param.TIMEOUT) * 6: - self.fail( - 'cluster not Starting -> Active, passed %d minutes' - % param.TIMEOUT) - get_data = self._get_object(get_url, object_id, 200) - get_data = get_data['cluster'] - del get_data[u'id'] - del get_data[u'service_urls'] - del get_data[u'nodes'] - eventlet.sleep(10) - i += 1 - self.assertEquals(get_data, get_body) diff --git a/savanna/tests/integration/parameters.py b/savanna/tests/integration/parameters.py deleted file mode 100644 index 88c13a37..00000000 --- a/savanna/tests/integration/parameters.py +++ /dev/null @@ -1,33 +0,0 @@ -import savanna.openstack.common.importutils as importutils - -_CONF = importutils.try_import('savanna.tests.integration.config') - - -def _get_conf(key, default): - return getattr(_CONF, key) if _CONF and hasattr(_CONF, key) else default - -OS_USERNAME = _get_conf('OS_USERNAME', 'admin') -OS_PASSWORD = _get_conf('OS_PASSWORD', 'password') -OS_TENANT_NAME = _get_conf('OS_TENANT_NAME', 'admin') -OS_AUTH_URL = _get_conf('OS_AUTH_URL', 'http://localhost:35357/v2.0/') - -SAVANNA_HOST = _get_conf('SAVANNA_HOST', '192.168.1.1') -SAVANNA_PORT = _get_conf('SAVANNA_PORT', '8080') - -IMAGE_ID = _get_conf('IMAGE_ID', '42') -FLAVOR_ID = _get_conf('FLAVOR_ID', 'abc') - -NODE_USERNAME = _get_conf('NODE_USERNAME', 'username') -NODE_PASSWORD = _get_conf('NODE_PASSWORD', 'password') - -CLUSTER_NAME_CRUD = _get_conf('CLUSTER_NAME_CRUD', 'cluster-crud') -CLUSTER_NAME_HADOOP = _get_conf('CLUSTER_NAME_HADOOP', 'cluster-hadoop') - -IP_PREFIX = _get_conf('IP_PREFIX', '10.') - -TIMEOUT = _get_conf('TIMEOUT', '15') - -HADOOP_VERSION = _get_conf('HADOOP_VERSION', '1.1.1') -HADOOP_DIRECTORY = _get_conf('HADOOP_DIRECTORY', '/usr/share/hadoop') -HADOOP_LOG_DIRECTORY = _get_conf('HADOOP_LOG_DIRECTORY', - '/mnt/log/hadoop/hadoop/userlogs') diff --git a/savanna/tests/integration/script.sh b/savanna/tests/integration/script.sh deleted file mode 100644 index af6073c1..00000000 --- a/savanna/tests/integration/script.sh +++ /dev/null @@ -1,170 +0,0 @@ -#!/bin/bash -#touch script.sh && chmod +x script.sh && vim script.sh - -dir=/outputTestMapReduce -log=$dir/log.txt - -case $1 in - mr) - FUNC="map_reduce" - ;; - pi) - FUNC="run_pi_job" - ;; - gn) - FUNC="get_job_name" - ;; - lt) - FUNC="get_list_active_trackers" - ;; - ld) - FUNC="get_list_active_datanodes" - ;; - ed) - FUNC=" check_exist_directory" - ;; -esac - -shift - -until [ -z $1 ] -do - if [ "$1" = "-nc" ] - then - NODE_COUNT="$2" - shift - fi - - if [ "$1" = "-jn" ] - then - JOB_NAME="$2" - shift - fi - - if [ "$1" = "-hv" ] - then - HADOOP_VERSION="$2" - shift - fi - - if [ "$1" = "-hd" ] - then - HADOOP_DIRECTORY="$2" - shift - fi - - if [ "$1" = "-hld" ] - then - HADOOP_LOG_DIRECTORY="$2" - shift - fi - - - shift -done - -f_var_check() { - case "$1" in - v_node_count) - if [ -z "$NODE_COUNT" ] - then - echo "count_of_node_not_specified" - exit 0 - fi - ;; - v_job_name) - if [ -z "$JOB_NAME" ] - then - echo "job_name_not_specified" - exit 0 - fi - ;; - v_hadoop_version) - if [ -z "$HADOOP_VERSION" ] - then - echo "hadoop_version_not_specified" - exit 0 - fi - ;; - v_hadoop_directory) - if [ -z "$HADOOP_DIRECTORY" ] - then - echo "hadoop_directory_not_specified" - exit 0 - fi - ;; - v_hadoop_log_directory) - if [ -z "$HADOOP_LOG_DIRECTORY" ] - then - echo "hadoop_log_directory_not_specified" - exit 0 - fi - ;; - esac -} - -f_create_log_dir() { -rm -r $dir 2>/dev/null -mkdir $dir -chmod -R 777 $dir -touch $log -} - -map_reduce() { -f_create_log_dir -f_var_check v_hadoop_version -f_var_check v_hadoop_directory -echo "[------ dpkg------]">>$log -echo `dpkg --get-selections | grep hadoop` >>$log -echo "[------jps------]">>$log -echo `jps | grep -v Jps` >>$log -echo "[------netstat------]">>$log -echo `sudo netstat -plten | grep java` &>>$log -echo "[------test for hdfs------]">>$log -echo `dmesg > $dir/input` 2>>$log -su -c "hadoop dfs -ls /" hadoop && -su -c "hadoop dfs -mkdir /test" hadoop && -su -c "hadoop dfs -copyFromLocal $dir/input /test/mydata" hadoop 2>>$log -echo "[------start job------]">>$log && -su -c "cd $HADOOP_DIRECTORY && hadoop jar hadoop-examples-$HADOOP_VERSION.jar wordcount /test/mydata /test/output" hadoop 2>>$log && -su -c "hadoop dfs -copyToLocal /test/output/ $dir/out/" hadoop 2>>$log && -su -c "hadoop dfs -rmr /test" hadoop 2>>$log -} - -run_pi_job() { -f_var_check v_node_count -f_var_check v_hadoop_version -f_var_check v_hadoop_directory -f_create_log_dir -directory=/usr/share/hadoop -logdir=/var/log/hadoop/hadoop/userlogs -su -c "cd $HADOOP_DIRECTORY && hadoop jar hadoop-examples-$HADOOP_VERSION.jar pi $[$NODE_COUNT*10] 1000" hadoop 2>>$log -} - -get_job_name() { -f_var_check v_hadoop_directory -su -c "cd $HADOOP_DIRECTORY && hadoop job -list all | tail -n1" hadoop | awk '{print $1}' 2>>$log -} - -get_list_active_trackers() { -f_create_log_dir -f_var_check v_hadoop_directory -sleep 30 && -su -c "cd $HADOOP_DIRECTORY && hadoop job -list-active-trackers" hadoop | wc -l 2>>$log -} - -get_list_active_datanodes() { -f_create_log_dir -f_var_check v_hadoop_directory -su -c "hadoop dfsadmin -report" hadoop | grep "Datanodes available:.*" | awk '{print $3}' 2>>$log -} - -check_exist_directory() { -f_var_check v_job_name -f_var_check v_hadoop_log_directory -if ! [ -d $HADOOP_LOG_DIRECTORY/$JOB_NAME ]; -then echo "directory_not_found" && exit 1 -fi -} - -$FUNC \ No newline at end of file diff --git a/savanna/tests/integration/test_clusters.py b/savanna/tests/integration/test_clusters.py deleted file mode 100644 index e0f5ba2b..00000000 --- a/savanna/tests/integration/test_clusters.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright (c) 2013 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from savanna.tests.integration.db import ITestCase -import savanna.tests.integration.parameters as param -from telnetlib import Telnet - - -class ITestClusterApi(ITestCase): - - def setUp(self): - super(ITestClusterApi, self).setUp() - Telnet(self.host, self.port) - - def test_cluster_crud_operations(self): - nt_body = self.make_nt('master-node', 'JT+NN', 1234, 2345) - data_nt_master = self._post_object(self.url_nt, nt_body, 202) - - nt_body = self.make_nt('worker-node', 'TT+DN', 1234, 2345) - data_nt_worker = self._post_object(self.url_nt, nt_body, 202) - - try: - cluster_body = self.make_cluster_body( - param.CLUSTER_NAME_CRUD, 'master-node', 'worker-node', 2) - get_cluster_body = self._get_body_cluster( - param.CLUSTER_NAME_CRUD, 'master-node', 'worker-node', 2) - - self._crud_object(cluster_body, get_cluster_body, self.url_cluster) - - finally: - self.delete_node_template(data_nt_master) - self.delete_node_template(data_nt_worker) diff --git a/savanna/tests/integration/test_hadoop.py b/savanna/tests/integration/test_hadoop.py deleted file mode 100644 index 0603a709..00000000 --- a/savanna/tests/integration/test_hadoop.py +++ /dev/null @@ -1,235 +0,0 @@ -# Copyright (c) 2013 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -from novaclient import client as nc -from os import getcwd -import paramiko -from re import search -from savanna.service.cluster_ops import _setup_ssh_connection -from savanna.tests.integration.db import ITestCase -import savanna.tests.integration.parameters as param -from telnetlib import Telnet - - -def _open_transport_chanel(transport): - transport.connect( - username=param.NODE_USERNAME, password=param.NODE_PASSWORD) - return paramiko.SFTPClient.from_transport(transport) - - -def _execute_transfer_to_node(host, locfile, nodefile): - try: - transport = paramiko.Transport(host) - sftp = _open_transport_chanel(transport) - sftp.put(locfile, nodefile) - - finally: - sftp.close() - transport.close() - - -def _execute_transfer_from_node(host, nodefile, localfile): - try: - transport = paramiko.Transport(host) - sftp = _open_transport_chanel(transport) - sftp.get(nodefile, localfile) - - finally: - sftp.close() - transport.close() - - -def _open_channel_and_execute(ssh, cmd, print_output): - chan = ssh.get_transport().open_session() - chan.exec_command(cmd) - stdout = chan.makefile('rb', -1) - chan.set_combine_stderr(True) - if print_output: - return stdout.read() - return chan.recv_exit_status() - - -def _execute_command_on_node(host, cmd, print_output=False): - ssh = paramiko.SSHClient() - try: - _setup_ssh_connection(host, ssh) - return _open_channel_and_execute(ssh, cmd, print_output) - finally: - ssh.close() - - -def _transfer_script_to_node(host, directory): - _execute_transfer_to_node( - str(host), '%s/integration/script.sh' % directory, 'script.sh') - _execute_command_on_node(str(host), 'chmod 777 script.sh') - - -class TestHadoop(ITestCase): - - def setUp(self): - super(TestHadoop, self).setUp() - Telnet(self.host, self.port) - - def _hadoop_testing(self, cluster_name, nt_name_master, - nt_name_worker, number_workers): - object_id = None - cluster_body = self.make_cluster_body( - cluster_name, nt_name_master, nt_name_worker, number_workers) - data = self._post_object(self.url_cluster, cluster_body, 202) - - try: - data = data['cluster'] - object_id = data.pop(u'id') - get_body = self._get_body_cluster( - cluster_name, nt_name_master, nt_name_worker, number_workers) - get_data = self._get_object(self.url_cl_with_slash, object_id, 200) - get_data = get_data['cluster'] - del get_data[u'id'] - self._await_cluster_active( - get_body, get_data, self.url_cl_with_slash, object_id) - - get_data = self._get_object( - self.url_cl_with_slash, object_id, 200, True) - get_data = get_data['cluster'] - namenode = get_data[u'service_urls'][u'namenode'] - jobtracker = get_data[u'service_urls'][u'jobtracker'] - nodes = get_data[u'nodes'] - worker_ips = [] - nova = nc.Client(version='2', - username=param.OS_USERNAME, - api_key=param.OS_PASSWORD, - auth_url=param.OS_AUTH_URL, - project_id=param.OS_TENANT_NAME) - for node in nodes: - if node[u'node_template'][u'name'] == '%s.%s'\ - % (nt_name_worker, param.FLAVOR_ID): - v = nova.servers.get('%s' % node[u'vm_id']) - for network, address in v.addresses.items(): - instance_ips = json.dumps(address) - instance_ips = json.loads(instance_ips) - for instance_ip in instance_ips: - if instance_ip[u'addr'][:len(param.IP_PREFIX)]\ - == param.IP_PREFIX: - worker_ips.append(instance_ip[u'addr']) - - p = '(?:http.*://)?(?P[^:/ ]+).?(?P[0-9]*).*' - m = search(p, namenode) - t = search(p, jobtracker) - - namenode_ip = m.group('host') - namenode_port = m.group('port') - jobtracker_ip = t.group('host') - jobtracker_port = t.group('port') - - try: - Telnet(str(namenode_ip), str(namenode_port)) - Telnet(str(jobtracker_ip), str(jobtracker_port)) - except Exception as e: - self.fail('telnet nn or jt is failure: ' + e.message) - - this_dir = getcwd() - - try: - _transfer_script_to_node(namenode_ip, this_dir) - for worker_ip in worker_ips: - _transfer_script_to_node(worker_ip, this_dir) - except Exception as e: - self.fail('failure in transfer script: ' + e.message) - - try: - self.assertEqual(int(_execute_command_on_node( - namenode_ip, './script.sh lt -hd %s' - % param.HADOOP_DIRECTORY, True)), - number_workers) - #TODO(vrovachev) delete sleep from script after fix bug 1183387 - except Exception as e: - self.fail('compare number active trackers is failure: ' - + e.message) - - try: - self.assertEqual(int(_execute_command_on_node( - namenode_ip, './script.sh ld -hd %s' - % param.HADOOP_DIRECTORY, True)), - number_workers) - except Exception as e: - self.fail('compare number active datanodes is failure: ' - + e.message) - - try: - _execute_command_on_node( - namenode_ip, './script.sh pi -nc %s -hv %s -hd %s' - % (number_workers, param.HADOOP_VERSION, - param.HADOOP_DIRECTORY)) - except Exception as e: - _execute_transfer_from_node( - namenode_ip, - '/outputTestMapReduce/log.txt', '%s/errorLog' % this_dir) - self.fail( - 'run pi script or get run in active trackers is failure: ' - + e.message) - - try: - job_name = _execute_command_on_node( - namenode_ip, './script.sh gn -hd %s' - % param.HADOOP_DIRECTORY, True) - except Exception as e: - self.fail('fail in get job name: ' + e.message) - - try: - for worker_ip in worker_ips: - self.assertEquals( - _execute_command_on_node( - worker_ip, - './script.sh ed -jn %s -hld %s' - % (job_name[:-1], param.HADOOP_LOG_DIRECTORY)), 0) - except Exception as e: - self.fail('fail in check run job in worker nodes: ' - + e.message) - - try: - self.assertEquals( - _execute_command_on_node( - namenode_ip, './script.sh mr -hv %s -hd %s' - % (param.HADOOP_VERSION, - param.HADOOP_DIRECTORY)), 0) - except Exception as e: - _execute_transfer_from_node( - namenode_ip, - '/outputTestMapReduce/log.txt', '%s/errorLog' % this_dir) - self.fail('run hdfs script is failure: ' + e.message) - except Exception as e: - self.fail(e.message) - - finally: - self._del_object(self.url_cl_with_slash, object_id, 204) - - def test_hadoop_single_master(self): - data_nt_master = self._post_object( - self.url_nt, self.make_nt('master_node', 'JT+NN', - 1234, 1234), 202) - data_nt_worker = self._post_object( - self.url_nt, self.make_nt('worker_node', 'TT+DN', - 1234, 1234), 202) - - try: - self._hadoop_testing( - param.CLUSTER_NAME_HADOOP, 'master_node', 'worker_node', 2) - except Exception as e: - self.fail(e.message) - - finally: - self.delete_node_template(data_nt_master) - self.delete_node_template(data_nt_worker) diff --git a/savanna/tests/integration/test_node_templates.py b/savanna/tests/integration/test_node_templates.py deleted file mode 100644 index 23012183..00000000 --- a/savanna/tests/integration/test_node_templates.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (c) 2013 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from savanna.tests.integration.db import ITestCase -from telnetlib import Telnet - - -class ITestNodeTemplateApi(ITestCase): - - def setUp(self): - super(ITestNodeTemplateApi, self).setUp() - Telnet(self.host, self.port) - - def test_crud_nt_jtnn(self): - nt_jtnn = self.make_nt('jtnn', 'JT+NN', 1024, 1024) - get_jtnn = self._get_body_nt('jtnn', 'JT+NN', 1024, 1024) - - self._crud_object(nt_jtnn, get_jtnn, self.url_nt) - - def test_crud_nt_ttdn(self): - nt_ttdn = self.make_nt('ttdn', 'TT+DN', 1024, 1024) - get_ttdn = self._get_body_nt('ttdn', 'TT+DN', 1024, 1024) - - self._crud_object(nt_ttdn, get_ttdn, self.url_nt) - - def test_crud_nt_nn(self): - nt_nn = self.make_nt('nn', 'NN', 1024, 1024) - get_nn = self._get_body_nt('nn', 'NN', 1024, 1024) - - self._crud_object(nt_nn, get_nn, self.url_nt) - - def test_crud_nt_jt(self): - nt_jt = self.make_nt('jt', 'JT', 1024, 1024) - get_jt = self._get_body_nt('jt', 'JT', 1024, 1024) - - self._crud_object(nt_jt, get_jt, self.url_nt) diff --git a/savanna/tests/unit/base.py b/savanna/tests/unit/base.py deleted file mode 100644 index 0c7c498d..00000000 --- a/savanna/tests/unit/base.py +++ /dev/null @@ -1,190 +0,0 @@ -# Copyright (c) 2013 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import tempfile -import unittest -import uuid - -import eventlet -from oslo.config import cfg - -import savanna.main -from savanna.openstack.common import log as logging -from savanna.service import api -from savanna.storage.db import DB -from savanna.storage.defaults import setup_defaults -from savanna.storage.models import Node, NodeTemplate -from savanna.utils.openstack import nova -from savanna.utils import scheduler - -LOG = logging.getLogger(__name__) - - -def _stub_vm_creation_job(template_id): - template = NodeTemplate.query.filter_by(id=template_id).first() - eventlet.sleep(2) - return 'ip-address', uuid.uuid4().hex, template.id - - -def _stub_launch_cluster(headers, cluster): - LOG.debug('stub launch_cluster called with %s, %s', headers, cluster) - pile = eventlet.GreenPile(scheduler.POOL) - - for elem in cluster.node_counts: - node_count = elem.count - for _ in xrange(0, node_count): - pile.spawn(_stub_vm_creation_job, elem.node_template_id) - - for (ip, vm_id, elem) in pile: - DB.session.add(Node(vm_id, cluster.id, elem)) - LOG.debug("VM '%s/%s/%s' created", ip, vm_id, elem) - - return True - - -def _stub_stop_cluster(headers, cluster): - LOG.debug("stub stop_cluster called with %s, %s", headers, cluster) - - -def _stub_auth_token(*args, **kwargs): - LOG.debug('stub token filter called with %s, %s', args, kwargs) - - def _filter(app): - def _handler(env, start_response): - env['HTTP_X_TENANT_ID'] = 'tenant-id-1' - return app(env, start_response) - - return _handler - - return _filter - - -def _stub_auth_valid(*args, **kwargs): - LOG.debug('stub token validation called with %s, %s', args, kwargs) - - def _filter(app): - def _handler(env, start_response): - return app(env, start_response) - - return _handler - - return _filter - - -def _stub_get_flavors(headers): - LOG.debug('Stub get_flavors called with %s', headers) - return [u'test_flavor', u'test_flavor_2'] - - -def _stub_get_images(headers): - LOG.debug('Stub get_images called with %s', headers) - return [u'base-image-id', u'base-image-id_2'] - - -def _stub_get_limits(headers): - limits = dict(maxTotalCores=100, - maxTotalRAMSize=51200, - maxTotalInstances=100, - totalCoresUsed=0, - totalRAMUsed=0, - totalInstancesUsed=0) - - LOG.debug('Stub get_limits called with headers %s and limits %s', - headers, limits) - - return limits - - -class StubFlavor: - def __init__(self, name, vcpus, ram): - self.name = name - self.vcpus = int(vcpus) - self.ram = int(ram) - - -def _stub_get_flavor(headers, **kwargs): - return StubFlavor('test_flavor', 1, 512) - - -CONF = cfg.CONF -CONF.import_opt('debug', 'savanna.openstack.common.log') -CONF.import_opt('allow_cluster_ops', 'savanna.config') -CONF.import_opt('database_uri', 'savanna.storage.db', group='sqlalchemy') -CONF.import_opt('echo', 'savanna.storage.db', group='sqlalchemy') - - -class SavannaTestCase(unittest.TestCase): - def setUp(self): - self.db_fd, self.db_path = tempfile.mkstemp() - self.maxDiff = 10000 - - # override configs - CONF.set_override('debug', True) - CONF.set_override('allow_cluster_ops', True) # stub process - CONF.set_override('database_uri', 'sqlite:///' + self.db_path, - group='sqlalchemy') - CONF.set_override('echo', False, group='sqlalchemy') - - # store functions that will be stubbed - self._prev_auth_token = savanna.main.auth_token - self._prev_auth_valid = savanna.main.auth_valid - self._prev_cluster_launch = api.cluster_ops.launch_cluster - self._prev_cluster_stop = api.cluster_ops.stop_cluster - self._prev_get_flavors = nova.get_flavors - self._prev_get_images = nova.get_images - self._prev_get_limits = nova.get_limits - self._prev_get_flavor = nova.get_flavor - - # stub functions - savanna.main.auth_token = _stub_auth_token - savanna.main.auth_valid = _stub_auth_valid - api.cluster_ops.launch_cluster = _stub_launch_cluster - api.cluster_ops.stop_cluster = _stub_stop_cluster - nova.get_flavors = _stub_get_flavors - nova.get_images = _stub_get_images - nova.get_limits = _stub_get_limits - nova.get_flavor = _stub_get_flavor - - app = savanna.main.make_app() - - DB.drop_all() - DB.create_all() - setup_defaults(True, True) - - LOG.debug('Test db path: %s', self.db_path) - LOG.debug('Test app.config: %s', app.config) - - self.app = app.test_client() - - def tearDown(self): - # unstub functions - savanna.main.auth_token = self._prev_auth_token - savanna.main.auth_valid = self._prev_auth_valid - api.cluster_ops.launch_cluster = self._prev_cluster_launch - api.cluster_ops.stop_cluster = self._prev_cluster_stop - nova.get_flavors = self._prev_get_flavors - nova.get_images = self._prev_get_images - nova.get_limits = self._prev_get_limits - nova.get_flavor = self._prev_get_flavor - - os.close(self.db_fd) - os.unlink(self.db_path) - - # place back default configs - CONF.clear_override('debug') - CONF.clear_override('allow_cluster_ops') - CONF.clear_override('database_uri', group='sqlalchemy') - CONF.clear_override('echo', group='sqlalchemy') diff --git a/savanna/tests/unit/db/__init__.py b/savanna/tests/unit/db/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/savanna/tests/unit/db/models/__init__.py b/savanna/tests/unit/db/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/savanna/tests/unit/db/models/base.py b/savanna/tests/unit/db/models/base.py new file mode 100644 index 00000000..7ada9c56 --- /dev/null +++ b/savanna/tests/unit/db/models/base.py @@ -0,0 +1,63 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import os +import tempfile +import unittest2 + +from savanna.context import Context +from savanna.context import set_ctx +from savanna.db.api import clear_db +from savanna.db.api import configure_db +from savanna.openstack.common.db.sqlalchemy import session +from savanna.openstack.common import timeutils +from savanna.openstack.common import uuidutils + + +class ModelTestCase(unittest2.TestCase): + def setUp(self): + set_ctx(Context('test_user', 'test_tenant', 'test_auth_token', {})) + self.db_fd, self.db_path = tempfile.mkstemp() + session.set_defaults('sqlite:///' + self.db_path, self.db_path) + configure_db() + + def tearDown(self): + clear_db() + os.close(self.db_fd) + os.unlink(self.db_path) + set_ctx(None) + + def assertIsValidModelObject(self, res): + self.assertIsNotNone(res) + self.assertIsNotNone(res.dict) + self.assertTrue(uuidutils.is_uuid_like(res.id)) + + # check created/updated + delta = datetime.timedelta(seconds=2) + now = timeutils.utcnow() + + self.assertAlmostEqual(res.created, now, delta=delta) + self.assertAlmostEqual(res.updated, now, delta=delta) + + def get_clean_dict(self, res): + res_dict = res.dict + del res_dict['created'] + del res_dict['updated'] + del res_dict['id'] + if 'tenant_id' in res_dict: + del res_dict['tenant_id'] + + return res_dict diff --git a/savanna/tests/unit/db/models/test_clusters.py b/savanna/tests/unit/db/models/test_clusters.py new file mode 100644 index 00000000..70aa08c8 --- /dev/null +++ b/savanna/tests/unit/db/models/test_clusters.py @@ -0,0 +1,45 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from savanna.context import ctx +import savanna.db.models as m +from savanna.tests.unit.db.models.base import ModelTestCase + + +class ClusterModelTest(ModelTestCase): + def testCreateCluster(self): + session = ctx().session + with session.begin(): + c = m.Cluster('c-1', 't-1', 'p-1', 'hv-1') + session.add(c) + + with session.begin(): + res = session.query(m.Cluster).filter_by().first() + + self.assertIsValidModelObject(res) + + def testCreateClusterFromDict(self): + c = m.Cluster('c-1', 't-1', 'p-1', 'hv-1') + c_dict = c.dict + del c_dict['created'] + del c_dict['updated'] + del c_dict['id'] + del c_dict['node_groups'] + + c_dict.update({ + 'tenant_id': 't-1' + }) + self.assertEqual(self.get_clean_dict(c), + self.get_clean_dict(m.Cluster(**c_dict))) diff --git a/savanna/tests/unit/db/models/test_templates.py b/savanna/tests/unit/db/models/test_templates.py new file mode 100644 index 00000000..7c1207fa --- /dev/null +++ b/savanna/tests/unit/db/models/test_templates.py @@ -0,0 +1,98 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from savanna.context import ctx + +import savanna.db.models as m +from savanna.tests.unit.db.models.base import ModelTestCase + + +SAMPLE_CONFIGS = { + 'a': 'av', + 'b': 123, + 'c': [1, '2', u"3"] +} + + +class TemplatesModelTest(ModelTestCase): + def testCreateNodeGroupTemplate(self): + session = ctx().session + with session.begin(): + ngt = m.NodeGroupTemplate('ngt-1', 't-1', 'f-1', 'p-1', 'hv-1', + ['np-1', 'np-2'], SAMPLE_CONFIGS, "d") + session.add(ngt) + + res = session.query(m.NodeGroupTemplate).filter_by().first() + + self.assertIsValidModelObject(res) + self.assertEquals(['np-1', 'np-2'], res.node_processes) + self.assertEquals(SAMPLE_CONFIGS, res.node_configs) + + res_dict = self.get_clean_dict(res) + + self.assertEqual(res_dict, { + 'description': 'd', + 'flavor_id': 'f-1', + 'hadoop_version': 'hv-1', + 'name': 'ngt-1', + 'node_configs': SAMPLE_CONFIGS, + 'node_processes': ['np-1', 'np-2'], + 'plugin_name': 'p-1' + }) + + def testCreateClusterTemplate(self): + session = ctx().session + with session.begin(): + c = m.ClusterTemplate('c-1', 't-1', 'p-1', 'hv-1', SAMPLE_CONFIGS, + "d") + session.add(c) + + res = session.query(m.ClusterTemplate).filter_by().first() + self.assertIsValidModelObject(res) + self.assertEqual(SAMPLE_CONFIGS, res.cluster_configs) + + res_dict = self.get_clean_dict(res) + + self.assertEqual(res_dict, { + 'cluster_configs': SAMPLE_CONFIGS, + 'description': 'd', + 'hadoop_version': 'hv-1', + 'name': 'c-1', + 'plugin_name': 'p-1', + 'node_group_templates': [] + }) + + def testCreateClusterTemplateWithNodeGroupTemplates(self): + session = ctx().session + with session.begin(): + ct = m.ClusterTemplate('ct', 't-1', 'p-1', 'hv-1') + session.add(ct) + + ngts = [] + for i in xrange(0, 3): + ngt = m.NodeGroupTemplate('ngt-%s' % i, 't-1', 'f-1', 'p-1', + 'hv-1', ['np-1', 'np-2']) + session.add(ngt) + session.flush() + rel = ct.add_node_group_template(ngt.id, 'group-%s' % i, 5 + i) + session.add(rel) + ngts.append(ngt) + + with session.begin(): + res = session.query(m.ClusterTemplate).filter_by().first() + self.assertIsValidModelObject(res) + + self.assertEqual(len(res.node_group_templates), 3) + self.assertEqual(set(t.name for t in res.node_group_templates), + set('ngt-%s' % i for i in xrange(0, 3))) diff --git a/savanna/tests/unit/test_api_v02.py b/savanna/tests/unit/test_api_v02.py deleted file mode 100644 index eac6101d..00000000 --- a/savanna/tests/unit/test_api_v02.py +++ /dev/null @@ -1,332 +0,0 @@ -# Copyright (c) 2013 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import eventlet -import json -import unittest - -from savanna.openstack.common import log as logging -from savanna.tests.unit.base import SavannaTestCase - -LOG = logging.getLogger(__name__) - - -class TestApiV02(SavannaTestCase): - - def test_list_node_templates(self): - rv = self.app.get('/v0.2/some-tenant-id/node-templates.json') - self.assertEquals(rv.status_code, 200) - data = json.loads(rv.data) - - # clean all ids - for idx in xrange(0, len(data.get(u'node_templates'))): - del data.get(u'node_templates')[idx][u'id'] - - self.assertEquals(data, _get_templates_stub_data()) - - def test_create_node_template(self): - rv = self.app.post('/v0.2/some-tenant-id/node-templates.json', - data=json.dumps(dict( - node_template=dict( - name='test-template', - node_type='JT+NN', - flavor_id='test_flavor', - job_tracker={ - 'heap_size': '1234' - }, - name_node={ - 'heap_size': '2345' - } - )))) - self.assertEquals(rv.status_code, 202) - data = json.loads(rv.data) - - data = data['node_template'] - - # clean all ids - del data[u'id'] - - self.assertEquals(data, { - u'job_tracker': { - u'heap_size': u'1234' - }, u'name': u'test-template', - u'node_type': { - u'processes': [ - u'job_tracker', u'name_node' - ], - u'name': u'JT+NN' - }, - u'flavor_id': u'test_flavor', - u'name_node': { - u'heap_size': u'2345' - } - }) - - def test_list_clusters(self): - rv = self.app.get('/v0.2/some-tenant-id/clusters.json') - self.assertEquals(rv.status_code, 200) - data = json.loads(rv.data) - - self.assertEquals(data, { - u'clusters': [] - }) - - def test_create_clusters(self): - rv = self.app.post('/v0.2/some-tenant-id/clusters.json', - data=json.dumps(dict( - cluster=dict( - name='test-cluster', - base_image_id='base-image-id', - node_templates={ - 'jt_nn.medium': 1, - 'tt_dn.small': 5 - } - )))) - self.assertEquals(rv.status_code, 202) - data = json.loads(rv.data) - - data = data['cluster'] - - cluster_id = data.pop(u'id') - - self.assertEquals(data, { - u'status': u'Starting', - u'service_urls': {}, - u'name': u'test-cluster', - u'base_image_id': u'base-image-id', - u'node_templates': { - u'jt_nn.medium': 1, - u'tt_dn.small': 5 - }, - u'nodes': [] - }) - - eventlet.sleep(4) - - rv = self.app.get('/v0.2/some-tenant-id/clusters/%s.json' % cluster_id) - self.assertEquals(rv.status_code, 200) - data = json.loads(rv.data) - - data = data['cluster'] - - self.assertEquals(data.pop(u'id'), cluster_id) - - # clean all ids - for idx in xrange(0, len(data.get(u'nodes'))): - del data.get(u'nodes')[idx][u'vm_id'] - del data.get(u'nodes')[idx][u'node_template'][u'id'] - - nodes = data.pop(u'nodes') - - self.assertEquals(data, { - u'status': u'Active', - u'service_urls': {}, - u'name': u'test-cluster', - u'base_image_id': u'base-image-id', - u'node_templates': { - u'jt_nn.medium': 1, - u'tt_dn.small': 5 - } - }) - - self.assertEquals(_sorted_nodes(nodes), _sorted_nodes([ - {u'node_template': {u'name': u'tt_dn.small'}}, - {u'node_template': {u'name': u'tt_dn.small'}}, - {u'node_template': {u'name': u'tt_dn.small'}}, - {u'node_template': {u'name': u'tt_dn.small'}}, - {u'node_template': {u'name': u'tt_dn.small'}}, - {u'node_template': {u'name': u'jt_nn.medium'}} - ])) - - def test_delete_node_template(self): - rv = self.app.post('/v0.2/some-tenant-id/node-templates.json', - data=json.dumps(dict( - node_template=dict( - name='test-template-2', - node_type='JT+NN', - flavor_id='test_flavor_2', - job_tracker={ - 'heap_size': '1234' - }, - name_node={ - 'heap_size': '2345' - } - )))) - self.assertEquals(rv.status_code, 202) - data = json.loads(rv.data) - - data = data['node_template'] - - node_template_id = data.pop(u'id') - - rv = self.app.get( - '/v0.2/some-tenant-id/node-templates/%s.json' % node_template_id) - self.assertEquals(rv.status_code, 200) - data = json.loads(rv.data) - - data = data['node_template'] - - # clean all ids - del data[u'id'] - - self.assertEquals(data, { - u'job_tracker': { - u'heap_size': u'1234' - }, u'name': u'test-template-2', - u'node_type': { - u'processes': [ - u'job_tracker', u'name_node' - ], - u'name': u'JT+NN' - }, - u'flavor_id': u'test_flavor_2', - u'name_node': { - u'heap_size': u'2345' - } - }) - - rv = self.app.delete( - '/v0.2/some-tenant-id/node-templates/%s.json' % node_template_id) - self.assertEquals(rv.status_code, 204) - - rv = self.app.get( - '/v0.2/some-tenant-id/node-templates/%s.json' % node_template_id) - self.assertEquals(rv.status_code, 404) - - def test_delete_cluster(self): - rv = self.app.post('/v0.2/some-tenant-id/clusters.json', - data=json.dumps(dict( - cluster=dict( - name='test-cluster-2', - base_image_id='base-image-id_2', - node_templates={ - 'jt_nn.medium': 1, - 'tt_dn.small': 5 - } - )))) - self.assertEquals(rv.status_code, 202) - data = json.loads(rv.data) - - data = data['cluster'] - - cluster_id = data.pop(u'id') - - rv = self.app.get('/v0.2/some-tenant-id/clusters/%s.json' % cluster_id) - self.assertEquals(rv.status_code, 200) - data = json.loads(rv.data) - - data = data['cluster'] - - # delete all ids - del data[u'id'] - - self.assertEquals(data, { - u'status': u'Starting', - u'service_urls': {}, - u'name': u'test-cluster-2', - u'base_image_id': u'base-image-id_2', - u'node_templates': { - u'jt_nn.medium': 1, - u'tt_dn.small': 5 - }, - u'nodes': [] - }) - - rv = self.app.delete( - '/v0.2/some-tenant-id/clusters/%s.json' % cluster_id) - self.assertEquals(rv.status_code, 204) - - eventlet.sleep(1) - - rv = self.app.get('/v0.2/some-tenant-id/clusters/%s.json' % cluster_id) - self.assertEquals(rv.status_code, 404) - - -def _sorted_nodes(nodes): - return sorted(nodes, key=lambda elem: elem[u'node_template'][u'name']) - - -def _get_templates_stub_data(): - return { - u'node_templates': [ - { - u'job_tracker': { - u'heap_size': u'896' - }, - u'name': u'jt_nn.small', - u'node_type': { - u'processes': [ - u'job_tracker', u'name_node' - ], - u'name': u'JT+NN' - }, - u'flavor_id': u'm1.small', - u'name_node': { - u'heap_size': u'896' - } - }, - { - u'job_tracker': { - u'heap_size': u'1792' - }, - u'name': u'jt_nn.medium', - u'node_type': { - u'processes': [ - u'job_tracker', u'name_node' - ], u'name': u'JT+NN' - }, - u'flavor_id': u'm1.medium', - u'name_node': { - u'heap_size': u'1792' - } - }, - { - u'name': u'tt_dn.small', - u'task_tracker': { - u'heap_size': u'896' - }, - u'data_node': { - u'heap_size': u'896' - }, - u'node_type': { - u'processes': [ - u'task_tracker', u'data_node' - ], - u'name': u'TT+DN' - }, - u'flavor_id': u'm1.small' - }, - { - u'name': u'tt_dn.medium', - u'task_tracker': { - u'heap_size': u'1792', - }, - u'data_node': { - u'heap_size': u'1792' - }, - u'node_type': { - u'processes': [ - u'task_tracker', u'data_node' - ], - u'name': u'TT+DN' - }, - u'flavor_id': u'm1.medium' - } - ] - } - - -if __name__ == '__main__': - unittest.main() diff --git a/savanna/tests/unit/test_cluster_ops.py b/savanna/tests/unit/test_cluster_ops.py deleted file mode 100644 index 420119bc..00000000 --- a/savanna/tests/unit/test_cluster_ops.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright (c) 2013 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from savanna.service.cluster_ops import _create_xml -import unittest - - -class ConfigGeneratorTest(unittest.TestCase): - def test_xml_generator(self): - config = { - 'key-1': 'value-1', - 'key-2': 'value-2', - 'key-3': 'value-3', - 'key-4': 'value-4', - 'key-5': 'value-5', - } - xml = _create_xml(config, config.keys()) - self.assertEqual(xml, """ - - - - key-3 - value-3 - - - key-2 - value-2 - - - key-1 - value-1 - - - key-5 - value-5 - - - key-4 - value-4 - - -""") diff --git a/savanna/tests/unit/test_service.py b/savanna/tests/unit/test_service.py deleted file mode 100644 index b236229c..00000000 --- a/savanna/tests/unit/test_service.py +++ /dev/null @@ -1,244 +0,0 @@ -# Copyright (c) 2013 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from mock import patch -import unittest - -import savanna.service.api as api - - -class TestServiceLayer(unittest.TestCase): - ## Node Template ops: - - @patch('savanna.storage.storage.get_node_template') - def test_get_node_template(self, m): - m.return_value = api.Resource("node_template", { - "id": "template-id", - "name": "jt_nn.small", - "node_type": api.Resource("node_type", { - "name": "JT+NN", - "processes": [ - api.Resource("process", {"name": "job_tracker"}), - api.Resource("process", {"name": "name_node"}) - ] - }), - "flavor_id": "flavor-id", - "node_template_configs": [ - api.Resource("conf", { - "node_process_property": api.Resource("prop", { - "name": "heap_size", - "node_process": api.Resource("process", { - "name": "job_tracker" - }) - }), - "value": "1234" - }), - api.Resource("conf", { - "node_process_property": api.Resource("prop", { - "name": "heap_size", - "node_process": api.Resource("process", { - "name": "name_node" - }) - }), - "value": "5678" - }) - ] - }) - - nt = api.get_node_template(id='template-id') - self.assertEqual(nt, api.Resource("node_template", { - 'id': 'template-id', - 'name': 'jt_nn.small', - 'node_type': { - 'processes': ['job_tracker', 'name_node'], - 'name': 'JT+NN' - }, - 'flavor_id': 'flavor-id', - 'job_tracker': {'heap_size': '1234'}, - 'name_node': {'heap_size': '5678'} - })) - m.assert_called_once_with(id='template-id') - - @patch('savanna.storage.storage.get_node_templates') - def test_get_node_templates(self, m): - # '_node_template' tested in 'test_get_node_template' - api.get_node_templates(node_type='JT+NN') - m.assert_called_once_with(node_type='JT+NN') - - @patch('savanna.service.api.get_node_template') - @patch('savanna.storage.storage.create_node_template') - @patch('savanna.storage.storage.get_node_type') - def test_create_node_template(self, get_n_type, create_tmpl, get_tmpl): - get_n_type.return_value = api.Resource( - "node_type", {"id": "node-type-1"}) - create_tmpl.return_value = api.Resource( - "node-template", {"id": "tmpl-1"}) - - api.create_node_template( - { - "node_template": { - "name": "nt-1", - "node_type": "JT+NN", - "flavor_id": "flavor-1" - } - }, {"X-Tenant-Id": "tenant-01"}) - - get_n_type.assert_called_once_with(name="JT+NN") - create_tmpl.assert_called_once_with("nt-1", "node-type-1", - "flavor-1", {}) - get_tmpl.assert_called_once_with(id="tmpl-1") - - @patch('savanna.storage.storage.terminate_node_template') - def test_terminate_node_template(self, m): - api.terminate_node_template(node_type='JT+NN') - m.assert_called_once_with(node_type='JT+NN') - - ## Cluster ops: - - @patch('savanna.storage.storage.get_cluster') - def test_get_cluster(self, m): - m.return_value = api.Resource("cluster", { - "id": "cluster-id", - "name": "cluster-name", - "base_image_id": "image-id", - "status": "Active", - "nodes": [ - api.Resource("node", { - "vm_id": "vm-1", - "node_template": api.Resource("node_template", { - "id": "jt_nn.small-id", - "name": "jt_nn.small" - }) - }), - api.Resource("node", { - "vm_id": "vm-2", - "node_template": api.Resource("node_template", { - "id": "tt_dn.small-id", - "name": "tt_dn.small" - }) - }), - api.Resource("node", { - "vm_id": "vm-3", - "node_template": api.Resource("node_template", { - "id": "tt_dn.small-id", - "name": "tt_dn.small" - }) - }) - ], - "node_counts": [ - api.Resource("node_count", { - "node_template": api.Resource("node_template", { - "name": "jt_nn.small" - }), - "count": "1" - }), - api.Resource("node_count", { - "node_template": api.Resource("node_template", { - "name": "tt_dn.small" - }), - "count": "2" - }) - ], - "service_urls": [ - api.Resource("service_url", { - "name": "job_tracker", - "url": "some-url" - }), - api.Resource("service_url", { - "name": "name_node", - "url": "some-url-2" - }) - ] - }) - - cluster = api.get_cluster(id="cluster-id") - self.assertEqual(cluster, api.Resource("cluster", { - 'id': 'cluster-id', - 'name': 'cluster-name', - 'base_image_id': "image-id", - 'status': 'Active', - 'node_templates': {'jt_nn.small': '1', 'tt_dn.small': '2'}, - 'nodes': [ - { - 'node_template': { - 'id': 'jt_nn.small-id', 'name': 'jt_nn.small' - }, 'vm_id': 'vm-1' - }, - { - 'node_template': { - 'id': 'tt_dn.small-id', 'name': 'tt_dn.small' - }, 'vm_id': 'vm-2' - }, - { - 'node_template': { - 'id': 'tt_dn.small-id', 'name': 'tt_dn.small' - }, 'vm_id': 'vm-3' - } - ], - 'service_urls': { - 'name_node': 'some-url-2', - 'job_tracker': 'some-url' - } - })) - m.assert_called_once_with(id="cluster-id") - - @patch('savanna.storage.storage.get_clusters') - def test_get_clusters(self, m): - # '_clusters' tested in 'test_get_clusters' - api.get_clusters(id="cluster-id") - m.assert_called_once_with(id="cluster-id") - - @patch('eventlet.spawn') - @patch('savanna.service.api.get_cluster') - @patch('savanna.storage.storage.create_cluster') - def test_create_cluster(self, create_c, get_c, spawn): - create_c.return_value = api.Resource("cluster", { - "id": "cluster-1" - }) - - api.create_cluster( - { - "cluster": { - "name": "cluster-1", - "base_image_id": "image-1", - "node_templates": { - "jt_nn.small": "1", - "tt_dn.small": "10" - } - } - }, {"X-Tenant-Id": "tenant-01"}) - - create_c.assert_called_once_with("cluster-1", "image-1", "tenant-01", { - "jt_nn.small": "1", - "tt_dn.small": "10" - }) - get_c.assert_called_once_with(id="cluster-1") - spawn.assert_called_once_with(api._cluster_creation_job, - {"X-Tenant-Id": "tenant-01"}, - "cluster-1") - - @patch('eventlet.spawn') - @patch('savanna.storage.storage.update_cluster_status') - def test_terminate_cluster(self, update_status, spawn): - update_status.return_value = api.Resource("cluster", { - "id": "cluster-id" - }) - - api.terminate_cluster({"X-Tenant-Id": "tenant-01"}, id="cluster-id") - - update_status.assert_called_once_with('Stopping', id="cluster-id") - spawn.assert_called_once_with(api._cluster_termination_job, - {"X-Tenant-Id": "tenant-01"}, - "cluster-id") diff --git a/savanna/tests/unit/test_validation.py b/savanna/tests/unit/test_validation.py deleted file mode 100644 index 7ea4d49f..00000000 --- a/savanna/tests/unit/test_validation.py +++ /dev/null @@ -1,544 +0,0 @@ -# Copyright (c) 2013 Mirantis Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from mock import patch, Mock -from oslo.config import cfg -import unittest - -from savanna.exceptions import NotFoundException, SavannaException -import savanna.openstack.common.exception as os_ex -from savanna.service.api import Resource -import savanna.service.validation as v - - -CONF = cfg.CONF -CONF.import_opt('allow_cluster_ops', 'savanna.config') - - -def _raise(ex): - def function(*args, **kwargs): - raise ex - - return function - - -def _cluster(base, **kwargs): - base['cluster'].update(**kwargs) - return base - - -def _template(base, **kwargs): - base['node_template'].update(**kwargs) - return base - - -class TestValidation(unittest.TestCase): - def setUp(self): - self._create_object_fun = None - CONF.set_override('allow_cluster_ops', False) - - def tearDown(self): - self._create_object_fun = None - CONF.clear_override('allow_cluster_ops') - - @patch("savanna.utils.api.bad_request") - @patch("savanna.utils.api.request_data") - def test_malformed_request_body(self, request_data, bad_request): - ex = os_ex.MalformedRequestBody() - request_data.side_effect = _raise(ex) - m_func = Mock() - m_func.__name__ = "m_func" - - v.validate(m_func)(m_func)() - - self._assert_calls(bad_request, - (1, 'MALFORMED_REQUEST_BODY', - 'Malformed message body: %(reason)s')) - - def _assert_exists_by_id(self, side_effect, assert_func=True): - m_checker = Mock() - m_checker.side_effect = side_effect - m_func = Mock() - m_func.__name__ = "m_func" - - v.exists_by_id(m_checker, "template_id")(m_func)(template_id="asd") - - m_checker.assert_called_once_with(id="asd") - - if assert_func: - m_func.assert_called_once_with(template_id="asd") - - @patch("savanna.utils.api.internal_error") - @patch("savanna.utils.api.not_found") - def test_exists_by_id_passed(self, not_found, internal_error): - self._assert_exists_by_id(None) - - self.assertEqual(not_found.call_count, 0) - self.assertEqual(internal_error.call_count, 0) - - @patch("savanna.utils.api.internal_error") - @patch("savanna.utils.api.not_found") - def test_exists_by_id_failed(self, not_found, internal_error): - self._assert_exists_by_id(_raise(NotFoundException("")), False) - self.assertEqual(not_found.call_count, 1) - self.assertEqual(internal_error.call_count, 0) - - self._assert_exists_by_id(_raise(SavannaException()), False) - self.assertEqual(not_found.call_count, 1) - self.assertEqual(internal_error.call_count, 1) - - self._assert_exists_by_id(_raise(AttributeError()), False) - self.assertEqual(not_found.call_count, 1) - self.assertEqual(internal_error.call_count, 2) - - def _assert_calls(self, mock, call_info): - print "_assert_calls for %s, \n\t actual: %s , \n\t expected: %s" \ - % (mock, mock.call_args, call_info) - if not call_info: - self.assertEqual(mock.call_count, 0) - else: - self.assertEqual(mock.call_count, call_info[0]) - self.assertEqual(mock.call_args[0][0].code, call_info[1]) - self.assertEqual(mock.call_args[0][0].message, call_info[2]) - - def _assert_create_object_validation( - self, data, bad_req_i=None, not_found_i=None, int_err_i=None): - - request_data_p = patch("savanna.utils.api.request_data") - bad_req_p = patch("savanna.utils.api.bad_request") - not_found_p = patch("savanna.utils.api.not_found") - int_err_p = patch("savanna.utils.api.internal_error") - get_clusters_p = patch("savanna.service.api.get_clusters") - get_templates_p = patch("savanna.service.api.get_node_templates") - get_template_p = patch("savanna.service.api.get_node_template") - get_types_p = patch("savanna.service.api.get_node_types") - get_node_type_required_params_p = \ - patch("savanna.service.api.get_node_type_required_params") - get_node_type_all_params_p = \ - patch("savanna.service.api.get_node_type_all_params") - patchers = (request_data_p, bad_req_p, not_found_p, int_err_p, - get_clusters_p, get_templates_p, get_template_p, - get_types_p, get_node_type_required_params_p, - get_node_type_all_params_p) - - request_data = request_data_p.start() - bad_req = bad_req_p.start() - not_found = not_found_p.start() - int_err = int_err_p.start() - get_clusters = get_clusters_p.start() - get_templates = get_templates_p.start() - get_template = get_template_p.start() - get_types = get_types_p.start() - get_node_type_required_params = get_node_type_required_params_p.start() - get_node_type_all_params = get_node_type_all_params_p.start() - - # stub clusters list - get_clusters.return_value = getattr(self, "_clusters_data", [ - Resource("cluster", { - "name": "some-cluster-1" - }) - ]) - - # stub node templates - get_templates.return_value = getattr(self, "_templates_data", [ - Resource("node_template", { - "name": "jt_nn.small", - "node_type": { - "name": "JT+NN", - "processes": ["job_tracker", "name_node"] - } - }), - Resource("node_template", { - "name": "nn.small", - "node_type": { - "name": "NN", - "processes": ["name_node"] - } - }) - ]) - - def _get_template(name): - for template in get_templates(): - if template.name == name: - return template - return None - - get_template.side_effect = _get_template - - get_types.return_value = getattr(self, "_types_data", [ - Resource("node_type", { - "name": "JT+NN", - "processes": ["job_tracker", "name_node"] - }) - ]) - - def _get_r_params(name): - if name == "JT+NN": - return {"job_tracker": ["jt_param"]} - return dict() - - get_node_type_required_params.side_effect = _get_r_params - - def _get_all_params(name): - if name == "JT+NN": - return {"job_tracker": ["jt_param"]} - return dict() - - get_node_type_all_params.side_effect = _get_all_params - - # mock function that should be validated - m_func = Mock() - m_func.__name__ = "m_func" - - # request data to validate - request_data.return_value = data - - v.validate(self._create_object_fun)(m_func)() - - self.assertEqual(request_data.call_count, 1) - - self._assert_calls(bad_req, bad_req_i) - self._assert_calls(not_found, not_found_i) - self._assert_calls(int_err, int_err_i) - - for patcher in patchers: - patcher.stop() - - def test_cluster_create_v_required(self): - self._create_object_fun = v.validate_cluster_create - - self._assert_create_object_validation( - {}, - bad_req_i=(1, "VALIDATION_ERROR", - u"'cluster' is a required property") - ) - self._assert_create_object_validation( - {"cluster": {}}, - bad_req_i=(1, "VALIDATION_ERROR", - u"'name' is a required property") - ) - self._assert_create_object_validation( - {"cluster": { - "name": "some-name" - }}, - bad_req_i=(1, "VALIDATION_ERROR", - u"'base_image_id' is a required property") - ) - self._assert_create_object_validation( - {"cluster": { - "name": "some-name", - "base_image_id": "some-image-id" - }}, - bad_req_i=(1, "VALIDATION_ERROR", - u"'node_templates' is a required property") - ) - - def test_cluster_create_v_name_base(self): - self._create_object_fun = v.validate_cluster_create - - cluster = { - "cluster": { - "base_image_id": "some-image-id", - "node_templates": {} - } - } - self._assert_create_object_validation( - _cluster(cluster, name=None), - bad_req_i=(1, "VALIDATION_ERROR", - u"None is not of type 'string'") - ) - self._assert_create_object_validation( - _cluster(cluster, name=""), - bad_req_i=(1, "VALIDATION_ERROR", - u"'' is too short") - ) - self._assert_create_object_validation( - _cluster(cluster, name="a" * 51), - bad_req_i=(1, "VALIDATION_ERROR", - u"'%s' is too long" % ('a' * 51)) - ) - - def test_cluster_create_v_name_pattern(self): - self._create_object_fun = v.validate_cluster_create - - cluster = { - "cluster": { - "base_image_id": "some-image-id", - "node_templates": {} - } - } - - def _assert_cluster_name_pattern(self, name): - cluster_schema = v.CLUSTER_CREATE_SCHEMA['properties']['cluster'] - name_p = cluster_schema['properties']['name']['pattern'] - self._assert_create_object_validation( - _cluster(cluster, name=name), - bad_req_i=(1, "VALIDATION_ERROR", - (u"'%s' does not match '%s'" % (name, name_p)) - .replace('\\', "\\\\")) - ) - - _assert_cluster_name_pattern(self, "asd_123") - _assert_cluster_name_pattern(self, "123") - _assert_cluster_name_pattern(self, "asd?") - - def test_cluster_create_v_name_exists(self): - self._create_object_fun = v.validate_cluster_create - - cluster = { - "cluster": { - "base_image_id": "some-image-id", - "node_templates": {} - } - } - - self._assert_create_object_validation( - _cluster(cluster, name="some-cluster-1"), - bad_req_i=(1, "CLUSTER_NAME_ALREADY_EXISTS", - u"Cluster with name 'some-cluster-1' already exists") - ) - - def test_cluster_create_v_templates(self): - self._create_object_fun = v.validate_cluster_create - - cluster = { - "cluster": { - "name": "some-cluster", - "base_image_id": "some-image-id" - } - } - self._assert_create_object_validation( - _cluster(cluster, node_templates={}), - bad_req_i=(1, "NOT_SINGLE_NAME_NODE", - u"Hadoop cluster should contain only 1 NameNode. " - u"Actual NN count is 0") - ) - self._assert_create_object_validation( - _cluster(cluster, node_templates={ - "nn.small": 1 - }), - bad_req_i=(1, "NOT_SINGLE_JOB_TRACKER", - u"Hadoop cluster should contain only 1 JobTracker. " - u"Actual JT count is 0") - ) - self._assert_create_object_validation( - _cluster(cluster, node_templates={ - "incorrect_template": 10 - }), - bad_req_i=(1, "NODE_TEMPLATE_NOT_FOUND", - u"NodeTemplate 'incorrect_template' not found") - ) - self._assert_create_object_validation( - _cluster(cluster, node_templates={ - "jt_nn.small": 1 - }) - ) - - def test_node_template_create_v_required(self): - self._create_object_fun = v.validate_node_template_create - - self._assert_create_object_validation( - {}, - bad_req_i=(1, "VALIDATION_ERROR", - u"'node_template' is a required property") - ) - self._assert_create_object_validation( - {"node_template": {}}, - bad_req_i=(1, "VALIDATION_ERROR", - u"'name' is a required property") - ) - self._assert_create_object_validation( - {"node_template": { - "name": "some-name" - }}, - bad_req_i=(1, "VALIDATION_ERROR", - u"'node_type' is a required property") - ) - self._assert_create_object_validation( - {"node_template": { - "name": "some-name", - "node_type": "some-node-type" - }}, - bad_req_i=(1, "VALIDATION_ERROR", - u"'flavor_id' is a required property") - ) - self._assert_create_object_validation( - {"node_template": { - "name": "some-name", - "node_type": "JT+NN", - "flavor_id": "flavor-1" - }}, - bad_req_i=(1, "VALIDATION_ERROR", - u"'name_node' is a required property") - ) - self._assert_create_object_validation( - {"node_template": { - "name": "some-name", - "node_type": "JT+NN", - "flavor_id": "flavor-1", - "name_node": {} - }}, - bad_req_i=(1, "VALIDATION_ERROR", - u"'job_tracker' is a required property") - ) - self._assert_create_object_validation( - {"node_template": { - "name": "some-name", - "node_type": "JT+NN", - "flavor_id": "flavor-1", - "name_node": {}, - "job_tracker": {} - }}, - bad_req_i=(1, "REQUIRED_PARAM_MISSED", - u"Required parameter 'jt_param' of process " - u"'job_tracker' should be specified") - ) - self._assert_create_object_validation( - {"node_template": { - "name": "some-name", - "node_type": "JT+NN", - "flavor_id": "flavor-1", - "name_node": {}, - "job_tracker": {"jt_param": ""} - }}, - bad_req_i=(1, "REQUIRED_PARAM_MISSED", - u"Required parameter 'jt_param' of process " - u"'job_tracker' should be specified") - ) - self._assert_create_object_validation( - {"node_template": { - "name": "some-name", - "node_type": "JT+NN", - "flavor_id": "flavor-1", - "name_node": {}, - "job_tracker": {"jt_param": "some value", "bad.parameter": "1"} - }}, - bad_req_i=(1, "PARAM_IS_NOT_ALLOWED", - u"Parameter 'bad.parameter' " - u"of process 'job_tracker' is not allowed to change") - ) - self._assert_create_object_validation( - {"node_template": { - "name": "some-name", - "node_type": "JT+NN", - "flavor_id": "flavor-1", - "name_node": {}, - "job_tracker": {"jt_param": "some value"} - }}, - ) - self._assert_create_object_validation( - {"node_template": { - "name": "some-name", - "node_type": "JT+NN", - "flavor_id": "flavor-1", - "name_node": {}, - "job_tracker": {}, - "task_tracker": {} - }}, - bad_req_i=(1, "NODE_PROCESS_DISCREPANCY", - u"Discrepancies in Node Processes. " - u"Required: ['name_node', 'job_tracker']") - ) - - def test_node_template_create_v_name_base(self): - self._create_object_fun = v.validate_node_template_create - - template = { - "node_template": { - "node_type": "JT+NN", - "flavor_id": "flavor-1", - "name_node": {}, - "job_tracker": {} - } - } - self._assert_create_object_validation( - _template(template, name=None), - bad_req_i=(1, "VALIDATION_ERROR", - u"None is not of type 'string'") - ) - self._assert_create_object_validation( - _template(template, name=""), - bad_req_i=(1, "VALIDATION_ERROR", - u"'' is too short") - ) - self._assert_create_object_validation( - _template(template, name="a" * 241), - bad_req_i=(1, "VALIDATION_ERROR", - u"'%s' is too long" % ('a' * 241)) - ) - - def test_node_template_create_v_name_pattern(self): - self._create_object_fun = v.validate_node_template_create - - template = { - "node_template": { - "node_type": "JT+NN", - "flavor_id": "flavor-1", - "name_node": {}, - "job_tracker": {} - } - } - - def _assert_template_name_pattern(self, name): - schema_props = v.TEMPLATE_CREATE_SCHEMA['properties'] - template_schema = schema_props['node_template'] - name_p = template_schema['properties']['name']['pattern'] - self._assert_create_object_validation( - _template(template, name=name), - bad_req_i=(1, "VALIDATION_ERROR", - (u"'%s' does not match '%s'" % (name, name_p)) - .replace('\\', "\\\\")) - ) - - _assert_template_name_pattern(self, "asd;123") - _assert_template_name_pattern(self, "123") - _assert_template_name_pattern(self, "asd?") - - def test_node_template_create_v_name_exists(self): - self._create_object_fun = v.validate_node_template_create - - template = { - "node_template": { - "node_type": "JT+NN", - "flavor_id": "flavor-1", - "name_node": {}, - "job_tracker": {} - } - } - - self._assert_create_object_validation( - _template(template, name="jt_nn.small"), - bad_req_i=(1, "NODE_TEMPLATE_ALREADY_EXISTS", - u"NodeTemplate with name 'jt_nn.small' already exists") - ) - - def test_node_template_create_v_types(self): - self._create_object_fun = v.validate_node_template_create - - self._assert_create_object_validation( - { - "node_template": { - "name": "some-name", - "node_type": "JJ", - "flavor_id": "flavor-1", - "name_node": {}, - "job_tracker": {} - } - }, - bad_req_i=(1, "NODE_TYPE_NOT_FOUND", - u"NodeType 'JJ' not found") - ) - -# TODO(slukjanov): add tests for allow_cluster_ops = True diff --git a/savanna/utils/api.py b/savanna/utils/api.py index de7602b3..1e7ad030 100644 --- a/savanna/utils/api.py +++ b/savanna/utils/api.py @@ -13,20 +13,21 @@ # See the License for the specific language governing permissions and # limitations under the License. +import flask as f +import inspect import mimetypes import traceback - -from flask import abort, request, Blueprint, Response from werkzeug.datastructures import MIMEAccept -from savanna.openstack.common.wsgi import JSONDictSerializer, \ - XMLDictSerializer, JSONDeserializer +from savanna.context import Context +from savanna.context import set_ctx from savanna.openstack.common import log as logging +from savanna.openstack.common import wsgi LOG = logging.getLogger(__name__) -class Rest(Blueprint): +class Rest(f.Blueprint): def get(self, rule, status_code=200): return self._mroute('GET', rule, status_code) @@ -52,24 +53,35 @@ class Rest(Blueprint): def handler(**kwargs): # extract response content type - resp_type = request.accept_mimetypes + resp_type = f.request.accept_mimetypes type_suffix = kwargs.pop('resp_type', None) if type_suffix: suffix_mime = mimetypes.guess_type("res." + type_suffix)[0] if suffix_mime: resp_type = MIMEAccept([(suffix_mime, 1)]) - request.resp_type = resp_type - - # extract fields (column selection) - fields = list(set(request.args.getlist('fields'))) - fields.sort() - request.fields_selector = fields + f.request.resp_type = resp_type + # update status code if status: - request.status_code = status + f.request.status_code = status kwargs.pop("tenant_id") + context = Context(f.request.headers['X-User-Id'], + f.request.headers['X-Tenant-Id'], + f.request.headers['X-Auth-Token'], + f.request.headers) + set_ctx(context) + + # set func implicit args + args = inspect.getargspec(func).args + + if 'ctx' in args: + kwargs['ctx'] = context + + if f.request.method in ['POST', 'PUT'] and 'data' in args: + kwargs['data'] = request_data() + return func(**kwargs) f_rule = "/" + rule @@ -77,7 +89,10 @@ class Rest(Blueprint): ext_rule = f_rule + '.' self.add_url_rule(ext_rule, endpoint, handler, **options) - return func + try: + return func + except Exception, e: + return internal_error(500, 'Exception in API call', e) return decorator @@ -86,6 +101,30 @@ RT_JSON = MIMEAccept([("application/json", 1)]) RT_XML = MIMEAccept([("application/xml", 1)]) +def _clean_nones(obj): + if not isinstance(obj, dict) and not isinstance(obj, list): + return obj + + if isinstance(obj, dict): + remove = [] + for key, value in obj.iteritems(): + if value is None: + remove.append(key) + for key in remove: + obj.pop(key) + for value in obj.values(): + _clean_nones(value) + elif isinstance(obj, list): + new_list = [] + for elem in obj: + elem = _clean_nones(elem) + if elem is not None: + new_list.append(elem) + return new_list + + return obj + + def render(res=None, resp_type=None, status=None, **kwargs): if not res: res = {} @@ -95,14 +134,16 @@ def render(res=None, resp_type=None, status=None, **kwargs): # can't merge kwargs into the non-dict res abort_and_log(500, "Non-dict and non-empty kwargs passed to render") - status_code = getattr(request, 'status_code', None) + res = _clean_nones(res) + + status_code = getattr(f.request, 'status_code', None) if status: status_code = status if not status_code: status_code = 200 if not resp_type: - resp_type = getattr(request, 'resp_type', RT_JSON) + resp_type = getattr(f.request, 'resp_type', RT_JSON) if not resp_type: resp_type = RT_JSON @@ -110,31 +151,31 @@ def render(res=None, resp_type=None, status=None, **kwargs): serializer = None if "application/json" in resp_type: resp_type = RT_JSON - serializer = JSONDictSerializer() + serializer = wsgi.JSONDictSerializer() elif "application/xml" in resp_type: resp_type = RT_XML - serializer = XMLDictSerializer() + serializer = wsgi.XMLDictSerializer() else: abort_and_log(400, "Content type '%s' isn't supported" % resp_type) body = serializer.serialize(res) resp_type = str(resp_type) - return Response(response=body, status=status_code, mimetype=resp_type) + return f.Response(response=body, status=status_code, mimetype=resp_type) def request_data(): - if hasattr(request, 'parsed_data'): - return request.parsed_data + if hasattr(f.request, 'parsed_data'): + return f.request.parsed_data - if not request.content_length > 0: + if not f.request.content_length > 0: LOG.debug("Empty body provided in request") return dict() deserializer = None - content_type = request.mimetype + content_type = f.request.mimetype if not content_type or content_type in RT_JSON: - deserializer = JSONDeserializer() + deserializer = wsgi.JSONDeserializer() elif content_type in RT_XML: abort_and_log(400, "XML requests are not supported yet") # deserializer = XMLDeserializer() @@ -142,9 +183,9 @@ def request_data(): abort_and_log(400, "Content type '%s' isn't supported" % content_type) # parsed request data to avoid unwanted re-parsings - request.parsed_data = deserializer.deserialize(request.data)['body'] + f.request.parsed_data = deserializer.deserialize(f.request.data)['body'] - return request.parsed_data + return f.request.parsed_data def abort_and_log(status_code, descr, exc=None): @@ -154,7 +195,7 @@ def abort_and_log(status_code, descr, exc=None): if exc is not None: LOG.error(traceback.format_exc()) - abort(status_code, description=descr) + f.abort(status_code, description=descr) def render_error_message(error_code, error_message, error_name): diff --git a/savanna/utils/openstack/images.py b/savanna/utils/openstack/images.py new file mode 100644 index 00000000..e4d04bf2 --- /dev/null +++ b/savanna/utils/openstack/images.py @@ -0,0 +1,101 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from novaclient.v1_1.images import Image +from novaclient.v1_1.images import ImageManager + + +PROP_DESCR = '_savanna_description' +PROP_USERNAME = '_savanna_username' +PROP_TAG = '_savanna_tag_' + + +def _iter_tags(meta): + for key in meta: + if key.startswith(PROP_TAG) and meta[key]: + yield key[len(PROP_TAG):] + + +def _ensure_tags(tags): + return [tags] if type(tags) in [str, unicode] else tags + + +class SavannaImage(Image): + def __init__(self, manager, info, loaded=False): + info['description'] = info.get('metadata', {}).get(PROP_DESCR) + info['username'] = info.get('metadata', {}).get(PROP_USERNAME) + info['tags'] = [tag for tag in _iter_tags(info.get('metadata', {}))] + super(SavannaImage, self).__init__(manager, info, loaded) + + def tag(self, tags): + self.manager.tag(self, tags) + + def untag(self, tags): + self.manager.untag(self, tags) + + def set_description(self, description=None, username=None): + self.manager.set_description(self, description, username) + + @property + def dict(self): + return self.to_dict() + + @property + def wrapped_dict(self): + return {'image': self.dict} + + def to_dict(self): + return self._info.copy() + + +class SavannaImageManager(ImageManager): + """Manage :class:`SavannaImage` resources. + + This is an extended version of nova client's ImageManager with support of + additional description and image tags stored in images' meta. + """ + resource_class = SavannaImage + + def set_description(self, image, description, username): + """Sets human-readable information for image. + + For example: + + Ubuntu 13.04 x64 with Java 1.7u21 and Apache Hadoop 1.1.1, ubuntu + """ + self.set_meta(image, { + PROP_DESCR: description, + PROP_USERNAME: username, + }) + + def tag(self, image, tags): + """Adds tags to the specified image.""" + tags = _ensure_tags(tags) + + self.set_meta(image, dict((PROP_TAG + tag, True) for tag in tags)) + + def untag(self, image, tags): + """Removes tags from the specified image.""" + tags = _ensure_tags(tags) + + self.delete_meta(image, [PROP_TAG + tag for tag in tags]) + + def list_by_tags(self, tags): + """Returns images having all of the specified tags.""" + tags = _ensure_tags(tags) + return [i for i in self.list() if set(tags).issubset(i.tags)] + + def list_registered(self): + return [i for i in self.list() if i.description and i.username] diff --git a/savanna/utils/openstack/nova.py b/savanna/utils/openstack/nova.py index 5734e663..4fc4afb9 100644 --- a/savanna/utils/openstack/nova.py +++ b/savanna/utils/openstack/nova.py @@ -16,11 +16,14 @@ import logging from novaclient.v1_1 import client as nova_client +from savanna.context import ctx import savanna.utils.openstack.base as base +from savanna.utils.openstack.images import SavannaImageManager -def novaclient(headers): +def novaclient(): + headers = ctx().headers username = headers['X-User-Name'] token = headers['X-Auth-Token'] tenant = headers['X-Tenant-Id'] @@ -35,26 +38,31 @@ def novaclient(headers): nova.client.auth_token = token nova.client.management_url = compute_url + nova.images = SavannaImageManager(nova) return nova -def get_flavors(headers): +def get_flavors(): + headers = ctx().headers flavors = [flavor.name for flavor in novaclient(headers).flavors.list()] return flavors -def get_flavor(headers, **kwargs): +def get_flavor(**kwargs): + headers = ctx().headers return novaclient(headers).flavors.find(**kwargs) -def get_images(headers): +def get_images(): + headers = ctx().headers images = [image.id for image in novaclient(headers).images.list()] return images -def get_limits(headers): +def get_limits(): + headers = ctx().headers limits = novaclient(headers).limits.get().absolute return dict((l.name, l.value) for l in limits) diff --git a/savanna/utils/patches.py b/savanna/utils/patches.py index 864ac067..026a7194 100644 --- a/savanna/utils/patches.py +++ b/savanna/utils/patches.py @@ -29,7 +29,7 @@ def patch_minidom_writexml(): if sys.version_info >= (2, 7, 3): return - from xml.dom.minidom import Element, Node, Text, _write_data + import xml.dom.minidom as md def writexml(self, writer, indent="", addindent="", newl=""): # indent = current indentation @@ -43,12 +43,12 @@ def patch_minidom_writexml(): for a_name in a_names: writer.write(" %s=\"" % a_name) - _write_data(writer, attrs[a_name].value) + md._write_data(writer, attrs[a_name].value) writer.write("\"") if self.childNodes: writer.write(">") if (len(self.childNodes) == 1 - and self.childNodes[0].nodeType == Node.TEXT_NODE): + and self.childNodes[0].nodeType == md.Node.TEXT_NODE): self.childNodes[0].writexml(writer, '', '', '') else: writer.write(newl) @@ -59,9 +59,9 @@ def patch_minidom_writexml(): else: writer.write("/>%s" % (newl)) - Element.writexml = writexml + md.Element.writexml = writexml def writexml(self, writer, indent="", addindent="", newl=""): - _write_data(writer, "%s%s%s" % (indent, self.data, newl)) + md._write_data(writer, "%s%s%s" % (indent, self.data, newl)) - Text.writexml = writexml + md.Text.writexml = writexml diff --git a/savanna/utils/resources.py b/savanna/utils/resources.py new file mode 100644 index 00000000..111b3107 --- /dev/null +++ b/savanna/utils/resources.py @@ -0,0 +1,70 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect + + +class BaseResource(object): + __resource_name__ = 'base' + __filter_cols__ = [] + + @property + def dict(self): + return self.to_dict() + + @property + def wrapped_dict(self): + return {self.__resource_name__: self.dict} + + @property + def __all_filter_cols__(self): + cls = self.__class__ + if not hasattr(cls, '__mro_filter_cols__'): + filter_cols = [] + for base_cls in inspect.getmro(cls): + filter_cols += getattr(base_cls, '__filter_cols__', []) + cls.__mro_filter_cols__ = set(filter_cols) + return cls.__mro_filter_cols__ + + def _filter_field(self, k): + return k == '_sa_instance_state' or k in self.__all_filter_cols__ + + def to_dict(self): + dictionary = self.__dict__.copy() + return dict([(k, v) for k, v in dictionary.iteritems() + if not self._filter_field(k)]) + + def as_resource(self): + return Resource(self.__resource_name__, self.to_dict()) + + +class Resource(BaseResource): + def __init__(self, _name, _info): + self._name = _name + self._info = _info + + def __getattr__(self, k): + if k not in self.__dict__: + return self._info.get(k) + return self.__dict__[k] + + def __repr__(self): + return '<%s %s>' % (self._name, self._info) + + def __eq__(self, other): + return self._name == other._name and self._info == other._info + + def to_dict(self): + return self._info.copy() diff --git a/savanna/utils/sqlatypes.py b/savanna/utils/sqlatypes.py new file mode 100644 index 00000000..2cb66876 --- /dev/null +++ b/savanna/utils/sqlatypes.py @@ -0,0 +1,110 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from sqlalchemy.ext.mutable import Mutable +from sqlalchemy.types import TypeDecorator, VARCHAR + +from savanna.openstack.common import jsonutils + + +class JSONEncoded(TypeDecorator): + """Represents an immutable structure as a json-encoded string.""" + + impl = VARCHAR + + def process_bind_param(self, value, dialect): + if value is not None: + value = jsonutils.dumps(value) + return value + + def process_result_value(self, value, dialect): + if value is not None: + value = jsonutils.loads(value) + return value + + +# todo verify this implementation +class MutableDict(Mutable, dict): + @classmethod + def coerce(cls, key, value): + """Convert plain dictionaries to MutableDict.""" + if not isinstance(value, MutableDict): + if isinstance(value, dict): + return MutableDict(value) + + # this call will raise ValueError + return Mutable.coerce(key, value) + else: + return value + + def update(self, e=None, **f): + """Detect dictionary update events and emit change events.""" + dict.update(self, e, **f) + self.changed() + + def __setitem__(self, key, value): + """Detect dictionary set events and emit change events.""" + dict.__setitem__(self, key, value) + self.changed() + + def __delitem__(self, key): + """Detect dictionary del events and emit change events.""" + dict.__delitem__(self, key) + self.changed() + + +# todo verify this implementation +class MutableList(Mutable, list): + @classmethod + def coerce(cls, key, value): + """Convert plain lists to MutableList.""" + if not isinstance(value, MutableList): + if isinstance(value, list): + return MutableList(value) + + # this call will raise ValueError + return Mutable.coerce(key, value) + else: + return value + + def __add__(self, value): + """Detect list add events and emit change events.""" + list.__add__(self, value) + self.changed() + + def append(self, value): + """Detect list add events and emit change events.""" + list.append(self, value) + self.changed() + + def __setitem__(self, key, value): + """Detect list set events and emit change events.""" + list.__setitem__(self, key, value) + self.changed() + + def __delitem__(self, i): + """Detect list del events and emit change events.""" + list.__delitem__(self, i) + self.changed() + + +def JsonDictType(): + """Returns an SQLAlchemy Column Type suitable to store a Json dict.""" + return MutableDict.as_mutable(JSONEncoded) + + +def JsonListType(): + """Returns an SQLAlchemy Column Type suitable to store a Json array.""" + return MutableList.as_mutable(JSONEncoded) diff --git a/setup.py b/setup.py index f20939fe..293b745a 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ project = 'savanna' setuptools.setup( name=project, - version=common_setup.get_version(project, '0.1.2'), + version=common_setup.get_version(project, '0.2'), description='Savanna project', author='Mirantis Inc.', author_email='savanna-team@mirantis.com', @@ -37,7 +37,7 @@ setuptools.setup( test_suite='nose.collector', scripts=[ 'bin/savanna-api', - 'bin/savanna-manage', + 'bin/savanna-db-manage', ], py_modules=[], data_files=[ diff --git a/tools/pip-requires b/tools/pip-requires index d67dd719..1bf5c7c8 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -1,4 +1,5 @@ # This file is managed by openstack-depends +alembic>=0.4.1 eventlet>=0.9.12 flask==0.9 jsonschema>=1.0.0 @@ -6,8 +7,6 @@ oslo.config>=1.1.0 paramiko>=1.8.0 python-keystoneclient>=0.2,<0.3 python-novaclient>=2.12.0,<3 +six sqlalchemy>=0.7,<=0.7.99 webob>=1.0.8 - -# Additional depends -flask-sqlalchemy diff --git a/tools/run_pyflakes b/tools/run_pyflakes deleted file mode 100755 index 30593ede..00000000 --- a/tools/run_pyflakes +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -echo "Please, use tools/run_pep8 instead of tools/run_pyflakes" diff --git a/tox.ini b/tox.ini index 83212177..87e49ef7 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26,py27,pep8,pyflakes +envlist = py26,py27,pep8 [testenv] setenv = @@ -31,10 +31,6 @@ deps = hacking commands = flake8 -[testenv:pyflakes] -deps = -commands = - [testenv:venv] commands = {posargs} @@ -49,9 +45,8 @@ commands = pylint --output-format=parseable --rcfile=.pylintrc bin/savanna-api bin/savanna-manage savanna | tee pylint-report.txt [flake8] -# H301 one import per line # H302 import only modules -ignore = H301,H302 +ignore = H302 show-source = true builtins = _ exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,tools