From e857447879bf83bf3b89f2d3fd39b726503643d1 Mon Sep 17 00:00:00 2001 From: Trevor McKay Date: Tue, 31 Mar 2015 12:53:43 -0400 Subject: [PATCH] Add a CLI tool for managing default templates This change adds a CLI called sahara-templates to manage default templates. Partial-implements: blueprint default-templates Change-Id: I4f30bd6bc378d90fda41b512c04afe8023d2b4b2 --- MANIFEST.in | 3 + sahara/db/templates/README.rst | 278 ++++++ sahara/db/templates/__init__.py | 0 sahara/db/templates/api.py | 803 ++++++++++++++++++ sahara/db/templates/cli.py | 208 +++++ sahara/db/templates/utils.py | 190 +++++ .../plugins/default_templates/template.conf | 16 + setup.cfg | 1 + 8 files changed, 1499 insertions(+) create mode 100644 sahara/db/templates/README.rst create mode 100644 sahara/db/templates/__init__.py create mode 100644 sahara/db/templates/api.py create mode 100644 sahara/db/templates/cli.py create mode 100644 sahara/db/templates/utils.py create mode 100644 sahara/plugins/default_templates/template.conf diff --git a/MANIFEST.in b/MANIFEST.in index c9ec2df8..566b07ad 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,9 +8,12 @@ include sahara/db/migration/alembic_migrations/env.py include sahara/db/migration/alembic_migrations/script.py.mako include sahara/db/migration/alembic_migrations/versions/*.py include sahara/db/migration/alembic_migrations/versions/README +include sahara/db/templates/README.rst recursive-include sahara/locale * +recursive-include sahara/plugins/default_templates *.json +include sahara/plugins/default_templates/template.conf include sahara/plugins/cdh/v5/resources/cdh_config.py include sahara/plugins/cdh/v5/resources/*.sh include sahara/plugins/cdh/v5/resources/*.json diff --git a/sahara/db/templates/README.rst b/sahara/db/templates/README.rst new file mode 100644 index 00000000..07e988a9 --- /dev/null +++ b/sahara/db/templates/README.rst @@ -0,0 +1,278 @@ +Sahara Default Template CLI +=========================== + +The *sahara-templates* application is a simple CLI for managing default +templates in Sahara. This document gives an overview of default templates +and explains how to use the CLI. + +Default Templates Overview +-------------------------- + +The goal of the default template facility in Sahara is to make cluster +launching quick and easy by providing users with a stable set of pre-generated +node group and cluster templates for each of the Sahara provisioning plugins. + +Template sets are defined in .json files grouped into directories. The CLI +reads these template sets and writes directly to the Sahara database. + +Default templates may only be created, modified, or deleted via the CLI -- +operations through the python-saharaclient or REST API are restricted. + +JSON Files +---------- + +Cluster and node group templates are defined in .json files. + +A very simple cluster template JSON file might look like this: + +.. code:: python + + { + "plugin_name": "vanilla", + "hadoop_version": "2.6.0", + "node_groups": [ + { + "name": "master", + "count": 1, + "node_group_template_id": "{master}" + }, + { + "name": "worker", + "count": 3, + "node_group_template_id": "{worker}" + } + ], + "name": "cluster-template" + } + +The values of the *node_group_template_id* fields are the +names of node group templates in set braces. In this example, +*master* and *worker* are the names of node group templates defined in +.json files in the same directory. When the CLI processes the +directory, it will create the node group templates first and +then substitute the appropriate id values for the name references +when it creates the cluster template. + +Configuration Files and Value Substitutions +------------------------------------------- + +The CLI supports value substitution for a limited set of fields. +For cluster templates, the following fields may use substitution: + +* default_image_id +* neutron_management_network + +For node group templates, the following fields may use substitution: + +* image_id +* flavor_id +* floating_ip_pool + +Substitution is indicated for one of these fields in a .json file +when the value is the name of the field in set braces. Here is an example +of a node group template file that uses substitution for *flavor_id*: + +.. code:: python + + { + "plugin_name": "vanilla", + "hadoop_version": "2.6.0", + "node_processes": [ + "namenode", + "resourcemanager", + "oozie", + "historyserver" + ], + "name": "master", + "flavor_id": "{flavor_id}", + "floating_ip_pool": "{floating_ip_pool}" + } + +The values for *flavor_id* and *floating_ip_pool* in this template +will come from a configuration file. + +If a configuration value is found for the substitution, the value will +be replaced. If a configuration value is not found, the field will be +omitted from the template. (In this example, *flavor_id* is a required +field of node group templates and the template will fail validation +if there is no substitution value specifed. However, *floating_ip_pool* +is not required and so the template will still pass validation if it +is omitted). + +The CLI will look for configuration sections with names based on +the *plugin_name*, *hadoop_version*, and *name* fields in the +template. It will look for sections in the following order: + +* **[]** + + May contain fields only for the type of the named template + + If templates are named in an **unambiguous** way, the template + name alone can be a used as the name of the config section. + This produces shorter names and aids readability when there + is a one-to-one mapping between template names and config + sections. + +* **[__]** + + May contain fields only for the type of the named template + + This form unambiguously applies to a specific template for + a specific plugin. + +* **[_]** + + May contain node group or cluster template fields + +* **[]** + + May contain node group or cluster template fields + +* **[DEFAULT]** + + May contain node group or cluster template fields + +If we have the following configuration file in our example +the CLI will find the value of *flavor_id* for the *master* template +in the first configuration section and the value for *floating_ip_pool* +in the third section: + +.. code:: python + + [vanilla_2.6.0_master] + # This is named for the plugin, version, and template. + # It may contain only node group template fields. + flavor_id = 5 + image_id = b7883f8a-9a7f-42cc-89a2-d3c8b1cc7b28 + + [vanilla_2.6.0] + # This is named for the plugin and version. + # It may contain fields for both node group and cluster templates. + flavor_id = 4 + neutron_mangement_network = 9973da0b-68eb-497d-bd48-d85aca37f088 + + [vanilla] + # This is named for the plugin. + # It may contain fields for both node group and cluster templates. + flavor_id = 3 + default_image_id = 89de8d21-9743-4d20-873e-7677973416dd + floating_ip_pool = my_pool + + [DEFAULT] + # This is the normal default section. + # It may contain fields for both node group and cluster templates. + flavor_id = 2 + +Sample Configuration File +------------------------- + +A sample configuration file is provided in +*sahara/plugins/default_templates/template.conf*. This +file sets the *flavor_id* for most of the node group templates +supplied with Sahara to 2 which indicates the *m1.small* +flavor in a default OpenStack deployment. + +The master node templates for the CDH plugin have the +*flavor_id* set to 4 which indicates the *m1.large* flavor, +since these nodes require more resources. + +This configuration file may be used with the CLI as is, or +it may be copied and modified. Note that multiple configuration +files may be passed to the CLI by repeating the *--config-file* +option. + +Other Special Configuration Parameters +-------------------------------------- + +The only configuration parameter that is strictly required is +the *connection* parameter in the *database* section. Without this +value the CLI will not be able to connect to the Sahara database. + +By default, the CLI will use the value of the *plugins* parameter +in the [DEFAULT] section on *update* to filter the templates that +will be created or updated. This parameter in Sahara defaults to +the set of fully supported plugins. To restrict the set of plugins +for the *update* operation set this parameter or use the +*--plugin-name* option. + +Directory Structure +------------------- + +The structure of the directory holding .json files for the CLI is +very flexible. The CLI will begin processing at the designated +starting directory and recurse through subdirectories. + +At each directory level, the CLI will look for .json files to +define a set of default templates. Cluster templates may reference +node group templates in the same set by name. Templates at different +levels in the directory structure are not in the same set. + +Plugin name and version are determined from the values in the .json +files, not by the file names or the directory structure. + +Recursion may be turned off with the "-n" option (see below). + +The default starting directory is *sahara/plugins/default_templates* + +Example CLI Commands +-------------------- + +For ``update``, ``delete``, ``node-group-template-delete``, and +``cluster-template-delete`` operations, the tenant must always be specified. +For ``node-group-template-delete-id`` and ``cluster-template-delete-id`` +tenant is not required. +All useful information about activity by the CLI is logged + +Create/update all of the default templates bundled with Sahara. Use the standard +Sahara configuration file in */etc/sahara/sahara.conf* to specify the plugin list +and the database connection string and another configuration file to supply +the *flavor_id* values:: + + $ sahara-templates --config-file /etc/sahara/sahara.conf --config-file myconfig update -t $TENANT_ID + +Create/update default templates from the directory *mypath*:: + + $ sahara-templates --config-file myconfig update -t $TENANT_ID -d mypath + +Create/update default templates from the directory *mypath* but do not descend +into subirectories:: + + $ sahara-templates --config-file myconfig update -t $TENANT_ID -d mypath -n + +Create/update default templates bundled with Sahara for just the vanilla plugin:: + + $ sahara-templates --config-file myconfig update -t $TENANT_ID -p vanilla + +Create/update default templates bundled with Sahara for just version 2.6.0 +of the vanilla plugin:: + + $ sahara-templates --config-file myconfig update -t $TENANT_ID -p vanilla -pv 2.6.0 + +Create/update default templates bundled with Sahara for just version 2.6.0 +of the vanilla plugin and version 2.0.6 of the hdp plugin:: + + $ sahara-templates --config-file myconfig update -t $TENANT_ID -p vanilla -pv vanilla.2.6.0 -p hdp -pv hdp.2.0.6 + +Delete default templates for the vanilla plugin:: + + $ sahara-templates --config-file myconfig delete -t $TENANT_ID -p vanilla + +Delete default templates for version 2.6.0 of the vanilla plugin:: + + $ sahara-templates --config-file myconfig delete -t $TENANT_ID -p vanilla -pv 2.6.0 + +Delete a specific node group template by ID:: + + $ sahara-templates --config-file myconfig node-group-template-delete-id --id ID + +Delete a specific cluster template by ID:: + + $ sahara-templates --config-file myconfig cluster-template-delete-id --id ID + +Delete a specific node group template by name:: + + $ sahara-templates --config-file myconfig node-group-template-delete --name NAME -t $TENANT_ID + +Delete a specific cluster template by name:: + + $ sahara-templates --config-file myconfig cluster-template-delete-id --name NAME -t $TENANT_ID diff --git a/sahara/db/templates/__init__.py b/sahara/db/templates/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/sahara/db/templates/api.py b/sahara/db/templates/api.py new file mode 100644 index 00000000..858b9b48 --- /dev/null +++ b/sahara/db/templates/api.py @@ -0,0 +1,803 @@ +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import json +import os +import uuid + +import jsonschema +from oslo_config import cfg +import six + +from sahara import conductor +from sahara.db.templates import utils as u +from sahara.service.validations import cluster_template_schema as clt +from sahara.service.validations import node_group_template_schema as ngt +from sahara.utils import api_validator + + +LOG = None +CONF = None + + +# This is broken out to support testability +def set_logger(log): + global LOG + LOG = log + + +# This is broken out to support testability +def set_conf(conf): + global CONF + CONF = conf + +ng_validator = api_validator.ApiValidator(ngt.NODE_GROUP_TEMPLATE_SCHEMA) +ct_validator = api_validator.ApiValidator(clt.CLUSTER_TEMPLATE_SCHEMA) + +# Options that we allow to be replaced in a node group template +node_group_template_opts = [ + cfg.StrOpt('image_id', + help='Image id field for a node group template.'), + + cfg.StrOpt('flavor_id', + help='Flavor id field for a node group template.'), + + cfg.StrOpt('floating_ip_pool', + help='Floating ip pool field for a node group template.') + ] + +# Options that we allow to be replaced in a cluster template +cluster_template_opts = [ + cfg.StrOpt('default_image_id', + help='Default image id field for a cluster template.'), + + cfg.StrOpt('neutron_management_network', + help='Neutron management network ' + 'field for a cluster template.')] + +all_template_opts = node_group_template_opts + cluster_template_opts + +node_group_template_opt_names = [o.name for o in node_group_template_opts] +cluster_template_opt_names = [o.name for o in cluster_template_opts] + + +# This is a local exception class that is used to exit routines +# in cases where error information has already been logged. +# It is caught and suppressed everywhere it is used. +class Handled(Exception): + pass + + +class Context(object): + '''Create a pseudo Context object + + Since this tool does not use the REST interface, we + do not have a request from which to build a Context. + ''' + def __init__(self, is_admin=False, tenant_id=None): + self.is_admin = is_admin + self.tenant_id = tenant_id + + +def check_usage_of_existing(ctx, ng_templates, cl_templates): + '''Determine if any of the specified templates are in use + + This method searches for the specified templates by name and + determines whether or not any existing templates are in use + by a cluster or cluster template. Returns True if any of + the templates are in use. + + :param ng_templates: A list of dictionaries. Each dictionary + has a "template" entry that represents + a node group template. + :param cl_templates: A list of dictionaries. Each dictionary + has a "template" entry that represents + a cluster template + :returns: True if any of the templates are in use, False otherwise + ''' + error = False + clusters = conductor.API.cluster_get_all(ctx) + + for ng_info in ng_templates: + ng = u.find_node_group_template_by_name(ctx, + ng_info["template"]["name"]) + if ng: + cluster_users, template_users = u.check_node_group_template_usage( + ng["id"], clusters) + + if cluster_users: + LOG.warning("Node group template {name} " + "in use by clusters {clusters}".format( + name=ng["name"], clusters=cluster_users)) + if template_users: + LOG.warning("Node group template {name} " + "in use by cluster templates {templates}".format( + name=ng["name"], templates=template_users)) + + if cluster_users or template_users: + LOG.warning("Update of node group template " + "{name} is not allowed".format(name=ng["name"])) + error = True + + for cl_info in cl_templates: + cl = u.find_cluster_template_by_name(ctx, cl_info["template"]["name"]) + if cl: + cluster_users = u.check_cluster_template_usage(cl["id"], clusters) + + if cluster_users: + LOG.warning("Cluster template {name} " + "in use by clusters {clusters}".format( + name=cl["name"], clusters=cluster_users)) + + LOG.warning("Update of cluster template " + "{name} is not allowed".format(name=cl["name"])) + error = True + + return error + + +def log_skipping_dir(path, reason=""): + if reason: + reason = ", " + reason + LOG.warning("Skipping processing for {dir}{reason}".format( + dir=path, reason=reason)) + + +def check_cluster_templates_valid(ng_templates, cl_templates): + # Check that if name references to node group templates + # are replaced with a uuid value that the cluster template + # passes JSON validation. We don't have the real uuid yet, + # but this will allow the validation test. + if ng_templates: + dummy_uuid = uuid.uuid4() + ng_ids = {ng["template"]["name"]: dummy_uuid for ng in ng_templates} + else: + ng_ids = {} + + for cl in cl_templates: + template = copy.deepcopy(cl["template"]) + u.substitute_ng_ids(template, ng_ids) + try: + ct_validator.validate(template) + except jsonschema.ValidationError as e: + LOG.warning("Validation for {path} failed, {reason}".format( + path=cl["path"], reason=e)) + return True + return False + + +def add_config_section(section_name, options): + if section_name and hasattr(CONF, section_name): + # It's already been added + return + + if section_name: + group = cfg.OptGroup(name=section_name) + CONF.register_group(group) + CONF.register_opts(options, group) + else: + # Add options to the default section + CONF.register_opts(options) + + +def add_config_section_for_template(template): + '''Register a config section based on the template values + + Check to see if the configuration files contain a section + that corresponds to the template. If an appropriate section + can be found, register options for the template so that the + config values can be read and applied to the template via + substitution (oslo supports registering groups and options + at any time, before or after the config files are parsed). + + Corresponding section names may be of the following forms: + + , example "hdp-2.0.6-master" + This is useful when a template naming convention is being used, + so that the template name is already unambiguous + + __, example "hdp_2.0.6_master" + This can be used if there is a name collision between templates + + _, example "hdp_2.0.6" + , example "hdp" + DEFAULT + + Sections are tried in the order given above. + + Since the first two section naming forms refer to a specific + template by name, options are added based on template type. + + However, the other section naming forms may map to node group templates + or cluster templates, so options for both are added. + ''' + sections = list(CONF.list_all_sections()) + + unique_name = "{name}".format(**template) + fullname = "{plugin_name}_{hadoop_version}_{name}".format(**template) + plugin_version = "{plugin_name}_{hadoop_version}".format(**template) + plugin = "{plugin_name}".format(**template) + + section_name = None + if unique_name in sections: + section_name = unique_name + elif fullname in sections: + section_name = fullname + + if section_name: + if u.is_node_group(template): + opts = node_group_template_opts + else: + opts = cluster_template_opts + else: + if plugin_version in sections: + section_name = plugin_version + elif plugin in sections: + section_name = plugin + opts = all_template_opts + + add_config_section(section_name, opts) + return section_name + + +def substitute_config_values(configs, template, path): + if u.is_node_group(template): + opt_names = node_group_template_opt_names + else: + opt_names = cluster_template_opt_names + + for opt, value in six.iteritems(configs): + if opt in opt_names and opt in template: + if value is None: + # TODO(tmckay): someday if we support 'null' in JSON + # we should replace this value with None. json.load + # will replace 'null' with None, and sqlalchemy will + # accept None as a value for a nullable field. + del template[opt] + LOG.debug("No replacement value specified for {opt} in " + "{path}, removing".format(opt=opt, path=path)) + else: + # Use args to allow for keyword arguments to format + args = {opt: value} + template[opt] = template[opt].format(**args) + + +def get_configs(section): + if section is None: + return dict(CONF) + return dict(CONF[section]) + + +def get_plugin_name(): + if CONF.command.name == "update" and ( + not CONF.command.plugin_name and ( + hasattr(CONF, "plugins") and CONF.plugins)): + return CONF.plugins + return CONF.command.plugin_name + + +def process_files(dirname, files): + + node_groups = [] + clusters = [] + plugin_name = get_plugin_name() + + try: + for fname in files: + if os.path.splitext(fname)[1] == ".json": + fpath = os.path.join(dirname, fname) + with open(fpath, 'r') as fp: + try: + template = json.load(fp) + except ValueError as e: + LOG.warning("Error processing {path}, {reason}".format( + path=fpath, reason=e)) + raise Handled("error processing files") + + # If this file doesn't contain basic fields, skip it. + # If we are filtering on plugin and version make + # sure the file is one that we want + if not u.check_basic_fields(template) or ( + not u.check_plugin_name_and_version( + template, + plugin_name, + CONF.command.plugin_version)): + continue + + # Look through the sections in CONF and register + # options for this template if we find a section + # related to the template (ie, plugin, version, name) + section = add_config_section_for_template(template) + LOG.debug("Using config section {section} " + "for {path}".format(section=section, path=fpath)) + + # Attempt to resolve substitutions using the config section + substitute_config_values(get_configs(section), + template, fpath) + + file_entry = {'template': template, + 'path': fpath} + + if u.is_node_group(template): + # JSON validator + try: + ng_validator.validate(template) + except jsonschema.ValidationError as e: + LOG.warning("Validation for {path} failed, " + "{reason}".format(path=fpath, + reason=e)) + raise Handled( + "node group template validation failed") + node_groups.append(file_entry) + LOG.debug("Added {path} to node group " + "template files".format(path=fpath)) + else: + clusters.append(file_entry) + LOG.debug("Added {path} to cluster template " + "files".format(path=fpath)) + + except Handled as e: + log_skipping_dir(dirname, e.message) + node_groups = [] + clusters = [] + + except Exception as e: + log_skipping_dir(dirname, + "unhandled exception, {reason}".format(reason=e)) + node_groups = [] + clusters = [] + + return node_groups, clusters + + +def delete_node_group_template(ctx, template, rollback=False): + rollback_msg = " on rollback" if rollback else "" + + # If we are not deleting something that we just created, + # do usage checks to ensure that the template is not in + # use by a cluster or a cluster template + if not rollback: + clusters = conductor.API.cluster_get_all(ctx) + cluster_templates = conductor.API.cluster_template_get_all(ctx) + cluster_users, template_users = u.check_node_group_template_usage( + template["id"], clusters, cluster_templates) + + if cluster_users: + LOG.warning("Node group template {info} " + "in use by clusters {clusters}".format( + info=u.name_and_id(template), + clusters=cluster_users)) + if template_users: + LOG.warning("Node group template {info} " + "in use by cluster templates {templates}".format( + info=u.name_and_id(template), + templates=template_users)) + + if cluster_users or template_users: + LOG.warning("Deletion of node group template " + "{info} failed".format(info=u.name_and_id(template))) + return + + try: + conductor.API.node_group_template_destroy(ctx, template["id"], + ignore_default=True) + except Exception as e: + LOG.warning("Deletion of node group template {info} failed{rollback}" + ", {reason}".format(info=u.name_and_id(template), + reason=e, + rollback=rollback_msg)) + else: + LOG.info("Deleted node group template {info}{rollback}".format( + info=u.name_and_id(template), rollback=rollback_msg)) + + +def reverse_node_group_template_creates(ctx, templates): + for template in templates: + delete_node_group_template(ctx, template, rollback=True) + + +def reverse_node_group_template_updates(ctx, update_info): + for template, values in update_info: + # values are the original values that we overwrote in the update + try: + conductor.API.node_group_template_update(ctx, + template["id"], values, + ignore_default=True) + except Exception as e: + LOG.warning("Rollback of update for node group " + "template {info} failed, {reason}".format( + info=u.name_and_id(template), + reason=e)) + else: + LOG.info("Rolled back update for " + "node group template {info}".format( + info=u.name_and_id(template))) + + +def add_node_group_templates(ctx, node_groups): + + error = False + ng_info = {"ids": {}, + "created": [], + "updated": []} + + def do_reversals(ng_info): + reverse_node_group_template_updates(ctx, ng_info["updated"]) + reverse_node_group_template_creates(ctx, ng_info["created"]) + return {}, True + + try: + for ng in node_groups: + template = ng['template'] + current = u.find_node_group_template_by_name(ctx, template['name']) + if current: + + # Track what we see in the current template that is different + # from our update values. Save it for possible rollback. + # Note, this is not perfect because it does not recurse through + # nested structures to get an exact diff, but it ensures that + # we track only fields that are valid in the JSON schema + updated_fields = u.value_diff(current.to_dict(), template) + + # Always attempt to update. Since the template value is a + # combination of JSON and config values, there is no useful + # timestamp we can use to skip an update. + # If sqlalchemy determines no change in fields, it will not + # mark it as updated. + try: + template = conductor.API.node_group_template_update( + ctx, current['id'], template, ignore_default=True) + except Exception as e: + LOG.warning("Update of node group template {info} " + "failed, {reason}".format( + info=u.name_and_id(current), + reason=e)) + raise Handled() + + if template['updated_at'] != current['updated_at']: + ng_info["updated"].append((template, updated_fields)) + LOG.info("Updated node group template {info} " + "from {path}".format( + info=u.name_and_id(template), + path=ng["path"])) + else: + LOG.debug("No change to node group template {info} " + "from {path}".format( + info=u.name_and_id(current), + path=ng['path'])) + else: + template['is_default'] = True + try: + template = conductor.API.node_group_template_create( + ctx, template) + except Exception as e: + LOG.warning("Creation of node group template " + "from {path} failed, {reason}".format( + path=ng['path'], reason=e)) + raise Handled() + + ng_info["created"].append(template) + LOG.info("Created node group template {info} " + "from {path}".format(info=u.name_and_id(template), + path=ng["path"])) + + # For the purposes of substituion we need a dict of id by name + ng_info["ids"][template['name']] = template['id'] + + except Handled: + ng_info, error = do_reversals(ng_info) + + except Exception as e: + LOG.warning("Unhandled exception while processing " + "node group templates, {reason}".format(reason=e)) + ng_info, error = do_reversals(ng_info) + + return ng_info, error + + +def delete_cluster_template(ctx, template, rollback=False): + rollback_msg = " on rollback" if rollback else "" + + # If we are not deleting something that we just created, + # do usage checks to ensure that the template is not in + # use by a cluster + if not rollback: + clusters = conductor.API.cluster_get_all(ctx) + cluster_users = u.check_cluster_template_usage(template["id"], + clusters) + + if cluster_users: + LOG.warning("Cluster template {info} " + "in use by clusters {clusters}".format( + info=u.name_and_id(template), + clusters=cluster_users)) + + LOG.warning("Deletion of cluster template " + "{info} failed".format(info=u.name_and_id(template))) + return + + try: + conductor.API.cluster_template_destroy(ctx, template["id"], + ignore_default=True) + except Exception as e: + LOG.warning("Deletion of cluster template {info} failed{rollback}" + ", {reason}".format(info=u.name_and_id(template), + reason=e, + rollback=rollback_msg)) + else: + LOG.info("Deleted cluster template {info}{rollback}".format( + info=u.name_and_id(template), rollback=rollback_msg)) + + +def reverse_cluster_template_creates(ctx, templates): + for template in templates: + delete_cluster_template(ctx, template, rollback=True) + + +def reverse_cluster_template_updates(ctx, update_info): + for template, values in update_info: + # values are the original values that we overwrote in the update + try: + conductor.API.cluster_template_update(ctx, + template["id"], values, + ignore_default=True) + except Exception as e: + LOG.warning("Rollback of update for cluster " + "template {info} failed, {reason}".format( + info=u.name_and_id(template), + reason=e)) + else: + LOG.info("Rolled back update for " + "cluster template {info}".format( + info=u.name_and_id(template))) + + +def add_cluster_templates(ctx, clusters, ng_dict): + '''Add cluster templates to the database. + + The value of any node_group_template_id fields in cluster + templates which reference a node group template in ng_dict by name + will be changed to the id of the node group template. + + If there is an error in creating or updating a template, any templates + that have already been created will be delete and any updates will + be reversed. + + :param clusters: a list of dictionaries. Each dictionary + has a "template" entry holding the cluster template + and a "path" entry holding the path of the file + from which the template was read. + :param ng_dict: a dictionary of node group template ids keyed + by node group template names + ''' + + error = False + created = [] + updated = [] + + def do_reversals(created, updated): + reverse_cluster_template_updates(ctx, updated) + reverse_cluster_template_creates(ctx, created) + return True + + try: + for cl in clusters: + template = cl['template'] + + # Fix up node_group_template_id fields + u.substitute_ng_ids(template, ng_dict) + + # Name + tenant_id is unique, so search by name + current = u.find_cluster_template_by_name(ctx, template['name']) + if current: + + # Track what we see in the current template that is different + # from our update values. Save it for possible rollback. + # Note, this is not perfect because it does not recurse through + # nested structures to get an exact diff, but it ensures that + # we track only fields that are valid in the JSON schema + updated_fields = u.value_diff(current.to_dict(), template) + + # Always attempt to update. Since the template value is a + # combination of JSON and config values, there is no useful + # timestamp we can use to skip an update. + # If sqlalchemy determines no change in fields, it will not + # mark it as updated. + + # TODO(tmckay): why when I change the count in an + # entry in node_groups does it not count as an update? + # Probably a bug + try: + template = conductor.API.cluster_template_update( + ctx, current['id'], template, ignore_default=True) + except Exception as e: + LOG.warning("Update of cluster template {info} " + "failed, {reason}".format( + info=u.name_and_id(current), reason=e)) + raise Handled() + + if template['updated_at'] != current['updated_at']: + updated.append((template, updated_fields)) + LOG.info("Updated cluster template {info} " + "from {path}".format( + info=u.name_and_id(template), + path=cl['path'])) + else: + LOG.debug("No change to cluster template {info} " + "from {path}".format(info=u.name_and_id(current), + path=cl["path"])) + else: + template["is_default"] = True + try: + template = conductor.API.cluster_template_create(ctx, + template) + except Exception as e: + LOG.warning("Creation of cluster template " + "from {path} failed, {reason}".format( + path=cl['path'], + reason=e)) + raise Handled() + + created.append(template) + LOG.info("Created cluster template {info} from {path}".format( + info=u.name_and_id(template), path=cl['path'])) + + except Handled: + error = do_reversals(created, updated) + + except Exception as e: + LOG.warning("Unhandled exception while processing " + "cluster templates, {reason}".format(reason=e)) + error = do_reversals(created, updated) + + return error + + +def do_update(): + '''Create or update default templates for the specified tenant. + + Looks for '.json' files beginning at the specified starting + directory (--directory CLI option) and descending + through subdirectories by default. + + The .json files represent cluster templates or node group + templates. All '.json' files at the same directory level are treated + as a set. Cluster templates may reference node group templates + in the same set. + + If an error occurs in processing a set, skip it and continue. + + If creation of cluster templates fails, any node group templates + in the set that were already created will be deleted. + ''' + + ctx = Context(tenant_id=CONF.command.tenant_id) + start_dir = os.path.abspath(CONF.command.directory) + + for root, dirs, files in os.walk(start_dir): + + # Find all the template files and identify them as node_group + # or cluster templates. If there is an exception in + # processing the set, return empty lists. + node_groups, clusters = process_files(root, files) + + # Now that we know what the valid node group templates are, + # we can validate the cluster templates as well. + if check_cluster_templates_valid(node_groups, clusters): + log_skipping_dir(root, "error processing cluster templates") + + # If there are existing default templates that match the names + # in the template files, do usage checks here to detect update + # failures early (we can't update a template in use) + elif check_usage_of_existing(ctx, node_groups, clusters): + log_skipping_dir(root, "templates in use") + else: + ng_info, error = add_node_group_templates(ctx, node_groups) + if error: + log_skipping_dir(root, "error processing node group templates") + + elif add_cluster_templates(ctx, clusters, ng_info["ids"]): + log_skipping_dir(root, "error processing cluster templates") + + # Cluster templates failed so remove the node group templates + reverse_node_group_template_updates(ctx, ng_info["updated"]) + reverse_node_group_template_creates(ctx, ng_info["created"]) + + if CONF.command.norecurse: + break + + +def do_delete(): + '''Delete default templates in the specified tenant + + Deletion uses the --plugin-name and --plugin-version options + as filters. + + Only templates with 'is_default=True' will be deleted. + ''' + + ctx = Context(tenant_id=CONF.command.tenant_id) + + for plugin in get_plugin_name(): + + kwargs = {'is_default': True} + kwargs['plugin_name'] = plugin + + # Delete cluster templates first for the sake of usage checks + lst = conductor.API.cluster_template_get_all(ctx, **kwargs) + for l in lst: + if not u.check_plugin_version(l, CONF.command.plugin_version): + continue + delete_cluster_template(ctx, l) + + lst = conductor.API.node_group_template_get_all(ctx, **kwargs) + for l in lst: + if not u.check_plugin_version(l, CONF.command.plugin_version): + continue + delete_node_group_template(ctx, l) + + +def do_node_group_template_delete(): + ctx = Context(tenant_id=CONF.command.tenant_id) + + t = u.find_node_group_template_by_name(ctx, CONF.command.template_name) + if t: + delete_node_group_template(ctx, t) + else: + LOG.warning("Deletion of node group template {name} failed, " + "no such template".format(name=CONF.command.template_name)) + + +def do_node_group_template_delete_by_id(): + ctx = Context(is_admin=True) + + # Make sure it's a default + t = conductor.API.node_group_template_get(ctx, CONF.command.id) + if t: + if t["is_default"]: + delete_node_group_template(ctx, t) + else: + LOG.warning("Deletion of node group template {info} skipped, " + "not a default template".format(info=u.name_and_id(t))) + else: + LOG.warning("Deletion of node group template {id} failed, " + "no such template".format(id=CONF.command.id)) + + +def do_cluster_template_delete(): + ctx = Context(tenant_id=CONF.command.tenant_id) + + t = u.find_cluster_template_by_name(ctx, CONF.command.template_name) + if t: + delete_cluster_template(ctx, t) + else: + LOG.warning("Deletion of cluster template {name} failed, " + "no such template".format(name=CONF.command.template_name)) + + +def do_cluster_template_delete_by_id(): + ctx = Context(is_admin=True) + + # Make sure it's a default + t = conductor.API.cluster_template_get(ctx, CONF.command.id) + if t: + if t["is_default"]: + delete_cluster_template(ctx, t) + else: + LOG.warning("Deletion of cluster template {info} skipped, " + "not a default template".format(info=u.name_and_id(t))) + else: + LOG.warning("Deletion of cluster template {id} failed, " + "no such template".format(id=CONF.command.id)) diff --git a/sahara/db/templates/cli.py b/sahara/db/templates/cli.py new file mode 100644 index 00000000..3f253fe7 --- /dev/null +++ b/sahara/db/templates/cli.py @@ -0,0 +1,208 @@ +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function +import sys + +from oslo_config import cfg +from oslo_log import log +import pkg_resources as pkg + +from sahara.db.templates import api +from sahara import version + +LOG = log.getLogger(__name__) + +CONF = cfg.CONF + + +def extra_option_checks(): + + if not CONF.database.connection: + print("No database connection string was specified in configuration", + file=sys.stderr) + sys.exit(1) + + if CONF.command.name in ['update', 'delete']: + if CONF.command.plugin_version and not CONF.command.plugin_name: + print("The --plugin-version option is not valid " + "without --plugin-name", file=sys.stderr) + sys.exit(-1) + + if CONF.command.name == "update": + # Options handling probably needs some refactoring in the future. + # For now, though, we touch the conductor which ultimately touches + # the plugins.base. Use the "plugins" option there as a default + # list of plugins to process, since those are the plugins that + # will be loaded by Sahara + if not CONF.command.plugin_name: + if "plugins" in CONF and CONF.plugins: + LOG.info("Using plugin list {plugins} from config".format( + plugins=CONF.plugins)) + else: + print("No plugins specified with --plugin-name " + "or config", file=sys.stderr) + sys.exit(-1) + + +def add_command_parsers(subparsers): + # Note, there is no 'list' command here because the client + # or REST can be used for list operations. Default templates + # will display, and templates will show the 'is_default' field. + + def add_id(parser): + parser.add_argument('--id', required=True, + help='The id of the default ' + 'template to delete') + + def add_tenant_id(parser): + parser.add_argument('-t', '--tenant_id', required=True, + help='Tenant ID for database operations.') + + def add_name_and_tenant_id(parser): + parser.add_argument('--name', dest='template_name', required=True, + help='Name of the default template') + add_tenant_id(parser) + + def add_plugin_name_and_version(parser, require_plugin_name=False): + + plugin_name_help = ('Only process templates containing ' + 'a "plugin_name" field matching ' + 'one of these values.') + + if not require_plugin_name: + extra = (' The default list of plugin names ' + 'is taken from the "plugins" parameter in ' + 'the [DEFAULT] config section.') + plugin_name_help += extra + + parser.add_argument('-p', '--plugin-name', nargs="*", + required=require_plugin_name, + help=plugin_name_help) + + parser.add_argument('-pv', '--plugin-version', nargs="*", + help='Only process templates containing a ' + '"hadoop_version" field matching one of ' + 'these values. This option is ' + 'only valid if --plugin-name is specified ' + 'as well. A version specified ' + 'here may optionally be prefixed with a ' + 'plugin name and a dot, for exmaple ' + '"vanilla.1.2.1". Dotted versions only ' + 'apply to the plugin named in the ' + 'prefix. Versions without a prefix apply to ' + 'all plugins.') + + fname = pkg.resource_filename(version.version_info.package, + "plugins/default_templates") + # update command + parser = subparsers.add_parser('update', + help='Update the default template set') + parser.add_argument('-d', '--directory', + default=fname, + help='Template directory. Default is %s' % fname) + parser.add_argument('-n', '--norecurse', action='store_true', + help='Do not descend into subdirectories') + + add_plugin_name_and_version(parser) + add_tenant_id(parser) + parser.set_defaults(func=api.do_update) + + # delete command + parser = subparsers.add_parser('delete', + help='Delete default templates ' + 'by plugin and version') + add_plugin_name_and_version(parser, require_plugin_name=True) + add_tenant_id(parser) + parser.set_defaults(func=api.do_delete) + + # node-group-template-delete command + parser = subparsers.add_parser('node-group-template-delete', + help='Delete a default ' + 'node group template by name') + add_name_and_tenant_id(parser) + parser.set_defaults(func=api.do_node_group_template_delete) + + # cluster-template-delete command + parser = subparsers.add_parser('cluster-template-delete', + help='Delete a default ' + 'cluster template by name') + add_name_and_tenant_id(parser) + parser.set_defaults(func=api.do_cluster_template_delete) + + # node-group-template-delete-id command + parser = subparsers.add_parser('node-group-template-delete-id', + help='Delete a default ' + 'node group template by id') + add_id(parser) + parser.set_defaults(func=api.do_node_group_template_delete_by_id) + + # cluster-template-delete-id command + parser = subparsers.add_parser('cluster-template-delete-id', + help='Delete a default ' + 'cluster template by id') + add_id(parser) + parser.set_defaults(func=api.do_cluster_template_delete_by_id) + + +command_opt = cfg.SubCommandOpt('command', + title='Command', + help='Available commands', + handler=add_command_parsers) +CONF.register_cli_opt(command_opt) + + +def unregister_extra_cli_opt(name): + try: + for cli in CONF._cli_opts: + if cli['opt'].name == name: + CONF.unregister_opt(cli['opt']) + except Exception: + pass + + +# Remove a few extra CLI opts that we picked up via imports +# Do this early so that they do not appear in the help +for extra_opt in ["log-exchange", "host", "port"]: + unregister_extra_cli_opt(extra_opt) + + +def main(): + # TODO(tmckay): Work on restricting the options + # pulled in by imports which show up in the help. + # If we find a nice way to do this the calls to + # unregister_extra_cli_opt() can be removed + CONF(project='sahara') + + # For some reason, this is necessary to clear cached values + # and re-read configs. For instance, if this is not done + # here the 'plugins' value will not reflect the value from + # the config file on the command line + CONF.reload_config_files() + log.setup(CONF, "sahara") + + # If we have to enforce extra option checks, like one option + # requires another, do it here + extra_option_checks() + + # Since this may be scripted, record the command in the log + # so a user can know exactly what was done + LOG.info("Command: {command}".format(command=' '.join(sys.argv))) + + api.set_logger(LOG) + api.set_conf(CONF) + + CONF.command.func() + + LOG.info("Finished {command}".format(command=CONF.command.name)) diff --git a/sahara/db/templates/utils.py b/sahara/db/templates/utils.py new file mode 100644 index 00000000..266576c1 --- /dev/null +++ b/sahara/db/templates/utils.py @@ -0,0 +1,190 @@ +# Copyright 2015 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy + +import six + +from sahara import conductor + + +def name_and_id(template): + return "{name} ({id})".format(name=template["name"], + id=template["id"]) + + +def is_node_group(template): + # Node group templates and cluster templates have + # different required fields in validation and neither + # allows additional fields. So, the presence of + # node_processes or flavor_id should suffice to + # identify a node group template. Check for both + # to be nice, in case someone made a typo. + return 'node_processes' in template or 'flavor_id' in template + + +def substitute_ng_ids(cl, ng_dict): + '''Substitute node group template ids for node group template names + + If the cluster template contains node group elements with + node_group_template_id fields that reference node group templates + by name, substitute the node group template id for the name. + The name reference is expected to be a string containing a format + specifier of the form "{name}", for example "{master}" + + :param cl: a cluster template + :param ng_dict: a dictionary of node group template ids keyed by + node group template names + ''' + for ng in cl["node_groups"]: + if "node_group_template_id" in ng: + val = ng["node_group_template_id"].format(**ng_dict) + ng["node_group_template_id"] = val + + +def check_basic_fields(template): + return "plugin_name" in template and ( + "hadoop_version" in template and ( + "name" in template)) + + +def check_plugin_version(template, plugin_versions): + '''Check that the template matches the plugin versions list + + Tests whether or not the plugin version indicated by the template + matches one of the versions specified in plugin_versions + + :param template: A node group or cluster template + :param plugin_versions: A list of plugin version strings. These + values may be regular version strings or may be + the name of the plugin followed by a + "." followed by a version string. + :returns: True if the plugin version specified in the template + matches a version in plugin_versions or plugin_versions + is an empty list. Otherwise False + ''' + def dotted_name(template): + return template['plugin_name'] + "." + template['hadoop_version'] + + version_matches = plugin_versions is None or ( + template['hadoop_version'] in plugin_versions) or ( + dotted_name(template) in plugin_versions) + + return version_matches + + +def check_plugin_name_and_version(template, plugin_names, plugin_versions): + '''Check that the template is for one of the specified plugins + + Tests whether or not the plugin name and version indicated by the template + matches one of the names and one of the versions specified in + plugin_names and plugin_versions + + :param template: A node group or cluster template + :param plugin_names: A list of plugin names + :param plugin_versions: A list of plugin version strings. These + values may be regular version strings or may be + the name of the plugin followed by a + "." followed by a version string. + :returns: True if the plugin name specified in the template matches + a name in plugin_names or plugin_names is an empty list, and if + the plugin version specified in the template matches a version + in plugin_versions or plugin_versions is an empty list. + Otherwise False + ''' + name_and_version_matches = (plugin_names is None or ( + template['plugin_name'] in plugin_names)) and ( + check_plugin_version(template, plugin_versions)) + + return name_and_version_matches + + +# TODO(tmckay): refactor the service validation code so +# that the node group template usage checks there can be reused +# without incurring unnecessary dependencies +def check_node_group_template_usage(node_group_template_id, + cluster_list, cluster_template_list=[]): + cluster_users = [] + template_users = [] + + for cluster in cluster_list: + if (node_group_template_id in + [node_group.node_group_template_id + for node_group in cluster.node_groups]): + cluster_users += [cluster.name] + + for cluster_template in cluster_template_list: + if (node_group_template_id in + [node_group.node_group_template_id + for node_group in cluster_template.node_groups]): + template_users += [cluster_template.name] + + return cluster_users, template_users + + +# TODO(tmckay): refactor the service validation code so +# that the cluster template usage checks there can be reused +# without incurring unnecessary dependencies +def check_cluster_template_usage(cluster_template_id, cluster_list): + cluster_users = [] + for cluster in cluster_list: + if cluster_template_id == cluster.cluster_template_id: + cluster_users.append(cluster.name) + + return cluster_users + + +def find_node_group_template_by_name(ctx, name): + t = conductor.API.node_group_template_get_all(ctx, + name=name, + is_default=True) + if t: + return t[0] + return None + + +def find_cluster_template_by_name(ctx, name): + t = conductor.API.cluster_template_get_all(ctx, + name=name, + is_default=True) + if t: + return t[0] + return None + + +def value_diff(current, new_values): + '''Return the entries in current that would be overwritten by new_values + + Returns the set of entries in current that would be overwritten + if current.update(new_values) was called. + + :param current: A dictionary whose key values are a superset + of the key values in new_values + :param new_values: A dictionary + ''' + # Current is an existing template from the db and + # template is a set of values that has been validated + # against the JSON schema for the template. + # Copy items from current if they are present in template. + + # In the case of "node_groups" the conductor does magic + # to set up template relations and insures that appropriate + # fields are cleaned (like "updated_at" and "id") so we + # trust the conductor in that case. + + diff_values = {} + for k, v in six.iteritems(new_values): + if k in current and current[k] != v: + diff_values[k] = copy.deepcopy(current[k]) + return diff_values diff --git a/sahara/plugins/default_templates/template.conf b/sahara/plugins/default_templates/template.conf new file mode 100644 index 00000000..0c1d51ee --- /dev/null +++ b/sahara/plugins/default_templates/template.conf @@ -0,0 +1,16 @@ +[DEFAULT] +# Set the flavor_id to 2 which is m1.small in the +# default flavor set +flavor_id = 2 + +[cdh-5-default-namenode] +# For the CDH plugin, version 5, set the flavor_id +# of the master node to 4 which is m1.large in the +# default flavor set +flavor_id = 4 + +[cdh-530-default-namenode] +# For the CDH plugin, version 5.3.0, set the flavor_id +# of the master node to 4 which is m1.large in the +# default flavor set +flavor_id = 4 diff --git a/setup.cfg b/setup.cfg index 9e18d540..2f1b9959 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,6 +35,7 @@ console_scripts = sahara-db-manage = sahara.db.migration.cli:main sahara-rootwrap = oslo_rootwrap.cmd:main _sahara-subprocess = sahara.cli.sahara_subprocess:main + sahara-templates = sahara.db.templates.cli:main sahara.cluster.plugins = vanilla = sahara.plugins.vanilla.plugin:VanillaProvider