deb-sahara/sahara/conductor/manager.py

# Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Handles database requests from other Sahara services."""

import copy

from sahara.db import base as db_base
from sahara.utils import configs
from sahara.utils import crypto


CLUSTER_DEFAULTS = {
    "cluster_configs": {},
    "status": "undefined",
    "anti_affinity": [],
    "status_description": "",
    "info": {},
    "rollback_info": {},
    "sahara_info": {},
    "is_public": False,
    "is_protected": False
}

NODE_GROUP_DEFAULTS = {
    "node_processes": [],
    "node_configs": {},
    "volumes_per_node": 0,
    "volumes_size": 0,
    "volumes_availability_zone": None,
    "volume_mount_prefix": "/volumes/disk",
    "volume_type": None,
    "floating_ip_pool": None,
    "security_groups": None,
    "auto_security_group": False,
    "availability_zone": None,
    "is_proxy_gateway": False,
    "volume_local_to_instance": False,
}

NODE_GROUP_TEMPLATE_DEFAULTS = copy.deepcopy(NODE_GROUP_DEFAULTS)
NODE_GROUP_TEMPLATE_DEFAULTS.update({"is_public": False,
                                     "is_protected": False})

INSTANCE_DEFAULTS = {
    "volumes": [],
    "storage_devices_number": 0
}

DATA_SOURCE_DEFAULTS = {
    "credentials": {},
    "is_public": False,
    "is_protected": False
}

JOB_DEFAULTS = {
    "is_public": False,
    "is_protected": False
}

JOB_BINARY_DEFAULTS = {
    "is_public": False,
    "is_protected": False
}

JOB_BINARY_INTERNAL_DEFAULTS = {
    "is_public": False,
    "is_protected": False
}

JOB_EXECUTION_DEFAULTS = {
    "is_public": False,
    "is_protected": False
}


def _apply_defaults(values, defaults):
    new_values = copy.deepcopy(defaults)
    new_values.update(values)
    return new_values


class ConductorManager(db_base.Base):
    """This class aimed to conduct things.

    The methods in the base API for sahara-conductor are various proxy
    operations that allows other services to get specific work done without
    locally accessing the database.

    Additionally it performs some template-to-object copying magic.
    """

    def __init__(self):
        super(ConductorManager, self).__init__()

    # Common helpers

    def _populate_node_groups(self, context, cluster):
        node_groups = cluster.get('node_groups')
        if not node_groups:
            return []

        populated_node_groups = []
        for node_group in node_groups:
            populated_node_group = self._populate_node_group(context,
                                                             node_group)
            self._cleanup_node_group(populated_node_group)
            populated_node_group["tenant_id"] = context.tenant_id
            populated_node_groups.append(
                populated_node_group)

        return populated_node_groups

    def _cleanup_node_group(self, node_group):
        node_group.pop('id', None)
        node_group.pop('created_at', None)
        node_group.pop('updated_at', None)

    def _populate_node_group(self, context, node_group):
        node_group_merged = copy.deepcopy(NODE_GROUP_DEFAULTS)

        ng_tmpl_id = node_group.get('node_group_template_id')
        ng_tmpl = None
        if ng_tmpl_id:
            ng_tmpl = self.node_group_template_get(context, ng_tmpl_id)

            self._cleanup_node_group(ng_tmpl)
            node_group_merged.update(ng_tmpl)

        node_group_merged.update(node_group)

        if ng_tmpl:
            node_group_merged['node_configs'] = configs.merge_configs(
                ng_tmpl.get('node_configs'),
                node_group.get('node_configs'))

        return node_group_merged

    # Cluster ops

    def cluster_get(self, context, cluster, show_progress=False):
        """Return the cluster or None if it does not exist."""
        return self.db.cluster_get(context, cluster, show_progress)

    def cluster_get_all(self, context, **kwargs):
        """Get all clusters filtered by **kwargs.

        e.g. cluster_get_all(plugin_name='vanilla', hadoop_version='1.1')
        """
        return self.db.cluster_get_all(context, **kwargs)

    def cluster_create(self, context, values):
        """Create a cluster from the values dictionary."""

        # loading defaults
        merged_values = copy.deepcopy(CLUSTER_DEFAULTS)
        merged_values['tenant_id'] = context.tenant_id

        private_key, public_key = crypto.generate_key_pair()
        merged_values['management_private_key'] = private_key
        merged_values['management_public_key'] = public_key

        cluster_template_id = values.get('cluster_template_id')
        c_tmpl = None

        if cluster_template_id:
            c_tmpl = self.cluster_template_get(context, cluster_template_id)

            del c_tmpl['created_at']
            del c_tmpl['updated_at']
            del c_tmpl['id']
            del c_tmpl['is_public']
            del c_tmpl['is_protected']

            # updating with cluster_template values
            merged_values.update(c_tmpl)

        # updating with values provided in request
        merged_values.update(values)

        if c_tmpl:
            merged_values['cluster_configs'] = configs.merge_configs(
                c_tmpl.get('cluster_configs'),
                values.get('cluster_configs'))

        merged_values['node_groups'] = self._populate_node_groups(
            context, merged_values)

        return self.db.cluster_create(context, merged_values)

    def cluster_update(self, context, cluster, values):
        """Set the given properties on cluster and update it."""
        values = copy.deepcopy(values)
        return self.db.cluster_update(context, cluster, values)

    def cluster_destroy(self, context, cluster):
        """Destroy the cluster or raise if it does not exist."""
        self.db.cluster_destroy(context, cluster)

    # Node Group ops

    def node_group_add(self, context, cluster, values):
        """Create a Node Group from the values dictionary."""
        values = copy.deepcopy(values)
        values = self._populate_node_group(context, values)
        values['tenant_id'] = context.tenant_id
        return self.db.node_group_add(context, cluster, values)

    def node_group_update(self, context, node_group, values):
        """Set the given properties on node_group and update it."""
        values = copy.deepcopy(values)
        self.db.node_group_update(context, node_group, values)

    def node_group_remove(self, context, node_group):
        """Destroy the node_group or raise if it does not exist."""
        self.db.node_group_remove(context, node_group)

    # Instance ops

    def instance_add(self, context, node_group, values):
        """Create an Instance from the values dictionary."""
        values = copy.deepcopy(values)
        values = _apply_defaults(values, INSTANCE_DEFAULTS)
        values['tenant_id'] = context.tenant_id
        return self.db.instance_add(context, node_group, values)

    def instance_update(self, context, instance, values):
        """Set the given properties on Instance and update it."""
        values = copy.deepcopy(values)
        self.db.instance_update(context, instance, values)

    def instance_remove(self, context, instance):
        """Destroy the Instance or raise if it does not exist."""
        self.db.instance_remove(context, instance)

    # Volumes ops

    def append_volume(self, context, instance, volume_id):
        """Append volume_id to instance."""
        self.db.append_volume(context, instance, volume_id)

    def remove_volume(self, context, instance, volume_id):
        """Remove volume_id in instance."""
        self.db.remove_volume(context, instance, volume_id)

    # Cluster Template ops

    def cluster_template_get(self, context, cluster_template):
        """Return the cluster_template or None if it does not exist."""
        return self.db.cluster_template_get(context, cluster_template)

    def cluster_template_get_all(self, context, **kwargs):
        """Get all cluster templates filtered by **kwargs.

        e.g.  cluster_template_get_all(plugin_name='vanilla',
                                       hadoop_version='1.1')
        """
        return self.db.cluster_template_get_all(context, **kwargs)

    def cluster_template_create(self, context, values):
        """Create a cluster_template from the values dictionary."""
        values = copy.deepcopy(values)
        values = _apply_defaults(values, CLUSTER_DEFAULTS)
        values['tenant_id'] = context.tenant_id

        values['node_groups'] = self._populate_node_groups(context, values)

        return self.db.cluster_template_create(context, values)

    def cluster_template_destroy(self, context, cluster_template,
                                 ignore_default=False):
        """Destroy the cluster_template or raise if it does not exist."""
        self.db.cluster_template_destroy(context, cluster_template,
                                         ignore_default)

    def cluster_template_update(self, context, id, values,
                                ignore_default=False):
        """Update a cluster_template from the values dictionary."""
        values = copy.deepcopy(values)
        values['tenant_id'] = context.tenant_id
        values['id'] = id

        if 'node_groups' in values:
            values['node_groups'] = self._populate_node_groups(context, values)

        return self.db.cluster_template_update(context, values, ignore_default)

    # Node Group Template ops

    def node_group_template_get(self, context, node_group_template):
        """Return the Node Group Template or None if it does not exist."""
        return self.db.node_group_template_get(context, node_group_template)

    def node_group_template_get_all(self, context, **kwargs):
        """Get all NodeGroupTemplates filtered by **kwargs.

        e.g.  node_group_template_get_all(plugin_name='vanilla',
                                          hadoop_version='1.1')
        """
        return self.db.node_group_template_get_all(context, **kwargs)

    def node_group_template_create(self, context, values):
        """Create a Node Group Template from the values dictionary."""
        values = copy.deepcopy(values)
        values = _apply_defaults(values, NODE_GROUP_TEMPLATE_DEFAULTS)
        values['tenant_id'] = context.tenant_id

        return self.db.node_group_template_create(context, values)

    def node_group_template_destroy(self, context, node_group_template,
                                    ignore_default=False):
        """Destroy the Node Group Template or raise if it does not exist."""
        self.db.node_group_template_destroy(context, node_group_template,
                                            ignore_default)

    def node_group_template_update(self, context, id, values,
                                   ignore_default=False):
        """Update a Node Group Template from the values dictionary."""
        values = copy.deepcopy(values)
        values['tenant_id'] = context.tenant_id
        values['id'] = id

        return self.db.node_group_template_update(context, values,
                                                  ignore_default)

    # Data Source ops

    def data_source_get(self, context, data_source):
        """Return the Data Source or None if it does not exist."""
        return self.db.data_source_get(context, data_source)

    def data_source_get_all(self, context, **kwargs):
        """Get all Data Sources filtered by **kwargs.

        e.g.  data_source_get_all(name='myfile', type='swift')
        """
        return self.db.data_source_get_all(context, **kwargs)

    def data_source_count(self, context, **kwargs):
        """Count Data Sources filtered by **kwargs.

        Uses sqlalchemy "in_" clause for any tuple values
        Uses sqlalchemy "like" clause for any string values containing %
        """
        return self.db.data_source_count(context, **kwargs)

    def data_source_create(self, context, values):
        """Create a Data Source from the values dictionary."""
        values = copy.deepcopy(values)
        values = _apply_defaults(values, DATA_SOURCE_DEFAULTS)
        values['tenant_id'] = context.tenant_id

        return self.db.data_source_create(context, values)

    def data_source_destroy(self, context, data_source):
        """Destroy the Data Source or raise if it does not exist."""
        return self.db.data_source_destroy(context, data_source)

    def data_source_update(self, context, id, values):
        """Update the Data Source or raise if it does not exist."""
        values = copy.deepcopy(values)
        values["id"] = id
        return self.db.data_source_update(context, values)

    # JobExecution ops

    def job_execution_get(self, context, job_execution):
        """Return the JobExecution or None if it does not exist."""
        return self.db.job_execution_get(context, job_execution)

    def job_execution_get_all(self, context, **kwargs):
        """Get all JobExecutions filtered by **kwargs.

        kwargs key values may be the names of fields in a JobExecution
        plus the following special values with the indicated meaning:

        'cluster.name' -- name of the Cluster referenced by the JobExecution
        'job.name' -- name of the Job referenced by the JobExecution
        'status' -- JobExecution['info']['status']

        e.g. job_execution_get_all(cluster_id=12, input_id=123)
             job_execution_get_all(**{'cluster.name': 'test',
                                      'job.name': 'wordcount'})
        """
        return self.db.job_execution_get_all(context, **kwargs)

    def job_execution_count(self, context, **kwargs):
        """Count number of JobExecutions filtered by **kwargs.

        e.g. job_execution_count(cluster_id=12, input_id=123)
        """
        return self.db.job_execution_count(context, **kwargs)

    def job_execution_create(self, context, values):
        """Create a JobExecution from the values dictionary."""
        values = copy.deepcopy(values)
        values = _apply_defaults(values, JOB_EXECUTION_DEFAULTS)
        values['tenant_id'] = context.tenant_id
        return self.db.job_execution_create(context, values)

    def job_execution_update(self, context, job_execution, values):
        """Updates a JobExecution from the values dictionary."""
        values = copy.deepcopy(values)
        return self.db.job_execution_update(context, job_execution, values)

    def job_execution_destroy(self, context, job_execution):
        """Destroy the JobExecution or raise if it does not exist."""
        return self.db.job_execution_destroy(context, job_execution)

    # Job ops

    def job_get(self, context, job):
        """Return the Job or None if it does not exist."""
        return self.db.job_get(context, job)

    def job_get_all(self, context, **kwargs):
        """Get all Jobs filtered by **kwargs.

        e.g.  job_get_all(name='myjob', type='MapReduce')
        """
        return self.db.job_get_all(context, **kwargs)

    def job_create(self, context, values):
        """Create a Job from the values dictionary."""
        values = copy.deepcopy(values)
        values = _apply_defaults(values, JOB_DEFAULTS)
        values['tenant_id'] = context.tenant_id
        return self.db.job_create(context, values)

    def job_update(self, context, job, values):
        """Updates a Job from the values dictionary."""
        return self.db.job_update(context, job, values)

    def job_destroy(self, context, job):
        """Destroy the Job or raise if it does not exist."""
        self.db.job_destroy(context, job)

    # JobBinary ops

    def job_binary_get_all(self, context, **kwargs):
        """Get all JobBinarys filtered by **kwargs.

        e.g.  job_binary_get_all(name='wordcount.jar')
        """
        return self.db.job_binary_get_all(context, **kwargs)

    def job_binary_get(self, context, job_binary_id):
        """Return the JobBinary or None if it does not exist."""
        return self.db.job_binary_get(context, job_binary_id)

    def job_binary_create(self, context, values):
        """Create a JobBinary from the values dictionary."""

        values = copy.deepcopy(values)
        values = _apply_defaults(values, JOB_BINARY_DEFAULTS)
        values['tenant_id'] = context.tenant_id
        return self.db.job_binary_create(context, values)

    def job_binary_destroy(self, context, job_binary):
        """Destroy the JobBinary or raise if it does not exist."""
        self.db.job_binary_destroy(context, job_binary)

    def job_binary_update(self, context, id, values):
        """Update a JobBinary from the values dictionary."""

        values = copy.deepcopy(values)
        values['id'] = id
        return self.db.job_binary_update(context, values)

    # JobBinaryInternal ops

    def job_binary_internal_get_all(self, context, **kwargs):
        """Get all JobBinaryInternals filtered by **kwargs.

        e.g.  cluster_get_all(name='wordcount.jar')

        The JobBinaryInternals returned do not contain a data field.
        """
        return self.db.job_binary_internal_get_all(context, **kwargs)

    def job_binary_internal_get(self, context, job_binary_internal_id):
        """Return the JobBinaryInternal or None if it does not exist

        The JobBinaryInternal returned does not contain a data field.
        """
        return self.db.job_binary_internal_get(context, job_binary_internal_id)

    def job_binary_internal_create(self, context, values):
        """Create a JobBinaryInternal from the values dictionary."""

        # Since values["data"] is (should be) encoded as a string
        # here the deepcopy of values only incs a reference count on data.
        # This is nice, since data could be big...
        values = copy.deepcopy(values)
        values = _apply_defaults(values, JOB_BINARY_INTERNAL_DEFAULTS)
        values['tenant_id'] = context.tenant_id
        return self.db.job_binary_internal_create(context, values)

    def job_binary_internal_destroy(self, context, job_binary_internal):
        """Destroy the JobBinaryInternal or raise if it does not exist."""
        self.db.job_binary_internal_destroy(context, job_binary_internal)

    def job_binary_internal_get_raw_data(self,
                                         context, job_binary_internal_id):
        """Return the binary data field from a JobBinaryInternal."""
        return self.db.job_binary_internal_get_raw_data(
            context,
            job_binary_internal_id)

    def job_binary_internal_update(self, context, id, values):
        """Updates a JobBinaryInternal from the values dictionary."""
        return self.db.job_binary_internal_update(context, id, values)

    # Events ops

    def cluster_provision_step_add(self, context, cluster_id, values):
        """Create a provisioning step assigned to cluster from values dict."""
        return self.db.cluster_provision_step_add(context, cluster_id, values)

    def cluster_provision_step_update(self, context, provision_step):
        """Update the cluster provisioning step."""
        return self.db.cluster_provision_step_update(context, provision_step)

    def cluster_provision_progress_update(self, context, cluster_id):
        """Return cluster with provision progress updated field."""
        return self.db.cluster_provision_progress_update(context, cluster_id)

    def cluster_event_add(self, context, provision_step, values):
        """Assign new event to the specified provision step."""
        return self.db.cluster_event_add(context, provision_step, values)