diff --git a/reddwarf/db/sqlalchemy/migrate_repo/versions/007_add_volume_flavor.py b/reddwarf/db/sqlalchemy/migrate_repo/versions/007_add_volume_flavor.py new file mode 100644 index 0000000000..84676ae69a --- /dev/null +++ b/reddwarf/db/sqlalchemy/migrate_repo/versions/007_add_volume_flavor.py @@ -0,0 +1,43 @@ +# Copyright 2012 OpenStack LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy.schema import Column +from sqlalchemy.schema import MetaData + +from reddwarf.db.sqlalchemy.migrate_repo.schema import String +from reddwarf.db.sqlalchemy.migrate_repo.schema import Table + + +def upgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + + # add column: + instances = Table('instances', meta, autoload=True) + volume_size = Column('volume_size', String(36)) + flavor_id = Column('flavor_id', String(36)) + + instances.create_column(flavor_id) + instances.create_column(volume_size) + + +def downgrade(migrate_engine): + meta = MetaData() + meta.bind = migrate_engine + + # drop column: + instances = Table('instances', meta, autoload=True) + + instances.drop_column('flavor_id') + instances.drop_column('volume_size') diff --git a/reddwarf/dns/manager.py b/reddwarf/dns/manager.py index b8f9cb5393..356af9826c 100644 --- a/reddwarf/dns/manager.py +++ b/reddwarf/dns/manager.py @@ -77,10 +77,12 @@ class DnsManager(object): Use instance by default """ dns_support = config.Config.get('reddwarf_dns_support', 'False') + LOG.debug(_("reddwarf dns support = %s") % dns_support) if utils.bool_from_string(dns_support): entry = self.entry_factory.create_entry(instance.id) instance.hostname = entry.name instance.save() + LOG.debug("Saved the hostname as %s " % instance.hostname) else: instance.hostname = instance.name instance.save() diff --git a/reddwarf/instance/models.py b/reddwarf/instance/models.py index eca2184a2a..4833ae086a 100644 --- a/reddwarf/instance/models.py +++ b/reddwarf/instance/models.py @@ -131,7 +131,119 @@ SERVER_INVALID_ACTION_STATUSES = ["BUILD", "REBOOT", "REBUILD"] VALID_ACTION_STATUSES = ["ACTIVE"] -class Instance(object): +class SimpleInstance(object): + """ + + Simple model is a quick hack for when server/volumes is not available, and + all we have is database info. Example is create instance response, when the + async call to server/volume may not have completed yet. + + """ + + def __init__(self, context, db_info, service_status): + self.context = context + self.db_info = db_info + self.service_status = service_status + self.volumes = [{'size': self.db_info.volume_size }] + + @staticmethod + def load(context, id): + if context is None: + raise TypeError("Argument context not defined.") + elif id is None: + raise TypeError("Argument id not defined.") + try: + db_info = DBInstance.find_by(id=id) + except rd_exceptions.NotFound: + raise rd_exceptions.NotFound(uuid=id) + task_status = db_info.task_status + service_status = InstanceServiceStatus.find_by(instance_id=id) + LOG.info("service status=%s" % service_status) + return SimpleInstance(context, db_info, service_status) + + @property + def name(self): + return self.db_info.name + + @property + def id(self): + return self.db_info.id + + @property + def hostname(self): + return self.db_info.hostname + + @property + def status(self): + # If the service status is NEW, then we are building. + if ServiceStatuses.NEW == self.service_status.status: + return InstanceStatus.BUILD + + @property + def created(self): + return self.db_info.created + + @property + def updated(self): + return self.db_info.updated + + @property + def addresses(self): + return None + + @property + def is_building(self): + return self.status in [InstanceStatus.BUILD] + + @property + def is_sql_running(self): + """True if the service status indicates MySQL is up and running.""" + return self.service_status.status in MYSQL_RESPONSIVE_STATUSES + + @property + def links(self): + """ + The links here are just used for structural format. The actual link is + created in the views by replacing the matching pieces from the request + """ + links = [ + { + "href": "https://localhost/v1.0/tenant_id/instances/instance_id", + "rel": "self" + }, + { + "href": "https://localhost/instances/instance_id", + "rel": "bookmark" + } + ] + return links + + @property + def flavor_links(self): + links = [ + { + "href": "https://localhost/v1.0/tenant_id/flavors/flavor_id", + "rel": "self" + }, + { + "href": "https://localhost/flavors/flavor_id", + "rel": "bookmark" + } + ] + return links + + def restart(self): + # Just so it doesnt blow if restart is accidentally called + raise rd_exceptions.UnprocessableEntity("Instance %s is not ready." + % self.id) + + def delete(self): + # Just so it doesnt blow if delete is accidentally called + raise rd_exceptions.UnprocessableEntity("Instance %s is not ready." + % self.id) + + +class Instance(SimpleInstance): """Represents an instance. The life span of this object should be limited. Do not store them or @@ -139,10 +251,8 @@ class Instance(object): """ def __init__(self, context, db_info, server, service_status, volumes): - self.context = context - self.db_info = db_info + super(Instance, self).__init__(context, db_info, service_status) self.server = server - self.service_status = service_status self.volumes = volumes @staticmethod @@ -155,11 +265,19 @@ class Instance(object): db_info = DBInstance.find_by(id=id) except rd_exceptions.NotFound: raise rd_exceptions.NotFound(uuid=id) - server, volumes = load_server_with_volumes(context, db_info.id, - db_info.compute_instance_id) + task_status = db_info.task_status service_status = InstanceServiceStatus.find_by(instance_id=id) LOG.info("service status=%s" % service_status) + if db_info.compute_instance_id is None: + LOG.debug("Missing server_id for instance %s " % db_info.id) + # TODO: Should it raise exception or return SimpleInstance? + # If I return SimpleInstance, somebody will invoke delete on it and + # it will be method not found. + return SimpleInstance(context, db_info, service_status) + + server, volumes = load_server_with_volumes(context, db_info.id, + db_info.compute_instance_id) return Instance(context, db_info, server, service_status, volumes) def delete(self, force=False): @@ -173,142 +291,33 @@ class Instance(object): self.db_info.save() task_api.API(self.context).delete_instance(self.id) - @classmethod - def _create_volume(cls, context, db_info, volume_size): - volume_support = config.Config.get("reddwarf_volume_support", 'False') - LOG.debug(_("reddwarf volume support = %s") % volume_support) - if utils.bool_from_string(volume_support): - LOG.debug(_("Starting to create the volume for the instance")) - volume_client = create_nova_volume_client(context) - volume_desc = ("mysql volume for %s" % db_info.id) - volume_ref = volume_client.volumes.create( - volume_size, - display_name="mysql-%s" % db_info.id, - display_description=volume_desc) - # Record the volume ID in case something goes wrong. - db_info.volume_id = volume_ref.id - db_info.save() - #TODO(cp16net) this is bad to wait here for the volume create - # before returning but this was a quick way to get it working - # for now we need this to go into the task manager - v_ref = volume_client.volumes.get(volume_ref.id) - while not v_ref.status in ['available', 'error']: - LOG.debug(_("waiting for volume [volume.status=%s]") % - v_ref.status) - greenthread.sleep(1) - v_ref = volume_client.volumes.get(volume_ref.id) - - if v_ref.status in ['error']: - raise rd_exceptions.VolumeCreationFailure() - LOG.debug(_("Created volume %s") % v_ref) - # The mapping is in the format: - # :[]:[]:[] - # setting the delete_on_terminate instance to true=1 - mapping = "%s:%s:%s:%s" % (v_ref.id, '', v_ref.size, 1) - bdm = CONFIG.get('block_device_mapping', 'vdb') - block_device = {bdm: mapping} - volumes = [{'id': v_ref.id, - 'size': v_ref.size}] - LOG.debug("block_device = %s" % block_device) - LOG.debug("volume = %s" % volumes) - - device_path = CONFIG.get('device_path', '/dev/vdb') - mount_point = CONFIG.get('mount_point', '/var/lib/mysql') - LOG.debug(_("device_path = %s") % device_path) - LOG.debug(_("mount_point = %s") % mount_point) - else: - LOG.debug(_("Skipping setting up the volume")) - block_device = None - device_path = None - mount_point = None - volumes = None - #end volume_support - #block_device = "" - #device_path = /dev/vdb - #mount_point = /var/lib/mysql - volume_info = {'block_device': block_device, - 'device_path': device_path, - 'mount_point': mount_point, - 'volumes': volumes} - return volume_info - @classmethod def create(cls, context, name, flavor_ref, image_id, databases, service_type, volume_size): - db_info = DBInstance.create(name=name, - task_status=InstanceTasks.NONE) + flavor_id = utils.get_id_from_href(flavor_ref) + db_info = DBInstance.create(name=name, volume_size=volume_size, + flavor_id =flavor_id, task_status=InstanceTasks.NONE) LOG.debug(_("Created new Reddwarf instance %s...") % db_info.id) - if volume_size: - volume_info = cls._create_volume(context, db_info, volume_size) - block_device_mapping = volume_info['block_device'] - device_path = volume_info['device_path'] - mount_point = volume_info['mount_point'] - volumes = volume_info['volumes'] - else: - block_device_mapping = None - device_path = None - mount_point = None - volumes = [] - - client = create_nova_client(context) - files = {"/etc/guest_info": "guest_id=%s\nservice_type=%s\n" % - (db_info.id, service_type)} - server = client.servers.create(name, image_id, flavor_ref, - files=files, - block_device_mapping=block_device_mapping) - LOG.debug(_("Created new compute instance %s.") % server.id) - - db_info.compute_instance_id = server.id - db_info.save() - service_status = InstanceServiceStatus.create(instance_id=db_info.id, - status=ServiceStatuses.NEW) - # Now wait for the response from the create to do additional work - - guest = create_guest_client(context, db_info.id) - - # populate the databases - model_schemas = populate_databases(databases) - guest.prepare(512, model_schemas, users=[], - device_path=device_path, - mount_point=mount_point) - - dns_support = config.Config.get("reddwarf_dns_support", 'False') - LOG.debug(_("reddwarf dns support = %s") % dns_support) + task_api.API(context).create_instance(db_info.id, name, + flavor_ref, image_id, databases, service_type, + volume_size) + # Defaults the hostname to instance name of dns is disabled. dns_client = create_dns_client(context) - # Default the hostname to instance name if no dns support dns_client.update_hostname(db_info) - if utils.bool_from_string(dns_support): - def get_server(): - return client.servers.get(server.id) + #Check to see if a New status has already been created + service_status = InstanceServiceStatus.get_by(instance_id=db_info.id) + if service_status is None: + service_status = InstanceServiceStatus.create( + instance_id=db_info.id, + status=ServiceStatuses.NEW) - def ip_is_available(server): - if server.addresses != {}: - return True - elif server.addresses == {} and\ - server.status != InstanceStatus.ERROR: - return False - elif server.addresses == {} and\ - server.status == InstanceStatus.ERROR: - LOG.error(_("Instance IP not available, instance (%s): server had " - " status (%s).") % (db_info['id'], server.status)) - raise rd_exceptions.ReddwarfError( - status=server.status) - poll_until(get_server, ip_is_available, sleep_time=1, time_out=60*2) - - dns_client.create_instance_entry(db_info['id'], - get_ip_address(server.addresses)) - - return Instance(context, db_info, server, service_status, volumes) + return SimpleInstance(context, db_info, service_status) def get_guest(self): return create_guest_client(self.context, self.db_info.id) - @property - def id(self): - return self.db_info.id - @property def is_building(self): return self.status in [InstanceStatus.BUILD] @@ -350,14 +359,6 @@ class Instance(object): # For everything else we can look at the service status mapping. return self.service_status.status.api_status - @property - def created(self): - return self.db_info.created - - @property - def updated(self): - return self.db_info.updated - @property def flavor(self): return self.server.flavor diff --git a/reddwarf/instance/views.py b/reddwarf/instance/views.py index c245e0f48d..833a6616a3 100644 --- a/reddwarf/instance/views.py +++ b/reddwarf/instance/views.py @@ -59,7 +59,7 @@ class InstanceView(object): } dns_support = config.Config.get("reddwarf_dns_support", 'False') if utils.bool_from_string(dns_support): - instance_dict['hostname'] = self.instance.db_info.hostname + instance_dict['hostname'] = self.instance.hostname if self.add_addresses and ip is not None and len(ip) > 0: instance_dict['ip'] = ip if self.add_volumes and volumes is not None: @@ -118,10 +118,51 @@ class InstanceDetailView(InstanceView): def data(self): result = super(InstanceDetailView, self).data() result['instance']['created'] = self.instance.created - result['instance']['flavor'] = self.instance.flavor + result['instance']['flavor'] = self._build_flavor() result['instance']['updated'] = self.instance.updated return result + def _build_flavor(self): + try: + return self.instance.flavor + except: + return { + 'id': self.instance.db_info.flavor_id, + 'links': self._build_flavor_links(), + } + + def _build_flavor_links(self): + result = [] + #scheme = self.req.scheme + scheme = 'https' # Forcing https + endpoint = self.req.host + splitpath = self.req.path.split('/') + detailed = '' + if splitpath[-1] == 'detail': + detailed = '/detail' + splitpath.pop(-1) + flavorid = self.instance.db_info.flavor_id + if str(splitpath[-1]) == str(flavorid): + splitpath.pop(-1) + href_template = "%(scheme)s://%(endpoint)s%(path)s/%(flavorid)s" + for link in self.instance.flavor_links: + rlink = link + href = rlink['href'] + if rlink['rel'] == 'self': + path = '/'.join(splitpath) + href = href_template % locals() + elif rlink['rel'] == 'bookmark': + splitpath.pop(2) # Remove the version. + splitpath.pop(1) # Remove the tenant id. + path = '/'.join(splitpath) + href = href_template % locals() + + rlink['href'] = href + result.append(rlink) + for link in result: + link['href'] = link['href'].replace('instances', 'flavors') + return result + class InstancesView(object): diff --git a/reddwarf/taskmanager/api.py b/reddwarf/taskmanager/api.py index b6f0b3a369..e73781817b 100644 --- a/reddwarf/taskmanager/api.py +++ b/reddwarf/taskmanager/api.py @@ -73,3 +73,11 @@ class API(object): def delete_instance(self, instance_id): LOG.debug("Making async call to delete instance: %s" % instance_id) self._cast("delete_instance", instance_id=instance_id) + + def create_instance(self, instance_id, name, flavor_ref, image_id, + databases, service_type, volume_size): + LOG.debug("Making async call to create instance %s " % instance_id) + self._cast("create_instance", instance_id=instance_id, name=name, + flavor_ref=flavor_ref, image_id=image_id, + databases=databases, service_type=service_type, + volume_size=volume_size) diff --git a/reddwarf/taskmanager/manager.py b/reddwarf/taskmanager/manager.py index ae6c2401f1..95ed1b97ce 100644 --- a/reddwarf/taskmanager/manager.py +++ b/reddwarf/taskmanager/manager.py @@ -23,6 +23,7 @@ from eventlet import greenthread from reddwarf.common import service from reddwarf.taskmanager import models +from reddwarf.taskmanager.models import InstanceTasks LOG = logging.getLogger(__name__) @@ -67,4 +68,11 @@ class TaskManager(service.Manager): instance_tasks = models.InstanceTasks.load(context, instance_id) instance_tasks.delete_instance() + def create_instance(self, context, instance_id, name, flavor_ref, + image_id, databases, service_type, volume_size): + instance_tasks = InstanceTasks(context) + instance_tasks.create_instance(instance_id, name, flavor_ref, + image_id, databases, + service_type, volume_size) + diff --git a/reddwarf/taskmanager/models.py b/reddwarf/taskmanager/models.py index 5814833c85..309903bf07 100644 --- a/reddwarf/taskmanager/models.py +++ b/reddwarf/taskmanager/models.py @@ -21,9 +21,21 @@ from reddwarf.common import config from reddwarf.common import remote from reddwarf.common import utils from reddwarf.common.exception import PollTimeOut +from reddwarf.common.exception import VolumeCreationFailure +from reddwarf.common.exception import NotFound from reddwarf.common.exception import ReddwarfError from reddwarf.common.remote import create_dns_client +from reddwarf.common.remote import create_nova_client +from reddwarf.common.remote import create_nova_volume_client +from reddwarf.common.remote import create_guest_client +from reddwarf.common.utils import poll_until from reddwarf.instance import models as inst_models +from reddwarf.instance.models import DBInstance +from reddwarf.instance.models import InstanceStatus +from reddwarf.instance.models import InstanceServiceStatus +from reddwarf.instance.models import populate_databases +from reddwarf.instance.models import ServiceStatuses +from reddwarf.instance.views import get_ip_address LOG = logging.getLogger(__name__) @@ -34,7 +46,7 @@ class InstanceTasks: Performs the various asynchronous instance related tasks. """ - def __init__(self, context, db_info, server, volumes, + def __init__(self, context, db_info=None, server=None, volumes=None, nova_client=None, volume_client=None, guest=None): self.context = context self.db_info = db_info @@ -179,6 +191,152 @@ class InstanceTasks: self.db_info.task_status = inst_models.InstanceTasks.NONE self.db_info.save() + def create_instance(self, instance_id, name, flavor_ref, + image_id, databases, service_type, volume_size): + LOG.info("Entering create_instance") + try: + db_info = DBInstance.find_by(id=instance_id) + volume_info = self._create_volume(instance_id, + volume_size) + block_device_mapping = volume_info['block_device'] + server = self._create_server(instance_id, name, + flavor_ref, image_id, service_type, block_device_mapping) + LOG.info("server id: %s" % server) + server_id = server.id + self._create_dns_entry(instance_id, server_id) + LOG.info("volume_info %s " % volume_info) + self._guest_prepare(server, db_info, volume_info, databases) + except Exception, e: + LOG.error(e) + self._log_service_status(instance_id, ServiceStatuses.UNKNOWN) + + def _create_volume(self, instance_id, volume_size): + LOG.info("Entering create_volume") + LOG.debug(_("Starting to create the volume for the instance")) + + volume_support = config.Config.get("reddwarf_volume_support", 'False') + LOG.debug(_("reddwarf volume support = %s") % volume_support) + if volume_size is None or \ + utils.bool_from_string(volume_support) is False: + volume_info = {'block_device': None, + 'device_path': None, + 'mount_point': None, + 'volumes': None} + return volume_info + + db_info = DBInstance.find_by(id=instance_id) + + volume_client = create_nova_volume_client(self.context) + volume_desc = ("mysql volume for %s" % instance_id) + volume_ref = volume_client.volumes.create( + volume_size, + display_name="mysql-%s" % db_info.id, + display_description=volume_desc) + + # Record the volume ID in case something goes wrong. + db_info.volume_id = volume_ref.id + db_info.save() + + utils.poll_until( + lambda: volume_client.volumes.get(volume_ref.id), + lambda v_ref: v_ref.status in ['available', 'error'], + sleep_time=2, + time_out=2 * 60) + + v_ref = volume_client.volumes.get(volume_ref.id) + if v_ref.status in ['error']: + raise VolumeCreationFailure() + LOG.debug(_("Created volume %s") % v_ref) + # The mapping is in the format: + # :[]:[]:[] + # setting the delete_on_terminate instance to true=1 + mapping = "%s:%s:%s:%s" % (v_ref.id, '', v_ref.size, 1) + bdm = config.Config.get('block_device_mapping', 'vdb') + block_device = {bdm: mapping} + volumes = [{'id': v_ref.id, + 'size': v_ref.size}] + LOG.debug("block_device = %s" % block_device) + LOG.debug("volume = %s" % volumes) + + device_path = config.Config.get('device_path', '/dev/vdb') + mount_point = config.Config.get('mount_point', '/var/lib/mysql') + LOG.debug(_("device_path = %s") % device_path) + LOG.debug(_("mount_point = %s") % mount_point) + + volume_info = {'block_device': block_device, + 'device_path': device_path, + 'mount_point': mount_point, + 'volumes': volumes} + return volume_info + + def _create_server(self, instance_id, name, flavor_ref, image_id, + service_type, block_device_mapping): + nova_client = create_nova_client(self.context) + files = {"/etc/guest_info": "guest_id=%s\nservice_type=%s\n" % + (instance_id, service_type)} + server = nova_client.servers.create(name, image_id, flavor_ref, + files=files, block_device_mapping=block_device_mapping) + LOG.debug(_("Created new compute instance %s.") % server.id) + return server + + def _guest_prepare(self, server, db_info, volume_info, databases): + LOG.info("Entering guest_prepare") + db_info.compute_instance_id = server.id + db_info.save() + self._log_service_status(db_info, ServiceStatuses.NEW) + + # Now wait for the response from the create to do additional work + guest = create_guest_client(self.context, db_info.id) + + # populate the databases + model_schemas = populate_databases(databases) + guest.prepare(512, model_schemas, users=[], + device_path=volume_info['device_path'], + mount_point=volume_info['mount_point']) + + def _create_dns_entry(self, instance_id, server_id): + LOG.debug("%s: Creating dns entry for instance: %s" + % (greenthread.getcurrent(), instance_id)) + dns_client = create_dns_client(self.context) + dns_support = config.Config.get("reddwarf_dns_support", 'False') + LOG.debug(_("reddwarf dns support = %s") % dns_support) + + nova_client = create_nova_client(self.context) + if utils.bool_from_string(dns_support): + def get_server(): + return nova_client.servers.get(server_id) + + def ip_is_available(server): + LOG.info("Polling for ip addresses: $%s " % server.addresses) + if server.addresses != {}: + return True + elif server.addresses == {} and\ + server.status != InstanceStatus.ERROR: + return False + elif server.addresses == {} and\ + server.status == InstanceStatus.ERROR: + LOG.error(_("Instance IP not available, instance (%s): " + "server had status (%s).") + % (instance_id, server.status)) + raise ReddwarfError(status=server.status) + poll_until(get_server, ip_is_available, + sleep_time=1, time_out=60 * 2) + server = nova_client.servers.get(server_id) + LOG.info("Creating dns entry...") + dns_client.create_instance_entry(instance_id, + get_ip_address(server.addresses)) + + def _log_service_status(self, instance_id, status): + LOG.info("Saving service status %s for instance %s " + % (status, instance_id)) + service_status = InstanceServiceStatus.get_by(instance_id=instance_id) + if service_status: + service_status.status = status + service_status.save() + else: + InstanceServiceStatus.create(instance_id=instance_id, + status=status) + def _refresh_compute_server_info(self): """Refreshes the compute server field.""" server = self.nova_client.servers.get(self.server.id)