From 457d6b59c79f592a7411db3073913e2f9fb2d27f Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 13 Sep 2013 07:03:54 -0500 Subject: [PATCH] Build images using diskimage-builder Create images locally using diskimage-builder, and then upload to glance. Change-Id: I8e96e9ea5b74ca640f483c9e1ad04a584b5660ed --- doc/source/configuration.rst | 22 +++++++ nodepool/nodepool.py | 107 ++++++++++++++++++++++++++++++++++- nodepool/provider_manager.py | 71 +++++++++++++++-------- requirements.txt | 1 + 4 files changed, 174 insertions(+), 27 deletions(-) diff --git a/doc/source/configuration.rst b/doc/source/configuration.rst index 631606ce1..63cd2d6e6 100644 --- a/doc/source/configuration.rst +++ b/doc/source/configuration.rst @@ -32,6 +32,28 @@ Example:: script-dir: /path/to/script/dir +elements-dir +------------ + +If an image is configured to use disk-image-builder and glance to locally +create and upload images, then a collection of disk-image-builder elements +must be present. The ``elements-dir`` parameter indicates a directory +that holds one or more elements. + +Example:: + + elements-dir: /path/to/elements/dir + +images-dir +---------- + +When we create location images they need to be written to somewhere. The +``images-dir`` parameter is the place to write them. + +Example:: + + images-dir: /path/to/images/dir + dburi ----- Indicates the URI for the database connection. See the `SQLAlchemy diff --git a/nodepool/nodepool.py b/nodepool/nodepool.py index 3e4f950ce..b51b9bad9 100644 --- a/nodepool/nodepool.py +++ b/nodepool/nodepool.py @@ -24,6 +24,8 @@ import logging import os.path import paramiko import re +import shlex +import subprocess import threading import time import yaml @@ -664,7 +666,6 @@ class SubNodeLauncher(threading.Thread): class ImageUpdater(threading.Thread): - log = logging.getLogger("nodepool.ImageUpdater") def __init__(self, nodepool, provider, image, snap_image_id): threading.Thread.__init__(self, name='ImageUpdater for %s' % @@ -674,6 +675,8 @@ class ImageUpdater(threading.Thread): self.snap_image_id = snap_image_id self.nodepool = nodepool self.scriptdir = self.nodepool.config.scriptdir + self.elementsdir = self.nodepool.config.elementsdir + self.imagesdir = self.nodepool.config.imagesdir def run(self): try: @@ -708,6 +711,81 @@ class ImageUpdater(threading.Thread): self.snap_image.id) return + +class DiskImageUpdater(ImageUpdater): + + log = logging.getLogger("nodepool.DiskImageUpdater") + + def updateImage(self, session): + start_time = time.time() + timestamp = int(start_time) + + # TODO(mordred) we can optimize by making an image once, not per + # provider + filename = os.path.join(self.imagesdir, + '%s-%s' % (self.image.name, str(timestamp))) + self.log.info("Creating image id: %s with filename %s for %s in %s" % + (self.snap_image.id, filename, self.image.name, + self.provider.name)) + + # TODO(mordred) abusing the hostname field + self.snap_image.hostname = filename + self.snap_image.version = timestamp + session.commit() + + self.buildImage(filename) + + image_id = self.manager.uploadImage(filename) + self.snap_image.external_id = image_id + session.commit() + self.log.debug("Image id: %s building image %s" % + (self.snap_image.id, image_id)) + # It can take a _very_ long time for Rackspace 1.0 to save an image + self.manager.waitForImage(image_id, IMAGE_TIMEOUT) + + if statsd: + dt = int((time.time() - start_time) * 1000) + key = 'nodepool.image_update.%s.%s' % (self.image.name, + self.provider.name) + statsd.timing(key, dt) + statsd.incr(key) + + self.snap_image.state = nodedb.READY + session.commit() + self.log.info("Image %s in %s is ready" % (hostname, + self.provider.name)) + + def buildImage(self, filename): + env = dict() + for k, v in os.environ.items(): + if k.startswith('NODEPOOL_'): + env[k]=v + env['ELEMENTS_PATH'] = self.elementsdir + env['DIB_RELEASE'] = self.image.release + + cmd = 'disk-image-create -n -o %s %s' % (filename, self.image.elements) + self.log.info('Running %s' % cmd) + + p = subprocess.Popen( + shlex.split(cmd), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + (stdout, stderr) = p.communicate() + for line in stdout: + if self.log: + self.log.info(line.rstrip()) + for line in stderr: + if self.log: + self.log.error(line.rstrip()) + ret = p.returncode + if ret: + raise Exception("Unable to create %s" % filename) + + +class SnapshotImageUpdater(ImageUpdater): + + log = logging.getLogger("nodepool.SnapshotImageUpdater") + def updateImage(self, session): start_time = time.time() timestamp = int(start_time) @@ -901,6 +979,10 @@ class GearmanServer(ConfigValue): pass +class DiskImage(ConfigValue): + pass + + class NodePool(threading.Thread): log = logging.getLogger("nodepool.NodePool") @@ -938,11 +1020,14 @@ class NodePool(threading.Thread): newconfig.targets = {} newconfig.labels = {} newconfig.scriptdir = config.get('script-dir') + newconfig.elementsdir = config.get('elements-dir') + newconfig.imagesdir = config.get('images-dir') newconfig.dburi = config.get('dburi') newconfig.provider_managers = {} newconfig.jenkins_managers = {} newconfig.zmq_publishers = {} newconfig.gearman_servers = {} + newconfig.diskimages = {} newconfig.crons = {} for name, default in [ @@ -983,6 +1068,13 @@ class NodePool(threading.Thread): p.name = provider['name'] l.providers[p.name] = p + for diskimage in config['diskimages']: + d = DiskImage() + d.name = diskimage['name'] + newconfig.diskimages[d.name] = d + d.elements = diskimage['elements'] + d.release = diskimage['release'] + for provider in config['providers']: p = Provider() p.name = provider['name'] @@ -1010,8 +1102,9 @@ class NodePool(threading.Thread): i.base_image = image['base-image'] i.min_ram = image['min-ram'] i.name_filter = image.get('name-filter', None) - i.setup = image.get('setup') + i.setup = image.get('setup', None) i.reset = image.get('reset') + i.diskimage = image.get('diskimage', None) i.username = image.get('username', 'jenkins') i.private_key = image.get('private-key', '/var/lib/jenkins/.ssh/id_rsa') @@ -1427,7 +1520,15 @@ class NodePool(threading.Thread): snap_image = session.createSnapshotImage( provider_name=provider.name, image_name=image.name) - t = ImageUpdater(self, provider, image, snap_image.id) + if image.setup: + t = SnapshotImageUpdater(self, provider, image, snap_image.id) + elif image.diskimage: + t = DiskImageUpdater(self, provider, image, snap_image.id) + else: + raise Exception( + "Invalid image config. Must specify either a setup script, or" + " a diskimage to use.") + t.start() # Enough time to give them different timestamps (versions) # Just to keep things clearer. diff --git a/nodepool/provider_manager.py b/nodepool/provider_manager.py index a2adb2284..3154bd574 100644 --- a/nodepool/provider_manager.py +++ b/nodepool/provider_manager.py @@ -90,14 +90,14 @@ class NotFound(Exception): class CreateServerTask(Task): def main(self, client): - server = client.servers.create(**self.args) + server = client.nova.servers.create(**self.args) return str(server.id) class GetServerTask(Task): def main(self, client): try: - server = client.servers.get(self.args['server_id']) + server = client.nova.servers.get(self.args['server_id']) except novaclient.exceptions.NotFound: raise NotFound() return make_server_dict(server) @@ -105,52 +105,52 @@ class GetServerTask(Task): class DeleteServerTask(Task): def main(self, client): - client.servers.delete(self.args['server_id']) + client.nova.servers.delete(self.args['server_id']) class ListServersTask(Task): def main(self, client): - servers = client.servers.list() + servers = client.nova.servers.list() return [make_server_dict(server) for server in servers] class AddKeypairTask(Task): def main(self, client): - client.keypairs.create(**self.args) + client.nova.keypairs.create(**self.args) class ListKeypairsTask(Task): def main(self, client): - keys = client.keypairs.list() + keys = client.nova.keypairs.list() return [dict(id=str(key.id), name=key.name) for key in keys] class DeleteKeypairTask(Task): def main(self, client): - client.keypairs.delete(self.args['name']) + client.nova.keypairs.delete(self.args['name']) class CreateFloatingIPTask(Task): def main(self, client): - ip = client.floating_ips.create(**self.args) + ip = client.nova.floating_ips.create(**self.args) return dict(id=str(ip.id), ip=ip.ip) class AddFloatingIPTask(Task): def main(self, client): - client.servers.add_floating_ip(**self.args) + client.nova.servers.add_floating_ip(**self.args) class GetFloatingIPTask(Task): def main(self, client): - ip = client.floating_ips.get(self.args['ip_id']) + ip = client.nova.floating_ips.get(self.args['ip_id']) return dict(id=str(ip.id), ip=ip.ip, instance_id=str(ip.instance_id)) class ListFloatingIPsTask(Task): def main(self, client): - ips = client.floating_ips.list() + ips = client.nova.floating_ips.list() return [dict(id=str(ip.id), ip=ip.ip, instance_id=str(ip.instance_id)) for ip in ips] @@ -158,24 +158,31 @@ class ListFloatingIPsTask(Task): class RemoveFloatingIPTask(Task): def main(self, client): - client.servers.remove_floating_ip(**self.args) + client.nova.servers.remove_floating_ip(**self.args) class DeleteFloatingIPTask(Task): def main(self, client): - client.floating_ips.delete(self.args['ip_id']) + client.nova.floating_ips.delete(self.args['ip_id']) class CreateImageTask(Task): def main(self, client): # This returns an id - return str(client.servers.create_image(**self.args)) + return str(client.nova.servers.create_image(**self.args)) + + +class UploadImageTask(Task): + def main(self, client): + # This returns an id + # EEK? + return str(client.glance.upload_image(**self.args)) class GetImageTask(Task): def main(self, client): try: - image = client.images.get(**self.args) + image = client.nova.images.get(**self.args) except novaclient.exceptions.NotFound: raise NotFound() # HP returns 404, rackspace can return a 'DELETED' image. @@ -187,7 +194,7 @@ class GetImageTask(Task): class ListExtensionsTask(Task): def main(self, client): try: - resp, body = client.client.get('/extensions') + resp, body = client.nova.client.get('/extensions') return [x['alias'] for x in body['extensions']] except novaclient.exceptions.NotFound: # No extensions present. @@ -196,26 +203,33 @@ class ListExtensionsTask(Task): class ListFlavorsTask(Task): def main(self, client): - flavors = client.flavors.list() + flavors = client.nova.flavors.list() return [dict(id=str(flavor.id), ram=flavor.ram, name=flavor.name) for flavor in flavors] class ListImagesTask(Task): def main(self, client): - images = client.images.list() + images = client.nova.images.list() return [make_image_dict(image) for image in images] class FindImageTask(Task): def main(self, client): - image = client.images.find(**self.args) + image = client.nova.images.find(**self.args) return dict(id=str(image.id)) class DeleteImageTask(Task): def main(self, client): - client.images.delete(**self.args) + client.nova.images.delete(**self.args) + + +class ClientContainer(object): + def __init__(self, nova, glance, neutron): + self.nova = nova + self.glance = glance + self.neutron = neutron class ProviderManager(TaskManager): @@ -251,9 +265,12 @@ class ProviderManager(TaskManager): self._cloud_metadata_read = True def _getClient(self): - args = ['1.1', self.provider.username, self.provider.password, - self.provider.project_id, self.provider.auth_url] - kwargs = {} + kwargs = dict( + username=self.provider.username, + password=self.provider.password, + tenant_id=self.provider.project_id, + auth_url=self.provider.auth_url) + # Help - need to have this per service if self.provider.service_type: kwargs['service_type'] = self.provider.service_type if self.provider.service_name: @@ -262,7 +279,10 @@ class ProviderManager(TaskManager): kwargs['region_name'] = self.provider.region_name if self.provider.auth_url == 'fake': return fakeprovider.FAKE_CLIENT - return novaclient.client.Client(*args, **kwargs) + nova = novaclient.client.Client('1.1'. *args, **kwargs) + glance = glanceclient.client.Client('1', *args, **kwargs) + neutron = neutronclient.v2_0.client.Client(*args, **kwargs) + return ClientContainer(nova, glance, neutron) def _getFlavors(self): flavors = self.listFlavors() @@ -415,6 +435,9 @@ class ProviderManager(TaskManager): def getImage(self, image_id): return self.submitTask(GetImageTask(image=image_id)) + def uploadImage(self, filename): + return self.submitTask(UploadImageTask(filename=filename)) + def listExtensions(self): return self.submitTask(ListExtensionsTask()) diff --git a/requirements.txt b/requirements.txt index e8def33bf..cbce176eb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,4 @@ python-novaclient MySQL-python PrettyTable>=0.6,<0.8 six>=1.4.1 +diskimage-builder