Build images using diskimage-builder
Create images locally using diskimage-builder, and then upload to glance. Change-Id: I8e96e9ea5b74ca640f483c9e1ad04a584b5660ed
This commit is contained in:
Monty Taylor
parent
7747eb461c
commit
457d6b59c7
|
|
@ -32,6 +32,28 @@ Example::
|
|||
|
||||
script-dir: /path/to/script/dir
|
||||
|
||||
elements-dir
|
||||
------------
|
||||
|
||||
If an image is configured to use disk-image-builder and glance to locally
|
||||
create and upload images, then a collection of disk-image-builder elements
|
||||
must be present. The ``elements-dir`` parameter indicates a directory
|
||||
that holds one or more elements.
|
||||
|
||||
Example::
|
||||
|
||||
elements-dir: /path/to/elements/dir
|
||||
|
||||
images-dir
|
||||
----------
|
||||
|
||||
When we create location images they need to be written to somewhere. The
|
||||
``images-dir`` parameter is the place to write them.
|
||||
|
||||
Example::
|
||||
|
||||
images-dir: /path/to/images/dir
|
||||
|
||||
dburi
|
||||
-----
|
||||
Indicates the URI for the database connection. See the `SQLAlchemy
|
||||
|
|
|
|||
|
|
@ -24,6 +24,8 @@ import logging
|
|||
import os.path
|
||||
import paramiko
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import yaml
|
||||
|
|
@ -664,7 +666,6 @@ class SubNodeLauncher(threading.Thread):
|
|||
|
||||
|
||||
class ImageUpdater(threading.Thread):
|
||||
log = logging.getLogger("nodepool.ImageUpdater")
|
||||
|
||||
def __init__(self, nodepool, provider, image, snap_image_id):
|
||||
threading.Thread.__init__(self, name='ImageUpdater for %s' %
|
||||
|
|
@ -674,6 +675,8 @@ class ImageUpdater(threading.Thread):
|
|||
self.snap_image_id = snap_image_id
|
||||
self.nodepool = nodepool
|
||||
self.scriptdir = self.nodepool.config.scriptdir
|
||||
self.elementsdir = self.nodepool.config.elementsdir
|
||||
self.imagesdir = self.nodepool.config.imagesdir
|
||||
|
||||
def run(self):
|
||||
try:
|
||||
|
|
@ -708,6 +711,81 @@ class ImageUpdater(threading.Thread):
|
|||
self.snap_image.id)
|
||||
return
|
||||
|
||||
|
||||
class DiskImageUpdater(ImageUpdater):
|
||||
|
||||
log = logging.getLogger("nodepool.DiskImageUpdater")
|
||||
|
||||
def updateImage(self, session):
|
||||
start_time = time.time()
|
||||
timestamp = int(start_time)
|
||||
|
||||
# TODO(mordred) we can optimize by making an image once, not per
|
||||
# provider
|
||||
filename = os.path.join(self.imagesdir,
|
||||
'%s-%s' % (self.image.name, str(timestamp)))
|
||||
self.log.info("Creating image id: %s with filename %s for %s in %s" %
|
||||
(self.snap_image.id, filename, self.image.name,
|
||||
self.provider.name))
|
||||
|
||||
# TODO(mordred) abusing the hostname field
|
||||
self.snap_image.hostname = filename
|
||||
self.snap_image.version = timestamp
|
||||
session.commit()
|
||||
|
||||
self.buildImage(filename)
|
||||
|
||||
image_id = self.manager.uploadImage(filename)
|
||||
self.snap_image.external_id = image_id
|
||||
session.commit()
|
||||
self.log.debug("Image id: %s building image %s" %
|
||||
(self.snap_image.id, image_id))
|
||||
# It can take a _very_ long time for Rackspace 1.0 to save an image
|
||||
self.manager.waitForImage(image_id, IMAGE_TIMEOUT)
|
||||
|
||||
if statsd:
|
||||
dt = int((time.time() - start_time) * 1000)
|
||||
key = 'nodepool.image_update.%s.%s' % (self.image.name,
|
||||
self.provider.name)
|
||||
statsd.timing(key, dt)
|
||||
statsd.incr(key)
|
||||
|
||||
self.snap_image.state = nodedb.READY
|
||||
session.commit()
|
||||
self.log.info("Image %s in %s is ready" % (hostname,
|
||||
self.provider.name))
|
||||
|
||||
def buildImage(self, filename):
|
||||
env = dict()
|
||||
for k, v in os.environ.items():
|
||||
if k.startswith('NODEPOOL_'):
|
||||
env[k]=v
|
||||
env['ELEMENTS_PATH'] = self.elementsdir
|
||||
env['DIB_RELEASE'] = self.image.release
|
||||
|
||||
cmd = 'disk-image-create -n -o %s %s' % (filename, self.image.elements)
|
||||
self.log.info('Running %s' % cmd)
|
||||
|
||||
p = subprocess.Popen(
|
||||
shlex.split(cmd),
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT)
|
||||
(stdout, stderr) = p.communicate()
|
||||
for line in stdout:
|
||||
if self.log:
|
||||
self.log.info(line.rstrip())
|
||||
for line in stderr:
|
||||
if self.log:
|
||||
self.log.error(line.rstrip())
|
||||
ret = p.returncode
|
||||
if ret:
|
||||
raise Exception("Unable to create %s" % filename)
|
||||
|
||||
|
||||
class SnapshotImageUpdater(ImageUpdater):
|
||||
|
||||
log = logging.getLogger("nodepool.SnapshotImageUpdater")
|
||||
|
||||
def updateImage(self, session):
|
||||
start_time = time.time()
|
||||
timestamp = int(start_time)
|
||||
|
|
@ -901,6 +979,10 @@ class GearmanServer(ConfigValue):
|
|||
pass
|
||||
|
||||
|
||||
class DiskImage(ConfigValue):
|
||||
pass
|
||||
|
||||
|
||||
class NodePool(threading.Thread):
|
||||
log = logging.getLogger("nodepool.NodePool")
|
||||
|
||||
|
|
@ -938,11 +1020,14 @@ class NodePool(threading.Thread):
|
|||
newconfig.targets = {}
|
||||
newconfig.labels = {}
|
||||
newconfig.scriptdir = config.get('script-dir')
|
||||
newconfig.elementsdir = config.get('elements-dir')
|
||||
newconfig.imagesdir = config.get('images-dir')
|
||||
newconfig.dburi = config.get('dburi')
|
||||
newconfig.provider_managers = {}
|
||||
newconfig.jenkins_managers = {}
|
||||
newconfig.zmq_publishers = {}
|
||||
newconfig.gearman_servers = {}
|
||||
newconfig.diskimages = {}
|
||||
newconfig.crons = {}
|
||||
|
||||
for name, default in [
|
||||
|
|
@ -983,6 +1068,13 @@ class NodePool(threading.Thread):
|
|||
p.name = provider['name']
|
||||
l.providers[p.name] = p
|
||||
|
||||
for diskimage in config['diskimages']:
|
||||
d = DiskImage()
|
||||
d.name = diskimage['name']
|
||||
newconfig.diskimages[d.name] = d
|
||||
d.elements = diskimage['elements']
|
||||
d.release = diskimage['release']
|
||||
|
||||
for provider in config['providers']:
|
||||
p = Provider()
|
||||
p.name = provider['name']
|
||||
|
|
@ -1010,8 +1102,9 @@ class NodePool(threading.Thread):
|
|||
i.base_image = image['base-image']
|
||||
i.min_ram = image['min-ram']
|
||||
i.name_filter = image.get('name-filter', None)
|
||||
i.setup = image.get('setup')
|
||||
i.setup = image.get('setup', None)
|
||||
i.reset = image.get('reset')
|
||||
i.diskimage = image.get('diskimage', None)
|
||||
i.username = image.get('username', 'jenkins')
|
||||
i.private_key = image.get('private-key',
|
||||
'/var/lib/jenkins/.ssh/id_rsa')
|
||||
|
|
@ -1427,7 +1520,15 @@ class NodePool(threading.Thread):
|
|||
snap_image = session.createSnapshotImage(
|
||||
provider_name=provider.name,
|
||||
image_name=image.name)
|
||||
t = ImageUpdater(self, provider, image, snap_image.id)
|
||||
if image.setup:
|
||||
t = SnapshotImageUpdater(self, provider, image, snap_image.id)
|
||||
elif image.diskimage:
|
||||
t = DiskImageUpdater(self, provider, image, snap_image.id)
|
||||
else:
|
||||
raise Exception(
|
||||
"Invalid image config. Must specify either a setup script, or"
|
||||
" a diskimage to use.")
|
||||
|
||||
t.start()
|
||||
# Enough time to give them different timestamps (versions)
|
||||
# Just to keep things clearer.
|
||||
|
|
|
|||
|
|
@ -90,14 +90,14 @@ class NotFound(Exception):
|
|||
|
||||
class CreateServerTask(Task):
|
||||
def main(self, client):
|
||||
server = client.servers.create(**self.args)
|
||||
server = client.nova.servers.create(**self.args)
|
||||
return str(server.id)
|
||||
|
||||
|
||||
class GetServerTask(Task):
|
||||
def main(self, client):
|
||||
try:
|
||||
server = client.servers.get(self.args['server_id'])
|
||||
server = client.nova.servers.get(self.args['server_id'])
|
||||
except novaclient.exceptions.NotFound:
|
||||
raise NotFound()
|
||||
return make_server_dict(server)
|
||||
|
|
@ -105,52 +105,52 @@ class GetServerTask(Task):
|
|||
|
||||
class DeleteServerTask(Task):
|
||||
def main(self, client):
|
||||
client.servers.delete(self.args['server_id'])
|
||||
client.nova.servers.delete(self.args['server_id'])
|
||||
|
||||
|
||||
class ListServersTask(Task):
|
||||
def main(self, client):
|
||||
servers = client.servers.list()
|
||||
servers = client.nova.servers.list()
|
||||
return [make_server_dict(server) for server in servers]
|
||||
|
||||
|
||||
class AddKeypairTask(Task):
|
||||
def main(self, client):
|
||||
client.keypairs.create(**self.args)
|
||||
client.nova.keypairs.create(**self.args)
|
||||
|
||||
|
||||
class ListKeypairsTask(Task):
|
||||
def main(self, client):
|
||||
keys = client.keypairs.list()
|
||||
keys = client.nova.keypairs.list()
|
||||
return [dict(id=str(key.id), name=key.name) for
|
||||
key in keys]
|
||||
|
||||
|
||||
class DeleteKeypairTask(Task):
|
||||
def main(self, client):
|
||||
client.keypairs.delete(self.args['name'])
|
||||
client.nova.keypairs.delete(self.args['name'])
|
||||
|
||||
|
||||
class CreateFloatingIPTask(Task):
|
||||
def main(self, client):
|
||||
ip = client.floating_ips.create(**self.args)
|
||||
ip = client.nova.floating_ips.create(**self.args)
|
||||
return dict(id=str(ip.id), ip=ip.ip)
|
||||
|
||||
|
||||
class AddFloatingIPTask(Task):
|
||||
def main(self, client):
|
||||
client.servers.add_floating_ip(**self.args)
|
||||
client.nova.servers.add_floating_ip(**self.args)
|
||||
|
||||
|
||||
class GetFloatingIPTask(Task):
|
||||
def main(self, client):
|
||||
ip = client.floating_ips.get(self.args['ip_id'])
|
||||
ip = client.nova.floating_ips.get(self.args['ip_id'])
|
||||
return dict(id=str(ip.id), ip=ip.ip, instance_id=str(ip.instance_id))
|
||||
|
||||
|
||||
class ListFloatingIPsTask(Task):
|
||||
def main(self, client):
|
||||
ips = client.floating_ips.list()
|
||||
ips = client.nova.floating_ips.list()
|
||||
return [dict(id=str(ip.id), ip=ip.ip,
|
||||
instance_id=str(ip.instance_id)) for
|
||||
ip in ips]
|
||||
|
|
@ -158,24 +158,31 @@ class ListFloatingIPsTask(Task):
|
|||
|
||||
class RemoveFloatingIPTask(Task):
|
||||
def main(self, client):
|
||||
client.servers.remove_floating_ip(**self.args)
|
||||
client.nova.servers.remove_floating_ip(**self.args)
|
||||
|
||||
|
||||
class DeleteFloatingIPTask(Task):
|
||||
def main(self, client):
|
||||
client.floating_ips.delete(self.args['ip_id'])
|
||||
client.nova.floating_ips.delete(self.args['ip_id'])
|
||||
|
||||
|
||||
class CreateImageTask(Task):
|
||||
def main(self, client):
|
||||
# This returns an id
|
||||
return str(client.servers.create_image(**self.args))
|
||||
return str(client.nova.servers.create_image(**self.args))
|
||||
|
||||
|
||||
class UploadImageTask(Task):
|
||||
def main(self, client):
|
||||
# This returns an id
|
||||
# EEK?
|
||||
return str(client.glance.upload_image(**self.args))
|
||||
|
||||
|
||||
class GetImageTask(Task):
|
||||
def main(self, client):
|
||||
try:
|
||||
image = client.images.get(**self.args)
|
||||
image = client.nova.images.get(**self.args)
|
||||
except novaclient.exceptions.NotFound:
|
||||
raise NotFound()
|
||||
# HP returns 404, rackspace can return a 'DELETED' image.
|
||||
|
|
@ -187,7 +194,7 @@ class GetImageTask(Task):
|
|||
class ListExtensionsTask(Task):
|
||||
def main(self, client):
|
||||
try:
|
||||
resp, body = client.client.get('/extensions')
|
||||
resp, body = client.nova.client.get('/extensions')
|
||||
return [x['alias'] for x in body['extensions']]
|
||||
except novaclient.exceptions.NotFound:
|
||||
# No extensions present.
|
||||
|
|
@ -196,26 +203,33 @@ class ListExtensionsTask(Task):
|
|||
|
||||
class ListFlavorsTask(Task):
|
||||
def main(self, client):
|
||||
flavors = client.flavors.list()
|
||||
flavors = client.nova.flavors.list()
|
||||
return [dict(id=str(flavor.id), ram=flavor.ram, name=flavor.name)
|
||||
for flavor in flavors]
|
||||
|
||||
|
||||
class ListImagesTask(Task):
|
||||
def main(self, client):
|
||||
images = client.images.list()
|
||||
images = client.nova.images.list()
|
||||
return [make_image_dict(image) for image in images]
|
||||
|
||||
|
||||
class FindImageTask(Task):
|
||||
def main(self, client):
|
||||
image = client.images.find(**self.args)
|
||||
image = client.nova.images.find(**self.args)
|
||||
return dict(id=str(image.id))
|
||||
|
||||
|
||||
class DeleteImageTask(Task):
|
||||
def main(self, client):
|
||||
client.images.delete(**self.args)
|
||||
client.nova.images.delete(**self.args)
|
||||
|
||||
|
||||
class ClientContainer(object):
|
||||
def __init__(self, nova, glance, neutron):
|
||||
self.nova = nova
|
||||
self.glance = glance
|
||||
self.neutron = neutron
|
||||
|
||||
|
||||
class ProviderManager(TaskManager):
|
||||
|
|
@ -251,9 +265,12 @@ class ProviderManager(TaskManager):
|
|||
self._cloud_metadata_read = True
|
||||
|
||||
def _getClient(self):
|
||||
args = ['1.1', self.provider.username, self.provider.password,
|
||||
self.provider.project_id, self.provider.auth_url]
|
||||
kwargs = {}
|
||||
kwargs = dict(
|
||||
username=self.provider.username,
|
||||
password=self.provider.password,
|
||||
tenant_id=self.provider.project_id,
|
||||
auth_url=self.provider.auth_url)
|
||||
# Help - need to have this per service
|
||||
if self.provider.service_type:
|
||||
kwargs['service_type'] = self.provider.service_type
|
||||
if self.provider.service_name:
|
||||
|
|
@ -262,7 +279,10 @@ class ProviderManager(TaskManager):
|
|||
kwargs['region_name'] = self.provider.region_name
|
||||
if self.provider.auth_url == 'fake':
|
||||
return fakeprovider.FAKE_CLIENT
|
||||
return novaclient.client.Client(*args, **kwargs)
|
||||
nova = novaclient.client.Client('1.1'. *args, **kwargs)
|
||||
glance = glanceclient.client.Client('1', *args, **kwargs)
|
||||
neutron = neutronclient.v2_0.client.Client(*args, **kwargs)
|
||||
return ClientContainer(nova, glance, neutron)
|
||||
|
||||
def _getFlavors(self):
|
||||
flavors = self.listFlavors()
|
||||
|
|
@ -415,6 +435,9 @@ class ProviderManager(TaskManager):
|
|||
def getImage(self, image_id):
|
||||
return self.submitTask(GetImageTask(image=image_id))
|
||||
|
||||
def uploadImage(self, filename):
|
||||
return self.submitTask(UploadImageTask(filename=filename))
|
||||
|
||||
def listExtensions(self):
|
||||
return self.submitTask(ListExtensionsTask())
|
||||
|
||||
|
|
|
|||
|
|
@ -16,3 +16,4 @@ python-novaclient
|
|||
MySQL-python
|
||||
PrettyTable>=0.6,<0.8
|
||||
six>=1.4.1
|
||||
diskimage-builder
|
||||
|
|
|
|||
Loading…
Reference in New Issue