x/anvil
Code Issues Proposed changes

Simplify image uploading and allow for qcow2 uploading

Browse Source
This commit is contained in:
Joshua Harlow 2012-03-29 15:45:06 -07:00
parent 1405685d2e
commit 44f1ed062f
2 changed files with 84 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

149
devstack/image/uploader.py
View File

@ -32,21 +32,17 @@ from devstack.components import keystone
LOG = log.getLogger("devstack.image.uploader") LOG = log.getLogger("devstack.image.uploader")
# These are used when looking inside archives # These are used when looking inside archives
KERNEL_FN_MATCH = re.compile(r"(.*)vmlinuz$", re.I) KERNEL_FN_MATCH = re.compile(r"(.*)-vmlinuz$", re.I)
RAMDISK_FN_MATCH = re.compile(r"(.*)initrd$", re.I) RAMDISK_FN_MATCH = re.compile(r"(.*)-initrd$", re.I)
IMAGE_FN_MATCH = re.compile(r"(.*)img$", re.I) IMAGE_FN_MATCH = re.compile(r"(.*)img$", re.I)
# Glance commands # Glance commands
KERNEL_ADD = ['glance', 'add', '-A', '%TOKEN%', '--silent-upload', IMAGE_ADD = ['glance', 'add', '-A', '%TOKEN%',
'name="%IMAGE_NAME%-kernel"', 'is_public=true', 'container_format=aki', '--silent-upload',
'disk_format=aki'] 'name="%NAME%"',
INITRD_ADD = ['glance', 'add', '-A', '%TOKEN%', '--silent-upload', 'is_public=true',
'name="%IMAGE_NAME%-ramdisk"', 'is_public=true', 'container_format=ari', 'container_format=%CONTAINER_FORMAT%',
'disk_format=ari'] 'disk_format=%DISK_FORMAT%']
IMAGE_ADD = ['glance', 'add', '-A', '%TOKEN%', '--silent-upload',
'name="%IMAGE_NAME%.img"',
'is_public=true', 'container_format=ami', 'disk_format=ami',
'kernel_id=%KERNEL_ID%', 'ramdisk_id=%INITRD_ID%']
DETAILS_SHOW = ['glance', '-A', '%TOKEN%', 'details'] DETAILS_SHOW = ['glance', '-A', '%TOKEN%', 'details']
# Extensions that tarfile knows how to work with # Extensions that tarfile knows how to work with
@ -58,6 +54,7 @@ TAR_EXTS = ['.tgz', '.gzip', '.gz', '.bz2', '.tar']
NAME_CLEANUPS = [ NAME_CLEANUPS = [
'.tar.gz', '.tar.gz',
'.img.gz', '.img.gz',
'.qcow2',
'.img', '.img',
] + TAR_EXTS ] + TAR_EXTS
NAME_CLEANUPS.sort() NAME_CLEANUPS.sort()
@ -96,26 +93,38 @@ class Unpacker(object):
LOG.info("Extracting %r to %r", file_location, extract_dir) LOG.info("Extracting %r to %r", file_location, extract_dir)
with contextlib.closing(tarfile.open(file_location, 'r')) as tfh: with contextlib.closing(tarfile.open(file_location, 'r')) as tfh:
tfh.extractall(extract_dir) tfh.extractall(extract_dir)
locations = dict() info = dict()
if kernel_fn: if kernel_fn:
locations['kernel'] = sh.joinpths(extract_dir, kernel_fn) info['kernel'] = {
'FILE_NAME': sh.joinpths(extract_dir, kernel_fn),
'DISK_FORMAT': 'aki',
'CONTAINER_FORMAT': 'aki',
}
if ramdisk_fn: if ramdisk_fn:
locations['ramdisk'] = sh.joinpths(extract_dir, ramdisk_fn) info['ramdisk'] = {
locations['image'] = sh.joinpths(extract_dir, root_img_fn) 'FILE_NAME': sh.joinpths(extract_dir, ramdisk_fn),
return locations 'DISK_FORMAT': 'ari',
'CONTAINER_FORMAT': 'ari',
def _unpack_image(self, file_name, file_location, tmp_dir): }
locations = dict() info['FILE_NAME'] = sh.joinpths(extract_dir, root_img_fn)
locations['image'] = file_location info['DISK_FORMAT'] = 'ami'
return locations info['CONTAINER_FORMAT'] = 'ami'
return info
def unpack(self, file_name, file_location, tmp_dir): def unpack(self, file_name, file_location, tmp_dir):
(_, fn_ext) = os.path.splitext(file_name) (_, fn_ext) = os.path.splitext(file_name)
fn_ext = fn_ext.lower() fn_ext = fn_ext.lower()
if fn_ext in TAR_EXTS: if fn_ext in TAR_EXTS:
return self._unpack_tar(file_name, file_location, tmp_dir) return self._unpack_tar(file_name, file_location, tmp_dir)
elif fn_ext in ['.img']: elif fn_ext in ['.img', '.qcow2']:
return self._unpack_image(file_name, file_location, tmp_dir) info = dict()
info['FILE_NAME'] = file_location,
if fn_ext == '.img':
info['DISK_FORMAT'] = 'raw'
else:
info['DISK_FORMAT'] = 'qcow2'
info['CONTAINER_FORMAT'] = 'bare'
return info
else: else:
msg = "Currently we do not know how to unpack %r" % (file_name) msg = "Currently we do not know how to unpack %r" % (file_name)
raise NotImplementedError(msg) raise NotImplementedError(msg)
@ -128,16 +137,18 @@ class Image(object):
self.token = token self.token = token
self._registry = Registry(token) self._registry = Registry(token)
def _register(self, image_name, locations): def _register(self, image_name, location):
# Upload the kernel, if we have one # Upload the kernel, if we have one
kernel = locations.get('kernel') kernel = location.pop('kernel', None)
kernel_id = '' kernel_id = ''
if kernel: if kernel:
LOG.info('Adding kernel %r to glance.', kernel) LOG.info('Adding kernel %s to glance.', kernel)
params = {'TOKEN': self.token, 'IMAGE_NAME': image_name} params = dict(kernel)
cmd = {'cmd': KERNEL_ADD} params['TOKEN'] = self.token
with open(kernel, 'r') as fh: params['NAME'] = "%s-vmlinuz" % (image_name)
cmd = {'cmd': IMAGE_ADD}
with open(params['FILE_NAME'], 'r') as fh:
res = utils.execute_template(cmd, res = utils.execute_template(cmd,
params=params, stdin_fh=fh, params=params, stdin_fh=fh,
close_stdin=True) close_stdin=True)
@ -146,13 +157,15 @@ class Image(object):
kernel_id = stdout.split(':')[1].strip() kernel_id = stdout.split(':')[1].strip()
# Upload the ramdisk, if we have one # Upload the ramdisk, if we have one
initrd = locations.get('ramdisk') initrd = location.pop('ramdisk', None)
initrd_id = '' initrd_id = ''
if initrd: if initrd:
LOG.info('Adding ramdisk %r to glance.', initrd) LOG.info('Adding ramdisk %s to glance.', initrd)
params = {'TOKEN': self.token, 'IMAGE_NAME': image_name} params = dict(initrd)
cmd = {'cmd': INITRD_ADD} params['TOKEN'] = self.token
with open(initrd, 'r') as fh: params['NAME'] = "%s-initrd" % (image_name)
cmd = {'cmd': IMAGE_ADD}
with open(params['FILE_NAME'], 'r') as fh:
res = utils.execute_template(cmd, res = utils.execute_template(cmd,
params=params, stdin_fh=fh, params=params, stdin_fh=fh,
close_stdin=True) close_stdin=True)
@ -161,16 +174,24 @@ class Image(object):
initrd_id = stdout.split(':')[1].strip() initrd_id = stdout.split(':')[1].strip()
# Upload the root, we must have one... # Upload the root, we must have one...
img_id = '' root_image = dict(location)
root_image = locations['image'] LOG.info('Adding image %s to glance.', root_image)
LOG.info('Adding image %r to glance.', root_image) add_cmd = list(IMAGE_ADD)
params = {'TOKEN': self.token, 'IMAGE_NAME': image_name, params = dict(root_image)
'KERNEL_ID': kernel_id, 'INITRD_ID': initrd_id} params['TOKEN'] = self.token
cmd = {'cmd': IMAGE_ADD} params['NAME'] = image_name
with open(root_image, 'r') as fh: if kernel_id:
add_cmd += ['kernel_id=%KERNEL_ID%']
params['KERNEL_ID'] = kernel_id
if initrd_id:
add_cmd += ['ramdisk_id=%INITRD_ID%']
params['INITRD_ID'] = initrd_id
cmd = {'cmd': add_cmd}
with open(params['FILE_NAME'], 'r') as fh:
res = utils.execute_template(cmd, res = utils.execute_template(cmd,
params=params, stdin_fh=fh, params=params, stdin_fh=fh,
close_stdin=True) close_stdin=True)
img_id = ''
if res: if res:
(stdout, _) = res[0] (stdout, _) = res[0]
img_id = stdout.split(':')[1].strip() img_id = stdout.split(':')[1].strip()
@ -223,9 +244,9 @@ class Image(object):
with utils.tempdir() as tdir: with utils.tempdir() as tdir:
fetch_fn = sh.joinpths(tdir, url_fn) fetch_fn = sh.joinpths(tdir, url_fn)
down.UrlLibDownloader(self.url, fetch_fn).download() down.UrlLibDownloader(self.url, fetch_fn).download()
locations = Unpacker().unpack(url_fn, fetch_fn, tdir) unpack_info = Unpacker().unpack(url_fn, fetch_fn, tdir)
tgt_image_name = self._generate_img_name(url_fn) tgt_image_name = self._generate_img_name(url_fn)
self._register(tgt_image_name, locations) self._register(tgt_image_name, unpack_info)
return tgt_image_name return tgt_image_name
else: else:
return None return None
@ -300,17 +321,16 @@ class Service:
}) })
# Prepare the request # Prepare the request
request = urllib2.Request(keystone_token_url) headers = {
'Content-Type': 'application/json'
# Post body }
request.add_data(data) request = urllib2.Request(keystone_token_url, data=data, headers=headers)
# Content type
request.add_header('Content-Type', 'application/json')
# Make the request # Make the request
LOG.info("Getting your token from url [%s], please wait..." % (keystone_token_url)) LOG.info("Getting your token from url %r, please wait..." % (keystone_token_url))
LOG.debug("With post json data %s" % (data)) LOG.debug("With post data %s" % (data))
LOG.debug("With headers %s" % (headers))
response = urllib2.urlopen(request) response = urllib2.urlopen(request)
token = json.loads(response.read()) token = json.loads(response.read())
@ -320,36 +340,33 @@ class Service:
not token.get('access') or not type(token.get('access')) is dict or not token.get('access') or not type(token.get('access')) is dict or
not token.get('access').get('token') or not type(token.get('access').get('token')) is dict or not token.get('access').get('token') or not type(token.get('access').get('token')) is dict or
not token.get('access').get('token').get('id')): not token.get('access').get('token').get('id')):
msg = "Response from url [%s] did not match expected json format." % (keystone_token_url) msg = "Response from url %r did not match expected json format." % (keystone_token_url)
raise IOError(msg) raise IOError(msg)
# Basic checks passed, extract it! # Basic checks passed, extract it!
tok = token['access']['token']['id'] tok = token['access']['token']['id']
LOG.debug("Got token %s" % (tok)) LOG.debug("Got token %r" % (tok))
return tok return tok
def install(self): def install(self):
LOG.info("Setting up any specified images in glance.") LOG.info("Setting up any specified images in glance.")
# Extract the urls from the config # Extract the urls from the config
urls = list() flat_locations = self.cfg.getdefaulted('img', 'image_urls', '')
flat_urls = self.cfg.getdefaulted('img', 'image_urls', []) locations = [loc.strip() for loc in flat_locations.split(',') if len(loc.strip())]
expanded_urls = [x.strip() for x in flat_urls.split(',')]
for url in expanded_urls:
if len(url):
urls.append(url)
# Install them in glance # Install them in glance
am_installed = 0 am_installed = 0
if urls: if locations:
LOG.info("Attempting to download & extract and upload (%s) images." % (", ".join(urls))) utils.log_iterable(locations, logger=LOG,
header="Attempting to download+extract+upload %s images." % len(locations))
token = self._get_token() token = self._get_token()
for url in urls: for uri in locations:
try: try:
name = Image(url, token).install() name = Image(uri, token).install()
if name: if name:
LOG.info("Installed image named %r" % (name)) LOG.info("Installed image named %r" % (name))
am_installed += 1 am_installed += 1
except (IOError, tarfile.TarError) as e: except (IOError, tarfile.TarError) as e:
LOG.exception('Installing %r failed due to: %s', url, e) LOG.exception('Installing %r failed due to: %s', uri, e)
return am_installed return am_installed

2
stack
View File

@ -118,7 +118,7 @@ def run(args):
loaded_rcs = True loaded_rcs = True
root_dir = env.get_key(env_rc.INSTALL_ROOT) root_dir = env.get_key(env_rc.INSTALL_ROOT)
if not root_dir: if not root_dir:
root_dir = utils.joinpths(sh.gethomedir(), 'openstack') root_dir = sh.joinpths(sh.gethomedir(), 'openstack')
root_dir = sh.abspth(root_dir) root_dir = sh.abspth(root_dir)
setup_root(root_dir) setup_root(root_dir)