Initial checkin and job config
Change-Id: Ifd9ff6d17a8a43683a26b68bf60e9ec781aeb2de
This commit is contained in:
parent
4ad9856d39
commit
a2cc0129c1
14
.zuul.yaml
Normal file
14
.zuul.yaml
Normal file
@ -0,0 +1,14 @@
|
|||||||
|
- job:
|
||||||
|
name: generate-test-images
|
||||||
|
parent: tox
|
||||||
|
description: |
|
||||||
|
Build sample images with "tox"
|
||||||
|
vars:
|
||||||
|
tox_envlist: generate
|
||||||
|
- project:
|
||||||
|
check:
|
||||||
|
jobs:
|
||||||
|
- generate-test-images
|
||||||
|
gate:
|
||||||
|
jobs:
|
||||||
|
- generate-test-images
|
10
README
Normal file
10
README
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
This repo contains instructions and scripts for a variety of sample image
|
||||||
|
formats. The images are described in manifest.yaml and generated using the
|
||||||
|
generate.py script.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
```
|
||||||
|
$ tox -e generate
|
||||||
|
```
|
||||||
|
|
||||||
|
Images will (by default) be created in `images/`.
|
5
bindep.txt
Normal file
5
bindep.txt
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
mkisofs [platform:rpm]
|
||||||
|
genisoimage [platform:dpkg]
|
||||||
|
parted
|
||||||
|
qemu-img [platform:rpm]
|
||||||
|
qemu-utils [platform:dpkg]
|
99
generate.py
Executable file
99
generate.py
Executable file
@ -0,0 +1,99 @@
|
|||||||
|
#!/usr/bin/python
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import struct
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
LOG = logging.getLogger('generate')
|
||||||
|
|
||||||
|
|
||||||
|
def footerify_vmdk(fn):
|
||||||
|
"""Convert a monolithicSparse to use a footer instead of just a header"""
|
||||||
|
GD_AT_END = 0xffffffffffffffff
|
||||||
|
|
||||||
|
with open(fn, 'rb+') as f:
|
||||||
|
header = f.read(512)
|
||||||
|
# Write the "expect a footer" sentinel into the header
|
||||||
|
f.seek(56)
|
||||||
|
f.write(struct.pack('<Q', GD_AT_END))
|
||||||
|
# Add room for the footer marker, footer, and EOS marker, but
|
||||||
|
# filled with zeroes (which is invalid)
|
||||||
|
f.seek(0, 2)
|
||||||
|
f.write(b'\x00' * 512 * 3)
|
||||||
|
# This is the footer marker (type=3)
|
||||||
|
f.seek(-512 * 3 + 12, 2)
|
||||||
|
f.write(b'\x03\x00\x00\x00')
|
||||||
|
# Second-to-last sector is the footer, which must be a copy of the
|
||||||
|
# header but with gdOffset set to something other than the flag.
|
||||||
|
f.seek(-512 * 2, 2)
|
||||||
|
f.write(header)
|
||||||
|
|
||||||
|
|
||||||
|
POSTPROCS = {
|
||||||
|
'footerify_vmdk': footerify_vmdk,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def generate_one(yamldef, output_dir):
|
||||||
|
vars = {
|
||||||
|
'name': yamldef['name'],
|
||||||
|
'filename': '%s.%s' % (yamldef['name'], yamldef['format']),
|
||||||
|
}
|
||||||
|
cmds = (yamldef['generated_by'] % vars).strip().split('\n')
|
||||||
|
for i, one_cmd in enumerate(cmds):
|
||||||
|
if one_cmd:
|
||||||
|
LOG.info('Generating %s step %i/%i with %r',
|
||||||
|
yamldef['name'], i + 1, len(cmds), one_cmd)
|
||||||
|
try:
|
||||||
|
output = subprocess.check_output(one_cmd, shell=True,
|
||||||
|
cwd=output_dir,
|
||||||
|
stderr=subprocess.STDOUT)
|
||||||
|
except subprocess.CalledProcessError as e:
|
||||||
|
LOG.error('Command %r failed with %i: %s',
|
||||||
|
one_cmd, e.returncode, e.output)
|
||||||
|
raise
|
||||||
|
|
||||||
|
LOG.debug('Command %r returned %s', one_cmd, output)
|
||||||
|
if 'postprocess' in yamldef:
|
||||||
|
postproc = POSTPROCS[yamldef['postprocess']]
|
||||||
|
LOG.info('Running postprocesser %s on %r',
|
||||||
|
yamldef['postprocess'], vars['filename'])
|
||||||
|
postproc(os.path.join(output_dir, vars['filename']))
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
p = argparse.ArgumentParser()
|
||||||
|
p.add_argument('manifest')
|
||||||
|
p.add_argument('--output', default='images',
|
||||||
|
help='Output directory for generated images')
|
||||||
|
p.add_argument('--only',
|
||||||
|
help='Only generate this named image')
|
||||||
|
p.add_argument('--debug', action='store_true', default=False)
|
||||||
|
args = p.parse_args()
|
||||||
|
|
||||||
|
logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO)
|
||||||
|
|
||||||
|
try:
|
||||||
|
os.mkdir(args.output)
|
||||||
|
except FileExistsError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
with open(args.manifest) as f:
|
||||||
|
yamldef = yaml.load(f, Loader=yaml.SafeLoader)
|
||||||
|
|
||||||
|
for image in yamldef['images']:
|
||||||
|
if args.only and args.only != image['name']:
|
||||||
|
continue
|
||||||
|
if 'generated_by' in image:
|
||||||
|
generate_one(image, args.output)
|
||||||
|
else:
|
||||||
|
LOG.error('Unknown source for image %s', image['name'])
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
sys.exit(main())
|
165
manifest.yaml
Normal file
165
manifest.yaml
Normal file
@ -0,0 +1,165 @@
|
|||||||
|
# Manifest of OpenStack test images
|
||||||
|
#
|
||||||
|
# Schema:
|
||||||
|
# name: The base (short) name of the image file, used to form the file name
|
||||||
|
# and the image name when uploaded to glance
|
||||||
|
# format: The name of the format that glance and qemu-img consider this image
|
||||||
|
# to be. Appended as an extension to the name above to generate the
|
||||||
|
# output filename.
|
||||||
|
# usable: Boolean indicating whether or not this should be accepted by glance,
|
||||||
|
# nova, cinder, etc. This is sort of a loose thing that is hard to
|
||||||
|
# define because it may depend on config. Probably mostly useful for
|
||||||
|
# humans, but perhaps also test automation.
|
||||||
|
# insecure: Boolean indicating if this image will contain some
|
||||||
|
# security-sensitive exploit or other violation. These should never
|
||||||
|
# be acceptable to services properly checking for them.
|
||||||
|
# generated_by: If present, a series of shell commands used to generate the
|
||||||
|
# image.
|
||||||
|
# postprocess: Some symbolic name of a python function in generate.py that
|
||||||
|
# must be run after the generated_by commands (if present) to
|
||||||
|
# finish creation of the image.
|
||||||
|
images:
|
||||||
|
- name: standard-qcow2v3
|
||||||
|
format: qcow2
|
||||||
|
usable: true
|
||||||
|
insecure: false
|
||||||
|
generated_by: |
|
||||||
|
qemu-img create -f qcow2 %(filename)s 10M
|
||||||
|
description: |
|
||||||
|
This is a standard blank qcow2 file without any external linkage
|
||||||
|
- name: qcow-with-backing
|
||||||
|
format: qcow2
|
||||||
|
usable: false
|
||||||
|
insecure: true
|
||||||
|
generated_by: |
|
||||||
|
qemu-img create -f qcow2 -F raw -b /etc/hosts %(filename)s 10M
|
||||||
|
description: |
|
||||||
|
This is a qcow2 file with a backing file, which can be manipulated to
|
||||||
|
expose files on a host machine if processed without checking what file
|
||||||
|
is being included.
|
||||||
|
- name: qcow-with-datafile
|
||||||
|
format: qcow2
|
||||||
|
usable: false
|
||||||
|
insecure: true
|
||||||
|
generated_by: |
|
||||||
|
qemu-img create -f qcow2 -o data_file=qcow-data-file,data_file_raw=on %(filename)s 10M
|
||||||
|
rm qcow-data-file
|
||||||
|
description: |
|
||||||
|
This is a qcow2 file with a data-file specified, which can both
|
||||||
|
reference external data like backing-file or embed a QMP JSON
|
||||||
|
specification for a more complex storage arrangement and do much more
|
||||||
|
damage to a host system.
|
||||||
|
- name: raw-blank
|
||||||
|
format: raw
|
||||||
|
usable: true
|
||||||
|
insecure: false
|
||||||
|
generated_by: |
|
||||||
|
qemu-img create -f raw %(filename)s 1M
|
||||||
|
description: |
|
||||||
|
A truly unformatted file, which should not match any other format. Since
|
||||||
|
qemu-img is used here, it's literally a file of zero bytes.
|
||||||
|
- name: gpt-blank
|
||||||
|
format: gpt
|
||||||
|
usable: true
|
||||||
|
insecure: false
|
||||||
|
generated_by: |
|
||||||
|
qemu-img create -f raw %(filename)s 1M
|
||||||
|
parted %(filename)s --script 'mklabel gpt'
|
||||||
|
description: |
|
||||||
|
An image of a whole disk commonly seen in the PC/x86 space, with a
|
||||||
|
protective MBR and GPT paritition table.
|
||||||
|
- name: standard
|
||||||
|
format: qed
|
||||||
|
usable: false
|
||||||
|
insecure: false
|
||||||
|
generated_by: |
|
||||||
|
qemu-img create -f qed %(filename)s 10M
|
||||||
|
description: |
|
||||||
|
A blank QED formatted image, similar to qcow, but should not be supported
|
||||||
|
by any service.
|
||||||
|
- name: standard-iso9660
|
||||||
|
format: iso
|
||||||
|
usable: true
|
||||||
|
insecure: false
|
||||||
|
generated_by: |
|
||||||
|
qemu-img create -f raw %(filename)s 1M
|
||||||
|
mkisofs -V %(name)s -o %(filename)s /etc/hosts
|
||||||
|
description: |
|
||||||
|
An ISO9660 image with a single file inside
|
||||||
|
- name: standard-udf
|
||||||
|
format: iso
|
||||||
|
usable: true
|
||||||
|
insecure: false
|
||||||
|
generated_by: |
|
||||||
|
qemu-img create -f raw %(filename)s 1M
|
||||||
|
mkisofs -udf -V %(name)s -o %(filename)s /etc/hosts
|
||||||
|
description: |
|
||||||
|
A UDF filesystem (similar to ISO9660) with a single file inside
|
||||||
|
- name: iso-with-qcow2-in-system
|
||||||
|
format: iso
|
||||||
|
usable: false
|
||||||
|
insecure: true
|
||||||
|
generated_by: |
|
||||||
|
qemu-img create -f raw %(filename)s 1M
|
||||||
|
mkisofs -udf -V %(name)s -o %(filename)s /etc/hosts
|
||||||
|
qemu-img create -f qcow2 tmp.qcow 10M
|
||||||
|
dd if=tmp.qcow of=%(filename)s bs=32k count=1 conv=notrunc
|
||||||
|
rm tmp.qcow
|
||||||
|
description: |
|
||||||
|
An ISO9660 image with a single file inside, but with a qcow2 header in
|
||||||
|
the "system area" which can fool tools (like qemu-img) into thinking it
|
||||||
|
is a valid qcow2 file.
|
||||||
|
- name: vmdk-monolithicSparse
|
||||||
|
format: vmdk
|
||||||
|
usable: true
|
||||||
|
insecure: false
|
||||||
|
generated_by: |
|
||||||
|
qemu-img create -f vmdk -o subformat=monolithicSparse %(filename)s 10M
|
||||||
|
description: |
|
||||||
|
A VMDK file in monolithicSparse format (i.e. has a sparse header,
|
||||||
|
embedded descriptor and extents)
|
||||||
|
- name: vmdk-streamOptimized
|
||||||
|
format: vmdk
|
||||||
|
usable: true
|
||||||
|
insecure: false
|
||||||
|
generated_by: |
|
||||||
|
qemu-img create -f vmdk -o subformat=streamOptimized %(filename)s 10M
|
||||||
|
description: |
|
||||||
|
A VMDK file in streamOptimized format, similar to monolithicSparse,
|
||||||
|
without a footer.
|
||||||
|
- name: vmdk-monolithicFlat
|
||||||
|
format: vmdk
|
||||||
|
usable: false
|
||||||
|
insecure: false
|
||||||
|
generated_by: |
|
||||||
|
qemu-img create -f vmdk -o subformat=monolithicFlat %(filename)s 10M
|
||||||
|
rm %(name)s-flat.vmdk
|
||||||
|
description: |
|
||||||
|
A VMDK file in monolithicFlat format, which is actually just a text
|
||||||
|
descriptor that references external extent files. Not usable in
|
||||||
|
OpenStack and thus should always be rejected by services.
|
||||||
|
- name: vmdk-sparse-with-url-backing
|
||||||
|
format: vmdk
|
||||||
|
usable: false
|
||||||
|
insecure: true
|
||||||
|
generated_by: |
|
||||||
|
qemu-img create -f vmdk -o subformat=monolithicSparse %(filename)s 10M
|
||||||
|
dd if=%(filename)s bs=512 count=1 skip=1 | sed 's#%(filename)s#/etc/hosts#' | dd of=%(filename)s conv=notrunc seek=1
|
||||||
|
description: |
|
||||||
|
A monolithicSparse VMDK file that references external extent files. This
|
||||||
|
is in an acceptable format, but is insafe as it would result in exposing
|
||||||
|
files on a host machine within the guest image.
|
||||||
|
- name: vmdk-sparse-with-footer
|
||||||
|
format: vmdk
|
||||||
|
usable: true
|
||||||
|
insecure: false
|
||||||
|
postprocess: footerify_vmdk
|
||||||
|
generated_by: |
|
||||||
|
qemu-img create -f vmdk -o subformat=monolithicSparse %(filename)s 10M
|
||||||
|
description: |
|
||||||
|
A VMDK file in monolithicSparse format with a footer that overrides the
|
||||||
|
header. This footer must be supported specifically and checked for
|
||||||
|
sanity to make sure it does not reference a descriptor or other
|
||||||
|
resources that would not have been inspected in the stream by the time
|
||||||
|
we read the footer. Apparently vmware tooling generates these with a
|
||||||
|
footer frequently and thus is a format we need to (carefully) support.
|
1
requirements.txt
Normal file
1
requirements.txt
Normal file
@ -0,0 +1 @@
|
|||||||
|
pyyaml
|
Loading…
x
Reference in New Issue
Block a user