Initial checkin and job config

Change-Id: Ifd9ff6d17a8a43683a26b68bf60e9ec781aeb2de

.zuul.yaml

@@ -0,0 +1,14 @@
+- job:
+    name: generate-test-images
+    parent: tox
+    description: |
+      Build sample images with "tox"
+    vars:
+      tox_envlist: generate
+- project:
+    check:
+      jobs:
+        - generate-test-images
+    gate:
+      jobs:
+        - generate-test-images

README

@@ -0,0 +1,10 @@
+This repo contains instructions and scripts for a variety of sample image
+formats. The images are described in manifest.yaml and generated using the
+generate.py script.
+
+Usage:
+```
+$ tox -e generate
+```
+
+Images will (by default) be created in `images/`.

bindep.txt

@@ -0,0 +1,5 @@
+mkisofs [platform:rpm]
+genisoimage [platform:dpkg]
+parted
+qemu-img [platform:rpm]
+qemu-utils [platform:dpkg]

generate.py

@@ -0,0 +1,99 @@
+#!/usr/bin/python
+
+import argparse
+import logging
+import os
+import struct
+import subprocess
+import sys
+
+import yaml
+
+LOG = logging.getLogger('generate')
+
+
+def footerify_vmdk(fn):
+    """Convert a monolithicSparse to use a footer instead of just a header"""
+    GD_AT_END = 0xffffffffffffffff
+
+    with open(fn, 'rb+') as f:
+        header = f.read(512)
+        # Write the "expect a footer" sentinel into the header
+        f.seek(56)
+        f.write(struct.pack('<Q', GD_AT_END))
+        # Add room for the footer marker, footer, and EOS marker, but
+        # filled with zeroes (which is invalid)
+        f.seek(0, 2)
+        f.write(b'\x00' * 512 * 3)
+        # This is the footer marker (type=3)
+        f.seek(-512 * 3 + 12, 2)
+        f.write(b'\x03\x00\x00\x00')
+        # Second-to-last sector is the footer, which must be a copy of the
+        # header but with gdOffset set to something other than the flag.
+        f.seek(-512 * 2, 2)
+        f.write(header)
+
+
+POSTPROCS = {
+    'footerify_vmdk': footerify_vmdk,
+}
+
+
+def generate_one(yamldef, output_dir):
+    vars = {
+        'name': yamldef['name'],
+        'filename': '%s.%s' % (yamldef['name'], yamldef['format']),
+    }
+    cmds = (yamldef['generated_by'] % vars).strip().split('\n')
+    for i, one_cmd in enumerate(cmds):
+        if one_cmd:
+            LOG.info('Generating %s step %i/%i with %r',
+                     yamldef['name'], i + 1, len(cmds), one_cmd)
+            try:
+                output = subprocess.check_output(one_cmd, shell=True,
+                                                 cwd=output_dir,
+                                                 stderr=subprocess.STDOUT)
+            except subprocess.CalledProcessError as e:
+                LOG.error('Command %r failed with %i: %s',
+                          one_cmd, e.returncode, e.output)
+                raise
+
+            LOG.debug('Command %r returned %s', one_cmd, output)
+        if 'postprocess' in yamldef:
+            postproc = POSTPROCS[yamldef['postprocess']]
+            LOG.info('Running postprocesser %s on %r',
+                     yamldef['postprocess'], vars['filename'])
+            postproc(os.path.join(output_dir, vars['filename']))
+
+
+def main():
+    p = argparse.ArgumentParser()
+    p.add_argument('manifest')
+    p.add_argument('--output', default='images',
+                   help='Output directory for generated images')
+    p.add_argument('--only',
+                   help='Only generate this named image')
+    p.add_argument('--debug', action='store_true', default=False)
+    args = p.parse_args()
+
+    logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO)
+
+    try:
+        os.mkdir(args.output)
+    except FileExistsError:
+        pass
+
+    with open(args.manifest) as f:
+        yamldef = yaml.load(f, Loader=yaml.SafeLoader)
+
+    for image in yamldef['images']:
+        if args.only and args.only != image['name']:
+            continue
+        if 'generated_by' in image:
+            generate_one(image, args.output)
+        else:
+            LOG.error('Unknown source for image %s', image['name'])
+
+
+if __name__ == '__main__':
+    sys.exit(main())

manifest.yaml

@@ -0,0 +1,165 @@
+# Manifest of OpenStack test images
+#
+# Schema:
+# name: The base (short) name of the image file, used to form the file name
+#       and the image name when uploaded to glance
+# format: The name of the format that glance and qemu-img consider this image
+#         to be. Appended as an extension to the name above to generate the
+#         output filename.
+# usable: Boolean indicating whether or not this should be accepted by glance,
+#         nova, cinder, etc. This is sort of a loose thing that is hard to
+#         define because it may depend on config. Probably mostly useful for
+#         humans, but perhaps also test automation.
+# insecure: Boolean indicating if this image will contain some
+#           security-sensitive exploit or other violation. These should never
+#           be acceptable to services properly checking for them.
+# generated_by: If present, a series of shell commands used to generate the
+#               image.
+# postprocess: Some symbolic name of a python function in generate.py that
+#              must be run after the generated_by commands (if present) to
+#              finish creation of the image.
+images:
+  - name: standard-qcow2v3
+    format: qcow2
+    usable: true
+    insecure: false
+    generated_by: |
+      qemu-img create -f qcow2 %(filename)s 10M
+    description: |
+      This is a standard blank qcow2 file without any external linkage
+  - name: qcow-with-backing
+    format: qcow2
+    usable: false
+    insecure: true
+    generated_by: |
+      qemu-img create -f qcow2 -F raw -b /etc/hosts %(filename)s 10M
+    description: |
+      This is a qcow2 file with a backing file, which can be manipulated to
+      expose files on a host machine if processed without checking what file
+      is being included.
+  - name: qcow-with-datafile
+    format: qcow2
+    usable: false
+    insecure: true
+    generated_by: |
+      qemu-img create -f qcow2 -o data_file=qcow-data-file,data_file_raw=on %(filename)s 10M
+      rm qcow-data-file
+    description: |
+      This is a qcow2 file with a data-file specified, which can both
+      reference external data like backing-file or embed a QMP JSON
+      specification for a more complex storage arrangement and do much more
+      damage to a host system.
+  - name: raw-blank
+    format: raw
+    usable: true
+    insecure: false
+    generated_by: |
+      qemu-img create -f raw %(filename)s 1M
+    description: |
+      A truly unformatted file, which should not match any other format. Since
+      qemu-img is used here, it's literally a file of zero bytes.
+  - name: gpt-blank
+    format: gpt
+    usable: true
+    insecure: false
+    generated_by: |
+      qemu-img create -f raw %(filename)s 1M
+      parted %(filename)s --script 'mklabel gpt'
+    description: |
+      An image of a whole disk commonly seen in the PC/x86 space, with a
+      protective MBR and GPT paritition table.
+  - name: standard
+    format: qed
+    usable: false
+    insecure: false
+    generated_by: |
+      qemu-img create -f qed %(filename)s 10M
+    description: |
+      A blank QED formatted image, similar to qcow, but should not be supported
+      by any service.
+  - name: standard-iso9660
+    format: iso
+    usable: true
+    insecure: false
+    generated_by: |
+      qemu-img create -f raw %(filename)s 1M
+      mkisofs -V %(name)s -o %(filename)s /etc/hosts
+    description: |
+      An ISO9660 image with a single file inside
+  - name: standard-udf
+    format: iso
+    usable: true
+    insecure: false
+    generated_by: |
+      qemu-img create -f raw %(filename)s 1M
+      mkisofs -udf -V %(name)s -o %(filename)s /etc/hosts
+    description: |
+      A UDF filesystem (similar to ISO9660) with a single file inside
+  - name: iso-with-qcow2-in-system
+    format: iso
+    usable: false
+    insecure: true
+    generated_by: |
+      qemu-img create -f raw %(filename)s 1M
+      mkisofs -udf -V %(name)s -o %(filename)s /etc/hosts
+      qemu-img create -f qcow2 tmp.qcow 10M
+      dd if=tmp.qcow of=%(filename)s bs=32k count=1 conv=notrunc
+      rm tmp.qcow
+    description: |
+      An ISO9660 image with a single file inside, but with a qcow2 header in
+      the "system area" which can fool tools (like qemu-img) into thinking it
+      is a valid qcow2 file.
+  - name: vmdk-monolithicSparse
+    format: vmdk
+    usable: true
+    insecure: false
+    generated_by: |
+      qemu-img create -f vmdk -o subformat=monolithicSparse %(filename)s 10M
+    description: |
+      A VMDK file in monolithicSparse format (i.e. has a sparse header,
+      embedded descriptor and extents)
+  - name: vmdk-streamOptimized
+    format: vmdk
+    usable: true
+    insecure: false
+    generated_by: |
+      qemu-img create -f vmdk -o subformat=streamOptimized %(filename)s 10M
+    description: |
+      A VMDK file in streamOptimized format, similar to monolithicSparse,
+      without a footer.
+  - name: vmdk-monolithicFlat
+    format: vmdk
+    usable: false
+    insecure: false
+    generated_by: |
+      qemu-img create -f vmdk -o subformat=monolithicFlat %(filename)s 10M
+      rm %(name)s-flat.vmdk
+    description: |
+      A VMDK file in monolithicFlat format, which is actually just a text
+      descriptor that references external extent files. Not usable in
+      OpenStack and thus should always be rejected by services.
+  - name: vmdk-sparse-with-url-backing
+    format: vmdk
+    usable: false
+    insecure: true
+    generated_by: |
+      qemu-img create -f vmdk -o subformat=monolithicSparse %(filename)s 10M
+      dd if=%(filename)s bs=512 count=1 skip=1 | sed 's#%(filename)s#/etc/hosts#' | dd of=%(filename)s conv=notrunc seek=1
+    description: |
+      A monolithicSparse VMDK file that references external extent files. This
+      is in an acceptable format, but is insafe as it would result in exposing
+      files on a host machine within the guest image.
+  - name: vmdk-sparse-with-footer
+    format: vmdk
+    usable: true
+    insecure: false
+    postprocess: footerify_vmdk
+    generated_by: |
+      qemu-img create -f vmdk -o subformat=monolithicSparse %(filename)s 10M
+    description: |
+      A VMDK file in monolithicSparse format with a footer that overrides the
+      header. This footer must be supported specifically and checked for
+      sanity to make sure it does not reference a descriptor or other
+      resources that would not have been inspected in the stream by the time
+      we read the footer. Apparently vmware tooling generates these with a
+      footer frequently and thus is a format we need to (carefully) support.

requirements.txt

@@ -0,0 +1 @@
+pyyaml

tox.ini

@@ -0,0 +1,9 @@
+[tox]
+minversion = 1.6
+skipsdist = True
+envlist = generate
+
+[testenv]
+basepython = python3
+deps = -r{toxinidir}/requirements.txt
+commands = python3 generate.py manifest.yaml