Merge "Add a dib-cmd option for diskimages"

This commit is contained in:
Zuul 2019-08-23 14:35:39 +00:00 committed by Gerrit Code Review
commit 8efd70d0a8
69 changed files with 110 additions and 25 deletions

View File

@ -315,6 +315,21 @@ Options
The path of the default python interpreter.
.. attr:: dib-cmd
:type: string
:default: disk-image-create
Configure the command called to create this disk image. By
default this just ``disk-image-create``; i.e. it will use the
first match in ``$PATH``. For example, you may want to override
this with a fully qualified path to an alternative executable if
a custom ``diskimage-builder`` is installed in another
virutalenv.
.. note:: Any wrapping scripts or similar should consider that
the command-line or environment arguments to
``disk-image-create`` are not considered an API and
may change.
.. attr:: providers
:type: list

View File

@ -558,12 +558,11 @@ class CleanupWorker(BaseWorker):
class BuildWorker(BaseWorker):
def __init__(self, name, builder_id, config_path, secure_path,
interval, zk, dib_cmd):
interval, zk):
super(BuildWorker, self).__init__(builder_id, config_path, secure_path,
interval, zk)
self.log = logging.getLogger("nodepool.builder.BuildWorker.%s" % name)
self.name = 'BuildWorker.%s' % name
self.dib_cmd = dib_cmd
def _getBuildLogRoot(self, name):
log_dir = self._config.build_log_dir
@ -777,9 +776,13 @@ class BuildWorker(BaseWorker):
if 'qcow2' in img_types:
qemu_img_options = DEFAULT_QEMU_IMAGE_COMPAT_OPTIONS
# A bit of a hack, but useful for CI to pick up the
# fake-image-create relative to this file easily
dib_cmd = diskimage.dib_cmd.replace("%p", os.path.dirname(__file__))
cmd = ('%s -x -t %s --checksum --no-tmpfs %s -o %s %s' %
(self.dib_cmd, img_types, qemu_img_options, filename,
img_elements))
(dib_cmd, img_types, qemu_img_options,
filename, img_elements))
self._pruneBuildLogs(diskimage.name)
log_fn = self._getBuildLog(diskimage.name, build_id)
@ -969,7 +972,6 @@ class BuildWorker(BaseWorker):
self._checkForZooKeeperChanges(new_config)
self._config = new_config
self._checkForScheduledImageUpdates()
self._checkForManualBuildRequest()
@ -1237,7 +1239,7 @@ class NodePoolBuilder(object):
log = logging.getLogger("nodepool.builder.NodePoolBuilder")
def __init__(self, config_path, secure_path=None,
num_builders=1, num_uploaders=4, fake=False):
num_builders=1, num_uploaders=4):
'''
Initialize the NodePoolBuilder object.
@ -1245,7 +1247,6 @@ class NodePoolBuilder(object):
:param str secure_path: Path to secure configuration file.
:param int num_builders: Number of build workers to start.
:param int num_uploaders: Number of upload workers to start.
:param bool fake: Whether to fake the image builds.
'''
self._config_path = config_path
self._secure_path = secure_path
@ -1259,11 +1260,6 @@ class NodePoolBuilder(object):
self.cleanup_interval = 60
self.build_interval = 10
self.upload_interval = 10
if fake:
self.dib_cmd = os.path.join(os.path.dirname(__file__), '..',
'nodepool/tests/fake-image-create')
else:
self.dib_cmd = 'disk-image-create'
self.zk = None
# This lock is needed because the run() method is started in a
@ -1330,7 +1326,7 @@ class NodePoolBuilder(object):
for i in range(self._num_builders):
w = BuildWorker(i, builder_id,
self._config_path, self._secure_path,
self.build_interval, self.zk, self.dib_cmd)
self.build_interval, self.zk)
w.start()
self._build_workers.append(w)

View File

@ -42,9 +42,6 @@ class NodePoolBuilderApp(nodepool.cmd.NodepoolDaemonApp):
parser.add_argument('--upload-workers', dest='upload_workers',
default=4, help='number of upload workers',
type=int)
parser.add_argument('--fake', action='store_true',
help='Do not actually run diskimage-builder '
'(used for testing)')
return parser
def parse_args(self):
@ -57,8 +54,7 @@ class NodePoolBuilderApp(nodepool.cmd.NodepoolDaemonApp):
self.config_file,
secure_path=self.secure_file,
num_builders=self.args.build_workers,
num_uploaders=self.args.upload_workers,
fake=self.args.fake)
num_uploaders=self.args.upload_workers)
signal.signal(signal.SIGINT, self.sigint_handler)

View File

@ -37,6 +37,7 @@ class ConfigValidator:
diskimage = {
'name': str,
'dib-cmd': str,
'pause': bool,
'elements': [str],
'formats': [str],

View File

@ -107,6 +107,7 @@ class Config(ConfigValue):
d.elements = u' '.join(diskimage['elements'])
else:
d.elements = ''
d.dib_cmd = str(diskimage.get('dib-cmd', 'disk-image-create'))
# must be a string, as it's passed as env-var to
# d-i-b, but might be untyped in the yaml and
# interpreted as a number (e.g. "21" for fedora)
@ -175,6 +176,7 @@ class DiskImage(ConfigValue):
def __init__(self):
self.name = None
self.elements = None
self.dib_path = None
self.release = None
self.rebuild_age = None
self.env_vars = None
@ -188,6 +190,7 @@ class DiskImage(ConfigValue):
if isinstance(other, DiskImage):
return (other.name == self.name and
other.elements == self.elements and
other.dib_path == self.dib_path and
other.release == self.release and
other.rebuild_age == self.rebuild_age and
other.env_vars == self.env_vars and

View File

@ -325,7 +325,6 @@ class BuilderFixture(fixtures.Fixture):
self.builder.cleanup_interval = self.cleanup_interval
self.builder.build_interval = .1
self.builder.upload_interval = .1
self.builder.dib_cmd = 'nodepool/tests/fake-image-create'
self.builder.start()
self.addCleanup(self.cleanup)

View File

@ -41,6 +41,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -20,6 +20,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -41,6 +41,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -42,6 +42,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -50,6 +50,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -33,6 +33,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -43,6 +43,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -47,6 +47,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -47,6 +47,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -57,6 +57,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -33,6 +33,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -33,6 +33,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -34,6 +34,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -49,6 +49,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -51,6 +51,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -61,6 +61,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -46,6 +46,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -45,6 +45,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -78,6 +78,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -39,6 +39,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -37,6 +37,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -42,6 +42,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -42,6 +42,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -37,6 +37,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -37,6 +37,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
SHOULD_FAIL: 'true'
TMPDIR: /opt/dib_tmp

View File

@ -49,6 +49,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache
@ -59,6 +60,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -19,6 +19,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache
@ -30,6 +31,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -43,6 +43,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache
@ -53,6 +54,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -33,6 +33,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -43,6 +43,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache
@ -53,6 +54,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -57,6 +57,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -43,6 +43,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -34,6 +34,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -35,6 +35,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -48,6 +48,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -43,6 +43,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -43,6 +43,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -41,6 +41,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -42,6 +42,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -41,6 +41,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -45,6 +45,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -47,6 +47,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
python-path: /usr/bin/python3
env-vars:
TMPDIR: /opt/dib_tmp

View File

@ -33,6 +33,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -33,6 +33,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -33,6 +33,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -33,6 +33,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -40,6 +40,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -48,6 +48,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -39,6 +39,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache
@ -49,6 +50,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -33,6 +33,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -47,6 +47,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -39,6 +39,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -50,6 +50,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -33,6 +33,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -48,6 +48,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -41,6 +41,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -41,6 +41,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -41,6 +41,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -47,6 +47,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -0,0 +1,7 @@
---
features:
- |
A ``diskimage`` can specify the full path to the diskimage-builder
command with the ``dib-cmd`` configuration parameter. The
``--fake`` parameter of ``nodepool-builder`` (only used by CI) has
been removed and replaced with explicit calls in testing fixtures.

View File

@ -1,6 +1,9 @@
- name: Install packages and run zuul-nodepool-integration/start.sh
- name: Install packages
shell:
cmd: |
sudo pip3 install .
./tools/zuul-nodepool-integration/start.sh
cmd: sudo pip3 install .
chdir: "{{ zuul.projects['opendev.org/zuul/nodepool'].src_dir }}"
- name: Run zuul-nodepool-integration/start.sh
shell:
cmd: ./tools/zuul-nodepool-integration/start.sh
chdir: "{{ zuul.projects['opendev.org/zuul/nodepool'].src_dir }}"

View File

@ -10,6 +10,7 @@ diskimages:
- fedora
- vm
release: 21
dib-cmd: '%p/../nodepool/tests/fake-image-create'
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache

View File

@ -1,4 +1,4 @@
#!/bin/bash -e
#!/bin/bash -ex
cd "$(dirname "$0")"
@ -7,5 +7,5 @@ mkdir -p /tmp/nodepool/log
export OS_CLIENT_CONFIG_FILE=`pwd`/clouds.yaml
nodepool-builder -c `pwd`/nodepool.yaml -l `pwd`/builder-logging.conf -p /tmp/nodepool/builder.pid --fake
nodepool-builder -c `pwd`/nodepool.yaml -l `pwd`/builder-logging.conf -p /tmp/nodepool/builder.pid
nodepool-launcher -c `pwd`/nodepool.yaml -s `pwd`/secure.conf -l `pwd`/launcher-logging.conf -p /tmp/nodepool/launcher.pid