openstack/kolla
Code Issues Proposed changes

Allow for image additions archives (as well as plugins)

Browse Source 
It is quite useful to be able to add additional items (such
as jenkins build information) into the image but the only
way to do this seems to be via plugins archives. Using these
for such information ends badly when building neutron (because
those *plugin* archives need to contain setup.py files) so this
proposes a new mechanism to put files that are not python plugins but
are auxilary useful data to put in the image (such as the jenkins
data mentioned above).

Change-Id: Ica8b5750bd239e023f14225190e68dab75519d36
This commit is contained in:
Joshua Harlow 2016-12-15 13:50:39 -08:00
parent 96a604aa11
commit 9b1e519267
2 changed files with 136 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
doc/image-building.rst
View File

@ -267,7 +267,7 @@ image, add the following to the ``template-override`` file::
&& pip --no-cache-dir install networking-cisco
{% endblock %}
Acute readers may notice there is one problem with this however. Assuming
Astute readers may notice there is one problem with this however. Assuming
nothing else in the Dockerfile changes for a period of time, the above ``RUN``
statement will be cached by Docker, meaning new commits added to the Git
repository may be missed on subsequent builds. To solve this the Kolla build
@ -309,6 +309,68 @@ The template now becomes::
pip --no-cache-dir install /plugins/*
{% endblock %}
Additions Functionality
-----------------------
The Dockerfile customisation mechanism is also useful for adding/installing
additions into images. An example of this is adding your jenkins job build
metadata (say formatted into a jenkins.json file) into the image.
The bottom of each Dockerfile contains two blocks, ``image_name_footer``, and
``footer``. The ``image_name_footer`` is intended for image specific
modifications, while the ``footer`` can be used to apply a common set of
modifications to every Dockerfile.
For example, to add the ``jenkins.json`` additions to the ``neutron_server``
image, add the following to the ``template-override`` file::
{% extends parent_template %}
{% block neutron_server_footer %}
RUN cp /additions/jenkins/jenkins.json /jenkins.json
{% endblock %}
Astute readers may notice there is one problem with this however. Assuming
nothing else in the Dockerfile changes for a period of time, the above ``RUN``
statement will be cached by Docker, meaning new commits added to the Git
repository may be missed on subsequent builds. To solve this the Kolla build
tool also supports cloning additional repositories at build time, which will be
automatically made available to the build, within an archive named
``additions-archive``.
.. note::
The following is available for source build types only.
To use this, add a section to ``/etc/kolla/kolla-build.conf`` in the following
format::
[<image>-additions-<additions-name>]
Where ``<image>`` is the image that the plugin should be installed into, and
``<additions-name>`` is the chosen additions identifier.
Continuing with the above example, add the following to
``/etc/kolla/kolla-build.conf``::
[neutron-server-jenkins]
type = local
location = /path/to/your/jenkins/data
The build will copy the directory, resulting in the following archive
structure::
additions-archive.tar
|__ additions
|__jenkins
The template now becomes::
{% block neutron_server_footer %}
ADD additions-archive /
RUN cp /additions/jenkins/jenkins.json /jenkins.json
{% endblock %}
Custom Repos
------------

98
kolla/image/build.py
View File

@ -94,6 +94,10 @@ STATUS_ERRORS = (STATUS_CONNECTION_ERROR, STATUS_PUSH_ERROR,
STATUS_ERROR, STATUS_PARENT_ERROR)
class ArchivingError(Exception):
pass
@contextlib.contextmanager
def join_many(threads):
try:
@ -148,6 +152,7 @@ class Image(object):
self.logger = logger
self.children = []
self.plugins = []
self.additions = []
def copy(self):
c = Image(self.name, self.canonical_name, self.path,
@ -159,6 +164,8 @@ class Image(object):
c.children = list(self.children)
if self.plugins:
c.plugins = list(self.plugins)
if self.additions:
c.additions = list(self.additions)
return c
def __repr__(self):
@ -354,6 +361,39 @@ class BuildTask(DockerTask):
return buildargs
def builder(self, image):
def make_an_archive(items, arcname, item_child_path=None):
if not item_child_path:
item_child_path = arcname
archives = list()
items_path = os.path.join(image.path, item_child_path)
for item in items:
archive_path = self.process_source(image, item)
if image.status in STATUS_ERRORS:
raise ArchivingError
archives.append(archive_path)
if archives:
for archive in archives:
with tarfile.open(archive, 'r') as archive_tar:
archive_tar.extractall(path=items_path)
else:
try:
os.mkdir(items_path)
except OSError as e:
if e.errno == errno.EEXIST:
self.logger.info(
'Directory %s already exist. Skipping.',
items_path)
else:
self.logger.error('Failed to create directory %s: %s',
items_path, e)
image.status = STATUS_CONNECTION_ERROR
raise ArchivingError
arc_path = os.path.join(image.path, '%s-archive' % arcname)
with tarfile.open(arc_path, 'w') as tar:
tar.add(items_path, arcname=arcname)
return len(os.listdir(items_path))
self.logger.debug('Processing')
if image.status == STATUS_UNMATCHED:
return
@ -373,32 +413,26 @@ class BuildTask(DockerTask):
if image.status in STATUS_ERRORS:
return
plugin_archives = list()
plugins_path = os.path.join(image.path, 'plugins')
for plugin in image.plugins:
archive_path = self.process_source(image, plugin)
if image.status in STATUS_ERRORS:
return
plugin_archives.append(archive_path)
if plugin_archives:
for plugin_archive in plugin_archives:
with tarfile.open(plugin_archive, 'r') as plugin_archive_tar:
plugin_archive_tar.extractall(path=plugins_path)
try:
plugins_am = make_an_archive(image.plugins, 'plugins')
except ArchivingError:
self.logger.error(
"Failed turning any plugins into a plugins archive")
return
else:
try:
os.mkdir(plugins_path)
except OSError as e:
if e.errno == errno.EEXIST:
self.logger.info('Directory %s already exist. Skipping.',
plugins_path)
else:
self.logger.error('Failed to create directory %s: %s',
plugins_path, e)
image.status = STATUS_CONNECTION_ERROR
return
with tarfile.open(os.path.join(image.path, 'plugins-archive'),
'w') as tar:
tar.add(plugins_path, arcname='plugins')
self.logger.debug(
"Turned %s plugins into plugins archive",
plugins_am)
try:
additions_am = make_an_archive(image.additions, 'additions')
except ArchivingError:
self.logger.error(
"Failed turning any additions into a additions archive")
return
else:
self.logger.debug(
"Turned %s additions into additions archive",
additions_am)
# Pull the latest image for the base distro only
pull = self.conf.pull if image.parent is None else False
@ -848,6 +882,20 @@ class KollaWorker(object):
plugin)
image.plugins.append(
process_source_installation(image, plugin))
for addition in [
match.group(0) for match in
(re.search('^{}-additions-.+'.format(image.name),
section) for section in all_sections) if match]:
try:
self.conf.register_opts(
common_config.get_source_opts(),
addition
)
except cfg.DuplicateOptError:
LOG.debug('Addition %s already registered in config',
addition)
image.additions.append(
process_source_installation(image, addition))
self.images.append(image)