Merge "Allow for image additions archives (as well as plugins)"

doc/image-building.rst

@@ -267,7 +267,7 @@ image, add the following to the ``template-override`` file::
         && pip --no-cache-dir install networking-cisco
     {% endblock %}
 
-Acute readers may notice there is one problem with this however. Assuming
+Astute readers may notice there is one problem with this however. Assuming
 nothing else in the Dockerfile changes for a period of time, the above ``RUN``
 statement will be cached by Docker, meaning new commits added to the Git
 repository may be missed on subsequent builds. To solve this the Kolla build
@@ -309,6 +309,68 @@ The template now becomes::
     pip --no-cache-dir install /plugins/*
     {% endblock %}
 
+Additions Functionality
+-----------------------
+
+The Dockerfile customisation mechanism is also useful for adding/installing
+additions into images. An example of this is adding your jenkins job build
+metadata (say formatted into a jenkins.json file) into the image.
+
+The bottom of each Dockerfile contains two blocks, ``image_name_footer``, and
+``footer``. The ``image_name_footer`` is intended for image specific
+modifications, while the ``footer`` can be used to apply a common set of
+modifications to every Dockerfile.
+
+For example, to add the ``jenkins.json`` additions to the ``neutron_server``
+image, add the following to the ``template-override`` file::
+
+    {% extends parent_template %}
+
+    {% block neutron_server_footer %}
+    RUN cp /additions/jenkins/jenkins.json /jenkins.json
+    {% endblock %}
+
+Astute readers may notice there is one problem with this however. Assuming
+nothing else in the Dockerfile changes for a period of time, the above ``RUN``
+statement will be cached by Docker, meaning new commits added to the Git
+repository may be missed on subsequent builds. To solve this the Kolla build
+tool also supports cloning additional repositories at build time, which will be
+automatically made available to the build, within an archive named
+``additions-archive``.
+
+.. note::
+
+    The following is available for source build types only.
+
+To use this, add a section to ``/etc/kolla/kolla-build.conf`` in the following
+format::
+
+    [<image>-additions-<additions-name>]
+
+Where ``<image>`` is the image that the plugin should be installed into, and
+``<additions-name>`` is the chosen additions identifier.
+
+Continuing with the above example, add the following to
+``/etc/kolla/kolla-build.conf``::
+
+    [neutron-server-jenkins]
+    type = local
+    location = /path/to/your/jenkins/data
+
+The build will copy the directory, resulting in the following archive
+structure::
+
+    additions-archive.tar
+    |__ additions
+        |__jenkins
+
+The template now becomes::
+
+    {% block neutron_server_footer %}
+    ADD additions-archive /
+    RUN cp /additions/jenkins/jenkins.json /jenkins.json
+    {% endblock %}
+
 Custom Repos
 ------------
 

kolla/image/build.py

@@ -94,6 +94,10 @@ STATUS_ERRORS = (STATUS_CONNECTION_ERROR, STATUS_PUSH_ERROR,
                  STATUS_ERROR, STATUS_PARENT_ERROR)
 
 
+class ArchivingError(Exception):
+    pass
+
+
 @contextlib.contextmanager
 def join_many(threads):
     try:
@@ -148,6 +152,7 @@ class Image(object):
         self.logger = logger
         self.children = []
         self.plugins = []
+        self.additions = []
 
     def copy(self):
         c = Image(self.name, self.canonical_name, self.path,
@@ -159,6 +164,8 @@ class Image(object):
             c.children = list(self.children)
         if self.plugins:
             c.plugins = list(self.plugins)
+        if self.additions:
+            c.additions = list(self.additions)
         return c
 
     def __repr__(self):
@@ -354,6 +361,39 @@ class BuildTask(DockerTask):
         return buildargs
 
     def builder(self, image):
+
+        def make_an_archive(items, arcname, item_child_path=None):
+            if not item_child_path:
+                item_child_path = arcname
+            archives = list()
+            items_path = os.path.join(image.path, item_child_path)
+            for item in items:
+                archive_path = self.process_source(image, item)
+                if image.status in STATUS_ERRORS:
+                    raise ArchivingError
+                archives.append(archive_path)
+            if archives:
+                for archive in archives:
+                    with tarfile.open(archive, 'r') as archive_tar:
+                        archive_tar.extractall(path=items_path)
+            else:
+                try:
+                    os.mkdir(items_path)
+                except OSError as e:
+                    if e.errno == errno.EEXIST:
+                        self.logger.info(
+                            'Directory %s already exist. Skipping.',
+                            items_path)
+                    else:
+                        self.logger.error('Failed to create directory %s: %s',
+                                          items_path, e)
+                        image.status = STATUS_CONNECTION_ERROR
+                        raise ArchivingError
+            arc_path = os.path.join(image.path, '%s-archive' % arcname)
+            with tarfile.open(arc_path, 'w') as tar:
+                tar.add(items_path, arcname=arcname)
+            return len(os.listdir(items_path))
+
         self.logger.debug('Processing')
         if image.status == STATUS_UNMATCHED:
             return
@@ -373,32 +413,26 @@ class BuildTask(DockerTask):
             if image.status in STATUS_ERRORS:
                 return
 
-        plugin_archives = list()
-        plugins_path = os.path.join(image.path, 'plugins')
-        for plugin in image.plugins:
-            archive_path = self.process_source(image, plugin)
-            if image.status in STATUS_ERRORS:
-                return
-            plugin_archives.append(archive_path)
-        if plugin_archives:
-            for plugin_archive in plugin_archives:
-                with tarfile.open(plugin_archive, 'r') as plugin_archive_tar:
-                    plugin_archive_tar.extractall(path=plugins_path)
-        else:
         try:
-                os.mkdir(plugins_path)
+            plugins_am = make_an_archive(image.plugins, 'plugins')
-            except OSError as e:
+        except ArchivingError:
-                if e.errno == errno.EEXIST:
+            self.logger.error(
-                    self.logger.info('Directory %s already exist. Skipping.',
+                "Failed turning any plugins into a plugins archive")
-                                     plugins_path)
-                else:
-                    self.logger.error('Failed to create directory %s: %s',
-                                      plugins_path, e)
-                    image.status = STATUS_CONNECTION_ERROR
             return
-        with tarfile.open(os.path.join(image.path, 'plugins-archive'),
+        else:
-                          'w') as tar:
+            self.logger.debug(
-            tar.add(plugins_path, arcname='plugins')
+                "Turned %s plugins into plugins archive",
+                plugins_am)
+        try:
+            additions_am = make_an_archive(image.additions, 'additions')
+        except ArchivingError:
+            self.logger.error(
+                "Failed turning any additions into a additions archive")
+            return
+        else:
+            self.logger.debug(
+                "Turned %s additions into additions archive",
+                additions_am)
 
         # Pull the latest image for the base distro only
         pull = self.conf.pull if image.parent is None else False
@@ -876,6 +910,20 @@ class KollaWorker(object):
                                   plugin)
                     image.plugins.append(
                         process_source_installation(image, plugin))
+                for addition in [
+                    match.group(0) for match in
+                    (re.search('^{}-additions-.+'.format(image.name),
+                     section) for section in all_sections) if match]:
+                    try:
+                        self.conf.register_opts(
+                            common_config.get_source_opts(),
+                            addition
+                        )
+                    except cfg.DuplicateOptError:
+                        LOG.debug('Addition %s already registered in config',
+                                  addition)
+                    image.additions.append(
+                        process_source_installation(image, addition))
 
             self.images.append(image)