From 9b1e519267f022ceb3f920976591a8f8233addc4 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 15 Dec 2016 13:50:39 -0800 Subject: [PATCH] Allow for image additions archives (as well as plugins) It is quite useful to be able to add additional items (such as jenkins build information) into the image but the only way to do this seems to be via plugins archives. Using these for such information ends badly when building neutron (because those *plugin* archives need to contain setup.py files) so this proposes a new mechanism to put files that are not python plugins but are auxilary useful data to put in the image (such as the jenkins data mentioned above). Change-Id: Ica8b5750bd239e023f14225190e68dab75519d36 --- doc/image-building.rst | 64 ++++++++++++++++++++++++++- kolla/image/build.py | 98 +++++++++++++++++++++++++++++++----------- 2 files changed, 136 insertions(+), 26 deletions(-) diff --git a/doc/image-building.rst b/doc/image-building.rst index 9e8ac58f2c..4fdea5c9db 100644 --- a/doc/image-building.rst +++ b/doc/image-building.rst @@ -267,7 +267,7 @@ image, add the following to the ``template-override`` file:: && pip --no-cache-dir install networking-cisco {% endblock %} -Acute readers may notice there is one problem with this however. Assuming +Astute readers may notice there is one problem with this however. Assuming nothing else in the Dockerfile changes for a period of time, the above ``RUN`` statement will be cached by Docker, meaning new commits added to the Git repository may be missed on subsequent builds. To solve this the Kolla build @@ -309,6 +309,68 @@ The template now becomes:: pip --no-cache-dir install /plugins/* {% endblock %} +Additions Functionality +----------------------- + +The Dockerfile customisation mechanism is also useful for adding/installing +additions into images. An example of this is adding your jenkins job build +metadata (say formatted into a jenkins.json file) into the image. + +The bottom of each Dockerfile contains two blocks, ``image_name_footer``, and +``footer``. The ``image_name_footer`` is intended for image specific +modifications, while the ``footer`` can be used to apply a common set of +modifications to every Dockerfile. + +For example, to add the ``jenkins.json`` additions to the ``neutron_server`` +image, add the following to the ``template-override`` file:: + + {% extends parent_template %} + + {% block neutron_server_footer %} + RUN cp /additions/jenkins/jenkins.json /jenkins.json + {% endblock %} + +Astute readers may notice there is one problem with this however. Assuming +nothing else in the Dockerfile changes for a period of time, the above ``RUN`` +statement will be cached by Docker, meaning new commits added to the Git +repository may be missed on subsequent builds. To solve this the Kolla build +tool also supports cloning additional repositories at build time, which will be +automatically made available to the build, within an archive named +``additions-archive``. + +.. note:: + + The following is available for source build types only. + +To use this, add a section to ``/etc/kolla/kolla-build.conf`` in the following +format:: + + [-additions-] + +Where ```` is the image that the plugin should be installed into, and +```` is the chosen additions identifier. + +Continuing with the above example, add the following to +``/etc/kolla/kolla-build.conf``:: + + [neutron-server-jenkins] + type = local + location = /path/to/your/jenkins/data + +The build will copy the directory, resulting in the following archive +structure:: + + additions-archive.tar + |__ additions + |__jenkins + +The template now becomes:: + + {% block neutron_server_footer %} + ADD additions-archive / + RUN cp /additions/jenkins/jenkins.json /jenkins.json + {% endblock %} + Custom Repos ------------ diff --git a/kolla/image/build.py b/kolla/image/build.py index 35029f198b..a2935a2b1f 100755 --- a/kolla/image/build.py +++ b/kolla/image/build.py @@ -94,6 +94,10 @@ STATUS_ERRORS = (STATUS_CONNECTION_ERROR, STATUS_PUSH_ERROR, STATUS_ERROR, STATUS_PARENT_ERROR) +class ArchivingError(Exception): + pass + + @contextlib.contextmanager def join_many(threads): try: @@ -148,6 +152,7 @@ class Image(object): self.logger = logger self.children = [] self.plugins = [] + self.additions = [] def copy(self): c = Image(self.name, self.canonical_name, self.path, @@ -159,6 +164,8 @@ class Image(object): c.children = list(self.children) if self.plugins: c.plugins = list(self.plugins) + if self.additions: + c.additions = list(self.additions) return c def __repr__(self): @@ -354,6 +361,39 @@ class BuildTask(DockerTask): return buildargs def builder(self, image): + + def make_an_archive(items, arcname, item_child_path=None): + if not item_child_path: + item_child_path = arcname + archives = list() + items_path = os.path.join(image.path, item_child_path) + for item in items: + archive_path = self.process_source(image, item) + if image.status in STATUS_ERRORS: + raise ArchivingError + archives.append(archive_path) + if archives: + for archive in archives: + with tarfile.open(archive, 'r') as archive_tar: + archive_tar.extractall(path=items_path) + else: + try: + os.mkdir(items_path) + except OSError as e: + if e.errno == errno.EEXIST: + self.logger.info( + 'Directory %s already exist. Skipping.', + items_path) + else: + self.logger.error('Failed to create directory %s: %s', + items_path, e) + image.status = STATUS_CONNECTION_ERROR + raise ArchivingError + arc_path = os.path.join(image.path, '%s-archive' % arcname) + with tarfile.open(arc_path, 'w') as tar: + tar.add(items_path, arcname=arcname) + return len(os.listdir(items_path)) + self.logger.debug('Processing') if image.status == STATUS_UNMATCHED: return @@ -373,32 +413,26 @@ class BuildTask(DockerTask): if image.status in STATUS_ERRORS: return - plugin_archives = list() - plugins_path = os.path.join(image.path, 'plugins') - for plugin in image.plugins: - archive_path = self.process_source(image, plugin) - if image.status in STATUS_ERRORS: - return - plugin_archives.append(archive_path) - if plugin_archives: - for plugin_archive in plugin_archives: - with tarfile.open(plugin_archive, 'r') as plugin_archive_tar: - plugin_archive_tar.extractall(path=plugins_path) + try: + plugins_am = make_an_archive(image.plugins, 'plugins') + except ArchivingError: + self.logger.error( + "Failed turning any plugins into a plugins archive") + return else: - try: - os.mkdir(plugins_path) - except OSError as e: - if e.errno == errno.EEXIST: - self.logger.info('Directory %s already exist. Skipping.', - plugins_path) - else: - self.logger.error('Failed to create directory %s: %s', - plugins_path, e) - image.status = STATUS_CONNECTION_ERROR - return - with tarfile.open(os.path.join(image.path, 'plugins-archive'), - 'w') as tar: - tar.add(plugins_path, arcname='plugins') + self.logger.debug( + "Turned %s plugins into plugins archive", + plugins_am) + try: + additions_am = make_an_archive(image.additions, 'additions') + except ArchivingError: + self.logger.error( + "Failed turning any additions into a additions archive") + return + else: + self.logger.debug( + "Turned %s additions into additions archive", + additions_am) # Pull the latest image for the base distro only pull = self.conf.pull if image.parent is None else False @@ -848,6 +882,20 @@ class KollaWorker(object): plugin) image.plugins.append( process_source_installation(image, plugin)) + for addition in [ + match.group(0) for match in + (re.search('^{}-additions-.+'.format(image.name), + section) for section in all_sections) if match]: + try: + self.conf.register_opts( + common_config.get_source_opts(), + addition + ) + except cfg.DuplicateOptError: + LOG.debug('Addition %s already registered in config', + addition) + image.additions.append( + process_source_installation(image, addition)) self.images.append(image)