diff --git a/doc/image-building.rst b/doc/image-building.rst index 9e8ac58f2c..4fdea5c9db 100644 --- a/doc/image-building.rst +++ b/doc/image-building.rst @@ -267,7 +267,7 @@ image, add the following to the ``template-override`` file:: && pip --no-cache-dir install networking-cisco {% endblock %} -Acute readers may notice there is one problem with this however. Assuming +Astute readers may notice there is one problem with this however. Assuming nothing else in the Dockerfile changes for a period of time, the above ``RUN`` statement will be cached by Docker, meaning new commits added to the Git repository may be missed on subsequent builds. To solve this the Kolla build @@ -309,6 +309,68 @@ The template now becomes:: pip --no-cache-dir install /plugins/* {% endblock %} +Additions Functionality +----------------------- + +The Dockerfile customisation mechanism is also useful for adding/installing +additions into images. An example of this is adding your jenkins job build +metadata (say formatted into a jenkins.json file) into the image. + +The bottom of each Dockerfile contains two blocks, ``image_name_footer``, and +``footer``. The ``image_name_footer`` is intended for image specific +modifications, while the ``footer`` can be used to apply a common set of +modifications to every Dockerfile. + +For example, to add the ``jenkins.json`` additions to the ``neutron_server`` +image, add the following to the ``template-override`` file:: + + {% extends parent_template %} + + {% block neutron_server_footer %} + RUN cp /additions/jenkins/jenkins.json /jenkins.json + {% endblock %} + +Astute readers may notice there is one problem with this however. Assuming +nothing else in the Dockerfile changes for a period of time, the above ``RUN`` +statement will be cached by Docker, meaning new commits added to the Git +repository may be missed on subsequent builds. To solve this the Kolla build +tool also supports cloning additional repositories at build time, which will be +automatically made available to the build, within an archive named +``additions-archive``. + +.. note:: + + The following is available for source build types only. + +To use this, add a section to ``/etc/kolla/kolla-build.conf`` in the following +format:: + + [-additions-] + +Where ```` is the image that the plugin should be installed into, and +```` is the chosen additions identifier. + +Continuing with the above example, add the following to +``/etc/kolla/kolla-build.conf``:: + + [neutron-server-jenkins] + type = local + location = /path/to/your/jenkins/data + +The build will copy the directory, resulting in the following archive +structure:: + + additions-archive.tar + |__ additions + |__jenkins + +The template now becomes:: + + {% block neutron_server_footer %} + ADD additions-archive / + RUN cp /additions/jenkins/jenkins.json /jenkins.json + {% endblock %} + Custom Repos ------------ diff --git a/kolla/image/build.py b/kolla/image/build.py index ad319148aa..f7719d54f9 100755 --- a/kolla/image/build.py +++ b/kolla/image/build.py @@ -94,6 +94,10 @@ STATUS_ERRORS = (STATUS_CONNECTION_ERROR, STATUS_PUSH_ERROR, STATUS_ERROR, STATUS_PARENT_ERROR) +class ArchivingError(Exception): + pass + + @contextlib.contextmanager def join_many(threads): try: @@ -148,6 +152,7 @@ class Image(object): self.logger = logger self.children = [] self.plugins = [] + self.additions = [] def copy(self): c = Image(self.name, self.canonical_name, self.path, @@ -159,6 +164,8 @@ class Image(object): c.children = list(self.children) if self.plugins: c.plugins = list(self.plugins) + if self.additions: + c.additions = list(self.additions) return c def __repr__(self): @@ -354,6 +361,39 @@ class BuildTask(DockerTask): return buildargs def builder(self, image): + + def make_an_archive(items, arcname, item_child_path=None): + if not item_child_path: + item_child_path = arcname + archives = list() + items_path = os.path.join(image.path, item_child_path) + for item in items: + archive_path = self.process_source(image, item) + if image.status in STATUS_ERRORS: + raise ArchivingError + archives.append(archive_path) + if archives: + for archive in archives: + with tarfile.open(archive, 'r') as archive_tar: + archive_tar.extractall(path=items_path) + else: + try: + os.mkdir(items_path) + except OSError as e: + if e.errno == errno.EEXIST: + self.logger.info( + 'Directory %s already exist. Skipping.', + items_path) + else: + self.logger.error('Failed to create directory %s: %s', + items_path, e) + image.status = STATUS_CONNECTION_ERROR + raise ArchivingError + arc_path = os.path.join(image.path, '%s-archive' % arcname) + with tarfile.open(arc_path, 'w') as tar: + tar.add(items_path, arcname=arcname) + return len(os.listdir(items_path)) + self.logger.debug('Processing') if image.status == STATUS_UNMATCHED: return @@ -373,32 +413,26 @@ class BuildTask(DockerTask): if image.status in STATUS_ERRORS: return - plugin_archives = list() - plugins_path = os.path.join(image.path, 'plugins') - for plugin in image.plugins: - archive_path = self.process_source(image, plugin) - if image.status in STATUS_ERRORS: - return - plugin_archives.append(archive_path) - if plugin_archives: - for plugin_archive in plugin_archives: - with tarfile.open(plugin_archive, 'r') as plugin_archive_tar: - plugin_archive_tar.extractall(path=plugins_path) + try: + plugins_am = make_an_archive(image.plugins, 'plugins') + except ArchivingError: + self.logger.error( + "Failed turning any plugins into a plugins archive") + return else: - try: - os.mkdir(plugins_path) - except OSError as e: - if e.errno == errno.EEXIST: - self.logger.info('Directory %s already exist. Skipping.', - plugins_path) - else: - self.logger.error('Failed to create directory %s: %s', - plugins_path, e) - image.status = STATUS_CONNECTION_ERROR - return - with tarfile.open(os.path.join(image.path, 'plugins-archive'), - 'w') as tar: - tar.add(plugins_path, arcname='plugins') + self.logger.debug( + "Turned %s plugins into plugins archive", + plugins_am) + try: + additions_am = make_an_archive(image.additions, 'additions') + except ArchivingError: + self.logger.error( + "Failed turning any additions into a additions archive") + return + else: + self.logger.debug( + "Turned %s additions into additions archive", + additions_am) # Pull the latest image for the base distro only pull = self.conf.pull if image.parent is None else False @@ -876,6 +910,20 @@ class KollaWorker(object): plugin) image.plugins.append( process_source_installation(image, plugin)) + for addition in [ + match.group(0) for match in + (re.search('^{}-additions-.+'.format(image.name), + section) for section in all_sections) if match]: + try: + self.conf.register_opts( + common_config.get_source_opts(), + addition + ) + except cfg.DuplicateOptError: + LOG.debug('Addition %s already registered in config', + addition) + image.additions.append( + process_source_installation(image, addition)) self.images.append(image)