Move KollaWorker stuff to separate file
As part of kolla/image/ refactoring we move KollaWorker related classes into own file. Based on patch by Konstantin Yarovoy <konstantin.yarovoy@tietoevry.com> Change-Id: I55c3f6b34631048b2463b02e6ea611d7e4c1a912
This commit is contained in:
Marcin Juszkiewicz
Radosław Piliszek
parent
a8ee3244bf
commit
145d096bf7
|
|
@ -13,20 +13,15 @@
|
|||
# limitations under the License.
|
||||
|
||||
import contextlib
|
||||
import datetime
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
|
||||
import docker
|
||||
import jinja2
|
||||
from oslo_config import cfg
|
||||
|
||||
|
||||
|
|
@ -40,11 +35,9 @@ if PROJECT_ROOT not in sys.path:
|
|||
|
||||
from kolla.common import config as common_config # noqa
|
||||
from kolla.common import utils # noqa
|
||||
from kolla.image.unbuildable import UNBUILDABLE_IMAGES # noqa
|
||||
from kolla.image.tasks import BuildTask # noqa
|
||||
from kolla.image.kolla_worker import KollaWorker # noqa
|
||||
from kolla.image.utils import LOG # noqa
|
||||
from kolla.image.utils import Status # noqa
|
||||
from kolla.image.utils import STATUS_ERRORS # noqa
|
||||
from kolla import exception # noqa
|
||||
from kolla.template import filters as jinja_filters # noqa
|
||||
from kolla.template import methods as jinja_methods # noqa
|
||||
|
|
@ -72,49 +65,6 @@ def join_many(threads):
|
|||
LOG.warning('Force exits')
|
||||
|
||||
|
||||
class Image(object):
|
||||
def __init__(self, name, canonical_name, path, parent_name='',
|
||||
status=Status.UNPROCESSED, parent=None,
|
||||
source=None, logger=None, docker_client=None):
|
||||
self.name = name
|
||||
self.canonical_name = canonical_name
|
||||
self.path = path
|
||||
self.status = status
|
||||
self.parent = parent
|
||||
self.source = source
|
||||
self.parent_name = parent_name
|
||||
if logger is None:
|
||||
logger = utils.make_a_logger(image_name=name)
|
||||
self.logger = logger
|
||||
self.children = []
|
||||
self.plugins = []
|
||||
self.additions = []
|
||||
self.dc = docker_client
|
||||
|
||||
def copy(self):
|
||||
c = Image(self.name, self.canonical_name, self.path,
|
||||
logger=self.logger, parent_name=self.parent_name,
|
||||
status=self.status, parent=self.parent)
|
||||
if self.source:
|
||||
c.source = self.source.copy()
|
||||
if self.children:
|
||||
c.children = list(self.children)
|
||||
if self.plugins:
|
||||
c.plugins = list(self.plugins)
|
||||
if self.additions:
|
||||
c.additions = list(self.additions)
|
||||
return c
|
||||
|
||||
def in_docker_cache(self):
|
||||
return len(self.dc.images(name=self.canonical_name, quiet=True)) == 1
|
||||
|
||||
def __repr__(self):
|
||||
return ("Image(%s, %s, %s, parent_name=%s,"
|
||||
" status=%s, parent=%s, source=%s)") % (
|
||||
self.name, self.canonical_name, self.path,
|
||||
self.parent_name, self.status, self.parent, self.source)
|
||||
|
||||
|
||||
class WorkerThread(threading.Thread):
|
||||
"""Thread that executes tasks until the queue provides a tombstone."""
|
||||
|
||||
|
|
@ -158,688 +108,6 @@ class WorkerThread(threading.Thread):
|
|||
self.queue.task_done()
|
||||
|
||||
|
||||
class KollaWorker(object):
|
||||
|
||||
def __init__(self, conf):
|
||||
self.conf = conf
|
||||
self.images_dir = self._get_images_dir()
|
||||
self.registry = conf.registry
|
||||
if self.registry:
|
||||
self.namespace = self.registry + '/' + conf.namespace
|
||||
else:
|
||||
self.namespace = conf.namespace
|
||||
self.base = conf.base
|
||||
self.use_dumb_init = conf.use_dumb_init
|
||||
self.base_tag = conf.base_tag
|
||||
self.tag = conf.tag
|
||||
self.repos_yaml = conf.repos_yaml
|
||||
self.base_arch = conf.base_arch
|
||||
self.debian_arch = self.base_arch
|
||||
if self.base_arch == 'aarch64':
|
||||
self.debian_arch = 'arm64'
|
||||
elif self.base_arch == 'x86_64':
|
||||
self.debian_arch = 'amd64'
|
||||
self.images = list()
|
||||
self.openstack_release = conf.openstack_release
|
||||
self.openstack_branch = conf.openstack_branch
|
||||
self.openstack_branch_slashed = conf.openstack_branch_slashed
|
||||
self.docker_healthchecks = conf.docker_healthchecks
|
||||
rpm_setup_config = ([repo_file for repo_file in
|
||||
conf.rpm_setup_config if repo_file is not None])
|
||||
self.rpm_setup = self.build_rpm_setup(rpm_setup_config)
|
||||
|
||||
if self.base in ['centos', 'rocky']:
|
||||
self.conf.distro_python_version = "3.9"
|
||||
self.distro_package_manager = 'dnf'
|
||||
self.base_package_type = 'rpm'
|
||||
elif self.base in ['debian']:
|
||||
self.conf.distro_python_version = "3.9"
|
||||
self.distro_package_manager = 'apt'
|
||||
self.base_package_type = 'deb'
|
||||
elif self.base in ['ubuntu']:
|
||||
self.conf.distro_python_version = "3.10"
|
||||
self.distro_package_manager = 'apt'
|
||||
self.base_package_type = 'deb'
|
||||
else:
|
||||
# Assume worst
|
||||
self.conf.distro_python_version = "3.8"
|
||||
|
||||
if self.conf.distro_package_manager is not None:
|
||||
self.distro_package_manager = self.conf.distro_package_manager
|
||||
|
||||
if self.conf.base_package_type:
|
||||
self.base_package_type = self.conf.base_package_type
|
||||
|
||||
self.clean_package_cache = self.conf.clean_package_cache
|
||||
|
||||
self.image_prefix = self.conf.image_name_prefix
|
||||
|
||||
self.regex = conf.regex
|
||||
self.image_statuses_bad = dict()
|
||||
self.image_statuses_good = dict()
|
||||
self.image_statuses_unmatched = dict()
|
||||
self.image_statuses_skipped = dict()
|
||||
self.image_statuses_unbuildable = dict()
|
||||
self.image_statuses_allowed_to_fail = dict()
|
||||
self.maintainer = conf.maintainer
|
||||
self.distro_python_version = conf.distro_python_version
|
||||
|
||||
docker_kwargs = docker.utils.kwargs_from_env()
|
||||
try:
|
||||
self.dc = docker.APIClient(version='auto', **docker_kwargs)
|
||||
except docker.errors.DockerException as e:
|
||||
self.dc = None
|
||||
if not (conf.template_only or
|
||||
conf.save_dependency or
|
||||
conf.list_images or
|
||||
conf.list_dependencies):
|
||||
LOG.error("Unable to connect to Docker, exiting")
|
||||
LOG.info("Exception caught: {0}".format(e))
|
||||
sys.exit(1)
|
||||
|
||||
def _get_images_dir(self):
|
||||
possible_paths = (
|
||||
PROJECT_ROOT,
|
||||
os.path.join(sys.prefix, 'share/kolla'),
|
||||
os.path.join(sys.prefix, 'local/share/kolla'),
|
||||
os.path.join(os.getenv('HOME', ''), '.local/share/kolla'),
|
||||
# NOTE(zioproto): When Kolla is used within a snap, the env var
|
||||
# $SNAP is the directory where the snap is mounted.
|
||||
# https://github.com/zioproto/snap-kolla
|
||||
# More info in snap packages https://snapcraft.io
|
||||
os.path.join(os.environ.get('SNAP', ''), 'share/kolla'))
|
||||
|
||||
for path in possible_paths:
|
||||
image_path = os.path.join(path, 'docker')
|
||||
# NOTE(SamYaple): We explicitly check for the base folder to ensure
|
||||
# this is the correct path
|
||||
# TODO(SamYaple): Improve this to make this safer
|
||||
if os.path.exists(os.path.join(image_path, 'base')):
|
||||
LOG.info('Found the docker image folder at %s', image_path)
|
||||
return image_path
|
||||
else:
|
||||
raise exception.KollaDirNotFoundException('Image dir can not '
|
||||
'be found')
|
||||
|
||||
def build_rpm_setup(self, rpm_setup_config):
|
||||
"""Generates a list of docker commands based on provided configuration.
|
||||
|
||||
:param rpm_setup_config: A list of .rpm or .repo paths or URLs
|
||||
(can be empty)
|
||||
:return: A list of docker commands
|
||||
"""
|
||||
rpm_setup = list()
|
||||
|
||||
for config in rpm_setup_config:
|
||||
if config.endswith('.rpm'):
|
||||
# RPM files can be installed with dnf from file path or url
|
||||
cmd = "RUN dnf -y install {}".format(config)
|
||||
elif config.endswith('.repo'):
|
||||
if config.startswith('http'):
|
||||
# Curl http://url/etc.repo to /etc/yum.repos.d/etc.repo
|
||||
name = config.split('/')[-1]
|
||||
cmd = "RUN curl -L {} -o /etc/yum.repos.d/{}".format(
|
||||
config, name)
|
||||
else:
|
||||
# Copy .repo file from filesystem
|
||||
cmd = "COPY {} /etc/yum.repos.d/".format(config)
|
||||
elif not config:
|
||||
cmd = ''
|
||||
else:
|
||||
raise exception.KollaRpmSetupUnknownConfig(
|
||||
'RPM setup must be provided as .rpm or .repo files.'
|
||||
' Attempted configuration was {}'.format(config)
|
||||
)
|
||||
|
||||
rpm_setup.append(cmd)
|
||||
|
||||
return rpm_setup
|
||||
|
||||
def copy_apt_files(self):
|
||||
if self.conf.apt_sources_list:
|
||||
shutil.copyfile(
|
||||
self.conf.apt_sources_list,
|
||||
os.path.join(self.working_dir, "base", "sources.list")
|
||||
)
|
||||
|
||||
if self.conf.apt_preferences:
|
||||
shutil.copyfile(
|
||||
self.conf.apt_preferences,
|
||||
os.path.join(self.working_dir, "base", "apt_preferences")
|
||||
)
|
||||
|
||||
def copy_dir(self, src, dest):
|
||||
shutil.copytree(src, dest, dirs_exist_ok=True)
|
||||
|
||||
def setup_working_dir(self):
|
||||
"""Creates a working directory for use while building."""
|
||||
if self.conf.work_dir:
|
||||
self.working_dir = os.path.join(self.conf.work_dir, 'docker')
|
||||
else:
|
||||
ts = time.time()
|
||||
ts = datetime.datetime.fromtimestamp(ts).strftime(
|
||||
'%Y-%m-%d_%H-%M-%S_')
|
||||
self.temp_dir = tempfile.mkdtemp(prefix='kolla-' + ts)
|
||||
self.working_dir = os.path.join(self.temp_dir, 'docker')
|
||||
self.copy_dir(self.images_dir, self.working_dir)
|
||||
for dir in self.conf.docker_dir:
|
||||
self.copy_dir(dir, self.working_dir)
|
||||
self.copy_apt_files()
|
||||
LOG.debug('Created working dir: %s', self.working_dir)
|
||||
|
||||
def set_time(self):
|
||||
for root, dirs, files in os.walk(self.working_dir):
|
||||
for file_ in files:
|
||||
os.utime(os.path.join(root, file_), (0, 0))
|
||||
for dir_ in dirs:
|
||||
os.utime(os.path.join(root, dir_), (0, 0))
|
||||
LOG.debug('Set atime and mtime to 0 for all content in working dir')
|
||||
|
||||
def _get_filters(self):
|
||||
filters = {
|
||||
'customizable': jinja_filters.customizable,
|
||||
}
|
||||
return filters
|
||||
|
||||
def _get_methods(self):
|
||||
"""Mapping of available Jinja methods.
|
||||
|
||||
return a dictionary that maps available function names and their
|
||||
corresponding python methods to make them available in jinja templates
|
||||
"""
|
||||
|
||||
return {
|
||||
'debian_package_install': jinja_methods.debian_package_install,
|
||||
'handle_repos': jinja_methods.handle_repos,
|
||||
}
|
||||
|
||||
def get_users(self):
|
||||
all_sections = (set(self.conf._groups.keys()) |
|
||||
set(self.conf.list_all_sections()))
|
||||
ret = dict()
|
||||
for section in all_sections:
|
||||
match = re.search('^.*-user$', section)
|
||||
if match:
|
||||
user = self.conf[match.group(0)]
|
||||
ret[match.group(0)[:-5]] = {
|
||||
'uid': user.uid,
|
||||
'gid': user.gid,
|
||||
'group': user.group,
|
||||
}
|
||||
return ret
|
||||
|
||||
def create_dockerfiles(self):
|
||||
kolla_version = version.git_info if len(version.git_info) != 0 else \
|
||||
version.version_info.cached_version_string()
|
||||
supported_distro_name = common_config.DISTRO_PRETTY_NAME.get(
|
||||
self.base)
|
||||
for path in self.docker_build_paths:
|
||||
template_name = "Dockerfile.j2"
|
||||
image_name = path.split("/")[-1]
|
||||
ts = time.time()
|
||||
build_date = datetime.datetime.fromtimestamp(ts).strftime(
|
||||
'%Y%m%d')
|
||||
values = {'base_distro': self.base,
|
||||
'base_image': self.conf.base_image,
|
||||
'base_distro_tag': self.base_tag,
|
||||
'base_arch': self.base_arch,
|
||||
'repos_yaml': self.repos_yaml,
|
||||
'use_dumb_init': self.use_dumb_init,
|
||||
'base_package_type': self.base_package_type,
|
||||
'debian_arch': self.debian_arch,
|
||||
'docker_healthchecks': self.docker_healthchecks,
|
||||
'supported_distro_name': supported_distro_name,
|
||||
'image_prefix': self.image_prefix,
|
||||
'namespace': self.namespace,
|
||||
'openstack_release': self.openstack_release,
|
||||
'openstack_branch': self.openstack_branch,
|
||||
'openstack_branch_slashed':
|
||||
self.openstack_branch_slashed,
|
||||
'tag': self.tag,
|
||||
'maintainer': self.maintainer,
|
||||
'kolla_version': kolla_version,
|
||||
'image_name': image_name,
|
||||
'users': self.get_users(),
|
||||
'distro_python_version': self.distro_python_version,
|
||||
'distro_package_manager': self.distro_package_manager,
|
||||
'rpm_setup': self.rpm_setup,
|
||||
'build_date': build_date,
|
||||
'clean_package_cache': self.clean_package_cache}
|
||||
env = jinja2.Environment( # nosec: not used to render HTML
|
||||
loader=jinja2.FileSystemLoader(self.working_dir))
|
||||
env.filters.update(self._get_filters())
|
||||
env.globals.update(self._get_methods())
|
||||
tpl_path = os.path.join(
|
||||
os.path.relpath(path, self.working_dir),
|
||||
template_name)
|
||||
|
||||
template = env.get_template(tpl_path)
|
||||
if self.conf.template_override:
|
||||
tpl_dict = self._merge_overrides(self.conf.template_override)
|
||||
template_name = os.path.basename(list(tpl_dict.keys())[0])
|
||||
values['parent_template'] = template
|
||||
env = jinja2.Environment( # nosec: not used to render HTML
|
||||
loader=jinja2.DictLoader(tpl_dict))
|
||||
env.filters.update(self._get_filters())
|
||||
env.globals.update(self._get_methods())
|
||||
template = env.get_template(template_name)
|
||||
content = template.render(values, env=os.environ)
|
||||
content_path = os.path.join(path, 'Dockerfile')
|
||||
with open(content_path, 'w') as f:
|
||||
LOG.debug("Rendered %s into:", tpl_path)
|
||||
LOG.debug(content)
|
||||
f.write(content)
|
||||
LOG.debug("Wrote it to %s", content_path)
|
||||
|
||||
def _merge_overrides(self, overrides):
|
||||
tpl_name = os.path.basename(overrides[0])
|
||||
with open(overrides[0], 'r') as f:
|
||||
tpl_content = f.read()
|
||||
for override in overrides[1:]:
|
||||
with open(override, 'r') as f:
|
||||
cont = f.read()
|
||||
# Remove extends header
|
||||
cont = re.sub(r'.*\{\%.*extends.*\n', '', cont)
|
||||
tpl_content += cont
|
||||
return {tpl_name: tpl_content}
|
||||
|
||||
def find_dockerfiles(self):
|
||||
"""Recursive search for Dockerfiles in the working directory."""
|
||||
self.docker_build_paths = list()
|
||||
path = self.working_dir
|
||||
filename = 'Dockerfile.j2'
|
||||
|
||||
for root, dirs, names in os.walk(path):
|
||||
if filename in names:
|
||||
self.docker_build_paths.append(root)
|
||||
LOG.debug('Found %s', root.split(self.working_dir)[1])
|
||||
|
||||
LOG.debug('Found %d Dockerfiles', len(self.docker_build_paths))
|
||||
|
||||
def cleanup(self):
|
||||
"""Remove temp files."""
|
||||
if not self.conf.work_dir:
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
def filter_images(self):
|
||||
"""Filter which images to build."""
|
||||
filter_ = list()
|
||||
|
||||
if self.regex:
|
||||
filter_ += self.regex
|
||||
elif self.conf.profile:
|
||||
for profile in self.conf.profile:
|
||||
if profile not in self.conf.profiles:
|
||||
self.conf.register_opt(cfg.ListOpt(profile,
|
||||
default=[]),
|
||||
'profiles')
|
||||
if len(self.conf.profiles[profile]) == 0:
|
||||
msg = 'Profile: {} does not exist'.format(profile)
|
||||
raise ValueError(msg)
|
||||
else:
|
||||
filter_ += self.conf.profiles[profile]
|
||||
|
||||
# mark unbuildable images and their children
|
||||
base = self.base
|
||||
|
||||
tag_element = r'(%s|%s)' % (base, self.base_arch)
|
||||
tag_re = re.compile(r'^%s(\+%s)*$' % (tag_element, tag_element))
|
||||
unbuildable_images = set()
|
||||
|
||||
if not self.conf.enable_unbuildable:
|
||||
for set_tag in UNBUILDABLE_IMAGES:
|
||||
if tag_re.match(set_tag):
|
||||
unbuildable_images.update(UNBUILDABLE_IMAGES[set_tag])
|
||||
|
||||
if unbuildable_images:
|
||||
for image in self.images:
|
||||
if image.name in unbuildable_images:
|
||||
image.status = Status.UNBUILDABLE
|
||||
else:
|
||||
# let's check ancestors
|
||||
# if any of them is unbuildable then we mark it
|
||||
# and then mark image
|
||||
build_image = True
|
||||
ancestor_image = image
|
||||
while (ancestor_image.parent is not None):
|
||||
ancestor_image = ancestor_image.parent
|
||||
if ancestor_image.name in unbuildable_images or \
|
||||
ancestor_image.status == Status.UNBUILDABLE:
|
||||
build_image = False
|
||||
ancestor_image.status = Status.UNBUILDABLE
|
||||
break
|
||||
if not build_image:
|
||||
image.status = Status.UNBUILDABLE
|
||||
|
||||
# When we want to build a subset of images then filter_ part kicks in.
|
||||
# Otherwise we just mark everything buildable as matched for build.
|
||||
|
||||
# First, determine which buildable images match.
|
||||
if filter_:
|
||||
patterns = re.compile(r"|".join(filter_).join('()'))
|
||||
for image in self.images:
|
||||
# as we now list not buildable/skipped images we need to
|
||||
# process them otherwise list will contain also not requested
|
||||
# entries
|
||||
if image.status in (Status.MATCHED, Status.UNBUILDABLE):
|
||||
continue
|
||||
if re.search(patterns, image.name):
|
||||
image.status = Status.MATCHED
|
||||
|
||||
ancestor_image = image
|
||||
while (ancestor_image.parent is not None and
|
||||
ancestor_image.parent.status != Status.MATCHED):
|
||||
ancestor_image = ancestor_image.parent
|
||||
# Parents of a buildable image must also be buildable.
|
||||
ancestor_image.status = Status.MATCHED
|
||||
LOG.debug('Image %s matched regex', image.name)
|
||||
else:
|
||||
image.status = Status.UNMATCHED
|
||||
else:
|
||||
for image in self.images:
|
||||
if image.status != Status.UNBUILDABLE:
|
||||
image.status = Status.MATCHED
|
||||
|
||||
# Next, mark any skipped images.
|
||||
for image in self.images:
|
||||
if image.status != Status.MATCHED:
|
||||
continue
|
||||
# Skip image if --skip-existing was given and image exists.
|
||||
if (self.conf.skip_existing and image.in_docker_cache()):
|
||||
LOG.debug('Skipping existing image %s', image.name)
|
||||
image.status = Status.SKIPPED
|
||||
# Skip image if --skip-parents was given and image has children.
|
||||
elif self.conf.skip_parents and image.children:
|
||||
LOG.debug('Skipping parent image %s', image.name)
|
||||
image.status = Status.SKIPPED
|
||||
|
||||
def summary(self):
|
||||
"""Walk the dictionary of images statuses and print results."""
|
||||
# For debug we print the logs again if the image error'd. This is to
|
||||
# help us debug and it will be extra helpful in the gate.
|
||||
for image in self.images:
|
||||
if image.status in STATUS_ERRORS:
|
||||
LOG.debug("Image %s failed", image.name)
|
||||
|
||||
self.get_image_statuses()
|
||||
results = {
|
||||
'built': [],
|
||||
'failed': [],
|
||||
'not_matched': [],
|
||||
'skipped': [],
|
||||
'unbuildable': [],
|
||||
}
|
||||
|
||||
if self.image_statuses_good:
|
||||
LOG.info("=========================")
|
||||
LOG.info("Successfully built images")
|
||||
LOG.info("=========================")
|
||||
for name in sorted(self.image_statuses_good.keys()):
|
||||
LOG.info(name)
|
||||
results['built'].append({
|
||||
'name': name,
|
||||
})
|
||||
|
||||
if self.image_statuses_bad or self.image_statuses_allowed_to_fail:
|
||||
LOG.info("===========================")
|
||||
LOG.info("Images that failed to build")
|
||||
LOG.info("===========================")
|
||||
all_bad_statuses = self.image_statuses_bad.copy()
|
||||
all_bad_statuses.update(self.image_statuses_allowed_to_fail)
|
||||
for name, status in sorted(all_bad_statuses.items()):
|
||||
if name in self.image_statuses_allowed_to_fail:
|
||||
LOG.error('%s Failed with status: %s (allowed to fail)',
|
||||
name, status.value)
|
||||
else:
|
||||
LOG.error('%s Failed with status: %s', name, status.value)
|
||||
|
||||
results['failed'].append({
|
||||
'name': name,
|
||||
'status': status.value,
|
||||
})
|
||||
if self.conf.logs_dir and status == Status.ERROR:
|
||||
linkname = os.path.join(self.conf.logs_dir,
|
||||
"000_FAILED_%s.log" % name)
|
||||
try:
|
||||
os.lstat(linkname)
|
||||
os.remove(linkname)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
os.symlink("%s.log" % name, linkname)
|
||||
|
||||
if self.image_statuses_unmatched:
|
||||
LOG.debug("=====================================")
|
||||
LOG.debug("Images not matched for build by regex")
|
||||
LOG.debug("=====================================")
|
||||
for name in sorted(self.image_statuses_unmatched.keys()):
|
||||
LOG.debug(name)
|
||||
results['not_matched'].append({
|
||||
'name': name,
|
||||
})
|
||||
|
||||
if self.image_statuses_skipped:
|
||||
LOG.info("===================================")
|
||||
LOG.info("Images skipped due to build options")
|
||||
LOG.info("===================================")
|
||||
for name in sorted(self.image_statuses_skipped.keys()):
|
||||
LOG.info(name)
|
||||
results['skipped'].append({
|
||||
'name': name,
|
||||
})
|
||||
|
||||
if self.image_statuses_unbuildable:
|
||||
LOG.info("=========================================")
|
||||
LOG.info("Images not buildable due to build options")
|
||||
LOG.info("=========================================")
|
||||
for name in sorted(self.image_statuses_unbuildable.keys()):
|
||||
LOG.info(name)
|
||||
results['unbuildable'].append({
|
||||
'name': name,
|
||||
})
|
||||
|
||||
return results
|
||||
|
||||
def get_image_statuses(self):
|
||||
if any([self.image_statuses_bad,
|
||||
self.image_statuses_good,
|
||||
self.image_statuses_unmatched,
|
||||
self.image_statuses_skipped,
|
||||
self.image_statuses_unbuildable,
|
||||
self.image_statuses_allowed_to_fail]):
|
||||
return (self.image_statuses_bad,
|
||||
self.image_statuses_good,
|
||||
self.image_statuses_unmatched,
|
||||
self.image_statuses_skipped,
|
||||
self.image_statuses_unbuildable,
|
||||
self.image_statuses_allowed_to_fail)
|
||||
for image in self.images:
|
||||
if image.status == Status.BUILT:
|
||||
self.image_statuses_good[image.name] = image.status
|
||||
elif image.status == Status.UNMATCHED:
|
||||
self.image_statuses_unmatched[image.name] = image.status
|
||||
elif image.status == Status.SKIPPED:
|
||||
self.image_statuses_skipped[image.name] = image.status
|
||||
elif image.status == Status.UNBUILDABLE:
|
||||
self.image_statuses_unbuildable[image.name] = image.status
|
||||
else:
|
||||
if image.name in self.conf.allowed_to_fail:
|
||||
self.image_statuses_allowed_to_fail[
|
||||
image.name] = image.status
|
||||
else:
|
||||
self.image_statuses_bad[image.name] = image.status
|
||||
return (self.image_statuses_bad,
|
||||
self.image_statuses_good,
|
||||
self.image_statuses_unmatched,
|
||||
self.image_statuses_skipped,
|
||||
self.image_statuses_unbuildable,
|
||||
self.image_statuses_allowed_to_fail)
|
||||
|
||||
def build_image_list(self):
|
||||
def process_source_installation(image, section):
|
||||
installation = dict()
|
||||
# NOTE(jeffrey4l): source is not needed when the type is None
|
||||
if self.conf._get('type', self.conf._get_group(section)) is None:
|
||||
if image.parent_name is None:
|
||||
LOG.debug('No source location found in section %s',
|
||||
section)
|
||||
else:
|
||||
installation['type'] = self.conf[section]['type']
|
||||
installation['source'] = self.conf[section]['location']
|
||||
installation['name'] = section
|
||||
if installation['type'] == 'git':
|
||||
installation['reference'] = self.conf[section]['reference']
|
||||
installation['enabled'] = self.conf[section]['enabled']
|
||||
return installation
|
||||
|
||||
all_sections = (set(self.conf._groups.keys()) |
|
||||
set(self.conf.list_all_sections()))
|
||||
|
||||
for path in self.docker_build_paths:
|
||||
# Reading parent image name
|
||||
with open(os.path.join(path, 'Dockerfile')) as f:
|
||||
content = f.read()
|
||||
|
||||
image_name = os.path.basename(path)
|
||||
canonical_name = (self.namespace + '/' + self.image_prefix +
|
||||
image_name + ':' + self.tag)
|
||||
parent_search_pattern = re.compile(r'^FROM.*$', re.MULTILINE)
|
||||
match = re.search(parent_search_pattern, content)
|
||||
if match:
|
||||
parent_name = match.group(0).split(' ')[1]
|
||||
else:
|
||||
parent_name = ''
|
||||
del match
|
||||
image = Image(image_name, canonical_name, path,
|
||||
parent_name=parent_name,
|
||||
logger=utils.make_a_logger(self.conf, image_name),
|
||||
docker_client=self.dc)
|
||||
|
||||
# NOTE(jeffrey4l): register the opts if the section didn't
|
||||
# register in the kolla/common/config.py file
|
||||
if image.name not in self.conf._groups:
|
||||
self.conf.register_opts(common_config.get_source_opts(),
|
||||
image.name)
|
||||
image.source = process_source_installation(image, image.name)
|
||||
for plugin in [match.group(0) for match in
|
||||
(re.search('^{}-plugin-.+'.format(image.name),
|
||||
section) for section in
|
||||
all_sections) if match]:
|
||||
try:
|
||||
self.conf.register_opts(
|
||||
common_config.get_source_opts(),
|
||||
plugin
|
||||
)
|
||||
except cfg.DuplicateOptError:
|
||||
LOG.debug('Plugin %s already registered in config',
|
||||
plugin)
|
||||
image.plugins.append(
|
||||
process_source_installation(image, plugin))
|
||||
for addition in [
|
||||
match.group(0) for match in
|
||||
(re.search('^{}-additions-.+'.format(image.name),
|
||||
section) for section in all_sections) if match]:
|
||||
try:
|
||||
self.conf.register_opts(
|
||||
common_config.get_source_opts(),
|
||||
addition
|
||||
)
|
||||
except cfg.DuplicateOptError:
|
||||
LOG.debug('Addition %s already registered in config',
|
||||
addition)
|
||||
image.additions.append(
|
||||
process_source_installation(image, addition))
|
||||
|
||||
self.images.append(image)
|
||||
|
||||
def save_dependency(self, to_file):
|
||||
try:
|
||||
import graphviz
|
||||
except ImportError:
|
||||
LOG.error('"graphviz" is required for save dependency')
|
||||
raise
|
||||
dot = graphviz.Digraph(comment='Docker Images Dependency')
|
||||
dot.body.extend(['rankdir=LR'])
|
||||
for image in self.images:
|
||||
if image.status not in [Status.MATCHED]:
|
||||
continue
|
||||
dot.node(image.name)
|
||||
if image.parent is not None:
|
||||
dot.edge(image.parent.name, image.name)
|
||||
|
||||
with open(to_file, 'w') as f:
|
||||
f.write(dot.source)
|
||||
|
||||
def list_images(self):
|
||||
for count, image in enumerate([
|
||||
image for image in self.images if image.status == Status.MATCHED
|
||||
]):
|
||||
print(count + 1, ':', image.name)
|
||||
|
||||
def list_dependencies(self):
|
||||
match = False
|
||||
for image in self.images:
|
||||
if image.status in [Status.MATCHED]:
|
||||
match = True
|
||||
if image.parent is None:
|
||||
base = image
|
||||
if not match:
|
||||
print('Nothing matched!')
|
||||
return
|
||||
|
||||
def list_children(images, ancestry):
|
||||
children = next(iter(ancestry.values()))
|
||||
for image in images:
|
||||
if image.status not in [Status.MATCHED]:
|
||||
continue
|
||||
if not image.children:
|
||||
children.append(image.name)
|
||||
else:
|
||||
newparent = {image.name: []}
|
||||
children.append(newparent)
|
||||
list_children(image.children, newparent)
|
||||
|
||||
ancestry = {base.name: []}
|
||||
list_children(base.children, ancestry)
|
||||
json.dump(ancestry, sys.stdout, indent=2)
|
||||
|
||||
def find_parents(self):
|
||||
"""Associate all images with parents and children."""
|
||||
sort_images = dict()
|
||||
|
||||
for image in self.images:
|
||||
sort_images[image.canonical_name] = image
|
||||
|
||||
for parent_name, parent in sort_images.items():
|
||||
for image in sort_images.values():
|
||||
if (image.parent_name == parent_name):
|
||||
parent.children.append(image)
|
||||
image.parent = parent
|
||||
|
||||
def build_queue(self, push_queue):
|
||||
"""Organizes Queue list.
|
||||
|
||||
Return a list of Queues that have been organized into a hierarchy
|
||||
based on dependencies
|
||||
"""
|
||||
build_queue = queue.Queue()
|
||||
|
||||
for image in self.images:
|
||||
if image.status in (Status.UNMATCHED, Status.SKIPPED,
|
||||
Status.UNBUILDABLE):
|
||||
# Don't bother queuing up build tasks for things that
|
||||
# were not matched in the first place... (not worth the
|
||||
# effort to run them, if they won't be used anyway).
|
||||
continue
|
||||
# Build all root nodes, where a root is defined as having no parent
|
||||
# or having a parent that is explicitly being skipped.
|
||||
if image.parent is None or image.parent.status == Status.SKIPPED:
|
||||
build_queue.put(BuildTask(self.conf, image, push_queue))
|
||||
LOG.info('Added image %s to queue', image.name)
|
||||
|
||||
return build_queue
|
||||
|
||||
|
||||
def run_build():
|
||||
"""Build container images.
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,764 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import datetime
|
||||
import docker
|
||||
import json
|
||||
import os
|
||||
import queue
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
import jinja2
|
||||
from kolla.common import config as common_config
|
||||
from kolla.common import utils
|
||||
from kolla import exception
|
||||
from kolla.image.tasks import BuildTask
|
||||
from kolla.image.unbuildable import UNBUILDABLE_IMAGES
|
||||
from kolla.image.utils import LOG
|
||||
from kolla.image.utils import Status
|
||||
from kolla.image.utils import STATUS_ERRORS
|
||||
from kolla.template import filters as jinja_filters
|
||||
from kolla.template import methods as jinja_methods
|
||||
from kolla import version
|
||||
from oslo_config import cfg
|
||||
|
||||
PROJECT_ROOT = os.path.abspath(os.path.join(
|
||||
os.path.dirname(os.path.realpath(__file__)), '../..'))
|
||||
|
||||
|
||||
class Image(object):
|
||||
def __init__(self, name, canonical_name, path, parent_name='',
|
||||
status=Status.UNPROCESSED, parent=None,
|
||||
source=None, logger=None, docker_client=None):
|
||||
self.name = name
|
||||
self.canonical_name = canonical_name
|
||||
self.path = path
|
||||
self.status = status
|
||||
self.parent = parent
|
||||
self.source = source
|
||||
self.parent_name = parent_name
|
||||
if logger is None:
|
||||
logger = utils.make_a_logger(image_name=name)
|
||||
self.logger = logger
|
||||
self.children = []
|
||||
self.plugins = []
|
||||
self.additions = []
|
||||
self.dc = docker_client
|
||||
|
||||
def copy(self):
|
||||
c = Image(self.name, self.canonical_name, self.path,
|
||||
logger=self.logger, parent_name=self.parent_name,
|
||||
status=self.status, parent=self.parent)
|
||||
if self.source:
|
||||
c.source = self.source.copy()
|
||||
if self.children:
|
||||
c.children = list(self.children)
|
||||
if self.plugins:
|
||||
c.plugins = list(self.plugins)
|
||||
if self.additions:
|
||||
c.additions = list(self.additions)
|
||||
return c
|
||||
|
||||
def in_docker_cache(self):
|
||||
return len(self.dc.images(name=self.canonical_name, quiet=True)) == 1
|
||||
|
||||
def __repr__(self):
|
||||
return ("Image(%s, %s, %s, parent_name=%s,"
|
||||
" status=%s, parent=%s, source=%s)") % (
|
||||
self.name, self.canonical_name, self.path,
|
||||
self.parent_name, self.status, self.parent, self.source)
|
||||
|
||||
|
||||
class KollaWorker(object):
|
||||
|
||||
def __init__(self, conf):
|
||||
self.conf = conf
|
||||
self.images_dir = self._get_images_dir()
|
||||
self.registry = conf.registry
|
||||
if self.registry:
|
||||
self.namespace = self.registry + '/' + conf.namespace
|
||||
else:
|
||||
self.namespace = conf.namespace
|
||||
self.base = conf.base
|
||||
self.use_dumb_init = conf.use_dumb_init
|
||||
self.base_tag = conf.base_tag
|
||||
self.tag = conf.tag
|
||||
self.repos_yaml = conf.repos_yaml
|
||||
self.base_arch = conf.base_arch
|
||||
self.debian_arch = self.base_arch
|
||||
if self.base_arch == 'aarch64':
|
||||
self.debian_arch = 'arm64'
|
||||
elif self.base_arch == 'x86_64':
|
||||
self.debian_arch = 'amd64'
|
||||
self.images = list()
|
||||
self.openstack_release = conf.openstack_release
|
||||
self.openstack_branch = conf.openstack_branch
|
||||
self.openstack_branch_slashed = conf.openstack_branch_slashed
|
||||
self.docker_healthchecks = conf.docker_healthchecks
|
||||
rpm_setup_config = ([repo_file for repo_file in
|
||||
conf.rpm_setup_config if repo_file is not None])
|
||||
self.rpm_setup = self.build_rpm_setup(rpm_setup_config)
|
||||
|
||||
if self.base in ['centos', 'rocky']:
|
||||
self.conf.distro_python_version = "3.9"
|
||||
self.distro_package_manager = 'dnf'
|
||||
self.base_package_type = 'rpm'
|
||||
elif self.base in ['debian']:
|
||||
self.conf.distro_python_version = "3.9"
|
||||
self.distro_package_manager = 'apt'
|
||||
self.base_package_type = 'deb'
|
||||
elif self.base in ['ubuntu']:
|
||||
self.conf.distro_python_version = "3.10"
|
||||
self.distro_package_manager = 'apt'
|
||||
self.base_package_type = 'deb'
|
||||
else:
|
||||
# Assume worst
|
||||
self.conf.distro_python_version = "3.8"
|
||||
|
||||
if self.conf.distro_package_manager is not None:
|
||||
self.distro_package_manager = self.conf.distro_package_manager
|
||||
|
||||
if self.conf.base_package_type:
|
||||
self.base_package_type = self.conf.base_package_type
|
||||
|
||||
self.clean_package_cache = self.conf.clean_package_cache
|
||||
|
||||
self.image_prefix = self.conf.image_name_prefix
|
||||
|
||||
self.regex = conf.regex
|
||||
self.image_statuses_bad = dict()
|
||||
self.image_statuses_good = dict()
|
||||
self.image_statuses_unmatched = dict()
|
||||
self.image_statuses_skipped = dict()
|
||||
self.image_statuses_unbuildable = dict()
|
||||
self.image_statuses_allowed_to_fail = dict()
|
||||
self.maintainer = conf.maintainer
|
||||
self.distro_python_version = conf.distro_python_version
|
||||
|
||||
docker_kwargs = docker.utils.kwargs_from_env()
|
||||
try:
|
||||
self.dc = docker.APIClient(version='auto', **docker_kwargs)
|
||||
except docker.errors.DockerException as e:
|
||||
self.dc = None
|
||||
if not (conf.template_only or
|
||||
conf.save_dependency or
|
||||
conf.list_images or
|
||||
conf.list_dependencies):
|
||||
LOG.error("Unable to connect to Docker, exiting")
|
||||
LOG.info("Exception caught: {0}".format(e))
|
||||
sys.exit(1)
|
||||
|
||||
def _get_images_dir(self):
|
||||
possible_paths = (
|
||||
PROJECT_ROOT,
|
||||
os.path.join(sys.prefix, 'share/kolla'),
|
||||
os.path.join(sys.prefix, 'local/share/kolla'),
|
||||
os.path.join(os.getenv('HOME', ''), '.local/share/kolla'),
|
||||
# NOTE(zioproto): When Kolla is used within a snap, the env var
|
||||
# $SNAP is the directory where the snap is mounted.
|
||||
# https://github.com/zioproto/snap-kolla
|
||||
# More info in snap packages https://snapcraft.io
|
||||
os.path.join(os.environ.get('SNAP', ''), 'share/kolla'))
|
||||
|
||||
for path in possible_paths:
|
||||
image_path = os.path.join(path, 'docker')
|
||||
# NOTE(SamYaple): We explicitly check for the base folder to ensure
|
||||
# this is the correct path
|
||||
# TODO(SamYaple): Improve this to make this safer
|
||||
if os.path.exists(os.path.join(image_path, 'base')):
|
||||
LOG.info('Found the docker image folder at %s', image_path)
|
||||
return image_path
|
||||
else:
|
||||
raise exception.KollaDirNotFoundException('Image dir can not '
|
||||
'be found')
|
||||
|
||||
def build_rpm_setup(self, rpm_setup_config):
|
||||
"""Generates a list of docker commands based on provided configuration.
|
||||
|
||||
:param rpm_setup_config: A list of .rpm or .repo paths or URLs
|
||||
(can be empty)
|
||||
:return: A list of docker commands
|
||||
"""
|
||||
rpm_setup = list()
|
||||
|
||||
for config in rpm_setup_config:
|
||||
if config.endswith('.rpm'):
|
||||
# RPM files can be installed with dnf from file path or url
|
||||
cmd = "RUN dnf -y install {}".format(config)
|
||||
elif config.endswith('.repo'):
|
||||
if config.startswith('http'):
|
||||
# Curl http://url/etc.repo to /etc/yum.repos.d/etc.repo
|
||||
name = config.split('/')[-1]
|
||||
cmd = "RUN curl -L {} -o /etc/yum.repos.d/{}".format(
|
||||
config, name)
|
||||
else:
|
||||
# Copy .repo file from filesystem
|
||||
cmd = "COPY {} /etc/yum.repos.d/".format(config)
|
||||
elif not config:
|
||||
cmd = ''
|
||||
else:
|
||||
raise exception.KollaRpmSetupUnknownConfig(
|
||||
'RPM setup must be provided as .rpm or .repo files.'
|
||||
' Attempted configuration was {}'.format(config)
|
||||
)
|
||||
|
||||
rpm_setup.append(cmd)
|
||||
|
||||
return rpm_setup
|
||||
|
||||
def copy_apt_files(self):
|
||||
if self.conf.apt_sources_list:
|
||||
shutil.copyfile(
|
||||
self.conf.apt_sources_list,
|
||||
os.path.join(self.working_dir, "base", "sources.list")
|
||||
)
|
||||
|
||||
if self.conf.apt_preferences:
|
||||
shutil.copyfile(
|
||||
self.conf.apt_preferences,
|
||||
os.path.join(self.working_dir, "base", "apt_preferences")
|
||||
)
|
||||
|
||||
def copy_dir(self, src, dest):
|
||||
shutil.copytree(src, dest, dirs_exist_ok=True)
|
||||
|
||||
def setup_working_dir(self):
|
||||
"""Creates a working directory for use while building."""
|
||||
if self.conf.work_dir:
|
||||
self.working_dir = os.path.join(self.conf.work_dir, 'docker')
|
||||
else:
|
||||
ts = time.time()
|
||||
ts = datetime.datetime.fromtimestamp(ts).strftime(
|
||||
'%Y-%m-%d_%H-%M-%S_')
|
||||
self.temp_dir = tempfile.mkdtemp(prefix='kolla-' + ts)
|
||||
self.working_dir = os.path.join(self.temp_dir, 'docker')
|
||||
self.copy_dir(self.images_dir, self.working_dir)
|
||||
for dir in self.conf.docker_dir:
|
||||
self.copy_dir(dir, self.working_dir)
|
||||
self.copy_apt_files()
|
||||
LOG.debug('Created working dir: %s', self.working_dir)
|
||||
|
||||
def set_time(self):
|
||||
for root, dirs, files in os.walk(self.working_dir):
|
||||
for file_ in files:
|
||||
os.utime(os.path.join(root, file_), (0, 0))
|
||||
for dir_ in dirs:
|
||||
os.utime(os.path.join(root, dir_), (0, 0))
|
||||
LOG.debug('Set atime and mtime to 0 for all content in working dir')
|
||||
|
||||
def _get_filters(self):
|
||||
filters = {
|
||||
'customizable': jinja_filters.customizable,
|
||||
}
|
||||
return filters
|
||||
|
||||
def _get_methods(self):
|
||||
"""Mapping of available Jinja methods.
|
||||
|
||||
return a dictionary that maps available function names and their
|
||||
corresponding python methods to make them available in jinja templates
|
||||
"""
|
||||
|
||||
return {
|
||||
'debian_package_install': jinja_methods.debian_package_install,
|
||||
'handle_repos': jinja_methods.handle_repos,
|
||||
}
|
||||
|
||||
def get_users(self):
|
||||
all_sections = (set(self.conf._groups.keys()) |
|
||||
set(self.conf.list_all_sections()))
|
||||
ret = dict()
|
||||
for section in all_sections:
|
||||
match = re.search('^.*-user$', section)
|
||||
if match:
|
||||
user = self.conf[match.group(0)]
|
||||
ret[match.group(0)[:-5]] = {
|
||||
'uid': user.uid,
|
||||
'gid': user.gid,
|
||||
'group': user.group,
|
||||
}
|
||||
return ret
|
||||
|
||||
def create_dockerfiles(self):
|
||||
kolla_version = version.git_info if len(version.git_info) != 0 else \
|
||||
version.version_info.cached_version_string()
|
||||
supported_distro_name = common_config.DISTRO_PRETTY_NAME.get(
|
||||
self.base)
|
||||
for path in self.docker_build_paths:
|
||||
template_name = "Dockerfile.j2"
|
||||
image_name = path.split("/")[-1]
|
||||
ts = time.time()
|
||||
build_date = datetime.datetime.fromtimestamp(ts).strftime(
|
||||
'%Y%m%d')
|
||||
values = {'base_distro': self.base,
|
||||
'base_image': self.conf.base_image,
|
||||
'base_distro_tag': self.base_tag,
|
||||
'base_arch': self.base_arch,
|
||||
'repos_yaml': self.repos_yaml,
|
||||
'use_dumb_init': self.use_dumb_init,
|
||||
'base_package_type': self.base_package_type,
|
||||
'debian_arch': self.debian_arch,
|
||||
'docker_healthchecks': self.docker_healthchecks,
|
||||
'supported_distro_name': supported_distro_name,
|
||||
'image_prefix': self.image_prefix,
|
||||
'namespace': self.namespace,
|
||||
'openstack_release': self.openstack_release,
|
||||
'openstack_branch': self.openstack_branch,
|
||||
'openstack_branch_slashed':
|
||||
self.openstack_branch_slashed,
|
||||
'tag': self.tag,
|
||||
'maintainer': self.maintainer,
|
||||
'kolla_version': kolla_version,
|
||||
'image_name': image_name,
|
||||
'users': self.get_users(),
|
||||
'distro_python_version': self.distro_python_version,
|
||||
'distro_package_manager': self.distro_package_manager,
|
||||
'rpm_setup': self.rpm_setup,
|
||||
'build_date': build_date,
|
||||
'clean_package_cache': self.clean_package_cache}
|
||||
env = jinja2.Environment( # nosec: not used to render HTML
|
||||
loader=jinja2.FileSystemLoader(self.working_dir))
|
||||
env.filters.update(self._get_filters())
|
||||
env.globals.update(self._get_methods())
|
||||
tpl_path = os.path.join(
|
||||
os.path.relpath(path, self.working_dir),
|
||||
template_name)
|
||||
|
||||
template = env.get_template(tpl_path)
|
||||
if self.conf.template_override:
|
||||
tpl_dict = self._merge_overrides(self.conf.template_override)
|
||||
template_name = os.path.basename(list(tpl_dict.keys())[0])
|
||||
values['parent_template'] = template
|
||||
env = jinja2.Environment( # nosec: not used to render HTML
|
||||
loader=jinja2.DictLoader(tpl_dict))
|
||||
env.filters.update(self._get_filters())
|
||||
env.globals.update(self._get_methods())
|
||||
template = env.get_template(template_name)
|
||||
content = template.render(values, env=os.environ)
|
||||
content_path = os.path.join(path, 'Dockerfile')
|
||||
with open(content_path, 'w') as f:
|
||||
LOG.debug("Rendered %s into:", tpl_path)
|
||||
LOG.debug(content)
|
||||
f.write(content)
|
||||
LOG.debug("Wrote it to %s", content_path)
|
||||
|
||||
def _merge_overrides(self, overrides):
|
||||
tpl_name = os.path.basename(overrides[0])
|
||||
with open(overrides[0], 'r') as f:
|
||||
tpl_content = f.read()
|
||||
for override in overrides[1:]:
|
||||
with open(override, 'r') as f:
|
||||
cont = f.read()
|
||||
# Remove extends header
|
||||
cont = re.sub(r'.*\{\%.*extends.*\n', '', cont)
|
||||
tpl_content += cont
|
||||
return {tpl_name: tpl_content}
|
||||
|
||||
def find_dockerfiles(self):
|
||||
"""Recursive search for Dockerfiles in the working directory."""
|
||||
self.docker_build_paths = list()
|
||||
path = self.working_dir
|
||||
filename = 'Dockerfile.j2'
|
||||
|
||||
for root, dirs, names in os.walk(path):
|
||||
if filename in names:
|
||||
self.docker_build_paths.append(root)
|
||||
LOG.debug('Found %s', root.split(self.working_dir)[1])
|
||||
|
||||
LOG.debug('Found %d Dockerfiles', len(self.docker_build_paths))
|
||||
|
||||
def cleanup(self):
|
||||
"""Remove temp files."""
|
||||
if not self.conf.work_dir:
|
||||
shutil.rmtree(self.temp_dir)
|
||||
|
||||
def filter_images(self):
|
||||
"""Filter which images to build."""
|
||||
filter_ = list()
|
||||
|
||||
if self.regex:
|
||||
filter_ += self.regex
|
||||
elif self.conf.profile:
|
||||
for profile in self.conf.profile:
|
||||
if profile not in self.conf.profiles:
|
||||
self.conf.register_opt(cfg.ListOpt(profile,
|
||||
default=[]),
|
||||
'profiles')
|
||||
if len(self.conf.profiles[profile]) == 0:
|
||||
msg = 'Profile: {} does not exist'.format(profile)
|
||||
raise ValueError(msg)
|
||||
else:
|
||||
filter_ += self.conf.profiles[profile]
|
||||
|
||||
# mark unbuildable images and their children
|
||||
base = self.base
|
||||
|
||||
tag_element = r'(%s|%s)' % (base, self.base_arch)
|
||||
tag_re = re.compile(r'^%s(\+%s)*$' % (tag_element, tag_element))
|
||||
unbuildable_images = set()
|
||||
|
||||
if not self.conf.enable_unbuildable:
|
||||
for set_tag in UNBUILDABLE_IMAGES:
|
||||
if tag_re.match(set_tag):
|
||||
unbuildable_images.update(UNBUILDABLE_IMAGES[set_tag])
|
||||
|
||||
if unbuildable_images:
|
||||
for image in self.images:
|
||||
if image.name in unbuildable_images:
|
||||
image.status = Status.UNBUILDABLE
|
||||
else:
|
||||
# let's check ancestors
|
||||
# if any of them is unbuildable then we mark it
|
||||
# and then mark image
|
||||
build_image = True
|
||||
ancestor_image = image
|
||||
while (ancestor_image.parent is not None):
|
||||
ancestor_image = ancestor_image.parent
|
||||
if ancestor_image.name in unbuildable_images or \
|
||||
ancestor_image.status == Status.UNBUILDABLE:
|
||||
build_image = False
|
||||
ancestor_image.status = Status.UNBUILDABLE
|
||||
break
|
||||
if not build_image:
|
||||
image.status = Status.UNBUILDABLE
|
||||
|
||||
# When we want to build a subset of images then filter_ part kicks in.
|
||||
# Otherwise we just mark everything buildable as matched for build.
|
||||
|
||||
# First, determine which buildable images match.
|
||||
if filter_:
|
||||
patterns = re.compile(r"|".join(filter_).join('()'))
|
||||
for image in self.images:
|
||||
# as we now list not buildable/skipped images we need to
|
||||
# process them otherwise list will contain also not requested
|
||||
# entries
|
||||
if image.status in (Status.MATCHED, Status.UNBUILDABLE):
|
||||
continue
|
||||
if re.search(patterns, image.name):
|
||||
image.status = Status.MATCHED
|
||||
|
||||
ancestor_image = image
|
||||
while (ancestor_image.parent is not None and
|
||||
ancestor_image.parent.status != Status.MATCHED):
|
||||
ancestor_image = ancestor_image.parent
|
||||
# Parents of a buildable image must also be buildable.
|
||||
ancestor_image.status = Status.MATCHED
|
||||
LOG.debug('Image %s matched regex', image.name)
|
||||
else:
|
||||
image.status = Status.UNMATCHED
|
||||
else:
|
||||
for image in self.images:
|
||||
if image.status != Status.UNBUILDABLE:
|
||||
image.status = Status.MATCHED
|
||||
|
||||
# Next, mark any skipped images.
|
||||
for image in self.images:
|
||||
if image.status != Status.MATCHED:
|
||||
continue
|
||||
# Skip image if --skip-existing was given and image exists.
|
||||
if (self.conf.skip_existing and image.in_docker_cache()):
|
||||
LOG.debug('Skipping existing image %s', image.name)
|
||||
image.status = Status.SKIPPED
|
||||
# Skip image if --skip-parents was given and image has children.
|
||||
elif self.conf.skip_parents and image.children:
|
||||
LOG.debug('Skipping parent image %s', image.name)
|
||||
image.status = Status.SKIPPED
|
||||
|
||||
def summary(self):
|
||||
"""Walk the dictionary of images statuses and print results."""
|
||||
# For debug we print the logs again if the image error'd. This is to
|
||||
# help us debug and it will be extra helpful in the gate.
|
||||
for image in self.images:
|
||||
if image.status in STATUS_ERRORS:
|
||||
LOG.debug("Image %s failed", image.name)
|
||||
|
||||
self.get_image_statuses()
|
||||
results = {
|
||||
'built': [],
|
||||
'failed': [],
|
||||
'not_matched': [],
|
||||
'skipped': [],
|
||||
'unbuildable': [],
|
||||
}
|
||||
|
||||
if self.image_statuses_good:
|
||||
LOG.info("=========================")
|
||||
LOG.info("Successfully built images")
|
||||
LOG.info("=========================")
|
||||
for name in sorted(self.image_statuses_good.keys()):
|
||||
LOG.info(name)
|
||||
results['built'].append({
|
||||
'name': name,
|
||||
})
|
||||
|
||||
if self.image_statuses_bad or self.image_statuses_allowed_to_fail:
|
||||
LOG.info("===========================")
|
||||
LOG.info("Images that failed to build")
|
||||
LOG.info("===========================")
|
||||
all_bad_statuses = self.image_statuses_bad.copy()
|
||||
all_bad_statuses.update(self.image_statuses_allowed_to_fail)
|
||||
for name, status in sorted(all_bad_statuses.items()):
|
||||
if name in self.image_statuses_allowed_to_fail:
|
||||
LOG.error('%s Failed with status: %s (allowed to fail)',
|
||||
name, status.value)
|
||||
else:
|
||||
LOG.error('%s Failed with status: %s', name, status.value)
|
||||
|
||||
results['failed'].append({
|
||||
'name': name,
|
||||
'status': status.value,
|
||||
})
|
||||
if self.conf.logs_dir and status == Status.ERROR:
|
||||
linkname = os.path.join(self.conf.logs_dir,
|
||||
"000_FAILED_%s.log" % name)
|
||||
try:
|
||||
os.lstat(linkname)
|
||||
os.remove(linkname)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
os.symlink("%s.log" % name, linkname)
|
||||
|
||||
if self.image_statuses_unmatched:
|
||||
LOG.debug("=====================================")
|
||||
LOG.debug("Images not matched for build by regex")
|
||||
LOG.debug("=====================================")
|
||||
for name in sorted(self.image_statuses_unmatched.keys()):
|
||||
LOG.debug(name)
|
||||
results['not_matched'].append({
|
||||
'name': name,
|
||||
})
|
||||
|
||||
if self.image_statuses_skipped:
|
||||
LOG.info("===================================")
|
||||
LOG.info("Images skipped due to build options")
|
||||
LOG.info("===================================")
|
||||
for name in sorted(self.image_statuses_skipped.keys()):
|
||||
LOG.info(name)
|
||||
results['skipped'].append({
|
||||
'name': name,
|
||||
})
|
||||
|
||||
if self.image_statuses_unbuildable:
|
||||
LOG.info("=========================================")
|
||||
LOG.info("Images not buildable due to build options")
|
||||
LOG.info("=========================================")
|
||||
for name in sorted(self.image_statuses_unbuildable.keys()):
|
||||
LOG.info(name)
|
||||
results['unbuildable'].append({
|
||||
'name': name,
|
||||
})
|
||||
|
||||
return results
|
||||
|
||||
def get_image_statuses(self):
|
||||
if any([self.image_statuses_bad,
|
||||
self.image_statuses_good,
|
||||
self.image_statuses_unmatched,
|
||||
self.image_statuses_skipped,
|
||||
self.image_statuses_unbuildable,
|
||||
self.image_statuses_allowed_to_fail]):
|
||||
return (self.image_statuses_bad,
|
||||
self.image_statuses_good,
|
||||
self.image_statuses_unmatched,
|
||||
self.image_statuses_skipped,
|
||||
self.image_statuses_unbuildable,
|
||||
self.image_statuses_allowed_to_fail)
|
||||
for image in self.images:
|
||||
if image.status == Status.BUILT:
|
||||
self.image_statuses_good[image.name] = image.status
|
||||
elif image.status == Status.UNMATCHED:
|
||||
self.image_statuses_unmatched[image.name] = image.status
|
||||
elif image.status == Status.SKIPPED:
|
||||
self.image_statuses_skipped[image.name] = image.status
|
||||
elif image.status == Status.UNBUILDABLE:
|
||||
self.image_statuses_unbuildable[image.name] = image.status
|
||||
else:
|
||||
if image.name in self.conf.allowed_to_fail:
|
||||
self.image_statuses_allowed_to_fail[
|
||||
image.name] = image.status
|
||||
else:
|
||||
self.image_statuses_bad[image.name] = image.status
|
||||
return (self.image_statuses_bad,
|
||||
self.image_statuses_good,
|
||||
self.image_statuses_unmatched,
|
||||
self.image_statuses_skipped,
|
||||
self.image_statuses_unbuildable,
|
||||
self.image_statuses_allowed_to_fail)
|
||||
|
||||
def build_image_list(self):
|
||||
def process_source_installation(image, section):
|
||||
installation = dict()
|
||||
# NOTE(jeffrey4l): source is not needed when the type is None
|
||||
if self.conf._get('type', self.conf._get_group(section)) is None:
|
||||
if image.parent_name is None:
|
||||
LOG.debug('No source location found in section %s',
|
||||
section)
|
||||
else:
|
||||
installation['type'] = self.conf[section]['type']
|
||||
installation['source'] = self.conf[section]['location']
|
||||
installation['name'] = section
|
||||
if installation['type'] == 'git':
|
||||
installation['reference'] = self.conf[section]['reference']
|
||||
installation['enabled'] = self.conf[section]['enabled']
|
||||
return installation
|
||||
|
||||
all_sections = (set(self.conf._groups.keys()) |
|
||||
set(self.conf.list_all_sections()))
|
||||
|
||||
for path in self.docker_build_paths:
|
||||
# Reading parent image name
|
||||
with open(os.path.join(path, 'Dockerfile')) as f:
|
||||
content = f.read()
|
||||
|
||||
image_name = os.path.basename(path)
|
||||
canonical_name = (self.namespace + '/' + self.image_prefix +
|
||||
image_name + ':' + self.tag)
|
||||
parent_search_pattern = re.compile(r'^FROM.*$', re.MULTILINE)
|
||||
match = re.search(parent_search_pattern, content)
|
||||
if match:
|
||||
parent_name = match.group(0).split(' ')[1]
|
||||
else:
|
||||
parent_name = ''
|
||||
del match
|
||||
image = Image(image_name, canonical_name, path,
|
||||
parent_name=parent_name,
|
||||
logger=utils.make_a_logger(self.conf, image_name),
|
||||
docker_client=self.dc)
|
||||
|
||||
# NOTE(jeffrey4l): register the opts if the section didn't
|
||||
# register in the kolla/common/config.py file
|
||||
if image.name not in self.conf._groups:
|
||||
self.conf.register_opts(common_config.get_source_opts(),
|
||||
image.name)
|
||||
image.source = process_source_installation(image, image.name)
|
||||
for plugin in [match.group(0) for match in
|
||||
(re.search('^{}-plugin-.+'.format(image.name),
|
||||
section) for section in
|
||||
all_sections) if match]:
|
||||
try:
|
||||
self.conf.register_opts(
|
||||
common_config.get_source_opts(),
|
||||
plugin
|
||||
)
|
||||
except cfg.DuplicateOptError:
|
||||
LOG.debug('Plugin %s already registered in config',
|
||||
plugin)
|
||||
image.plugins.append(
|
||||
process_source_installation(image, plugin))
|
||||
for addition in [
|
||||
match.group(0) for match in
|
||||
(re.search('^{}-additions-.+'.format(image.name),
|
||||
section) for section in all_sections) if match]:
|
||||
try:
|
||||
self.conf.register_opts(
|
||||
common_config.get_source_opts(),
|
||||
addition
|
||||
)
|
||||
except cfg.DuplicateOptError:
|
||||
LOG.debug('Addition %s already registered in config',
|
||||
addition)
|
||||
image.additions.append(
|
||||
process_source_installation(image, addition))
|
||||
|
||||
self.images.append(image)
|
||||
|
||||
def save_dependency(self, to_file):
|
||||
try:
|
||||
import graphviz
|
||||
except ImportError:
|
||||
LOG.error('"graphviz" is required for save dependency')
|
||||
raise
|
||||
dot = graphviz.Digraph(comment='Docker Images Dependency')
|
||||
dot.body.extend(['rankdir=LR'])
|
||||
for image in self.images:
|
||||
if image.status not in [Status.MATCHED]:
|
||||
continue
|
||||
dot.node(image.name)
|
||||
if image.parent is not None:
|
||||
dot.edge(image.parent.name, image.name)
|
||||
|
||||
with open(to_file, 'w') as f:
|
||||
f.write(dot.source)
|
||||
|
||||
def list_images(self):
|
||||
for count, image in enumerate([
|
||||
image for image in self.images if image.status == Status.MATCHED
|
||||
]):
|
||||
print(count + 1, ':', image.name)
|
||||
|
||||
def list_dependencies(self):
|
||||
match = False
|
||||
for image in self.images:
|
||||
if image.status in [Status.MATCHED]:
|
||||
match = True
|
||||
if image.parent is None:
|
||||
base = image
|
||||
if not match:
|
||||
print('Nothing matched!')
|
||||
return
|
||||
|
||||
def list_children(images, ancestry):
|
||||
children = next(iter(ancestry.values()))
|
||||
for image in images:
|
||||
if image.status not in [Status.MATCHED]:
|
||||
continue
|
||||
if not image.children:
|
||||
children.append(image.name)
|
||||
else:
|
||||
newparent = {image.name: []}
|
||||
children.append(newparent)
|
||||
list_children(image.children, newparent)
|
||||
|
||||
ancestry = {base.name: []}
|
||||
list_children(base.children, ancestry)
|
||||
json.dump(ancestry, sys.stdout, indent=2)
|
||||
|
||||
def find_parents(self):
|
||||
"""Associate all images with parents and children."""
|
||||
sort_images = dict()
|
||||
|
||||
for image in self.images:
|
||||
sort_images[image.canonical_name] = image
|
||||
|
||||
for parent_name, parent in sort_images.items():
|
||||
for image in sort_images.values():
|
||||
if (image.parent_name == parent_name):
|
||||
parent.children.append(image)
|
||||
image.parent = parent
|
||||
|
||||
def build_queue(self, push_queue):
|
||||
"""Organizes Queue list.
|
||||
|
||||
Return a list of Queues that have been organized into a hierarchy
|
||||
based on dependencies
|
||||
"""
|
||||
build_queue = queue.Queue()
|
||||
|
||||
for image in self.images:
|
||||
if image.status in (Status.UNMATCHED, Status.SKIPPED,
|
||||
Status.UNBUILDABLE):
|
||||
# Don't bother queuing up build tasks for things that
|
||||
# were not matched in the first place... (not worth the
|
||||
# effort to run them, if they won't be used anyway).
|
||||
continue
|
||||
# Build all root nodes, where a root is defined as having no parent
|
||||
# or having a parent that is explicitly being skipped.
|
||||
if image.parent is None or image.parent.status == Status.SKIPPED:
|
||||
build_queue.put(BuildTask(self.conf, image, push_queue))
|
||||
LOG.info('Added image %s to queue', image.name)
|
||||
|
||||
return build_queue
|
||||
|
|
@ -18,32 +18,33 @@ from unittest import mock
|
|||
|
||||
from kolla.cmd import build as build_cmd
|
||||
from kolla.image import build
|
||||
from kolla.image.kolla_worker import Image
|
||||
from kolla.image import tasks
|
||||
from kolla.image import utils
|
||||
from kolla.tests import base
|
||||
|
||||
|
||||
FAKE_IMAGE = build.Image(
|
||||
FAKE_IMAGE = Image(
|
||||
'image-base', 'image-base:latest',
|
||||
'/fake/path', parent_name=None,
|
||||
parent=None, status=utils.Status.MATCHED)
|
||||
FAKE_IMAGE_CHILD = build.Image(
|
||||
FAKE_IMAGE_CHILD = Image(
|
||||
'image-child', 'image-child:latest',
|
||||
'/fake/path2', parent_name='image-base',
|
||||
parent=FAKE_IMAGE, status=utils.Status.MATCHED)
|
||||
FAKE_IMAGE_CHILD_UNMATCHED = build.Image(
|
||||
FAKE_IMAGE_CHILD_UNMATCHED = Image(
|
||||
'image-child-unmatched', 'image-child-unmatched:latest',
|
||||
'/fake/path3', parent_name='image-base',
|
||||
parent=FAKE_IMAGE, status=utils.Status.UNMATCHED)
|
||||
FAKE_IMAGE_CHILD_ERROR = build.Image(
|
||||
FAKE_IMAGE_CHILD_ERROR = Image(
|
||||
'image-child-error', 'image-child-error:latest',
|
||||
'/fake/path4', parent_name='image-base',
|
||||
parent=FAKE_IMAGE, status=utils.Status.ERROR)
|
||||
FAKE_IMAGE_CHILD_BUILT = build.Image(
|
||||
FAKE_IMAGE_CHILD_BUILT = Image(
|
||||
'image-child-built', 'image-child-built:latest',
|
||||
'/fake/path5', parent_name='image-base',
|
||||
parent=FAKE_IMAGE, status=utils.Status.BUILT)
|
||||
FAKE_IMAGE_GRANDCHILD = build.Image(
|
||||
FAKE_IMAGE_GRANDCHILD = Image(
|
||||
'image-grandchild', 'image-grandchild:latest',
|
||||
'/fake/path6', parent_name='image-child',
|
||||
parent=FAKE_IMAGE_CHILD, status=utils.Status.MATCHED)
|
||||
|
|
@ -492,7 +493,7 @@ class KollaWorkerTest(base.TestCase):
|
|||
self.assertEqual(utils.Status.SKIPPED, kolla.images[2].parent.status)
|
||||
self.assertEqual(utils.Status.SKIPPED, kolla.images[1].parent.status)
|
||||
|
||||
@mock.patch.object(build.Image, 'in_docker_cache')
|
||||
@mock.patch.object(Image, 'in_docker_cache')
|
||||
def test_skip_existing(self, mock_in_cache):
|
||||
mock_in_cache.side_effect = [True, False]
|
||||
self.conf.set_override('skip_existing', True)
|
||||
|
|
|
|||
Loading…
Reference in New Issue