385 lines
14 KiB
Python
385 lines
14 KiB
Python
# Copyright 2019 Red Hat, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
#
|
|
|
|
|
|
from concurrent import futures
|
|
import os
|
|
import pathlib
|
|
import six
|
|
import tenacity
|
|
|
|
from oslo_concurrency import processutils
|
|
from oslo_config import cfg
|
|
from oslo_log import log as logging
|
|
|
|
from tripleo_common import constants
|
|
from tripleo_common.image.builder import base
|
|
from tripleo_common.utils import process
|
|
|
|
CONF = cfg.CONF
|
|
LOG = logging.getLogger(__name__ + ".BuildahBuilder")
|
|
|
|
|
|
class BuildahBuilder(base.BaseBuilder):
|
|
"""Builder to build container images with Buildah."""
|
|
|
|
log = LOG
|
|
|
|
def __init__(self, work_dir, deps, base='fedora', img_type='binary',
|
|
tag='latest', namespace='master',
|
|
registry_address='127.0.0.1:8787', push_containers=True,
|
|
volumes=[], excludes=[], build_timeout=None, debug=False):
|
|
"""Setup the parameters to build with Buildah.
|
|
|
|
:params work_dir: Directory where the Dockerfiles or Containerfiles
|
|
are generated by Kolla.
|
|
:params deps: Dictionary defining the container images
|
|
dependencies.
|
|
:params base: Base image on which the containers are built.
|
|
Default to fedora.
|
|
:params img_type: Method used to build the image. All TripleO images
|
|
are built from binary method. Can be set to false to remove it
|
|
from image name.
|
|
:params tag: Tag used to identify the images that we build.
|
|
Default to latest.
|
|
:params namespace: Namespace used to build the containers.
|
|
Default to master.
|
|
:params registry_address: IP + port of the registry where we push
|
|
the images. Default is 127.0.0.1:8787.
|
|
:params push: Flag to bypass registry push if False. Default is True
|
|
:params volumes: Bind mount volumes used during buildah bud.
|
|
Default to [].
|
|
:params excludes: List of images to skip. Default to [].
|
|
:params build_timeout: Timeout. Default to constants.BUILD_TIMEOUT
|
|
:params debug: Enable debug flag. Default to False.
|
|
"""
|
|
|
|
logging.register_options(CONF)
|
|
if debug:
|
|
CONF.debug = True
|
|
logging.setup(CONF, '')
|
|
|
|
super(BuildahBuilder, self).__init__()
|
|
if build_timeout is None:
|
|
self.build_timeout = constants.BUILD_TIMEOUT
|
|
else:
|
|
self.build_timeout = build_timeout
|
|
self.work_dir = work_dir
|
|
self.deps = deps
|
|
self.base = base
|
|
self.img_type = img_type
|
|
self.tag = tag
|
|
self.namespace = namespace
|
|
self.registry_address = registry_address
|
|
self.push_containers = push_containers
|
|
self.volumes = volumes
|
|
self.excludes = excludes
|
|
self.debug = debug
|
|
# Each container image has a Dockerfile or a Containerfile.
|
|
# Buildah needs to know the base directory later.
|
|
self.cont_map = {os.path.basename(root): root for root, dirs,
|
|
fnames in os.walk(self.work_dir)
|
|
if 'Dockerfile' in fnames or
|
|
'Containerfile' in fnames}
|
|
# Building images with root so overlayfs is used, and not fuse-overlay
|
|
# from userspace, which would be slower.
|
|
self.buildah_cmd = ['sudo', 'buildah']
|
|
if self.debug:
|
|
self.buildah_cmd.append('--log-level=debug')
|
|
|
|
def _find_container_dir(self, container_name):
|
|
"""Return the path of the Dockerfile/Containerfile directory.
|
|
|
|
:params container_name: Name of the container.
|
|
"""
|
|
|
|
if container_name not in self.cont_map:
|
|
self.log.error('Container not found in Kolla '
|
|
'deps: %s' % container_name)
|
|
return self.cont_map.get(container_name, '')
|
|
|
|
def _get_destination(self, container_name):
|
|
"""Return the destination of a container image to push.
|
|
|
|
:params container_name: Name of the container.
|
|
"""
|
|
|
|
destination = "{}/{}/{}".format(
|
|
self.registry_address,
|
|
self.namespace,
|
|
self.base,
|
|
)
|
|
if self.img_type:
|
|
destination += '-' + self.img_type
|
|
destination += '-' + container_name + ':' + self.tag
|
|
return destination
|
|
|
|
def _generate_container(self, container_name):
|
|
"""Generate a container image by building and pushing the image.
|
|
|
|
:params container_name: Name of the container.
|
|
"""
|
|
|
|
if container_name in self.excludes:
|
|
return
|
|
|
|
# NOTE(mwhahaha): Use a try catch block so we can better log issues
|
|
# as this is called in a multiprocess fashion so the exception
|
|
# loses some information when it reaches _multi_build
|
|
try:
|
|
self.build(container_name,
|
|
self._find_container_dir(container_name))
|
|
if self.push_containers:
|
|
self.push(self._get_destination(container_name))
|
|
except Exception as e:
|
|
self.log.exception(e)
|
|
raise
|
|
|
|
@tenacity.retry(
|
|
# Retry up to 5 times: 0, 1, 5, 21, 85
|
|
# http://exponentialbackoffcalculator.com/
|
|
reraise=True,
|
|
wait=tenacity.wait_random_exponential(multiplier=4, max=60),
|
|
stop=tenacity.stop_after_attempt(5),
|
|
before_sleep=tenacity.after_log(LOG, logging.WARNING)
|
|
)
|
|
def build(self, container_name, container_build_path):
|
|
"""Build an image from a given directory.
|
|
|
|
:params container_name: Name of the container.
|
|
:params container_build_path: Directory where the Dockerfile or
|
|
Containerfile and other files are located to build the image.
|
|
"""
|
|
|
|
# 'buildah bud' is the command we want because Kolla uses Dockefile to
|
|
# build images.
|
|
# TODO(emilien): Stop ignoring TLS. The deployer should either secure
|
|
# the registry or add it to insecure_registries.
|
|
logfile = container_build_path + '/' + container_name + '-build.log'
|
|
|
|
# TODO(ramishra) Hack to make the logfile readable by current user,
|
|
# as we're running buildah as root. This would be removed once we
|
|
# move to rootless buildah.
|
|
pathlib.Path(logfile).touch()
|
|
|
|
bud_args = ['bud']
|
|
for v in self.volumes:
|
|
bud_args.extend(['--volume', v])
|
|
if self.debug:
|
|
# TODO(bogdando): add --log-rusage for newer buildah
|
|
bud_args.extend(['--loglevel=3'])
|
|
# TODO(aschultz): drop --format docker when oci format is properly
|
|
# supported by the undercloud registry
|
|
bud_args.extend(['--format', 'docker', '--tls-verify=False',
|
|
'--logfile', logfile, '-t',
|
|
self._get_destination(container_name),
|
|
container_build_path])
|
|
args = self.buildah_cmd + bud_args
|
|
self.log.info("Building %s image with: %s" %
|
|
(container_name, ' '.join(args)))
|
|
process.execute(
|
|
*args,
|
|
check_exit_code=True,
|
|
run_as_root=False,
|
|
use_standard_locale=True
|
|
)
|
|
|
|
@tenacity.retry( # Retry up to 10 times with jittered exponential backoff
|
|
reraise=True,
|
|
wait=tenacity.wait_random_exponential(multiplier=1, max=15),
|
|
stop=tenacity.stop_after_attempt(10),
|
|
before_sleep=tenacity.after_log(LOG, logging.WARNING)
|
|
)
|
|
def push(self, destination):
|
|
"""Push an image to a container registry.
|
|
|
|
:params destination: URL to used to push the container. It contains
|
|
the registry address, namespace, base, img_type (optional),
|
|
container name and tag.
|
|
"""
|
|
# TODO(emilien): Stop ignoring TLS. The deployer should either secure
|
|
# the registry or add it to insecure_registries.
|
|
# TODO(emilien) We need to figure out how we can push to something
|
|
# else than a Docker registry.
|
|
args = self.buildah_cmd + ['push', '--tls-verify=False', destination,
|
|
'docker://' + destination]
|
|
self.log.info("Pushing %s image with: %s" %
|
|
(destination, ' '.join(args)))
|
|
if self.debug:
|
|
# buildah push logs to stderr, since there is no --log* opt
|
|
# so we'll use the current logging context for that
|
|
process.execute(*args, log_stdout=True, run_as_root=False,
|
|
use_standard_locale=True, logger=self.log,
|
|
loglevel=logging.DEBUG)
|
|
else:
|
|
process.execute(*args, run_as_root=False,
|
|
use_standard_locale=True)
|
|
|
|
def build_all(self, deps=None):
|
|
"""Build all containers.
|
|
|
|
This function will thread the build process allowing it to complete
|
|
in the shortest possible time.
|
|
|
|
:params deps: Dictionary defining the container images
|
|
dependencies.
|
|
"""
|
|
|
|
if deps is None:
|
|
deps = self.deps
|
|
|
|
container_deps = self._generate_deps(deps=deps, containers=list())
|
|
self.log.debug("All container deps: {}".format(container_deps))
|
|
for containers in container_deps:
|
|
self.log.info("Processing containers: {}".format(containers))
|
|
if isinstance(deps, (list,)):
|
|
self._multi_build(containers=containers)
|
|
else:
|
|
self._multi_build(containers=[containers])
|
|
|
|
def _generate_deps(self, deps, containers, prio_list=None):
|
|
"""Browse containers dependencies and return an an array.
|
|
|
|
When the dependencies are generated they're captured in an array,
|
|
which contains additional arrays. This data structure is later
|
|
used in a futures queue.
|
|
|
|
:params deps: Dictionary defining the container images
|
|
dependencies.
|
|
:params containers: List used to keep track of dependent containers.
|
|
:params prio_list: List used to keep track of nested dependencies.
|
|
:returns: list
|
|
"""
|
|
|
|
self.log.debug("Process deps: {}".format(deps))
|
|
if isinstance(deps, (six.string_types,)):
|
|
if prio_list:
|
|
prio_list.append(deps)
|
|
else:
|
|
containers.append([deps])
|
|
|
|
elif isinstance(deps, (dict,)):
|
|
parents = list(deps.keys())
|
|
if prio_list:
|
|
prio_list.extend(parents)
|
|
else:
|
|
containers.append(parents)
|
|
for value in deps.values():
|
|
self.log.debug("Recursing with: {}".format(value))
|
|
self._generate_deps(
|
|
deps=value,
|
|
containers=containers
|
|
)
|
|
|
|
elif isinstance(deps, (list,)):
|
|
dep_list = list()
|
|
dep_rehash_list = list()
|
|
for item in deps:
|
|
if isinstance(item, (six.string_types,)):
|
|
dep_list.append(item)
|
|
else:
|
|
dep_rehash_list.append(item)
|
|
|
|
if dep_list:
|
|
containers.append(dep_list)
|
|
|
|
for item in dep_rehash_list:
|
|
self.log.debug("Recursing with: {}".format(item))
|
|
self._generate_deps(
|
|
deps=item,
|
|
containers=containers,
|
|
prio_list=dep_list
|
|
)
|
|
|
|
self.log.debug("Constructed containers: {}".format(containers))
|
|
return containers
|
|
|
|
def _multi_build(self, containers):
|
|
"""Build mutliple containers.
|
|
|
|
Multi-thread the build process for all containers defined within
|
|
the containers list.
|
|
|
|
:params containers: List defining the container images.
|
|
"""
|
|
|
|
# Workers will use the processor core count with a max of 8. If
|
|
# the containers array has a length less-than the expected processor
|
|
# count, the workers will be adjusted to meet the expectations of the
|
|
# work being processed.
|
|
workers = min(
|
|
min(
|
|
8,
|
|
max(
|
|
2,
|
|
processutils.get_worker_count()
|
|
)
|
|
),
|
|
len(containers)
|
|
)
|
|
with futures.ThreadPoolExecutor(max_workers=workers) as executor:
|
|
future_to_build = {
|
|
executor.submit(
|
|
self._generate_container, container_name
|
|
): container_name for container_name in containers
|
|
}
|
|
done, not_done = futures.wait(
|
|
future_to_build,
|
|
timeout=self.build_timeout,
|
|
return_when=futures.FIRST_EXCEPTION
|
|
)
|
|
|
|
# NOTE(cloudnull): Once the job has been completed all completed
|
|
# jobs are checked for exceptions. If any jobs
|
|
# failed a SystemError will be raised using the
|
|
# exception information. If any job was loaded
|
|
# but not executed a SystemError will be raised.
|
|
exceptions = list()
|
|
for job in done:
|
|
if job._exception:
|
|
exceptions.append(
|
|
"\nException information: {exception}".format(
|
|
exception=job._exception
|
|
)
|
|
)
|
|
|
|
if exceptions:
|
|
raise RuntimeError(
|
|
'\nThe following errors were detected during '
|
|
'container build(s):\n{exceptions}'.format(
|
|
exceptions='\n'.join(exceptions)
|
|
)
|
|
)
|
|
|
|
if not_done:
|
|
error_msg = (
|
|
'The following jobs were incomplete: {}'.format(
|
|
[future_to_build[job] for job in not_done]
|
|
)
|
|
)
|
|
|
|
jobs_with_exceptions = [{
|
|
'container': future_to_build[job],
|
|
'exception': job._exception}
|
|
for job in not_done if job._exception]
|
|
if jobs_with_exceptions:
|
|
for job_with_exception in jobs_with_exceptions:
|
|
error_msg = error_msg + os.linesep + (
|
|
"%(container)s raised the following "
|
|
"exception: %(exception)s" %
|
|
job_with_exception)
|
|
|
|
raise SystemError(error_msg)
|