Merge "Add a way to add other container engines"
This commit is contained in:
Zuul
Gerrit Code Review
commit
35088cf281
|
|
@ -156,7 +156,8 @@ _CLI_OPTS = [
|
|||
cfg.BoolOpt('skip-parents', default=False,
|
||||
help='Do not rebuild parents of matched images'),
|
||||
cfg.BoolOpt('skip-existing', default=False,
|
||||
help='Do not rebuild images present in the docker cache'),
|
||||
help='Do not rebuild images present in the container engine '
|
||||
'cache'),
|
||||
cfg.DictOpt('build-args',
|
||||
help='Set docker build time variables'),
|
||||
cfg.BoolOpt('keep', default=False,
|
||||
|
|
@ -172,7 +173,7 @@ _CLI_OPTS = [
|
|||
cfg.StrOpt('network_mode', default='host',
|
||||
help='The network mode for Docker build. Example: host'),
|
||||
cfg.BoolOpt('cache', default=True,
|
||||
help='Use the Docker cache when building'),
|
||||
help='Use the container engine cache when building'),
|
||||
cfg.MultiOpt('profile', types.String(), short='p',
|
||||
help=('Build a pre-defined set of images, see [profiles]'
|
||||
' section in config. The default profiles are:'
|
||||
|
|
@ -190,14 +191,14 @@ _CLI_OPTS = [
|
|||
help=('Build only images matching regex and its'
|
||||
' dependencies')),
|
||||
cfg.StrOpt('registry',
|
||||
help=('The docker registry host. The default registry host'
|
||||
' is Docker Hub')),
|
||||
help=('The container image registry host. The default registry'
|
||||
' host is Docker Hub')),
|
||||
cfg.StrOpt('save-dependency',
|
||||
help=('Path to the file to store the docker image'
|
||||
' dependency in Graphviz dot format')),
|
||||
cfg.StrOpt('format', short='f', default='json',
|
||||
choices=['json', 'none'],
|
||||
help='Format to write the final results in'),
|
||||
help='Format to write the final results in.'),
|
||||
cfg.StrOpt('tarballs-base', default=TARBALLS_BASE,
|
||||
help='Base url to OpenStack tarballs'),
|
||||
cfg.IntOpt('threads', short='T', default=8, min=1,
|
||||
|
|
@ -205,7 +206,7 @@ _CLI_OPTS = [
|
|||
' (Note: setting to one will allow real time'
|
||||
' logging)')),
|
||||
cfg.StrOpt('tag', default=version.cached_version_string(),
|
||||
help='The Docker tag'),
|
||||
help='The container image tag'),
|
||||
cfg.BoolOpt('template-only', default=False,
|
||||
help="Don't build images. Generate Dockerfile only"),
|
||||
cfg.IntOpt('timeout', default=120,
|
||||
|
|
@ -246,6 +247,8 @@ _CLI_OPTS = [
|
|||
help='Prefix prepended to image names'),
|
||||
cfg.StrOpt('repos-yaml', default='',
|
||||
help='Path to alternative repos.yaml file'),
|
||||
cfg.StrOpt('engine', default='docker', choices=['docker'],
|
||||
help='Container engine to build images on.')
|
||||
]
|
||||
|
||||
_BASE_OPTS = [
|
||||
|
|
|
|||
|
|
@ -0,0 +1,57 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from distutils.version import StrictVersion
|
||||
from enum import Enum
|
||||
from kolla.image.utils import LOG
|
||||
|
||||
try:
|
||||
import docker
|
||||
except (ImportError):
|
||||
LOG.debug("Docker python library was not found")
|
||||
|
||||
|
||||
class Engine(Enum):
|
||||
|
||||
DOCKER = "docker"
|
||||
|
||||
|
||||
class UnsupportedEngineError(ValueError):
|
||||
|
||||
def __init__(self, engine_name):
|
||||
super().__init__()
|
||||
self.engine_name = engine_name
|
||||
|
||||
def __str__(self):
|
||||
return f'Unsupported engine name given: "{self.engine_name}"'
|
||||
|
||||
|
||||
def getEngineException(conf):
|
||||
if conf.engine == Engine.DOCKER.value:
|
||||
return (docker.errors.DockerException)
|
||||
else:
|
||||
raise UnsupportedEngineError(conf.engine)
|
||||
|
||||
|
||||
def getEngineClient(conf):
|
||||
if conf.engine == Engine.DOCKER.value:
|
||||
kwargs_env = docker.utils.kwargs_from_env()
|
||||
return docker.APIClient(version='auto', **kwargs_env)
|
||||
else:
|
||||
raise UnsupportedEngineError(conf.engine)
|
||||
|
||||
|
||||
def getEngineVersion(conf):
|
||||
if conf.engine == Engine.DOCKER.value:
|
||||
return StrictVersion(docker.version)
|
||||
else:
|
||||
raise UnsupportedEngineError(conf.engine)
|
||||
|
|
@ -21,6 +21,7 @@ import time
|
|||
|
||||
from kolla.common import config as common_config
|
||||
from kolla.common import utils
|
||||
from kolla.engine_adapter import engine
|
||||
from kolla.image.kolla_worker import KollaWorker
|
||||
from kolla.image.utils import LOG
|
||||
from kolla.image.utils import Status
|
||||
|
|
@ -104,10 +105,28 @@ def run_build():
|
|||
if conf.debug:
|
||||
LOG.setLevel(logging.DEBUG)
|
||||
|
||||
if conf.squash:
|
||||
squash_version = utils.get_docker_squash_version()
|
||||
LOG.info('Image squash is enabled and "docker-squash" version is %s',
|
||||
squash_version)
|
||||
if conf.engine not in (engine.Engine.DOCKER.value,):
|
||||
LOG.error(f'Unsupported engine name "{conf.engine}", exiting.')
|
||||
sys.exit(1)
|
||||
LOG.info(f'Using engine: {conf.engine}')
|
||||
|
||||
if conf.engine == engine.Engine.DOCKER.value:
|
||||
try:
|
||||
import docker
|
||||
docker.version
|
||||
except ImportError:
|
||||
LOG.error("Error, you have set Docker as container engine, "
|
||||
"but the Python library is not found."
|
||||
"Try running 'pip install docker'")
|
||||
sys.exit(1)
|
||||
except AttributeError:
|
||||
LOG.error("Error, Docker Python library is too old, "
|
||||
"Try running 'pip install docker --upgrade'")
|
||||
|
||||
if conf.squash:
|
||||
squash_version = utils.get_docker_squash_version()
|
||||
LOG.info('Image squash is enabled and "docker-squash" version '
|
||||
'is %s', squash_version)
|
||||
|
||||
kolla = KollaWorker(conf)
|
||||
kolla.setup_working_dir()
|
||||
|
|
@ -133,7 +152,7 @@ def run_build():
|
|||
|
||||
if conf.save_dependency:
|
||||
kolla.save_dependency(conf.save_dependency)
|
||||
LOG.info('Docker images dependency are saved in %s',
|
||||
LOG.info('Container images dependency are saved in %s',
|
||||
conf.save_dependency)
|
||||
return
|
||||
if conf.list_images:
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@
|
|||
# limitations under the License.
|
||||
|
||||
import datetime
|
||||
import docker
|
||||
import json
|
||||
import os
|
||||
import queue
|
||||
|
|
@ -24,6 +23,7 @@ import time
|
|||
import jinja2
|
||||
from kolla.common import config as common_config
|
||||
from kolla.common import utils
|
||||
from kolla.engine_adapter import engine
|
||||
from kolla import exception
|
||||
from kolla.image.tasks import BuildTask
|
||||
from kolla.image.unbuildable import UNBUILDABLE_IMAGES
|
||||
|
|
@ -42,7 +42,7 @@ PROJECT_ROOT = os.path.abspath(os.path.join(
|
|||
class Image(object):
|
||||
def __init__(self, name, canonical_name, path, parent_name='',
|
||||
status=Status.UNPROCESSED, parent=None,
|
||||
source=None, logger=None, docker_client=None):
|
||||
source=None, logger=None, engine_client=None):
|
||||
self.name = name
|
||||
self.canonical_name = canonical_name
|
||||
self.path = path
|
||||
|
|
@ -56,7 +56,7 @@ class Image(object):
|
|||
self.children = []
|
||||
self.plugins = []
|
||||
self.additions = []
|
||||
self.dc = docker_client
|
||||
self.engine_client = engine_client
|
||||
|
||||
def copy(self):
|
||||
c = Image(self.name, self.canonical_name, self.path,
|
||||
|
|
@ -72,8 +72,9 @@ class Image(object):
|
|||
c.additions = list(self.additions)
|
||||
return c
|
||||
|
||||
def in_docker_cache(self):
|
||||
return len(self.dc.images(name=self.canonical_name, quiet=True)) == 1
|
||||
def in_engine_cache(self):
|
||||
return len(self.engine_client.images(name=self.canonical_name,
|
||||
quiet=True)) == 1
|
||||
|
||||
def __repr__(self):
|
||||
return ("Image(%s, %s, %s, parent_name=%s,"
|
||||
|
|
@ -148,16 +149,16 @@ class KollaWorker(object):
|
|||
self.maintainer = conf.maintainer
|
||||
self.distro_python_version = conf.distro_python_version
|
||||
|
||||
docker_kwargs = docker.utils.kwargs_from_env()
|
||||
try:
|
||||
self.dc = docker.APIClient(version='auto', **docker_kwargs)
|
||||
except docker.errors.DockerException as e:
|
||||
self.dc = None
|
||||
self.engine_client = engine.getEngineClient(self.conf)
|
||||
except engine.getEngineException(self.conf) as e:
|
||||
self.engine_client = None
|
||||
if not (conf.template_only or
|
||||
conf.save_dependency or
|
||||
conf.list_images or
|
||||
conf.list_dependencies):
|
||||
LOG.error("Unable to connect to Docker, exiting")
|
||||
LOG.error("Unable to connect to container engine daemon, "
|
||||
"exiting")
|
||||
LOG.info("Exception caught: {0}".format(e))
|
||||
sys.exit(1)
|
||||
|
||||
|
|
@ -179,18 +180,18 @@ class KollaWorker(object):
|
|||
# this is the correct path
|
||||
# TODO(SamYaple): Improve this to make this safer
|
||||
if os.path.exists(os.path.join(image_path, 'base')):
|
||||
LOG.info('Found the docker image folder at %s', image_path)
|
||||
LOG.info('Found the container image folder at %s', image_path)
|
||||
return image_path
|
||||
else:
|
||||
raise exception.KollaDirNotFoundException('Image dir can not '
|
||||
'be found')
|
||||
|
||||
def build_rpm_setup(self, rpm_setup_config):
|
||||
"""Generates a list of docker commands based on provided configuration.
|
||||
"""Generates a list of engine commands based on provided configuration
|
||||
|
||||
:param rpm_setup_config: A list of .rpm or .repo paths or URLs
|
||||
(can be empty)
|
||||
:return: A list of docker commands
|
||||
:return: A list of engine commands
|
||||
"""
|
||||
rpm_setup = list()
|
||||
|
||||
|
|
@ -470,7 +471,7 @@ class KollaWorker(object):
|
|||
if image.status != Status.MATCHED:
|
||||
continue
|
||||
# Skip image if --skip-existing was given and image exists.
|
||||
if (self.conf.skip_existing and image.in_docker_cache()):
|
||||
if (self.conf.skip_existing and image.in_engine_cache()):
|
||||
LOG.debug('Skipping existing image %s', image.name)
|
||||
image.status = Status.SKIPPED
|
||||
# Skip image if --skip-parents was given and image has children.
|
||||
|
|
@ -638,7 +639,7 @@ class KollaWorker(object):
|
|||
image = Image(image_name, canonical_name, path,
|
||||
parent_name=parent_name,
|
||||
logger=utils.make_a_logger(self.conf, image_name),
|
||||
docker_client=self.dc)
|
||||
engine_client=self.engine_client)
|
||||
|
||||
# NOTE(jeffrey4l): register the opts if the section didn't
|
||||
# register in the kolla/common/config.py file
|
||||
|
|
@ -683,7 +684,7 @@ class KollaWorker(object):
|
|||
except ImportError:
|
||||
LOG.error('"graphviz" is required for save dependency')
|
||||
raise
|
||||
dot = graphviz.Digraph(comment='Docker Images Dependency')
|
||||
dot = graphviz.Digraph(comment='Container Images Dependency')
|
||||
dot.body.extend(['rankdir=LR'])
|
||||
for image in self.images:
|
||||
if image.status not in [Status.MATCHED]:
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@
|
|||
# limitations under the License.
|
||||
|
||||
import datetime
|
||||
import docker
|
||||
import errno
|
||||
import os
|
||||
import shutil
|
||||
|
|
@ -23,6 +22,7 @@ from requests import exceptions as requests_exc
|
|||
|
||||
from kolla.common import task # noqa
|
||||
from kolla.common import utils # noqa
|
||||
from kolla.engine_adapter import engine
|
||||
from kolla.image.utils import Status
|
||||
from kolla.image.utils import STATUS_ERRORS
|
||||
|
||||
|
|
@ -31,21 +31,18 @@ class ArchivingError(Exception):
|
|||
pass
|
||||
|
||||
|
||||
class DockerTask(task.Task):
|
||||
|
||||
docker_kwargs = docker.utils.kwargs_from_env()
|
||||
|
||||
def __init__(self):
|
||||
super(DockerTask, self).__init__()
|
||||
self._dc = None
|
||||
class EngineTask(task.Task):
|
||||
def __init__(self, conf):
|
||||
super(EngineTask, self).__init__()
|
||||
self._ec = None
|
||||
self.conf = conf
|
||||
|
||||
@property
|
||||
def dc(self):
|
||||
if self._dc is not None:
|
||||
return self._dc
|
||||
docker_kwargs = self.docker_kwargs.copy()
|
||||
self._dc = docker.APIClient(version='auto', **docker_kwargs)
|
||||
return self._dc
|
||||
def engine_client(self):
|
||||
if self._ec is not None:
|
||||
return self._ec
|
||||
self._ec = engine.getEngineClient(self.conf)
|
||||
return self._ec
|
||||
|
||||
|
||||
class PushIntoQueueTask(task.Task):
|
||||
|
|
@ -70,11 +67,11 @@ class PushError(Exception):
|
|||
pass
|
||||
|
||||
|
||||
class PushTask(DockerTask):
|
||||
"""Task that pushes an image to a docker repository."""
|
||||
class PushTask(EngineTask):
|
||||
"""Task that pushes an image to a container image repository."""
|
||||
|
||||
def __init__(self, conf, image):
|
||||
super(PushTask, self).__init__()
|
||||
super(PushTask, self).__init__(conf)
|
||||
self.conf = conf
|
||||
self.image = image
|
||||
self.logger = image.logger
|
||||
|
|
@ -89,9 +86,10 @@ class PushTask(DockerTask):
|
|||
try:
|
||||
self.push_image(image)
|
||||
except requests_exc.ConnectionError:
|
||||
self.logger.exception('Make sure Docker is running and that you'
|
||||
' have the correct privileges to run Docker'
|
||||
' (root)')
|
||||
self.logger.exception('Make sure container engine daemon is '
|
||||
'running and that you have the correct '
|
||||
'privileges to run the '
|
||||
'container engine (root)')
|
||||
image.status = Status.CONNECTION_ERROR
|
||||
except PushError as exception:
|
||||
self.logger.error(exception)
|
||||
|
|
@ -110,7 +108,8 @@ class PushTask(DockerTask):
|
|||
def push_image(self, image):
|
||||
kwargs = dict(stream=True, decode=True)
|
||||
|
||||
for response in self.dc.push(image.canonical_name, **kwargs):
|
||||
for response in self.engine_client.push(
|
||||
image.canonical_name, **kwargs):
|
||||
if 'stream' in response:
|
||||
self.logger.info(response['stream'])
|
||||
elif 'errorDetail' in response:
|
||||
|
|
@ -120,11 +119,11 @@ class PushTask(DockerTask):
|
|||
image.status = Status.BUILT
|
||||
|
||||
|
||||
class BuildTask(DockerTask):
|
||||
class BuildTask(EngineTask):
|
||||
"""Task that builds out an image."""
|
||||
|
||||
def __init__(self, conf, image, push_queue):
|
||||
super(BuildTask, self).__init__()
|
||||
super(BuildTask, self).__init__(conf)
|
||||
self.conf = conf
|
||||
self.image = image
|
||||
self.push_queue = push_queue
|
||||
|
|
@ -145,8 +144,9 @@ class BuildTask(DockerTask):
|
|||
followups = []
|
||||
if self.conf.push and self.success:
|
||||
followups.extend([
|
||||
# If we are supposed to push the image into a docker
|
||||
# repository, then make sure we do that...
|
||||
# If we are supposed to push the image into a
|
||||
# container image repository,
|
||||
# then make sure we do that...
|
||||
PushIntoQueueTask(
|
||||
PushTask(self.conf, self.image),
|
||||
self.push_queue),
|
||||
|
|
@ -358,15 +358,16 @@ class BuildTask(DockerTask):
|
|||
|
||||
buildargs = self.update_buildargs()
|
||||
try:
|
||||
for stream in self.dc.build(path=image.path,
|
||||
tag=image.canonical_name,
|
||||
nocache=not self.conf.cache,
|
||||
rm=True,
|
||||
decode=True,
|
||||
network_mode=self.conf.network_mode,
|
||||
pull=pull,
|
||||
forcerm=self.forcerm,
|
||||
buildargs=buildargs):
|
||||
for stream in \
|
||||
self.engine_client.build(path=image.path,
|
||||
tag=image.canonical_name,
|
||||
nocache=not self.conf.cache,
|
||||
rm=True,
|
||||
decode=True,
|
||||
network_mode=self.conf.network_mode,
|
||||
pull=pull,
|
||||
forcerm=self.forcerm,
|
||||
buildargs=buildargs):
|
||||
if 'stream' in stream:
|
||||
for line in stream['stream'].split('\n'):
|
||||
if line:
|
||||
|
|
@ -379,11 +380,13 @@ class BuildTask(DockerTask):
|
|||
self.logger.error('%s', line)
|
||||
return
|
||||
|
||||
if image.status != Status.ERROR and self.conf.squash:
|
||||
if image.status != Status.ERROR and self.conf.squash and \
|
||||
self.conf.engine == engine.Engine.DOCKER.value:
|
||||
self.squash()
|
||||
except docker.errors.DockerException:
|
||||
except engine.getEngineException(self.conf):
|
||||
image.status = Status.ERROR
|
||||
self.logger.exception('Unknown docker error when building')
|
||||
self.logger.exception('Unknown container engine '
|
||||
'error when building')
|
||||
except Exception:
|
||||
image.status = Status.ERROR
|
||||
self.logger.exception('Unknown error when building')
|
||||
|
|
@ -395,9 +398,9 @@ class BuildTask(DockerTask):
|
|||
|
||||
def squash(self):
|
||||
image_tag = self.image.canonical_name
|
||||
image_id = self.dc.inspect_image(image_tag)['Id']
|
||||
image_id = self.engine_client.inspect_image(image_tag)['Id']
|
||||
|
||||
parent_history = self.dc.history(self.image.parent_name)
|
||||
parent_history = self.engine_client.history(self.image.parent_name)
|
||||
parent_last_layer = parent_history[0]['Id']
|
||||
self.logger.info('Parent lastest layer is: %s' % parent_last_layer)
|
||||
|
||||
|
|
|
|||
|
|
@ -493,7 +493,7 @@ class KollaWorkerTest(base.TestCase):
|
|||
self.assertEqual(utils.Status.SKIPPED, kolla.images[2].parent.status)
|
||||
self.assertEqual(utils.Status.SKIPPED, kolla.images[1].parent.status)
|
||||
|
||||
@mock.patch.object(Image, 'in_docker_cache')
|
||||
@mock.patch.object(Image, 'in_engine_cache')
|
||||
def test_skip_existing(self, mock_in_cache):
|
||||
mock_in_cache.side_effect = [True, False]
|
||||
self.conf.set_override('skip_existing', True)
|
||||
|
|
|
|||
Loading…
Reference in New Issue