Support squash docker images layers
Implements: blueprint squash-layers Change-Id: Ic9a144e50440ccb37f7781d8955815a64282e687
This commit is contained in:
Jeffrey Zhang
parent
628af2f8b0
commit
d98a102162
@ -258,6 +258,10 @@ _CLI_OPTS = [
|
||||
help='Attempt to pull a newer version of the base image'),
|
||||
cfg.StrOpt('work-dir', help=('Path to be used as working directory.'
|
||||
' By default, a temporary dir is created')),
|
||||
cfg.BoolOpt('squash', default=False,
|
||||
help=('Squash the image layers. WARNING: it will consume lots'
|
||||
' of disk IO. "docker-squash" tool is required, install'
|
||||
' it by "pip install docker-squash"')),
|
||||
]
|
||||
|
||||
_BASE_OPTS = [
|
||||
@ -269,7 +273,11 @@ _BASE_OPTS = [
|
||||
help=('Comma separated list of .rpm or .repo file(s) '
|
||||
'or URL(s) to install before building containers')),
|
||||
cfg.StrOpt('apt_sources_list', help=('Path to custom sources.list')),
|
||||
cfg.StrOpt('apt_preferences', help=('Path to custom apt/preferences'))
|
||||
cfg.StrOpt('apt_preferences', help=('Path to custom apt/preferences')),
|
||||
cfg.BoolOpt('squash-cleanup', default=True,
|
||||
help='Remove source image from Docker after squashing'),
|
||||
cfg.StrOpt('squash-tmp-dir',
|
||||
help='Temporary directory to be used during squashing')
|
||||
]
|
||||
|
||||
|
||||
|
||||
73
kolla/common/utils.py
Normal file
73
kolla/common/utils.py
Normal file
@ -0,0 +1,73 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import logging
|
||||
import os
|
||||
import subprocess # nosec
|
||||
import sys
|
||||
|
||||
|
||||
def make_a_logger(conf=None, image_name=None):
|
||||
if image_name:
|
||||
log = logging.getLogger(".".join([__name__, image_name]))
|
||||
else:
|
||||
log = logging.getLogger(__name__)
|
||||
if not log.handlers:
|
||||
if conf is None or not conf.logs_dir or not image_name:
|
||||
handler = logging.StreamHandler(sys.stderr)
|
||||
log.propagate = False
|
||||
else:
|
||||
filename = os.path.join(conf.logs_dir, "%s.log" % image_name)
|
||||
handler = logging.FileHandler(filename, delay=True)
|
||||
handler.setFormatter(logging.Formatter(logging.BASIC_FORMAT))
|
||||
log.addHandler(handler)
|
||||
if conf is not None and conf.debug:
|
||||
log.setLevel(logging.DEBUG)
|
||||
else:
|
||||
log.setLevel(logging.INFO)
|
||||
return log
|
||||
|
||||
|
||||
LOG = make_a_logger()
|
||||
|
||||
|
||||
def get_docker_squash_version():
|
||||
|
||||
try:
|
||||
stdout = subprocess.check_output( # nosec
|
||||
['docker-squash', '--version'], stderr=subprocess.STDOUT)
|
||||
return stdout.split()[0]
|
||||
except OSError as ex:
|
||||
if ex.errno == 2:
|
||||
LOG.error(('"docker-squash" command is not found.'
|
||||
' try to install it by "pip install docker-squash"'))
|
||||
raise
|
||||
|
||||
|
||||
def squash(old_image, new_image,
|
||||
from_layer=None,
|
||||
cleanup=False,
|
||||
tmp_dir=None):
|
||||
|
||||
cmds = ['docker-squash', '--tag', new_image, old_image]
|
||||
if cleanup:
|
||||
cmds += ['--cleanup']
|
||||
if from_layer:
|
||||
cmds += ['--from-layer', from_layer]
|
||||
if tmp_dir:
|
||||
cmds += ['--tmp-dir', tmp_dir]
|
||||
try:
|
||||
subprocess.check_output(cmds, stderr=subprocess.STDOUT) # nosec
|
||||
except subprocess.CalledProcessError as ex:
|
||||
LOG.exception('Get error during squashing image: %s',
|
||||
ex.stdout)
|
||||
raise
|
||||
@ -36,6 +36,7 @@ from oslo_config import cfg
|
||||
from requests import exceptions as requests_exc
|
||||
import six
|
||||
|
||||
|
||||
# NOTE(SamYaple): Update the search path to prefer PROJECT_ROOT as the source
|
||||
# of packages to import if we are using local tools instead of
|
||||
# pip installed kolla tools
|
||||
@ -46,34 +47,14 @@ if PROJECT_ROOT not in sys.path:
|
||||
|
||||
from kolla.common import config as common_config
|
||||
from kolla.common import task
|
||||
from kolla.common import utils
|
||||
from kolla import exception
|
||||
from kolla.template import filters as jinja_filters
|
||||
from kolla.template import methods as jinja_methods
|
||||
from kolla import version
|
||||
|
||||
|
||||
def make_a_logger(conf=None, image_name=None):
|
||||
if image_name:
|
||||
log = logging.getLogger(".".join([__name__, image_name]))
|
||||
else:
|
||||
log = logging.getLogger(__name__)
|
||||
if not log.handlers:
|
||||
if conf is None or not conf.logs_dir or not image_name:
|
||||
handler = logging.StreamHandler(sys.stderr)
|
||||
log.propagate = False
|
||||
else:
|
||||
filename = os.path.join(conf.logs_dir, "%s.log" % image_name)
|
||||
handler = logging.FileHandler(filename, delay=True)
|
||||
handler.setFormatter(logging.Formatter(logging.BASIC_FORMAT))
|
||||
log.addHandler(handler)
|
||||
if conf is not None and conf.debug:
|
||||
log.setLevel(logging.DEBUG)
|
||||
else:
|
||||
log.setLevel(logging.INFO)
|
||||
return log
|
||||
|
||||
|
||||
LOG = make_a_logger()
|
||||
LOG = utils.make_a_logger()
|
||||
|
||||
# Image status constants.
|
||||
#
|
||||
@ -261,7 +242,7 @@ class Image(object):
|
||||
self.source = source
|
||||
self.parent_name = parent_name
|
||||
if logger is None:
|
||||
logger = make_a_logger(image_name=name)
|
||||
logger = utils.make_a_logger(image_name=name)
|
||||
self.logger = logger
|
||||
self.children = []
|
||||
self.plugins = []
|
||||
@ -586,6 +567,9 @@ class BuildTask(DockerTask):
|
||||
if line:
|
||||
self.logger.error('%s', line)
|
||||
return
|
||||
|
||||
if image.status != STATUS_ERROR and self.conf.squash:
|
||||
self.squash()
|
||||
except docker.errors.DockerException:
|
||||
image.status = STATUS_ERROR
|
||||
self.logger.exception('Unknown docker error when building')
|
||||
@ -596,6 +580,19 @@ class BuildTask(DockerTask):
|
||||
image.status = STATUS_BUILT
|
||||
self.logger.info('Built')
|
||||
|
||||
def squash(self):
|
||||
image_tag = self.image.canonical_name
|
||||
image_id = self.dc.inspect_image(image_tag)['Id']
|
||||
|
||||
parent_history = self.dc.history(self.image.parent_name)
|
||||
parent_last_layer = parent_history[0]['Id']
|
||||
self.logger.info('Parent lastest layer is: %s' % parent_last_layer)
|
||||
|
||||
utils.squash(image_id, image_tag, from_layer=parent_last_layer,
|
||||
cleanup=self.conf.squash_cleanup,
|
||||
tmp_dir=self.conf.squash_tmp_dir)
|
||||
self.logger.info('Image is squashed successfully')
|
||||
|
||||
|
||||
class WorkerThread(threading.Thread):
|
||||
"""Thread that executes tasks until the queue provides a tombstone."""
|
||||
@ -1084,7 +1081,7 @@ class KollaWorker(object):
|
||||
del match
|
||||
image = Image(image_name, canonical_name, path,
|
||||
parent_name=parent_name,
|
||||
logger=make_a_logger(self.conf, image_name),
|
||||
logger=utils.make_a_logger(self.conf, image_name),
|
||||
docker_client=self.dc)
|
||||
|
||||
if self.install_type == 'source':
|
||||
@ -1226,6 +1223,13 @@ def run_build():
|
||||
if conf.debug:
|
||||
LOG.setLevel(logging.DEBUG)
|
||||
|
||||
if conf.squash:
|
||||
squash_version = utils.get_docker_squash_version()
|
||||
LOG.info('Image squash is enabled and "docker-squash" version is %s',
|
||||
squash_version)
|
||||
else:
|
||||
LOG.info('Image squash is disabled')
|
||||
|
||||
kolla = KollaWorker(conf)
|
||||
kolla.setup_working_dir()
|
||||
kolla.find_dockerfiles()
|
||||
|
||||
@ -0,0 +1,5 @@
|
||||
---
|
||||
features:
|
||||
- |
|
||||
support --squash parameter which leverage docker-squash tool to squash
|
||||
newly built layers into a single new layer
|
||||
Loading…
Reference in New Issue
Block a user