5a451f88cc
This allows the prepare command to set modify_role and modify_vars in the call to container_images_prepare. The result will be an images file that has extra options set in every entry. The actual modification on the images is done during the upload command, driven by these extra options. Change-Id: Ia1dc7a1b7f777ac3e000f0c2e155be6330c05a2c Blueprint: container-prepare-workflow Depends-On: I2c877a96264b351b4fc8527a3e40b87ddcb4f9a5
479 lines
18 KiB
Python
479 lines
18 KiB
Python
# Copyright 2015 Red Hat, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
#
|
|
|
|
import datetime
|
|
import json
|
|
import logging
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
|
|
from heatclient.common import template_utils
|
|
from heatclient.common import utils as heat_utils
|
|
from osc_lib import exceptions as oscexc
|
|
from osc_lib.i18n import _
|
|
from six.moves.urllib import request
|
|
import yaml
|
|
|
|
from tripleo_common.image import image_uploader
|
|
from tripleo_common.image import kolla_builder
|
|
|
|
from tripleoclient import command
|
|
from tripleoclient import constants
|
|
from tripleoclient import utils
|
|
|
|
|
|
class UploadImage(command.Command):
|
|
"""Push overcloud container images to registries."""
|
|
|
|
auth_required = False
|
|
log = logging.getLogger(__name__ + ".UploadImage")
|
|
|
|
def get_parser(self, prog_name):
|
|
parser = super(UploadImage, self).get_parser(prog_name)
|
|
parser.add_argument(
|
|
"--config-file",
|
|
dest="config_files",
|
|
metavar='<yaml config file>',
|
|
default=[],
|
|
action="append",
|
|
required=True,
|
|
help=_("YAML config file specifying the image build. May be "
|
|
"specified multiple times. Order is preserved, and later "
|
|
"files will override some options in previous files. "
|
|
"Other options will append."),
|
|
)
|
|
return parser
|
|
|
|
def take_action(self, parsed_args):
|
|
self.log.debug("take_action(%s)" % parsed_args)
|
|
uploader = image_uploader.ImageUploadManager(
|
|
parsed_args.config_files)
|
|
try:
|
|
uploader.upload()
|
|
except KeyboardInterrupt: # ctrl-c
|
|
self.log.warning('Upload was interrupted by ctrl-c.')
|
|
|
|
|
|
class BuildImage(command.Command):
|
|
"""Build overcloud container images with kolla-build."""
|
|
|
|
auth_required = False
|
|
log = logging.getLogger(__name__ + ".BuildImage")
|
|
|
|
@staticmethod
|
|
def images_from_deps(images, dep):
|
|
'''Builds a list from the dependencies depth-first. '''
|
|
if isinstance(dep, list):
|
|
for v in dep:
|
|
BuildImage.images_from_deps(images, v)
|
|
elif isinstance(dep, dict):
|
|
for k, v in dep.items():
|
|
images.append(k)
|
|
BuildImage.images_from_deps(images, v)
|
|
else:
|
|
images.append(dep)
|
|
|
|
def get_parser(self, prog_name):
|
|
default_kolla_conf = os.path.join(
|
|
sys.prefix, 'share', 'tripleo-common', 'container-images',
|
|
'tripleo_kolla_config_overrides.conf')
|
|
parser = super(BuildImage, self).get_parser(prog_name)
|
|
parser.add_argument(
|
|
"--config-file",
|
|
dest="config_files",
|
|
metavar='<yaml config file>',
|
|
default=[],
|
|
action="append",
|
|
required=True,
|
|
help=_("YAML config file specifying the images to build. May be "
|
|
"specified multiple times. Order is preserved, and later "
|
|
"files will override some options in previous files. "
|
|
"Other options will append."),
|
|
)
|
|
parser.add_argument(
|
|
"--kolla-config-file",
|
|
dest="kolla_config_files",
|
|
metavar='<config file>',
|
|
default=[default_kolla_conf],
|
|
action="append",
|
|
required=True,
|
|
help=_("Path to a Kolla config file to use. Multiple config files "
|
|
"can be specified, with values in later files taking "
|
|
"precedence. By default, tripleo kolla conf file {conf} "
|
|
"is added.").format(conf=default_kolla_conf),
|
|
)
|
|
parser.add_argument(
|
|
'--list-images',
|
|
dest='list_images',
|
|
action='store_true',
|
|
default=False,
|
|
help=_('Show the images which would be built instead of '
|
|
'building them.')
|
|
)
|
|
parser.add_argument(
|
|
'--list-dependencies',
|
|
dest='list_dependencies',
|
|
action='store_true',
|
|
default=False,
|
|
help=_('Show the image build dependencies instead of '
|
|
'building them.')
|
|
)
|
|
return parser
|
|
|
|
def take_action(self, parsed_args):
|
|
self.log.debug("take_action(%s)" % parsed_args)
|
|
|
|
fd, path = tempfile.mkstemp(prefix='kolla_conf_')
|
|
with os.fdopen(fd, 'w') as tmp:
|
|
tmp.write('[DEFAULT]\n')
|
|
if parsed_args.list_images or parsed_args.list_dependencies:
|
|
tmp.write('list_dependencies=true')
|
|
kolla_config_files = list(parsed_args.kolla_config_files)
|
|
kolla_config_files.append(path)
|
|
|
|
try:
|
|
builder = kolla_builder.KollaImageBuilder(parsed_args.config_files)
|
|
result = builder.build_images(kolla_config_files)
|
|
if parsed_args.list_dependencies:
|
|
deps = json.loads(result)
|
|
yaml.safe_dump(deps, self.app.stdout, indent=2,
|
|
default_flow_style=False)
|
|
elif parsed_args.list_images:
|
|
deps = json.loads(result)
|
|
images = []
|
|
BuildImage.images_from_deps(images, deps)
|
|
yaml.safe_dump(images, self.app.stdout,
|
|
default_flow_style=False)
|
|
elif result:
|
|
self.app.stdout.write(result)
|
|
finally:
|
|
os.remove(path)
|
|
|
|
|
|
class PrepareImageFiles(command.Command):
|
|
"""Generate files defining the images, tags and registry."""
|
|
|
|
auth_required = False
|
|
log = logging.getLogger(__name__ + ".PrepareImageFiles")
|
|
|
|
def get_parser(self, prog_name):
|
|
parser = super(PrepareImageFiles, self).get_parser(prog_name)
|
|
roles_file = os.path.join(constants.TRIPLEO_HEAT_TEMPLATES,
|
|
constants.OVERCLOUD_ROLES_FILE)
|
|
defaults = kolla_builder.container_images_prepare_defaults()
|
|
|
|
parser.add_argument(
|
|
"--template-file",
|
|
dest="template_file",
|
|
default=kolla_builder.DEFAULT_TEMPLATE_FILE,
|
|
metavar='<yaml template file>',
|
|
help=_("YAML template file which the images config file will be "
|
|
"built from.\n"
|
|
"Default: %s") % kolla_builder.DEFAULT_TEMPLATE_FILE,
|
|
)
|
|
parser.add_argument(
|
|
"--pull-source",
|
|
dest="pull_source",
|
|
metavar='<location>',
|
|
help=_("Location of image registry to pull images from. "
|
|
"(DEPRECATED. Include the registry in --namespace)"),
|
|
)
|
|
parser.add_argument(
|
|
"--push-destination",
|
|
dest="push_destination",
|
|
metavar='<location>',
|
|
help=_("Location of image registry to push images to. "
|
|
"If specified, a push_destination will be set for every "
|
|
"image entry."),
|
|
)
|
|
parser.add_argument(
|
|
"--tag",
|
|
dest="tag",
|
|
default=defaults['tag'],
|
|
metavar='<tag>',
|
|
help=_("Override the default tag substitution. "
|
|
"If --tag-from-label is specified, "
|
|
"start discovery with this tag.\n"
|
|
"Default: %s") % defaults['tag'],
|
|
)
|
|
parser.add_argument(
|
|
"--tag-from-label",
|
|
dest="tag_from_label",
|
|
metavar='<image label>',
|
|
help=_("Use the value of the specified label(s) to discover the "
|
|
"tag. Labels can be combined in a template format, "
|
|
"for example: {version}-{release}"),
|
|
)
|
|
parser.add_argument(
|
|
"--namespace",
|
|
dest="namespace",
|
|
default=defaults['namespace'],
|
|
metavar='<namespace>',
|
|
help=_("Override the default namespace substitution.\n"
|
|
"Default: %s") % defaults['namespace'],
|
|
)
|
|
parser.add_argument(
|
|
"--prefix",
|
|
dest="prefix",
|
|
default=defaults['name_prefix'],
|
|
metavar='<prefix>',
|
|
help=_("Override the default name prefix substitution.\n"
|
|
"Default: %s") % defaults['name_prefix'],
|
|
)
|
|
parser.add_argument(
|
|
"--suffix",
|
|
dest="suffix",
|
|
default=defaults['name_suffix'],
|
|
metavar='<suffix>',
|
|
help=_("Override the default name suffix substitution.\n"
|
|
"Default: %s") % defaults['name_suffix'],
|
|
)
|
|
parser.add_argument(
|
|
'--set',
|
|
metavar='<variable=value>',
|
|
action='append',
|
|
help=_('Set the value of a variable in the template, even if it '
|
|
'has no dedicated argument such as "--suffix".')
|
|
)
|
|
parser.add_argument(
|
|
"--exclude",
|
|
dest="excludes",
|
|
metavar='<regex>',
|
|
default=[],
|
|
action="append",
|
|
help=_("Pattern to match against resulting imagename entries to "
|
|
"exclude from the final output. Can be specified multiple "
|
|
"times."),
|
|
)
|
|
parser.add_argument(
|
|
"--images-file",
|
|
dest="output_images_file",
|
|
metavar='<file path>',
|
|
help=_("File to write resulting image entries to, as well as "
|
|
"stdout. Any existing file will be overwritten."
|
|
"(DEPRECATED. Use --output-images-file instead)"),
|
|
)
|
|
parser.add_argument(
|
|
"--output-images-file",
|
|
dest="output_images_file",
|
|
metavar='<file path>',
|
|
help=_("File to write resulting image entries to, as well as "
|
|
"stdout. Any existing file will be overwritten."),
|
|
)
|
|
parser.add_argument(
|
|
'--service-environment-file', metavar='<file path>',
|
|
action='append', dest='environment_files',
|
|
help=_('Environment files specifying which services are '
|
|
'containerized. Entries will be filtered to only contain '
|
|
'images used by containerized services. (Can be specified '
|
|
'more than once.)'
|
|
"(DEPRECATED. Use --environment-file instead)"),
|
|
)
|
|
parser.add_argument(
|
|
'--environment-file', '-e', metavar='<file path>',
|
|
action='append', dest='environment_files',
|
|
help=_('Environment files specifying which services are '
|
|
'containerized. Entries will be filtered to only contain '
|
|
'images used by containerized services. (Can be specified '
|
|
'more than once.)')
|
|
)
|
|
parser.add_argument(
|
|
'--environment-directory', metavar='<HEAT ENVIRONMENT DIRECTORY>',
|
|
action='append', dest='environment_directories',
|
|
default=[os.path.expanduser(constants.DEFAULT_ENV_DIRECTORY)],
|
|
help=_('Environment file directories that are automatically '
|
|
'added to the update command. Entries will be filtered '
|
|
'to only contain images used by containerized services. '
|
|
'Can be specified more than once. Files in directories are '
|
|
'loaded in ascending sort order.')
|
|
)
|
|
parser.add_argument(
|
|
"--env-file",
|
|
dest="output_env_file",
|
|
metavar='<file path>',
|
|
help=_("File to write heat environment file which specifies all "
|
|
"image parameters. Any existing file will be overwritten."
|
|
"(DEPRECATED. Use --output-env-file instead)"),
|
|
)
|
|
parser.add_argument(
|
|
"--output-env-file",
|
|
dest="output_env_file",
|
|
metavar='<file path>',
|
|
help=_("File to write heat environment file which specifies all "
|
|
"image parameters. Any existing file will be overwritten."),
|
|
)
|
|
parser.add_argument(
|
|
'--roles-file', '-r', dest='roles_file',
|
|
default=roles_file,
|
|
help=_('Roles file, overrides the default %s'
|
|
) % constants.OVERCLOUD_ROLES_FILE
|
|
)
|
|
parser.add_argument(
|
|
'--modify-role',
|
|
dest='modify_role',
|
|
help=_('Name of ansible role to run between every image upload '
|
|
'pull and push.')
|
|
)
|
|
parser.add_argument(
|
|
'--modify-vars',
|
|
dest='modify_vars',
|
|
help=_('Ansible variable file containing variables to use when '
|
|
'invoking the role --modify-role.')
|
|
)
|
|
return parser
|
|
|
|
def parse_set_values(self, subs, set_values):
|
|
if not set_values:
|
|
return
|
|
for s in set_values:
|
|
try:
|
|
(n, v) = s.split(('='), 1)
|
|
subs[n] = v
|
|
except ValueError:
|
|
msg = _('Malformed --set(%s). '
|
|
'Use the variable=value format.') % s
|
|
raise oscexc.CommandError(msg)
|
|
|
|
def write_env_file(self, params, env_file):
|
|
|
|
with os.fdopen(os.open(env_file,
|
|
os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o666),
|
|
'w') as f:
|
|
f.write('# Generated with the following on %s\n#\n' %
|
|
datetime.datetime.now().isoformat())
|
|
f.write('# openstack %s\n#\n\n' %
|
|
' '.join(self.app.command_options))
|
|
|
|
yaml.safe_dump({'parameter_defaults': params}, f,
|
|
default_flow_style=False)
|
|
|
|
def take_action(self, parsed_args):
|
|
self.log.debug("take_action(%s)" % parsed_args)
|
|
|
|
env_files = []
|
|
|
|
try:
|
|
roles_data = yaml.safe_load(open(parsed_args.roles_file).read())
|
|
except IOError:
|
|
roles_data = set()
|
|
|
|
if parsed_args.environment_directories:
|
|
env_files.extend(utils.load_environment_directories(
|
|
parsed_args.environment_directories))
|
|
if parsed_args.environment_files:
|
|
env_files.extend(parsed_args.environment_files)
|
|
|
|
def get_env_file(method, path):
|
|
if not os.path.exists(path):
|
|
return '{}'
|
|
env_url = heat_utils.normalise_file_path_to_url(path)
|
|
return request.urlopen(env_url).read()
|
|
|
|
env_f, env = (
|
|
template_utils.process_multiple_environments_and_files(
|
|
env_files, env_path_is_object=lambda path: True,
|
|
object_request=get_env_file))
|
|
|
|
if env_files:
|
|
service_filter = kolla_builder.build_service_filter(
|
|
env, roles_data)
|
|
else:
|
|
service_filter = None
|
|
|
|
mapping_args = {
|
|
'tag': parsed_args.tag,
|
|
'namespace': parsed_args.namespace,
|
|
'name_prefix': parsed_args.prefix,
|
|
'name_suffix': parsed_args.suffix,
|
|
}
|
|
self.parse_set_values(mapping_args, parsed_args.set)
|
|
|
|
output_images_file = (parsed_args.output_images_file
|
|
or 'container_images.yaml')
|
|
modify_role = None
|
|
modify_vars = None
|
|
append_tag = None
|
|
if parsed_args.modify_role:
|
|
modify_role = parsed_args.modify_role
|
|
append_tag = time.strftime('-modified-%Y%m%d%H%M%S')
|
|
if parsed_args.modify_vars:
|
|
modify_vars = yaml.safe_load(open(parsed_args.modify_vars).read())
|
|
|
|
prepare_data = kolla_builder.container_images_prepare(
|
|
excludes=parsed_args.excludes,
|
|
service_filter=service_filter,
|
|
pull_source=parsed_args.pull_source,
|
|
push_destination=parsed_args.push_destination,
|
|
mapping_args=mapping_args,
|
|
output_env_file=parsed_args.output_env_file,
|
|
output_images_file=output_images_file,
|
|
tag_from_label=parsed_args.tag_from_label,
|
|
modify_role=modify_role,
|
|
modify_vars=modify_vars,
|
|
append_tag=append_tag
|
|
)
|
|
if parsed_args.output_env_file:
|
|
params = prepare_data[parsed_args.output_env_file]
|
|
self.write_env_file(params, parsed_args.output_env_file)
|
|
|
|
result = prepare_data[output_images_file]
|
|
result_str = yaml.safe_dump({'container_images': result},
|
|
default_flow_style=False)
|
|
sys.stdout.write(result_str)
|
|
|
|
if parsed_args.output_images_file:
|
|
with os.fdopen(os.open(parsed_args.output_images_file,
|
|
os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o666),
|
|
'w') as f:
|
|
f.write(result_str)
|
|
|
|
|
|
class DiscoverImageTag(command.Command):
|
|
"""Discover the versioned tag for an image."""
|
|
|
|
auth_required = False
|
|
log = logging.getLogger(__name__ + ".DiscoverImageTag")
|
|
|
|
def get_parser(self, prog_name):
|
|
parser = super(DiscoverImageTag, self).get_parser(prog_name)
|
|
parser.add_argument(
|
|
"--image",
|
|
dest="image",
|
|
metavar='<container image>',
|
|
required=True,
|
|
help=_("Fully qualified name of the image to discover the tag for "
|
|
"(Including registry and stable tag)."),
|
|
)
|
|
parser.add_argument(
|
|
"--tag-from-label",
|
|
dest="tag_from_label",
|
|
metavar='<image label>',
|
|
help=_("Use the value of the specified label(s) to discover the "
|
|
"tag. Labels can be combined in a template format, "
|
|
"for example: {version}-{release}"),
|
|
)
|
|
return parser
|
|
|
|
def take_action(self, parsed_args):
|
|
self.log.debug("take_action(%s)" % parsed_args)
|
|
|
|
uploader = image_uploader.ImageUploadManager([])
|
|
print(uploader.discover_image_tag(
|
|
image=parsed_args.image,
|
|
tag_from_label=parsed_args.tag_from_label
|
|
))
|