566 lines
21 KiB
Python
566 lines
21 KiB
Python
# Copyright 2015 Red Hat, Inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
#
|
|
|
|
import datetime
|
|
import json
|
|
import logging
|
|
import os
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
|
|
from heatclient.common import template_utils
|
|
from heatclient.common import utils as heat_utils
|
|
from osc_lib import exceptions as oscexc
|
|
from osc_lib.i18n import _
|
|
from six.moves.urllib import request
|
|
import yaml
|
|
|
|
from tripleo_common.image import image_uploader
|
|
from tripleo_common.image import kolla_builder
|
|
|
|
from tripleoclient import command
|
|
from tripleoclient import constants
|
|
from tripleoclient import utils
|
|
|
|
|
|
class UploadImage(command.Command):
|
|
"""Push overcloud container images to registries."""
|
|
|
|
auth_required = False
|
|
log = logging.getLogger(__name__ + ".UploadImage")
|
|
|
|
def get_parser(self, prog_name):
|
|
parser = super(UploadImage, self).get_parser(prog_name)
|
|
parser.add_argument(
|
|
"--config-file",
|
|
dest="config_files",
|
|
metavar='<yaml config file>',
|
|
default=[],
|
|
action="append",
|
|
required=True,
|
|
help=_("YAML config file specifying the image build. May be "
|
|
"specified multiple times. Order is preserved, and later "
|
|
"files will override some options in previous files. "
|
|
"Other options will append."),
|
|
)
|
|
return parser
|
|
|
|
def take_action(self, parsed_args):
|
|
self.log.debug("take_action(%s)" % parsed_args)
|
|
uploader = image_uploader.ImageUploadManager(
|
|
parsed_args.config_files)
|
|
uploader.upload()
|
|
|
|
|
|
class BuildImage(command.Command):
|
|
"""Build overcloud container images with kolla-build."""
|
|
|
|
auth_required = False
|
|
log = logging.getLogger(__name__ + ".BuildImage")
|
|
|
|
@staticmethod
|
|
def images_from_deps(images, dep):
|
|
'''Builds a list from the dependencies depth-first. '''
|
|
if isinstance(dep, list):
|
|
for v in dep:
|
|
BuildImage.images_from_deps(images, v)
|
|
elif isinstance(dep, dict):
|
|
for k, v in dep.items():
|
|
images.append(k)
|
|
BuildImage.images_from_deps(images, v)
|
|
else:
|
|
images.append(dep)
|
|
|
|
def get_parser(self, prog_name):
|
|
default_kolla_conf = os.path.join(
|
|
sys.prefix, 'share', 'tripleo-common', 'container-images',
|
|
'tripleo_kolla_config_overrides.conf')
|
|
parser = super(BuildImage, self).get_parser(prog_name)
|
|
parser.add_argument(
|
|
"--config-file",
|
|
dest="config_files",
|
|
metavar='<yaml config file>',
|
|
default=[],
|
|
action="append",
|
|
required=True,
|
|
help=_("YAML config file specifying the images to build. May be "
|
|
"specified multiple times. Order is preserved, and later "
|
|
"files will override some options in previous files. "
|
|
"Other options will append."),
|
|
)
|
|
parser.add_argument(
|
|
"--kolla-config-file",
|
|
dest="kolla_config_files",
|
|
metavar='<config file>',
|
|
default=[default_kolla_conf],
|
|
action="append",
|
|
required=True,
|
|
help=_("Path to a Kolla config file to use. Multiple config files "
|
|
"can be specified, with values in later files taking "
|
|
"precedence. By default, tripleo kolla conf file {conf} "
|
|
"is added.").format(conf=default_kolla_conf),
|
|
)
|
|
parser.add_argument(
|
|
'--list-images',
|
|
dest='list_images',
|
|
action='store_true',
|
|
default=False,
|
|
help=_('Show the images which would be built instead of '
|
|
'building them.')
|
|
)
|
|
parser.add_argument(
|
|
'--list-dependencies',
|
|
dest='list_dependencies',
|
|
action='store_true',
|
|
default=False,
|
|
help=_('Show the image build dependencies instead of '
|
|
'building them.')
|
|
)
|
|
return parser
|
|
|
|
def take_action(self, parsed_args):
|
|
self.log.debug("take_action(%s)" % parsed_args)
|
|
|
|
fd, path = tempfile.mkstemp(prefix='kolla_conf_')
|
|
with os.fdopen(fd, 'w') as tmp:
|
|
tmp.write('[DEFAULT]\n')
|
|
if parsed_args.list_images or parsed_args.list_dependencies:
|
|
tmp.write('list_dependencies=true')
|
|
kolla_config_files = list(parsed_args.kolla_config_files)
|
|
kolla_config_files.append(path)
|
|
|
|
try:
|
|
builder = kolla_builder.KollaImageBuilder(parsed_args.config_files)
|
|
result = builder.build_images(kolla_config_files)
|
|
if parsed_args.list_dependencies:
|
|
deps = json.loads(result)
|
|
yaml.safe_dump(deps, self.app.stdout, indent=2,
|
|
default_flow_style=False)
|
|
elif parsed_args.list_images:
|
|
deps = json.loads(result)
|
|
images = []
|
|
BuildImage.images_from_deps(images, deps)
|
|
yaml.safe_dump(images, self.app.stdout,
|
|
default_flow_style=False)
|
|
elif result:
|
|
self.app.stdout.write(result)
|
|
finally:
|
|
os.remove(path)
|
|
|
|
|
|
class PrepareImageFiles(command.Command):
|
|
"""Generate files defining the images, tags and registry."""
|
|
|
|
auth_required = False
|
|
log = logging.getLogger(__name__ + ".PrepareImageFiles")
|
|
|
|
def get_parser(self, prog_name):
|
|
parser = super(PrepareImageFiles, self).get_parser(prog_name)
|
|
roles_file = os.path.join(constants.TRIPLEO_HEAT_TEMPLATES,
|
|
constants.OVERCLOUD_ROLES_FILE)
|
|
defaults = kolla_builder.container_images_prepare_defaults()
|
|
|
|
parser.add_argument(
|
|
"--template-file",
|
|
dest="template_file",
|
|
default=kolla_builder.DEFAULT_TEMPLATE_FILE,
|
|
metavar='<yaml template file>',
|
|
help=_("YAML template file which the images config file will be "
|
|
"built from.\n"
|
|
"Default: %s") % kolla_builder.DEFAULT_TEMPLATE_FILE,
|
|
)
|
|
parser.add_argument(
|
|
"--pull-source",
|
|
dest="pull_source",
|
|
metavar='<location>',
|
|
help=_("Location of image registry to pull images from. "
|
|
"(DEPRECATED. Include the registry in --namespace)"),
|
|
)
|
|
parser.add_argument(
|
|
"--push-destination",
|
|
dest="push_destination",
|
|
metavar='<location>',
|
|
help=_("Location of image registry to push images to. "
|
|
"If specified, a push_destination will be set for every "
|
|
"image entry."),
|
|
)
|
|
parser.add_argument(
|
|
"--tag",
|
|
dest="tag",
|
|
default=defaults['tag'],
|
|
metavar='<tag>',
|
|
help=_("Override the default tag substitution. "
|
|
"If --tag-from-label is specified, "
|
|
"start discovery with this tag.\n"
|
|
"Default: %s") % defaults['tag'],
|
|
)
|
|
parser.add_argument(
|
|
"--tag-from-label",
|
|
dest="tag_from_label",
|
|
metavar='<image label>',
|
|
help=_("Use the value of the specified label to discover the "
|
|
"versioned tag."),
|
|
)
|
|
parser.add_argument(
|
|
"--namespace",
|
|
dest="namespace",
|
|
default=defaults['namespace'],
|
|
metavar='<namespace>',
|
|
help=_("Override the default namespace substitution.\n"
|
|
"Default: %s") % defaults['namespace'],
|
|
)
|
|
parser.add_argument(
|
|
"--prefix",
|
|
dest="prefix",
|
|
default=defaults['name_prefix'],
|
|
metavar='<prefix>',
|
|
help=_("Override the default name prefix substitution.\n"
|
|
"Default: %s") % defaults['name_prefix'],
|
|
)
|
|
parser.add_argument(
|
|
"--suffix",
|
|
dest="suffix",
|
|
default=defaults['name_suffix'],
|
|
metavar='<suffix>',
|
|
help=_("Override the default name suffix substitution.\n"
|
|
"Default: %s") % defaults['name_suffix'],
|
|
)
|
|
parser.add_argument(
|
|
'--set',
|
|
metavar='<variable=value>',
|
|
action='append',
|
|
help=_('Set the value of a variable in the template, even if it '
|
|
'has no dedicated argument such as "--suffix".')
|
|
)
|
|
parser.add_argument(
|
|
"--exclude",
|
|
dest="excludes",
|
|
metavar='<regex>',
|
|
default=[],
|
|
action="append",
|
|
help=_("Pattern to match against resulting imagename entries to "
|
|
"exclude from the final output. Can be specified multiple "
|
|
"times."),
|
|
)
|
|
parser.add_argument(
|
|
"--images-file",
|
|
dest="output_images_file",
|
|
metavar='<file path>',
|
|
help=_("File to write resulting image entries to, as well as "
|
|
"stdout. Any existing file will be overwritten."
|
|
"(DEPRECATED. Use --output-images-file instead)"),
|
|
)
|
|
parser.add_argument(
|
|
"--output-images-file",
|
|
dest="output_images_file",
|
|
metavar='<file path>',
|
|
help=_("File to write resulting image entries to, as well as "
|
|
"stdout. Any existing file will be overwritten."),
|
|
)
|
|
parser.add_argument(
|
|
'--service-environment-file', metavar='<file path>',
|
|
action='append', dest='environment_files',
|
|
help=_('Environment files specifying which services are '
|
|
'containerized. Entries will be filtered to only contain '
|
|
'images used by containerized services. (Can be specified '
|
|
'more than once.)'
|
|
"(DEPRECATED. Use --environment-file instead)"),
|
|
)
|
|
parser.add_argument(
|
|
'--environment-file', '-e', metavar='<file path>',
|
|
action='append', dest='environment_files',
|
|
help=_('Environment files specifying which services are '
|
|
'containerized. Entries will be filtered to only contain '
|
|
'images used by containerized services. (Can be specified '
|
|
'more than once.)')
|
|
)
|
|
parser.add_argument(
|
|
'--environment-directory', metavar='<HEAT ENVIRONMENT DIRECTORY>',
|
|
action='append', dest='environment_directories',
|
|
default=[os.path.expanduser(constants.DEFAULT_ENV_DIRECTORY)],
|
|
help=_('Environment file directories that are automatically '
|
|
'added to the update command. Entries will be filtered '
|
|
'to only contain images used by containerized services. '
|
|
'Can be specified more than once. Files in directories are '
|
|
'loaded in ascending sort order.')
|
|
)
|
|
parser.add_argument(
|
|
"--env-file",
|
|
dest="output_env_file",
|
|
metavar='<file path>',
|
|
help=_("File to write heat environment file which specifies all "
|
|
"image parameters. Any existing file will be overwritten."
|
|
"(DEPRECATED. Use --output-env-file instead)"),
|
|
)
|
|
parser.add_argument(
|
|
"--output-env-file",
|
|
dest="output_env_file",
|
|
metavar='<file path>',
|
|
help=_("File to write heat environment file which specifies all "
|
|
"image parameters. Any existing file will be overwritten."),
|
|
)
|
|
parser.add_argument(
|
|
'--roles-file', '-r', dest='roles_file',
|
|
default=roles_file,
|
|
help=_('Roles file, overrides the default %s'
|
|
) % constants.OVERCLOUD_ROLES_FILE
|
|
)
|
|
return parser
|
|
|
|
def parse_set_values(self, subs, set_values):
|
|
if not set_values:
|
|
return
|
|
for s in set_values:
|
|
try:
|
|
(n, v) = s.split(('='), 1)
|
|
subs[n] = v
|
|
except ValueError:
|
|
msg = _('Malformed --set(%s). '
|
|
'Use the variable=value format.') % s
|
|
raise oscexc.CommandError(msg)
|
|
|
|
def write_env_file(self, params, env_file):
|
|
|
|
with os.fdopen(os.open(env_file,
|
|
os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o666),
|
|
'w') as f:
|
|
f.write('# Generated with the following on %s\n#\n' %
|
|
datetime.datetime.now().isoformat())
|
|
f.write('# openstack %s\n#\n\n' %
|
|
' '.join(self.app.command_options))
|
|
|
|
yaml.safe_dump({'parameter_defaults': params}, f,
|
|
default_flow_style=False)
|
|
|
|
def get_enabled_services(self, environment, roles_file):
|
|
enabled_services = set()
|
|
try:
|
|
roles_data = yaml.safe_load(open(roles_file).read())
|
|
except IOError:
|
|
return enabled_services
|
|
|
|
parameter_defaults = environment.get('parameter_defaults', {})
|
|
|
|
for role in roles_data:
|
|
count = parameter_defaults.get('%sCount' % role['name'],
|
|
role.get('CountDefault', 0))
|
|
if count > 0:
|
|
enabled_services.update(
|
|
parameter_defaults.get('%sServices' % role['name'],
|
|
role.get('ServicesDefault', [])))
|
|
|
|
return enabled_services
|
|
|
|
def build_service_filter(self, environment_files, roles_file):
|
|
# Do not filter unless asked for it
|
|
if not environment_files:
|
|
return None
|
|
|
|
def get_env_file(method, path):
|
|
if not os.path.exists(path):
|
|
return '{}'
|
|
env_url = heat_utils.normalise_file_path_to_url(path)
|
|
return request.urlopen(env_url).read()
|
|
|
|
env_files, env = (
|
|
template_utils.process_multiple_environments_and_files(
|
|
environment_files, env_path_is_object=lambda path: True,
|
|
object_request=get_env_file))
|
|
enabled_services = self.get_enabled_services(env, roles_file)
|
|
containerized_services = set()
|
|
for service, env_path in env.get('resource_registry', {}).items():
|
|
# Use the template path to determine if it represents a
|
|
# containerized service
|
|
if '/docker/services/' in env_path:
|
|
containerized_services.add(service)
|
|
|
|
return containerized_services.intersection(enabled_services)
|
|
|
|
def take_action(self, parsed_args):
|
|
self.log.debug("take_action(%s)" % parsed_args)
|
|
|
|
env_files = []
|
|
|
|
if parsed_args.environment_directories:
|
|
env_files.extend(utils.load_environment_directories(
|
|
parsed_args.environment_directories))
|
|
if parsed_args.environment_files:
|
|
env_files.extend(parsed_args.environment_files)
|
|
|
|
service_filter = self.build_service_filter(
|
|
env_files, parsed_args.roles_file)
|
|
|
|
neutron_driver = None
|
|
if service_filter:
|
|
if 'OS::TripleO::Services::OpenDaylightApi' in service_filter:
|
|
neutron_driver = 'odl'
|
|
elif 'OS::TripleO::Services::OVNController' in service_filter:
|
|
neutron_driver = 'ovn'
|
|
|
|
subs = {
|
|
'tag': parsed_args.tag,
|
|
'namespace': parsed_args.namespace,
|
|
'name_prefix': parsed_args.prefix,
|
|
'name_suffix': parsed_args.suffix,
|
|
'neutron_driver': neutron_driver,
|
|
}
|
|
self.parse_set_values(subs, parsed_args.set)
|
|
|
|
output_images_file = (parsed_args.output_images_file
|
|
or 'container_images.yaml')
|
|
|
|
prepare_data = kolla_builder.container_images_prepare(
|
|
excludes=parsed_args.excludes,
|
|
service_filter=service_filter,
|
|
pull_source=parsed_args.pull_source,
|
|
push_destination=parsed_args.push_destination,
|
|
mapping_args=subs,
|
|
output_env_file=parsed_args.output_env_file,
|
|
output_images_file=output_images_file,
|
|
tag_from_label=parsed_args.tag_from_label,
|
|
template_file=parsed_args.template_file
|
|
)
|
|
if parsed_args.output_env_file:
|
|
params = prepare_data[parsed_args.output_env_file]
|
|
self.write_env_file(params, parsed_args.output_env_file)
|
|
|
|
result = prepare_data[output_images_file]
|
|
result_str = yaml.safe_dump({'container_images': result},
|
|
default_flow_style=False)
|
|
sys.stdout.write(result_str)
|
|
|
|
if parsed_args.output_images_file:
|
|
with os.fdopen(os.open(parsed_args.output_images_file,
|
|
os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o666),
|
|
'w') as f:
|
|
f.write(result_str)
|
|
|
|
|
|
class DiscoverImageTag(command.Command):
|
|
"""Discover the versioned tag for an image."""
|
|
|
|
auth_required = False
|
|
log = logging.getLogger(__name__ + ".DiscoverImageTag")
|
|
|
|
def get_parser(self, prog_name):
|
|
parser = super(DiscoverImageTag, self).get_parser(prog_name)
|
|
parser.add_argument(
|
|
"--image",
|
|
dest="image",
|
|
metavar='<container image>',
|
|
required=True,
|
|
help=_("Fully qualified name of the image to discover the tag for "
|
|
"(Including registry and stable tag)."),
|
|
)
|
|
parser.add_argument(
|
|
"--tag-from-label",
|
|
dest="tag_from_label",
|
|
metavar='<image label>',
|
|
help=_("Use the value of the specified label to discover the "
|
|
"tag."),
|
|
)
|
|
return parser
|
|
|
|
def take_action(self, parsed_args):
|
|
self.log.debug("take_action(%s)" % parsed_args)
|
|
|
|
uploader = image_uploader.ImageUploadManager([])
|
|
print(uploader.discover_image_tag(
|
|
image=parsed_args.image,
|
|
tag_from_label=parsed_args.tag_from_label
|
|
))
|
|
|
|
|
|
class HotFix(command.Command):
|
|
"""Hotfix tripleo container images with Ansible."""
|
|
|
|
def get_parser(self, prog_name):
|
|
parser = super(HotFix, self).get_parser(prog_name)
|
|
parser.add_argument(
|
|
"--image",
|
|
dest="images",
|
|
metavar="<images>",
|
|
default=[],
|
|
action="append",
|
|
required=True,
|
|
help=_(
|
|
"Fully qualified reference to the source image to be "
|
|
"modified. Can be specified multiple times (one per "
|
|
"image) (default: %(default)s)."
|
|
),
|
|
)
|
|
parser.add_argument(
|
|
"--rpms-path",
|
|
dest="rpms_path",
|
|
metavar="<rpms-path>",
|
|
required=True,
|
|
help=_("Path containing RPMs to install (default: %(default)s)."),
|
|
)
|
|
parser.add_argument(
|
|
"--tag",
|
|
dest="tag",
|
|
metavar="<image-tag>",
|
|
default="latest",
|
|
help=_("Image hotfix tag (default: %(default)s)"),
|
|
)
|
|
return parser
|
|
|
|
def take_action(self, parsed_args):
|
|
tmp_modify_dir = tempfile.mkdtemp()
|
|
tasks = list()
|
|
try:
|
|
for image in parsed_args.images:
|
|
tasks.append(
|
|
{
|
|
"name": "include ansible-role-tripleo-modify-image",
|
|
"import_role": {"name": "tripleo-modify-image"},
|
|
"vars": {
|
|
"container_build_tool": "docker",
|
|
"tasks_from": "rpm_install.yml",
|
|
"source_image": image,
|
|
"rpms_path": parsed_args.rpms_path,
|
|
"modified_append_tag": "-{}".format(
|
|
parsed_args.tag
|
|
),
|
|
"modify_dir_path": tmp_modify_dir,
|
|
},
|
|
}
|
|
)
|
|
|
|
playbook = os.path.join(tmp_modify_dir,
|
|
"tripleo-hotfix-playbook.yaml")
|
|
playdata = {
|
|
"name": "Generate hotfixs",
|
|
"connection": "local",
|
|
"hosts": "localhost",
|
|
"gather_facts": False,
|
|
"tasks": tasks,
|
|
}
|
|
|
|
with open(playbook, "w") as f:
|
|
yaml.safe_dump(
|
|
[playdata], f, default_flow_style=False, width=4096
|
|
)
|
|
|
|
utils.run_ansible_playbook(
|
|
logger=self.log,
|
|
workdir=tmp_modify_dir,
|
|
playbook=playbook,
|
|
inventory="localhost"
|
|
)
|
|
finally:
|
|
shutil.rmtree(tmp_modify_dir)
|