diff --git a/releasenotes/notes/tripleo-prepare-d57bbccb2a44e8b2.yaml b/releasenotes/notes/tripleo-prepare-d57bbccb2a44e8b2.yaml new file mode 100644 index 000000000..2ebaf6c3f --- /dev/null +++ b/releasenotes/notes/tripleo-prepare-d57bbccb2a44e8b2.yaml @@ -0,0 +1,14 @@ +--- +features: + - | + The new command `openstack tripleo container image prepare` will do the + same container image preperation which happens during undercloud and + overcloud deploy, but in a standalone command. The prepare operations are + driven by a heat environment file containing the parameter + `ContainerImagePrepare`. This parameter allows multiple upload and + modification operations to be specified, and the result will be a list of + image parameters to use during a tripleo deployment. + + The command `openstack tripleo container image prepare default` will + generate a `ContainerImagePrepare` with the recommended defaults to use for + `openstack tripleo container image prepare`. diff --git a/setup.cfg b/setup.cfg index c626f6fc2..8d83ac637 100644 --- a/setup.cfg +++ b/setup.cfg @@ -94,6 +94,8 @@ openstack.tripleoclient.v1 = overcloud_ffwd-upgrade_converge = tripleoclient.v1.overcloud_ffwd_upgrade:FFWDUpgradeConverge overcloud_execute = tripleoclient.v1.overcloud_execute:RemoteExecute overcloud_generate_fencing = tripleoclient.v1.overcloud_parameters:GenerateFencingParameters + tripleo_container_image_prepare = tripleoclient.v1.container_image:TripleOImagePrepare + tripleo_container_image_prepare_default = tripleoclient.v1.container_image:TripleOImagePrepareDefault undercloud_deploy = tripleoclient.v1.undercloud_deploy:DeployUndercloud undercloud_install = tripleoclient.v1.undercloud:InstallUndercloud undercloud_upgrade = tripleoclient.v1.undercloud:UpgradeUndercloud diff --git a/tripleoclient/tests/v1/test_container_image.py b/tripleoclient/tests/v1/test_container_image.py index c45de021d..9a299c4b7 100644 --- a/tripleoclient/tests/v1/test_container_image.py +++ b/tripleoclient/tests/v1/test_container_image.py @@ -279,6 +279,148 @@ class TestContainerImagePrepare(TestPluginV1): self.assertEqual(env_data, yaml.safe_load(f)) +class TestTripleoImagePrepare(TestPluginV1): + + def setUp(self): + super(TestTripleoImagePrepare, self).setUp() + # Get the command object to test + self.cmd = container_image.TripleOImagePrepare(self.app, None) + + self.temp_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.temp_dir) + self.prepare_default_file = os.path.join( + self.temp_dir, 'prepare_env.yaml') + default_param = kolla_builder.CONTAINER_IMAGE_PREPARE_PARAM + self.default_env = { + 'parameter_defaults': { + 'ContainerImagePrepare': default_param + } + } + with open(self.prepare_default_file, 'w') as f: + yaml.safe_dump(self.default_env, f) + + self.roles_yaml = ''' + - name: EnabledRole + CountDefault: 1 + ServicesDefault: + - OS::TripleO::Services::AodhEvaluator + - name: RoleDisabledViaRolesData + CountDefault: 0 + ServicesDefault: + - OS::TripleO::Services::AodhApi + - name: RoleDisabledViaEnvironment + CountDefault: 1 + ServicesDefault: + - OS::TripleO::Services::Disabled + - name: RoleOverwrittenViaEnvironment + CountDefault: 1 + ServicesDefault: + - OS::TripleO::Services::Overwritten + ''' + self.roles_data_file = os.path.join( + self.temp_dir, 'roles_data.yaml') + with open(self.roles_data_file, 'w') as f: + f.write(self.roles_yaml) + + @mock.patch('tripleo_common.image.kolla_builder.' + 'container_images_prepare_multi') + def test_tripleo_container_image_prepare(self, prepare_multi): + + env_file = os.path.join(self.temp_dir, 'containers_env.yaml') + + arglist = [ + '--environment-file', self.prepare_default_file, + '--roles-file', self.roles_data_file, + '--output-env-file', env_file + ] + verifylist = [] + + self.app.command_options = [ + 'tripleo', 'container', 'image', 'prepare', 'default' + ] + arglist + + prepare_multi.return_value = { + 'DockerAodhApiImage': + '192.0.2.0:8787/t/os-aodh-apifoo:passed-ci', + 'DockerAodhConfigImage': + '192.0.2.0:8787/t/os-aodh-apifoo:passed-ci', + } + + parsed_args = self.check_parser(self.cmd, arglist, verifylist) + self.cmd.take_action(parsed_args) + + prepare_multi.assert_called_once_with( + self.default_env, yaml.safe_load(self.roles_yaml)) + + with open(env_file) as f: + result = yaml.safe_load(f) + + self.assertEqual({ + 'parameter_defaults': { + 'DockerAodhApiImage': + '192.0.2.0:8787/t/os-aodh-apifoo:passed-ci', + 'DockerAodhConfigImage': + '192.0.2.0:8787/t/os-aodh-apifoo:passed-ci', + } + }, result) + + +class TestTripleoImagePrepareDefault(TestPluginV1): + + def setUp(self): + super(TestTripleoImagePrepareDefault, self).setUp() + # Get the command object to test + self.cmd = container_image.TripleOImagePrepareDefault(self.app, None) + + def test_prepare_default(self): + arglist = [] + verifylist = [] + + self.app.command_options = [ + 'tripleo', 'container', 'image', 'prepare', 'default' + ] + arglist + self.cmd.app.stdout = six.StringIO() + cmd = container_image.TripleOImagePrepareDefault(self.app, None) + + parsed_args = self.check_parser(cmd, arglist, verifylist) + cmd.take_action(parsed_args) + + result = self.app.stdout.getvalue() + expected_param = kolla_builder.CONTAINER_IMAGE_PREPARE_PARAM + expected = { + 'parameter_defaults': { + 'ContainerImagePrepare': expected_param + } + } + self.assertEqual(expected, yaml.safe_load(result)) + + @mock.patch('tripleo_common.image.image_uploader.get_undercloud_registry') + def test_prepare_default_local_registry(self, mock_gur): + temp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, temp) + env_file = os.path.join(temp, 'containers_env.yaml') + + arglist = ['--local-push-destination', '--output-env-file', env_file] + verifylist = [] + mock_gur.return_value = '192.0.2.1:8787' + + self.app.command_options = [ + 'tripleo', 'container', 'image', 'prepare', 'default' + ] + arglist + cmd = container_image.TripleOImagePrepareDefault(self.app, None) + parsed_args = self.check_parser(cmd, arglist, verifylist) + + cmd.take_action(parsed_args) + + with open(env_file) as f: + result = yaml.safe_load(f) + self.assertEqual( + '192.0.2.1:8787', + result['parameter_defaults']['ContainerImagePrepare'] + [0]['push_destination'] + ) + + class TestContainerImageBuild(TestPluginV1): def setUp(self): diff --git a/tripleoclient/utils.py b/tripleoclient/utils.py index 7eeaf7783..2d55ec729 100644 --- a/tripleoclient/utils.py +++ b/tripleoclient/utils.py @@ -35,6 +35,7 @@ import yaml from heatclient.common import event_utils from heatclient.common import template_utils +from heatclient.common import utils as heat_utils from heatclient.exc import HTTPNotFound from osc_lib.i18n import _ from oslo_concurrency import processutils @@ -1099,3 +1100,36 @@ def ffwd_upgrade_operator_confirm(parsed_args_yes, log): log.debug(_("Fast forward upgrade cancelled on user request")) print(_("Cancelling fast forward upgrade")) sys.exit(1) + + +def build_prepare_env(environment_files, environment_directories): + '''Build the environment for container image prepare + + :param environment_files: List of environment files to build + environment from + :type environment_files: list + + :param environment_directories: List of environment directories to build + environment from + :type environment_directories: list + ''' + env_files = [] + + if environment_directories: + env_files.extend(load_environment_directories( + environment_directories)) + if environment_files: + env_files.extend(environment_files) + + def get_env_file(method, path): + if not os.path.exists(path): + return '{}' + env_url = heat_utils.normalise_file_path_to_url(path) + return request.urlopen(env_url).read() + + env_f, env = ( + template_utils.process_multiple_environments_and_files( + env_files, env_path_is_object=lambda path: True, + object_request=get_env_file)) + + return env diff --git a/tripleoclient/v1/container_image.py b/tripleoclient/v1/container_image.py index 4e3ffceb4..5028f7c75 100644 --- a/tripleoclient/v1/container_image.py +++ b/tripleoclient/v1/container_image.py @@ -13,6 +13,7 @@ # under the License. # +import copy import datetime import json import logging @@ -21,11 +22,9 @@ import sys import tempfile import time -from heatclient.common import template_utils -from heatclient.common import utils as heat_utils from osc_lib import exceptions as oscexc from osc_lib.i18n import _ -from six.moves.urllib import request +import six import yaml from tripleo_common.image import image_uploader @@ -36,6 +35,19 @@ from tripleoclient import constants from tripleoclient import utils +def build_env_file(params, command_options): + + f = six.StringIO() + f.write('# Generated with the following on %s\n#\n' % + datetime.datetime.now().isoformat()) + f.write('# openstack %s\n#\n\n' % + ' '.join(command_options)) + + yaml.safe_dump({'parameter_defaults': params}, f, + default_flow_style=False) + return f.getvalue() + + class UploadImage(command.Command): """Push overcloud container images to registries.""" @@ -174,6 +186,8 @@ class PrepareImageFiles(command.Command): parser = super(PrepareImageFiles, self).get_parser(prog_name) roles_file = os.path.join(constants.TRIPLEO_HEAT_TEMPLATES, constants.OVERCLOUD_ROLES_FILE) + if not os.path.isfile(roles_file): + roles_file = None defaults = kolla_builder.container_images_prepare_defaults() parser.add_argument( @@ -348,52 +362,24 @@ class PrepareImageFiles(command.Command): 'Use the variable=value format.') % s raise oscexc.CommandError(msg) - def write_env_file(self, params, env_file): - - with os.fdopen(os.open(env_file, - os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o666), - 'w') as f: - f.write('# Generated with the following on %s\n#\n' % - datetime.datetime.now().isoformat()) - f.write('# openstack %s\n#\n\n' % - ' '.join(self.app.command_options)) - - yaml.safe_dump({'parameter_defaults': params}, f, - default_flow_style=False) - def take_action(self, parsed_args): self.log.debug("take_action(%s)" % parsed_args) - env_files = [] - - try: + if parsed_args.roles_file: roles_data = yaml.safe_load(open(parsed_args.roles_file).read()) - except IOError: + else: roles_data = set() - if parsed_args.environment_directories: - env_files.extend(utils.load_environment_directories( - parsed_args.environment_directories)) - if parsed_args.environment_files: - env_files.extend(parsed_args.environment_files) + env = utils.build_prepare_env( + parsed_args.environment_files, + parsed_args.environment_directories + ) - def get_env_file(method, path): - if not os.path.exists(path): - return '{}' - env_url = heat_utils.normalise_file_path_to_url(path) - return request.urlopen(env_url).read() - - env_f, env = ( - template_utils.process_multiple_environments_and_files( - env_files, env_path_is_object=lambda path: True, - object_request=get_env_file)) - - if env_files: + if roles_data: service_filter = kolla_builder.build_service_filter( env, roles_data) else: service_filter = None - mapping_args = { 'tag': parsed_args.tag, 'namespace': parsed_args.namespace, @@ -428,7 +414,10 @@ class PrepareImageFiles(command.Command): ) if parsed_args.output_env_file: params = prepare_data[parsed_args.output_env_file] - self.write_env_file(params, parsed_args.output_env_file) + with os.fdopen(os.open(parsed_args.output_env_file, + os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o666), + 'w') as f: + f.write(build_env_file(params, self.app.command_options)) result = prepare_data[output_images_file] result_str = yaml.safe_dump({'container_images': result}, @@ -476,3 +465,119 @@ class DiscoverImageTag(command.Command): image=parsed_args.image, tag_from_label=parsed_args.tag_from_label )) + + +class TripleOImagePrepareDefault(command.Command): + """Generate a default ContainerImagePrepare parameter.""" + + auth_required = False + log = logging.getLogger(__name__ + ".TripleoImagePrepare") + + def get_parser(self, prog_name): + parser = super(TripleOImagePrepareDefault, self).get_parser(prog_name) + parser.add_argument( + "--output-env-file", + dest="output_env_file", + metavar='', + help=_("File to write environment file containing default " + "ContainerImagePrepare value."), + ) + parser.add_argument( + '--local-push-destination', + dest='push_destination', + action='store_true', + default=False, + help=_('Include a push_destination to trigger upload to a local ' + 'registry.') + ) + return parser + + def take_action(self, parsed_args): + self.log.debug("take_action(%s)" % parsed_args) + + cip = copy.deepcopy(kolla_builder.CONTAINER_IMAGE_PREPARE_PARAM) + if parsed_args.push_destination: + local_registry = image_uploader.get_undercloud_registry() + for entry in cip: + entry['push_destination'] = local_registry + params = { + 'ContainerImagePrepare': cip + } + env_data = build_env_file(params, self.app.command_options) + self.app.stdout.write(env_data) + if parsed_args.output_env_file: + with os.fdopen(os.open(parsed_args.output_env_file, + os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o666), + 'w') as f: + f.write(env_data) + + +class TripleOImagePrepare(command.Command): + """Prepare and upload containers from a single command.""" + + auth_required = False + log = logging.getLogger(__name__ + ".TripleoImagePrepare") + + def get_parser(self, prog_name): + parser = super(TripleOImagePrepare, self).get_parser(prog_name) + roles_file = os.path.join(constants.TRIPLEO_HEAT_TEMPLATES, + constants.OVERCLOUD_ROLES_FILE) + if not os.path.isfile(roles_file): + roles_file = None + parser.add_argument( + '--environment-file', '-e', metavar='', + action='append', dest='environment_files', + help=_('Environment file containing the ContainerImagePrepare ' + 'parameter which specifies all prepare actions. ' + 'Also, environment files specifying which services are ' + 'containerized. Entries will be filtered to only contain ' + 'images used by containerized services. (Can be specified ' + 'more than once.)') + ) + parser.add_argument( + '--environment-directory', metavar='', + action='append', dest='environment_directories', + default=[os.path.expanduser(constants.DEFAULT_ENV_DIRECTORY)], + help=_('Environment file directories that are automatically ' + 'added to the environment. ' + 'Can be specified more than once. Files in directories are ' + 'loaded in ascending sort order.') + ) + parser.add_argument( + '--roles-file', '-r', dest='roles_file', + default=roles_file, + help=_('Roles file to filter images by, overrides the default %s' + ) % constants.OVERCLOUD_ROLES_FILE + ) + parser.add_argument( + "--output-env-file", + dest="output_env_file", + metavar='', + help=_("File to write heat environment file which specifies all " + "image parameters. Any existing file will be overwritten."), + ) + return parser + + def take_action(self, parsed_args): + self.log.debug("take_action(%s)" % parsed_args) + + if parsed_args.roles_file: + roles_data = yaml.safe_load(open(parsed_args.roles_file).read()) + else: + roles_data = None + + env = utils.build_prepare_env( + parsed_args.environment_files, + parsed_args.environment_directories + ) + + params = kolla_builder.container_images_prepare_multi( + env, roles_data) + env_data = build_env_file(params, self.app.command_options) + if parsed_args.output_env_file: + with os.fdopen(os.open(parsed_args.output_env_file, + os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o666), + 'w') as f: + f.write(env_data) + else: + self.app.stdout.write(env_data)