openstack/python-tripleoclient
Code Issues Proposed changes

Implement TripleoImagePrepare command

Browse Source 
Running:

  $ openstack tripleo container image prepare default \
      --output-env-file prepare-default.yaml \
      --local-push-destination
  $ openstack tripleo container image prepare -e prepare-default.yaml

Will read a Heat template with ContainerImagePrepare, prepare containers
and upload to a registry if needed.

The idea is to replace the other commands used by the overcloud for any
use case: undercloud, overcloud or any cloud.

One of the goals here is to execute this process before starting the
containers while deploying OpenStack on any cloud.

Change-Id: Ie4b7951147f5a1aec654982e21296a749fdd865c
Blueprint: container-prepare-workflow
This commit is contained in:
Emilien Macchi 2018-04-23 17:07:55 -07:00 committed by Steve Baker
parent e9a6843040
commit 6f0136dbb8
5 changed files with 336 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
releasenotes/notes/tripleo-prepare-d57bbccb2a44e8b2.yaml Normal file
View File

@ -0,0 +1,14 @@
---
features:
- |
The new command `openstack tripleo container image prepare` will do the
same container image preperation which happens during undercloud and
overcloud deploy, but in a standalone command. The prepare operations are
driven by a heat environment file containing the parameter
`ContainerImagePrepare`. This parameter allows multiple upload and
modification operations to be specified, and the result will be a list of
image parameters to use during a tripleo deployment.
The command `openstack tripleo container image prepare default` will
generate a `ContainerImagePrepare` with the recommended defaults to use for
`openstack tripleo container image prepare`.

2
setup.cfg
View File

@ -94,6 +94,8 @@ openstack.tripleoclient.v1 =
overcloud_ffwd-upgrade_converge = tripleoclient.v1.overcloud_ffwd_upgrade:FFWDUpgradeConverge
overcloud_execute = tripleoclient.v1.overcloud_execute:RemoteExecute
overcloud_generate_fencing = tripleoclient.v1.overcloud_parameters:GenerateFencingParameters
tripleo_container_image_prepare = tripleoclient.v1.container_image:TripleOImagePrepare
tripleo_container_image_prepare_default = tripleoclient.v1.container_image:TripleOImagePrepareDefault
undercloud_deploy = tripleoclient.v1.undercloud_deploy:DeployUndercloud
undercloud_install = tripleoclient.v1.undercloud:InstallUndercloud
undercloud_upgrade = tripleoclient.v1.undercloud:UpgradeUndercloud

142
tripleoclient/tests/v1/test_container_image.py
View File

@ -279,6 +279,148 @@ class TestContainerImagePrepare(TestPluginV1):
self.assertEqual(env_data, yaml.safe_load(f))
class TestTripleoImagePrepare(TestPluginV1):
def setUp(self):
super(TestTripleoImagePrepare, self).setUp()
# Get the command object to test
self.cmd = container_image.TripleOImagePrepare(self.app, None)
self.temp_dir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, self.temp_dir)
self.prepare_default_file = os.path.join(
self.temp_dir, 'prepare_env.yaml')
default_param = kolla_builder.CONTAINER_IMAGE_PREPARE_PARAM
self.default_env = {
'parameter_defaults': {
'ContainerImagePrepare': default_param
}
}
with open(self.prepare_default_file, 'w') as f:
yaml.safe_dump(self.default_env, f)
self.roles_yaml = '''
- name: EnabledRole
CountDefault: 1
ServicesDefault:
- OS::TripleO::Services::AodhEvaluator
- name: RoleDisabledViaRolesData
CountDefault: 0
ServicesDefault:
- OS::TripleO::Services::AodhApi
- name: RoleDisabledViaEnvironment
CountDefault: 1
ServicesDefault:
- OS::TripleO::Services::Disabled
- name: RoleOverwrittenViaEnvironment
CountDefault: 1
ServicesDefault:
- OS::TripleO::Services::Overwritten
'''
self.roles_data_file = os.path.join(
self.temp_dir, 'roles_data.yaml')
with open(self.roles_data_file, 'w') as f:
f.write(self.roles_yaml)
@mock.patch('tripleo_common.image.kolla_builder.'
'container_images_prepare_multi')
def test_tripleo_container_image_prepare(self, prepare_multi):
env_file = os.path.join(self.temp_dir, 'containers_env.yaml')
arglist = [
'--environment-file', self.prepare_default_file,
'--roles-file', self.roles_data_file,
'--output-env-file', env_file
]
verifylist = []
self.app.command_options = [
'tripleo', 'container', 'image', 'prepare', 'default'
] + arglist
prepare_multi.return_value = {
'DockerAodhApiImage':
'192.0.2.0:8787/t/os-aodh-apifoo:passed-ci',
'DockerAodhConfigImage':
'192.0.2.0:8787/t/os-aodh-apifoo:passed-ci',
}
parsed_args = self.check_parser(self.cmd, arglist, verifylist)
self.cmd.take_action(parsed_args)
prepare_multi.assert_called_once_with(
self.default_env, yaml.safe_load(self.roles_yaml))
with open(env_file) as f:
result = yaml.safe_load(f)
self.assertEqual({
'parameter_defaults': {
'DockerAodhApiImage':
'192.0.2.0:8787/t/os-aodh-apifoo:passed-ci',
'DockerAodhConfigImage':
'192.0.2.0:8787/t/os-aodh-apifoo:passed-ci',
}
}, result)
class TestTripleoImagePrepareDefault(TestPluginV1):
def setUp(self):
super(TestTripleoImagePrepareDefault, self).setUp()
# Get the command object to test
self.cmd = container_image.TripleOImagePrepareDefault(self.app, None)
def test_prepare_default(self):
arglist = []
verifylist = []
self.app.command_options = [
'tripleo', 'container', 'image', 'prepare', 'default'
] + arglist
self.cmd.app.stdout = six.StringIO()
cmd = container_image.TripleOImagePrepareDefault(self.app, None)
parsed_args = self.check_parser(cmd, arglist, verifylist)
cmd.take_action(parsed_args)
result = self.app.stdout.getvalue()
expected_param = kolla_builder.CONTAINER_IMAGE_PREPARE_PARAM
expected = {
'parameter_defaults': {
'ContainerImagePrepare': expected_param
}
}
self.assertEqual(expected, yaml.safe_load(result))
@mock.patch('tripleo_common.image.image_uploader.get_undercloud_registry')
def test_prepare_default_local_registry(self, mock_gur):
temp = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, temp)
env_file = os.path.join(temp, 'containers_env.yaml')
arglist = ['--local-push-destination', '--output-env-file', env_file]
verifylist = []
mock_gur.return_value = '192.0.2.1:8787'
self.app.command_options = [
'tripleo', 'container', 'image', 'prepare', 'default'
] + arglist
cmd = container_image.TripleOImagePrepareDefault(self.app, None)
parsed_args = self.check_parser(cmd, arglist, verifylist)
cmd.take_action(parsed_args)
with open(env_file) as f:
result = yaml.safe_load(f)
self.assertEqual(
'192.0.2.1:8787',
result['parameter_defaults']['ContainerImagePrepare']
[0]['push_destination']
)
class TestContainerImageBuild(TestPluginV1):
def setUp(self):

34
tripleoclient/utils.py
View File

@ -35,6 +35,7 @@ import yaml
from heatclient.common import event_utils
from heatclient.common import template_utils
from heatclient.common import utils as heat_utils
from heatclient.exc import HTTPNotFound
from osc_lib.i18n import _
from oslo_concurrency import processutils
@ -1099,3 +1100,36 @@ def ffwd_upgrade_operator_confirm(parsed_args_yes, log):
log.debug("Fast forward upgrade cancelled on user request")
print("Cancelling fast forward upgrade")
sys.exit(1)
def build_prepare_env(environment_files, environment_directories):
'''Build the environment for container image prepare
:param environment_files: List of environment files to build
environment from
:type environment_files: list
:param environment_directories: List of environment directories to build
environment from
:type environment_directories: list
'''
env_files = []
if environment_directories:
env_files.extend(load_environment_directories(
environment_directories))
if environment_files:
env_files.extend(environment_files)
def get_env_file(method, path):
if not os.path.exists(path):
return '{}'
env_url = heat_utils.normalise_file_path_to_url(path)
return request.urlopen(env_url).read()
env_f, env = (
template_utils.process_multiple_environments_and_files(
env_files, env_path_is_object=lambda path: True,
object_request=get_env_file))
return env

183
tripleoclient/v1/container_image.py
View File

@ -13,6 +13,7 @@
# under the License.
#
import copy
import datetime
import json
import logging
@ -21,11 +22,9 @@ import sys
import tempfile
import time
from heatclient.common import template_utils
from heatclient.common import utils as heat_utils
from osc_lib import exceptions as oscexc
from osc_lib.i18n import _
from six.moves.urllib import request
import six
import yaml
from tripleo_common.image import image_uploader
@ -36,6 +35,19 @@ from tripleoclient import constants
from tripleoclient import utils
def build_env_file(params, command_options):
f = six.StringIO()
f.write('# Generated with the following on %s\n#\n' %
datetime.datetime.now().isoformat())
f.write('# openstack %s\n#\n\n' %
' '.join(command_options))
yaml.safe_dump({'parameter_defaults': params}, f,
default_flow_style=False)
return f.getvalue()
class UploadImage(command.Command):
"""Push overcloud container images to registries."""
@ -174,6 +186,8 @@ class PrepareImageFiles(command.Command):
parser = super(PrepareImageFiles, self).get_parser(prog_name)
roles_file = os.path.join(constants.TRIPLEO_HEAT_TEMPLATES,
constants.OVERCLOUD_ROLES_FILE)
if not os.path.isfile(roles_file):
roles_file = None
defaults = kolla_builder.container_images_prepare_defaults()
parser.add_argument(
@ -348,52 +362,24 @@ class PrepareImageFiles(command.Command):
'Use the variable=value format.') % s
raise oscexc.CommandError(msg)
def write_env_file(self, params, env_file):
with os.fdopen(os.open(env_file,
os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o666),
'w') as f:
f.write('# Generated with the following on %s\n#\n' %
datetime.datetime.now().isoformat())
f.write('# openstack %s\n#\n\n' %
' '.join(self.app.command_options))
yaml.safe_dump({'parameter_defaults': params}, f,
default_flow_style=False)
def take_action(self, parsed_args):
self.log.debug("take_action(%s)" % parsed_args)
env_files = []
try:
if parsed_args.roles_file:
roles_data = yaml.safe_load(open(parsed_args.roles_file).read())
except IOError:
else:
roles_data = set()
if parsed_args.environment_directories:
env_files.extend(utils.load_environment_directories(
parsed_args.environment_directories))
if parsed_args.environment_files:
env_files.extend(parsed_args.environment_files)
env = utils.build_prepare_env(
parsed_args.environment_files,
parsed_args.environment_directories
)
def get_env_file(method, path):
if not os.path.exists(path):
return '{}'
env_url = heat_utils.normalise_file_path_to_url(path)
return request.urlopen(env_url).read()
env_f, env = (
template_utils.process_multiple_environments_and_files(
env_files, env_path_is_object=lambda path: True,
object_request=get_env_file))
if env_files:
if roles_data:
service_filter = kolla_builder.build_service_filter(
env, roles_data)
else:
service_filter = None
mapping_args = {
'tag': parsed_args.tag,
'namespace': parsed_args.namespace,
@ -428,7 +414,10 @@ class PrepareImageFiles(command.Command):
)
if parsed_args.output_env_file:
params = prepare_data[parsed_args.output_env_file]
self.write_env_file(params, parsed_args.output_env_file)
with os.fdopen(os.open(parsed_args.output_env_file,
os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o666),
'w') as f:
f.write(build_env_file(params, self.app.command_options))
result = prepare_data[output_images_file]
result_str = yaml.safe_dump({'container_images': result},
@ -476,3 +465,119 @@ class DiscoverImageTag(command.Command):
image=parsed_args.image,
tag_from_label=parsed_args.tag_from_label
))
class TripleOImagePrepareDefault(command.Command):
"""Generate a default ContainerImagePrepare parameter."""
auth_required = False
log = logging.getLogger(__name__ + ".TripleoImagePrepare")
def get_parser(self, prog_name):
parser = super(TripleOImagePrepareDefault, self).get_parser(prog_name)
parser.add_argument(
"--output-env-file",
dest="output_env_file",
metavar='<file path>',
help=_("File to write environment file containing default "
"ContainerImagePrepare value."),
)
parser.add_argument(
'--local-push-destination',
dest='push_destination',
action='store_true',
default=False,
help=_('Include a push_destination to trigger upload to a local '
'registry.')
)
return parser
def take_action(self, parsed_args):
self.log.debug("take_action(%s)" % parsed_args)
cip = copy.deepcopy(kolla_builder.CONTAINER_IMAGE_PREPARE_PARAM)
if parsed_args.push_destination:
local_registry = image_uploader.get_undercloud_registry()
for entry in cip:
entry['push_destination'] = local_registry
params = {
'ContainerImagePrepare': cip
}
env_data = build_env_file(params, self.app.command_options)
self.app.stdout.write(env_data)
if parsed_args.output_env_file:
with os.fdopen(os.open(parsed_args.output_env_file,
os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o666),
'w') as f:
f.write(env_data)
class TripleOImagePrepare(command.Command):
"""Prepare and upload containers from a single command."""
auth_required = False
log = logging.getLogger(__name__ + ".TripleoImagePrepare")
def get_parser(self, prog_name):
parser = super(TripleOImagePrepare, self).get_parser(prog_name)
roles_file = os.path.join(constants.TRIPLEO_HEAT_TEMPLATES,
constants.OVERCLOUD_ROLES_FILE)
if not os.path.isfile(roles_file):
roles_file = None
parser.add_argument(
'--environment-file', '-e', metavar='<file path>',
action='append', dest='environment_files',
help=_('Environment file containing the ContainerImagePrepare '
'parameter which specifies all prepare actions. '
'Also, environment files specifying which services are '
'containerized. Entries will be filtered to only contain '
'images used by containerized services. (Can be specified '
'more than once.)')
)
parser.add_argument(
'--environment-directory', metavar='<HEAT ENVIRONMENT DIRECTORY>',
action='append', dest='environment_directories',
default=[os.path.expanduser(constants.DEFAULT_ENV_DIRECTORY)],
help=_('Environment file directories that are automatically '
'added to the environment. '
'Can be specified more than once. Files in directories are '
'loaded in ascending sort order.')
)
parser.add_argument(
'--roles-file', '-r', dest='roles_file',
default=roles_file,
help=_('Roles file to filter images by, overrides the default %s'
) % constants.OVERCLOUD_ROLES_FILE
)
parser.add_argument(
"--output-env-file",
dest="output_env_file",
metavar='<file path>',
help=_("File to write heat environment file which specifies all "
"image parameters. Any existing file will be overwritten."),
)
return parser
def take_action(self, parsed_args):
self.log.debug("take_action(%s)" % parsed_args)
if parsed_args.roles_file:
roles_data = yaml.safe_load(open(parsed_args.roles_file).read())
else:
roles_data = None
env = utils.build_prepare_env(
parsed_args.environment_files,
parsed_args.environment_directories
)
params = kolla_builder.container_images_prepare_multi(
env, roles_data)
env_data = build_env_file(params, self.app.command_options)
if parsed_args.output_env_file:
with os.fdopen(os.open(parsed_args.output_env_file,
os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o666),
'w') as f:
f.write(env_data)
else:
self.app.stdout.write(env_data)