openstack/python-tripleoclient
Code Issues Proposed changes

Move more prepare logic into kolla_builder

Browse Source 
Change-Id: I50fcfaab48bc3baeaefbaf9bb35f3e3b324cd267
Blueprint: container-prepare-workflow
This commit is contained in:
Steve Baker
2018-01-24 11:32:18 +13:00
parent 11b659aea0
commit 4449ee7d45
2 changed files with 143 additions and 225 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tripleoclient/tests/v1/overcloud_image/test_container_image.py
+127 -165
View File
@@ -23,6 +23,7 @@ import sys
import tempfile
import yaml
from tripleo_common.image import kolla_builder
from tripleoclient.tests.v1.test_plugin import TestPluginV1
from tripleoclient.v1 import container_image
@@ -75,18 +76,10 @@ class TestContainerImagePrepare(TestPluginV1):
# Get the command object to test
self.cmd = container_image.PrepareImageFiles(self.app, None)
def test_get_enabled_services(self):
temp = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, temp)
resource_registry = {'parameter_defaults': {
'RoleDisabledViaEnvironmentCount': 0,
'RoleOverwrittenViaEnvironmentServices': [
'OS::TripleO::Services::FromResourceRegistry'
]
}}
roles_file = '/foo/roles_data.yaml'
roles_yaml = '''
kolla_builder.DEFAULT_TEMPLATE_FILE = os.path.join(
'/tmp/overcloud_containers.yaml.j2'
)
self.roles_yaml = '''
- name: EnabledRole
CountDefault: 1
ServicesDefault:
@@ -104,7 +97,18 @@ class TestContainerImagePrepare(TestPluginV1):
ServicesDefault:
- OS::TripleO::Services::Overwritten
'''
mock_open_context = mock.mock_open(read_data=roles_yaml)
def test_get_enabled_services(self):
temp = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, temp)
resource_registry = {'parameter_defaults': {
'RoleDisabledViaEnvironmentCount': 0,
'RoleOverwrittenViaEnvironmentServices': [
'OS::TripleO::Services::FromResourceRegistry'
]
}}
roles_file = '/foo/roles_data.yaml'
mock_open_context = mock.mock_open(read_data=self.roles_yaml)
with mock.patch('six.moves.builtins.open', mock_open_context):
enabled_services = self.cmd.get_enabled_services(resource_registry,
roles_file)
@@ -113,44 +117,56 @@ class TestContainerImagePrepare(TestPluginV1):
'OS::TripleO::Services::FromResourceRegistry']),
enabled_services)
@mock.patch('tripleo_common.image.kolla_builder.KollaImageBuilder')
@mock.patch('tripleo_common.image.kolla_builder.'
'container_images_prepare_defaults', create=True)
@mock.patch('tripleo_common.image.kolla_builder.'
'container_images_prepare', create=True)
@mock.patch('requests.get')
def test_container_image_prepare_noargs(self, mock_get, mock_builder):
def test_container_image_prepare_noargs(self, mock_get, mock_cip,
mock_cipd):
mock_cipd.return_value = {
'neutron_driver': None,
'name_suffix': '',
'tag': 'latest',
'namespace': 'docker.io/tripleomaster',
'name_prefix':
'centos-binary-'
}
arglist = []
verifylist = []
cift = mock.MagicMock()
cift.return_value = {}
citi = mock.MagicMock()
citi.return_value = {
'name_prefix': 'centos-binary-',
'name_suffix': '',
'namespace': 'docker.io/tripleomaster',
'neutron_driver': None,
'tag': 'latest'
}
mock_builder.return_value.container_images_from_template = cift
mock_builder.return_value.container_images_template_inputs = citi
mock_cip.return_value = {'container_images.yaml': {}}
parsed_args = self.check_parser(self.cmd, arglist, verifylist)
self.cmd.take_action(parsed_args)
mock_builder.assert_called_with([parsed_args.template_file])
cift.assert_called_with(
filter=mock.ANY,
name_prefix='centos-binary-',
name_suffix='',
namespace='docker.io/tripleomaster',
neutron_driver=None,
tag='latest'
mock_cip.assert_called_with(
excludes=[],
mapping_args={
'neutron_driver': None,
'name_suffix': '',
'tag': 'latest',
'namespace': 'docker.io/tripleomaster',
'name_prefix':
'centos-binary-'
},
output_env_file=None,
output_images_file='container_images.yaml',
pull_source=None,
push_destination=None,
service_filter=None
)
@mock.patch('tripleo_common.image.kolla_builder.KollaImageBuilder')
@mock.patch('tripleo_common.image.kolla_builder.'
'container_images_prepare_defaults', create=True)
@mock.patch('tripleo_common.image.kolla_builder.'
'container_images_prepare', create=True)
@mock.patch('heatclient.common.template_utils.'
'process_multiple_environments_and_files')
@mock.patch('requests.get')
def test_container_image_prepare(self, mock_get, pmef, mock_builder):
def test_container_image_prepare(self, mock_get, pmef, mock_cip,
mock_cipd):
temp = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, temp)
@@ -158,6 +174,9 @@ class TestContainerImagePrepare(TestPluginV1):
env_file = os.path.join(temp, 'containers_env.yaml')
tmpl_file = os.path.join(temp, 'overcloud_containers.yaml.j2')
aodh_file = os.path.join(temp, 'docker', 'services', 'aodh.yaml')
roles_file = os.path.join(temp, 'roles_data.yaml')
with open(roles_file, 'w') as f:
f.write(self.roles_yaml)
mock_get.side_effect = requests.exceptions.SSLError('ouch')
resource_registry = {'resource_registry': {
@@ -188,46 +207,53 @@ class TestContainerImagePrepare(TestPluginV1):
'--set',
'ceph_tag=mytag',
'-e',
'environment/docker.yaml'
'environment/docker.yaml',
'--roles-file',
roles_file
]
self.cmd.app.command_options = arglist
verifylist = []
cift = mock.MagicMock()
cift.return_value = [{
mock_cip.return_value = {images_file: [{
'imagename': '192.0.2.0:8787/t/os-aodh-apifoo:passed-ci',
'params': ['DockerAodhApiImage', 'DockerAodhConfigImage'],
'services': [
'OS::TripleO::Services::AodhApi',
'OS::TripleO::Services::AodhEvaluator',
],
}, {
'imagename': '192.0.2.0:8787/t/os-aodh-evaluatorfoo:passed-ci',
'params': ['DockerAodhEvaluatorImage'],
'services': [
'OS::TripleO::Services::AodhEvaluator',
],
}]
mock_builder.return_value.container_images_from_template = cift
}], env_file: {
'DockerAodhApiImage':
'192.0.2.0:8787/t/os-aodh-apifoo:passed-ci',
'DockerAodhConfigImage':
'192.0.2.0:8787/t/os-aodh-apifoo:passed-ci',
'DockerAodhEvaluatorImage':
'192.0.2.0:8787/t/os-aodh-evaluatorfoo:passed-ci',
'DockerInsecureRegistryAddress': ['192.0.2.0:8787']
}
}
parsed_args = self.check_parser(self.cmd, arglist, verifylist)
self.cmd.take_action(parsed_args)
mock_builder.assert_called_with([tmpl_file])
pmef.assert_called_once_with(['environment/docker.yaml'],
env_path_is_object=mock.ANY,
object_request=mock.ANY)
cift.assert_called_once_with(
filter=mock.ANY,
name_prefix='os-',
name_suffix='foo',
namespace='192.0.2.0:8787/t',
tag='passed-ci',
ceph_image='mydaemon',
ceph_namespace='myceph',
ceph_tag='mytag',
neutron_driver=None
mock_cip.assert_called_once_with(
excludes=[],
mapping_args={
'namespace': '192.0.2.0:8787/t',
'name_suffix': 'foo',
'ceph_tag': 'mytag',
'ceph_image': 'mydaemon',
'tag': 'passed-ci',
'neutron_driver': None,
'ceph_namespace': 'myceph',
'name_prefix': 'os-'
},
output_env_file=env_file,
output_images_file=images_file,
pull_source=None,
push_destination=None,
service_filter=set([
'OS::TripleO::Services::AodhEvaluator',
])
)
ci_data = {
'container_images': [{
@@ -253,9 +279,9 @@ class TestContainerImagePrepare(TestPluginV1):
self.assertEqual(env_data, yaml.safe_load(f))
def _test_container_image_prepare_helper(
self, pmef, mock_builder, pmef_call_args,
arg_list, neutron_driver, expected_container_template_params,
expected_oc_yaml_contents, expected_env_contents):
self, pmef, mock_cip, pmef_call_args,
arg_list, neutron_driver, expected_oc_yaml_contents,
expected_env_contents):
temp = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, temp)
images_file = os.path.join(temp, 'overcloud_containers.yaml')
@@ -263,7 +289,9 @@ class TestContainerImagePrepare(TestPluginV1):
tmpl_file = os.path.join(temp, 'overcloud_containers.yaml.j2')
resource_registry = {}
service_filter = set()
if neutron_driver == 'odl':
service_filter.add('OS::TripleO::Services::OpenDaylightApi')
odlapi_file = os.path.join(temp, 'docker', 'services',
'opendaylight.yaml')
@@ -271,6 +299,7 @@ class TestContainerImagePrepare(TestPluginV1):
'OS::TripleO::Services::OpenDaylightApi': odlapi_file
}}
elif neutron_driver == 'ovn':
service_filter.add('OS::TripleO::Services::OVNController')
ovnapi_file = os.path.join(temp, 'docker', 'services',
'overcloud_containers.yaml.j2')
@@ -299,24 +328,31 @@ class TestContainerImagePrepare(TestPluginV1):
cmd_arglist.extend(arg_list)
self.cmd.app.command_options = cmd_arglist
verifylist = []
cift = mock.MagicMock()
cift.return_value = expected_container_template_params
mock_builder.return_value.container_images_from_template = cift
mock_cip.return_value = {
images_file: expected_oc_yaml_contents['container_images'],
env_file: expected_env_contents['parameter_defaults']
}
parsed_args = self.check_parser(self.cmd, cmd_arglist, verifylist)
self.cmd.take_action(parsed_args)
mock_builder.assert_called_with([tmpl_file])
pmef.assert_called_once_with(pmef_call_args,
env_path_is_object=mock.ANY,
object_request=mock.ANY)
cift.assert_called_once_with(
filter=mock.ANY,
name_prefix='os-',
name_suffix='foo',
namespace='tripleo',
tag='passed-ci',
neutron_driver=neutron_driver,
mock_cip.assert_called_once_with(
excludes=[],
mapping_args={
'neutron_driver': neutron_driver,
'name_suffix': 'foo',
'tag': 'passed-ci',
'namespace': 'tripleo',
'name_prefix': 'os-'
},
output_env_file=env_file,
output_images_file=images_file,
pull_source=None,
push_destination=None,
service_filter=service_filter
)
with open(images_file) as f:
@@ -324,42 +360,22 @@ class TestContainerImagePrepare(TestPluginV1):
with open(env_file) as f:
self.assertEqual(expected_env_contents, yaml.safe_load(f))
@mock.patch('tripleo_common.image.kolla_builder.KollaImageBuilder')
@mock.patch('tripleo_common.image.kolla_builder.'
'container_images_prepare_defaults', create=True)
@mock.patch('tripleo_common.image.kolla_builder.'
'container_images_prepare', create=True)
@mock.patch('heatclient.common.template_utils.'
'process_multiple_environments_and_files')
@mock.patch('tripleoclient.v1.container_image.PrepareImageFiles.'
'get_enabled_services')
@mock.patch('requests.get')
def test_container_image_prepare_for_odl(self, mock_get, ges, pmef,
mock_builder):
mock_cip, mock_cipd):
arglist = [
'-e',
'environments/services-docker/neutron-opendaylight.yaml',
]
expected_container_template_params = [{
'imagename':
'tripleo/os-neutron-server-opendaylightfoo:passed-ci',
'params':
['DockerNeutronApiImage', 'DockerNeutronConfigImage'],
'services': [
'OS::TripleO::Services::NeutronApi',
'OS::TripleO::Services::NeutronDhcpAgent',
'OS::TripleO::Services::NeutronMetadataAgent',
'OS::TripleO::Services::NeutronServer',
'OS::TripleO::Services::OpenDaylightApi',
],
}, {
'imagename': 'tripleo/os-opendaylightfoo:passed-ci',
'params':
['DockerOpendaylightApiImage',
'DockerOpendaylightConfigImage'],
'services': [
'OS::TripleO::Services::OpenDaylightApi',
],
}
]
ges.return_value = (
set(['OS::TripleO::Services::NeutronApi',
'OS::TripleO::Services::NeutronDhcpAgent',
@@ -392,45 +408,25 @@ class TestContainerImagePrepare(TestPluginV1):
}
self._test_container_image_prepare_helper(
pmef, mock_builder, pmef_call_args, arglist, 'odl',
expected_container_template_params, expected_oc_yaml_contents,
expected_env_contents)
pmef, mock_cip, pmef_call_args, arglist, 'odl',
expected_oc_yaml_contents, expected_env_contents)
@mock.patch('tripleo_common.image.kolla_builder.KollaImageBuilder')
@mock.patch('tripleo_common.image.kolla_builder.'
'container_images_prepare_defaults', create=True)
@mock.patch('tripleo_common.image.kolla_builder.'
'container_images_prepare', create=True)
@mock.patch('heatclient.common.template_utils.'
'process_multiple_environments_and_files')
@mock.patch('tripleoclient.v1.container_image.PrepareImageFiles.'
'get_enabled_services')
@mock.patch('requests.get')
def test_container_image_prepare_for_ovn(self, mock_get, ges, pmef,
mock_builder):
mock_cip, mock_cipd):
arglist = [
'-e',
'environments/services-docker/neutron-ovn.yaml',
]
expected_container_template_params = [{
'imagename':
'tripleo/os-neutron-server-ovnfoo:passed-ci',
'params':
['DockerNeutronApiImage', 'DockerNeutronConfigImage'],
'services': [
'OS::TripleO::Services::NeutronApi',
'OS::TripleO::Services::NeutronServer',
'OS::TripleO::Services::OVNController',
'OS::TripleO::Services::OVNDBs',
],
}, {
'imagename': 'tripleo/os-ovn-controllerfoo:passed-ci',
'params':
['DockerOvnControllerImage',
'DockerOvnControllerConfigImage'],
'services': [
'OS::TripleO::Services::OVNController',
],
}
]
ges.return_value = (
set(['OS::TripleO::Services::NeutronApi',
'OS::TripleO::Services::NeutronServer',
@@ -462,42 +458,8 @@ class TestContainerImagePrepare(TestPluginV1):
}
self._test_container_image_prepare_helper(
pmef, mock_builder, pmef_call_args, arglist, 'ovn',
expected_container_template_params, expected_oc_yaml_contents,
expected_env_contents)
@mock.patch('requests.get')
def test_detect_insecure_registry(self, mock_get):
self.assertEqual(
{},
self.cmd.detect_insecure_registries(
{'foo': 'docker.io/tripleo'}))
self.assertEqual(
{},
self.cmd.detect_insecure_registries(
{'foo': 'tripleo'}))
mock_get.side_effect = requests.exceptions.ReadTimeout('ouch')
self.assertEqual(
{},
self.cmd.detect_insecure_registries(
{'foo': '192.0.2.0:8787/tripleo'}))
mock_get.side_effect = requests.exceptions.SSLError('ouch')
self.assertEqual(
{'DockerInsecureRegistryAddress': ['192.0.2.0:8787']},
self.cmd.detect_insecure_registries(
{'foo': '192.0.2.0:8787/tripleo'}))
self.assertEqual(
{'DockerInsecureRegistryAddress': [
'192.0.2.0:8787',
'192.0.2.1:8787']},
self.cmd.detect_insecure_registries({
'foo': '192.0.2.0:8787/tripleo/foo',
'bar': '192.0.2.0:8787/tripleo/bar',
'baz': '192.0.2.1:8787/tripleo/baz',
}))
pmef, mock_cip, pmef_call_args, arglist, 'ovn',
expected_oc_yaml_contents, expected_env_contents)
class TestContainerImageBuild(TestPluginV1):
tripleoclient/v1/container_image.py
+16 -60
View File
@@ -17,7 +17,6 @@ import datetime
import json
import logging
import os
import re
import sys
import tempfile
@@ -25,7 +24,6 @@ from heatclient.common import template_utils
from heatclient.common import utils as heat_utils
from osc_lib import exceptions as oscexc
from osc_lib.i18n import _
import requests
from six.moves.urllib import request
import yaml
@@ -170,23 +168,18 @@ class PrepareImageFiles(command.Command):
def get_parser(self, prog_name):
parser = super(PrepareImageFiles, self).get_parser(prog_name)
template_file = os.path.join(sys.prefix, 'share', 'tripleo-common',
'container-images',
'overcloud_containers.yaml.j2')
roles_file = os.path.join(constants.TRIPLEO_HEAT_TEMPLATES,
constants.OVERCLOUD_ROLES_FILE)
builder = kolla_builder.KollaImageBuilder([template_file])
defaults = builder.container_images_template_inputs()
defaults = kolla_builder.container_images_prepare_defaults()
parser.add_argument(
"--template-file",
dest="template_file",
default=template_file,
default=kolla_builder.DEFAULT_TEMPLATE_FILE,
metavar='<yaml template file>',
help=_("YAML template file which the images config file will be "
"built from.\n"
"Default: %s") % template_file,
"Default: %s") % kolla_builder.DEFAULT_TEMPLATE_FILE,
)
parser.add_argument(
"--pull-source",
@@ -329,27 +322,6 @@ class PrepareImageFiles(command.Command):
'Use the variable=value format.') % s
raise oscexc.CommandError(msg)
def detect_insecure_registries(self, params):
insecure = []
hosts = set()
for image in params.values():
hosts.add(image.split('/')[0])
for host in hosts:
try:
requests.get('https://%s/' % host)
except requests.exceptions.SSLError:
insecure.append(host)
except Exception:
# for any other error assume it is a secure registry, because:
# - it is secure registry
# - the host is not accessible
# - the namespace doesn't include a host name
pass
if not insecure:
return {}
return {'DockerInsecureRegistryAddress': sorted(insecure)}
def write_env_file(self, params, env_file):
with os.fdopen(os.open(env_file,
@@ -437,39 +409,23 @@ class PrepareImageFiles(command.Command):
}
self.parse_set_values(subs, parsed_args.set)
def ffunc(entry):
imagename = entry.get('imagename', '')
for p in parsed_args.excludes:
if re.search(p, imagename):
return None
if service_filter is not None:
# check the entry is for a service being deployed
image_services = set(entry.get('services', []))
if not service_filter.intersection(image_services):
return None
if parsed_args.pull_source:
entry['pull_source'] = parsed_args.pull_source
if parsed_args.push_destination:
entry['push_destination'] = parsed_args.push_destination
return entry
builder = kolla_builder.KollaImageBuilder([parsed_args.template_file])
result = builder.container_images_from_template(filter=ffunc, **subs)
params = {}
for entry in result:
imagename = entry.get('imagename', '')
if 'params' in entry:
for p in entry.pop('params'):
params[p] = imagename
if 'services' in entry:
del(entry['services'])
output_images_file = (parsed_args.output_images_file
or 'container_images.yaml')
prepare_data = kolla_builder.container_images_prepare(
excludes=parsed_args.excludes,
service_filter=service_filter,
pull_source=parsed_args.pull_source,
push_destination=parsed_args.push_destination,
mapping_args=subs,
output_env_file=parsed_args.output_env_file,
output_images_file=output_images_file
)
if parsed_args.output_env_file:
params.update(
self.detect_insecure_registries(params))
params = prepare_data[parsed_args.output_env_file]
self.write_env_file(params, parsed_args.output_env_file)
result = prepare_data[output_images_file]
result_str = yaml.safe_dump({'container_images': result},
default_flow_style=False)
sys.stdout.write(result_str)