Merge "Make overcloud_containers.yaml template driven"

This commit is contained in:
Jenkins 2017-06-23 11:24:57 +00:00 committed by Gerrit Code Review
commit 4ba7d564db
4 changed files with 237 additions and 21 deletions

View File

@ -0,0 +1,80 @@
{% set namespace=namespace or "tripleoupstream" %}
{% set name_prefix=name_prefix or "centos-binary-" %}
{% set name_suffix=name_suffix or "" %}
{% set tag=tag or "latest" %}
container_images_template:
- imagename: "{{namespace}}/{{name_prefix}}aodh-api{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}aodh-evaluator{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}aodh-listener{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}aodh-notifier{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}ceilometer-central{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}ceilometer-compute{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}ceilometer-notification{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}ceilometer-ipmi{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}cinder-api{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}cinder-backup{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}cinder-scheduler{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}cinder-volume{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}collectd{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}congress-api{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}ec2-api{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}etcd{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}glance-api{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}gnocchi-api{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}gnocchi-metricd{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}gnocchi-statsd{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}haproxy{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}heat-api-cfn{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}heat-api{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}heat-engine{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}horizon{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}ironic-api{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}ironic-conductor{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}ironic-pxe{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}iscsid{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}keystone{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}manila-api{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}manila-base{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}manila-scheduler{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}mariadb{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}memcached{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}mistral-api{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}mistral-engine{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}mistral-executor{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}mistral-event-engine{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}mongodb{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}multipathd{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}neutron-dhcp-agent{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}neutron-l3-agent{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}neutron-metadata-agent{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}neutron-openvswitch-agent{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}neutron-sriov-agent{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}neutron-server{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}nova-api{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}nova-base{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}nova-compute-ironic{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}nova-compute{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}nova-conductor{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}nova-consoleauth{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}nova-libvirt{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}nova-novncproxy{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}nova-placement-api{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}nova-scheduler{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}octavia-base{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}octavia-api{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}octavia-health-manager{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}octavia-housekeeping{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}octavia-worker{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}panko-api{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}rabbitmq{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}redis{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}sahara-api{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}sahara-engine{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}sensu-client{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}swift-account{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}swift-container{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}swift-object{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}swift-proxy-server{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}tacker{{name_suffix}}:{{tag}}"
- imagename: "{{namespace}}/{{name_prefix}}zaqar{{name_suffix}}:{{tag}}"
- imagename: "ceph/daemon:tag-build-master-jewel-centos-7"

View File

@ -12,6 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import collections
import json
import os
import yaml
@ -25,9 +26,11 @@ class BaseImageManager(object):
logger = log.getLogger(__name__ + '.BaseImageManager')
APPEND_ATTRIBUTES = ['elements', 'options', 'packages']
CONFIG_SECTIONS = (
DISK_IMAGES, UPLOADS, CONTAINER_IMAGES
DISK_IMAGES, UPLOADS, CONTAINER_IMAGES,
CONTAINER_IMAGES_TEMPLATE
) = (
'disk_images', 'uploads', 'container_images'
'disk_images', 'uploads', 'container_images',
'container_images_template'
)
def __init__(self, config_files, images=None):
@ -45,7 +48,7 @@ class BaseImageManager(object):
existing_image[attribute_name] = attribute
def load_config_files(self, section):
config_data = {}
config_data = collections.OrderedDict()
for config_file in self.config_files:
if os.path.isfile(config_file):
with open(config_file) as cf:

View File

@ -14,19 +14,16 @@
#
import jinja2
import logging
import os
import re
import subprocess
import sys
import yaml
from tripleo_common.image import base
if sys.version_info[0] < 3:
import codecs
_open = open
open = codecs.open
class KollaImageBuilder(base.BaseImageManager):
"""Build images using kolla-build"""
@ -53,6 +50,44 @@ class KollaImageBuilder(base.BaseImageManager):
# what results should be acceptable as a regex to build one image
return imagename
def container_images_from_template(self, filter=None, **kwargs):
'''Build container_images data from container_images_template.
Any supplied keyword arguments are used for the substitution mapping to
transform the data in the config file container_images_template
section.
The resulting data resembles a config file which contains a valid
populated container_images section.
If a function is passed to the filter argument, this will be used to
modify the entry after substitution. If the filter function returns
None then the entry will not be added to the resulting list.
Defaults are applied so that when no arguments are provided the
resulting entries have the form:
- imagename: tripleoupstream/centos-binary-<name>:latest
'''
mapping = dict(kwargs)
result = []
if len(self.config_files) != 1:
raise ValueError('A single config file must be specified')
config_file = self.config_files[0]
with open(config_file) as cf:
template = jinja2.Template(cf.read())
rendered = template.render(mapping)
rendered_dict = yaml.safe_load(rendered)
for i in rendered_dict[self.CONTAINER_IMAGES_TEMPLATE]:
entry = dict(i)
if filter:
entry = filter(entry)
if entry is not None:
result.append(entry)
return result
def build_images(self, kolla_config_files=None):
cmd = ['kolla-build']

View File

@ -18,24 +18,45 @@ import mock
import os
import six
import subprocess
import sys
import tempfile
import yaml
from tripleo_common.image import kolla_builder as kb
from tripleo_common.tests import base
filedata = six.u(
"""container_images:
- imagename: tripleoupstream/heat-docker-agents-centos:latest
push_destination: localhost:8787
- imagename: tripleoupstream/centos-binary-nova-compute:liberty
uploader: docker
pull_source: docker.io
push_destination: localhost:8787
- imagename: tripleoupstream/centos-binary-nova-libvirt:liberty
uploader: docker
pull_source: docker.io
- imagename: tripleoupstream/image-with-missing-tag
push_destination: localhost:8787
filedata = six.u("""container_images:
- imagename: tripleoupstream/heat-docker-agents-centos:latest
push_destination: localhost:8787
- imagename: tripleoupstream/centos-binary-nova-compute:liberty
uploader: docker
pull_source: docker.io
push_destination: localhost:8787
- imagename: tripleoupstream/centos-binary-nova-libvirt:liberty
uploader: docker
pull_source: docker.io
- imagename: tripleoupstream/image-with-missing-tag
push_destination: localhost:8787
""")
template_filedata = six.u("""
{% set namespace=namespace or "tripleoupstream" %}
{% set name_prefix=name_prefix or "centos-binary-" %}
{% set name_suffix=name_suffix or "" %}
{% set tag=tag or "latest" %}
container_images_template:
- imagename: "{{namespace}}/heat-docker-agents-centos:latest"
push_destination: "{{push_destination}}"
- imagename: "{{namespace}}/{{name_prefix}}nova-compute{{name_suffix}}:{{tag}}"
uploader: "docker"
pull_source: "{{pull_source}}"
push_destination: "{{push_destination}}"
- imagename: "{{namespace}}/{{name_prefix}}nova-libvirt{{name_suffix}}:{{tag}}"
uploader: "docker"
pull_source: "{{pull_source}}"
- imagename: "{{namespace}}/image-with-missing-tag"
push_destination: "{{push_destination}}"
""")
@ -108,3 +129,80 @@ class TestKollaImageBuilder(base.TestCase):
mock_popen.assert_called_once_with([
'kolla-build',
], env=env)
class TestKollaImageBuilderTemplate(base.TestCase):
def setUp(self):
super(TestKollaImageBuilderTemplate, self).setUp()
with tempfile.NamedTemporaryFile(delete=False) as imagefile:
self.addCleanup(os.remove, imagefile.name)
self.filelist = [imagefile.name]
with open(imagefile.name, 'w') as f:
f.write(template_filedata)
def test_container_images_from_template(self):
builder = kb.KollaImageBuilder(self.filelist)
result = builder.container_images_from_template(
pull_source='docker.io',
push_destination='localhost:8787',
tag='liberty'
)
# template substitution on the container_images_template section should
# be identical to the container_images section
container_images = yaml.safe_load(filedata)['container_images']
self.assertEqual(container_images, result)
def test_container_images_from_template_filter(self):
builder = kb.KollaImageBuilder(self.filelist)
def filter(entry):
# do not want heat-agents image
if 'heat-docker-agents' in entry.get('imagename'):
return
# set source and destination on all entries
entry['pull_source'] = 'docker.io'
entry['push_destination'] = 'localhost:8787'
return entry
result = builder.container_images_from_template(
filter=filter,
tag='liberty'
)
container_images = [{
'imagename': 'tripleoupstream/centos-binary-nova-compute:liberty',
'pull_source': 'docker.io',
'push_destination': 'localhost:8787',
'uploader': 'docker'
}, {
'imagename': 'tripleoupstream/centos-binary-nova-libvirt:liberty',
'pull_source': 'docker.io',
'push_destination': 'localhost:8787',
'uploader': 'docker'
}, {
'imagename': 'tripleoupstream/image-with-missing-tag',
'pull_source': 'docker.io',
'push_destination': 'localhost:8787'
}]
self.assertEqual(container_images, result)
def test_container_images_yaml_in_sync(self):
'''Confirm overcloud_containers.tpl.yaml equals overcloud_containers.yaml
TODO(sbaker) remove when overcloud_containers.yaml is deleted
'''
mod_dir = os.path.dirname(sys.modules[__name__].__file__)
project_dir = os.path.abspath(os.path.join(mod_dir, '../../../'))
files_dir = os.path.join(project_dir, 'container-images')
oc_tmpl_file = os.path.join(files_dir, 'overcloud_containers.yaml.j2')
tmpl_builder = kb.KollaImageBuilder([oc_tmpl_file])
result = tmpl_builder.container_images_from_template()
oc_yaml_file = os.path.join(files_dir, 'overcloud_containers.yaml')
yaml_builder = kb.KollaImageBuilder([oc_yaml_file])
container_images = yaml_builder.load_config_files(
yaml_builder.CONTAINER_IMAGES)
self.assertSequenceEqual(container_images, result)