openstack/kayobe
Code Issues Proposed changes

Merge pull request #81 from markgoddard/container-image-builders

Browse Source 
Improve container image build workflow
This commit is contained in:
Mark Goddard 2017-11-24 15:37:43 +00:00 committed by GitHub
commit 5f6c181c69
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 203 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
ansible/container-image-build.yml
View File

@ -1,6 +1,6 @@
---
- name: Ensure Kolla container images are built
hosts: seed:controllers
hosts: container-image-builders
vars:
# Set this to True to push images to the registry when built.
push_images: False
@ -38,7 +38,9 @@
username: "{{ kolla_docker_registry_username }}"
password: "{{ kolla_docker_registry_password }}"
reauthorize: yes
when: kolla_docker_registry_username is not none and kolla_docker_registry_password is not none
when:
- kolla_docker_registry_username is not none
- kolla_docker_registry_password is not none
- name: Ensure Kolla container images are built
shell: >
@ -47,7 +49,7 @@
kolla-build \
--config-dir {{ kolla_build_config_path }} \
{% if item.type is defined %}--type {{ item.type }}{% endif %} \
{% if kolla_docker_registry != "" and kolla_docker_registry != None %}--registry {{ kolla_docker_registry }}{% endif %} \
{% if kolla_docker_registry is not none %}--registry {{ kolla_docker_registry }}{% endif %} \
{% if push_images | bool %}--push{% endif %} \
{{ item.regexes }} 2>&1 | tee --append {{ kolla_build_log_path }}
with_items: "{{ container_image_sets }}"

12
ansible/container-image-builders-check.yml Normal file
View File

@ -0,0 +1,12 @@
---
- name: Ensure the container-image-builders group exists
hosts: localhost
gather_facts: False
tasks:
- name: Ensure the container-image-builders group exists
fail:
msg: >
Container images are now built by hosts in the
container-image-builders group. Ensure that this group is present in
your inventory.
when: groups.get('container-image-builders', []) | length == 0

44
ansible/group_vars/all/kolla
View File

@ -104,8 +104,8 @@ seed_container_image_sets:
regexes: "{{ seed_container_image_regexes | join(' ') }}"
# List of regular expressions matching names of container images to build for
# controllers.
controller_container_image_regex_map:
# overcloud hosts.
overcloud_container_image_regex_map:
- regex: aodh
enabled: "{{ kolla_enable_aodh | bool }}"
- regex: barbican
@ -113,33 +113,35 @@ controller_container_image_regex_map:
- regex: ceilometer
enabled: "{{ kolla_enable_ceilometer | bool }}"
- regex: cinder
enabled: True
enabled: "{{ kolla_enable_cinder | bool }}"
- regex: cron
enabled: True
- regex: designate
enabled: "{{ kolla_enable_designate | bool }}"
- regex: dnsmasq
enabled: True
enabled: "{{ kolla_enable_ironic | bool }}"
- regex: elasticsearch
enabled: "{{ kolla_enable_elasticsearch | bool }}"
- regex: etcd
enabled: "{{ kolla_enable_etcd | bool }}"
- regex: fluentd
enabled: True
- regex: glance
enabled: "{{ kolla_enable_glance | bool }}"
- regex: gnocchi
enabled: "{{ kolla_enable_gnocchi | bool }}"
- regex: haproxy
enabled: True
- regex: ^haproxy$
enabled: "{{ kolla_enable_haproxy | bool }}"
- regex: heat
enabled: True
enabled: "{{ kolla_enable_heat | bool }}"
- regex: horizon
enabled: True
enabled: "{{ kolla_enable_horizon | bool }}"
- regex: ironic
enabled: "{{ kolla_enable_ironic | bool }}"
- regex: iscsid
enabled: True
enabled: "{{ kolla_enable_cinder | bool or kolla_enable_ironic | bool }}"
- regex: keepalived
enabled: True
enabled: "{{ kolla_enable_haproxy | bool }}"
- regex: keystone
enabled: True
- regex: kibana
@ -157,14 +159,14 @@ controller_container_image_regex_map:
- regex: memcached
enabled: True
- regex: neutron-server
enabled: True
enabled: "{{ kolla_enable_neutron | bool }}"
# Neutron SFC agent not currently supported on CentOS binary builds.
- regex: "neutron-\\(dhcp\\|l3\\|metadata\\|openvswitch\\)-agent"
enabled: True
enabled: "{{ kolla_enable_neutron | bool }}"
- regex: nova
enabled: True
enabled: "{{ kolla_enable_nova | bool }}"
- regex: openvswitch
enabled: True
enabled: "{{ kolla_enable_neutron | bool }}"
- regex: rabbitmq
enabled: True
- regex: sahara
@ -174,16 +176,16 @@ controller_container_image_regex_map:
- regex: swift
enabled: "{{ kolla_enable_swift | bool }}"
- regex: tgtd
enabled: True
enabled: "{{ kolla_enable_cinder | bool or kolla_enable_ironic | bool }}"
# List of regular expressions matching names of container images to build for
# controllers.
controller_container_image_regexes: "{{ controller_container_image_regex_map | selectattr('enabled') | map(attribute='regex') | list }}"
# overcloud hosts.
overcloud_container_image_regexes: "{{ overcloud_container_image_regex_map | selectattr('enabled') | map(attribute='regex') | list }}"
# List of container image sets for controllers. This is used when building
# List of container image sets for overcloud hosts. This is used when building
# container images to determine which images to build.
controller_container_image_sets:
- regexes: "{{ controller_container_image_regexes | join(' ') }}"
overcloud_container_image_sets:
- regexes: "{{ overcloud_container_image_regexes | join(' ') }}"
# Dict mapping Jinja2 block names in kolla's Docker images to their contents.
kolla_build_blocks: {}
@ -266,8 +268,10 @@ kolla_enable_barbican: "no"
kolla_enable_central_logging: "no"
kolla_enable_ceph: "no"
kolla_enable_ceilometer: "no"
kolla_enable_cinder: "no"
kolla_enable_designate: "no"
kolla_enable_elasticsearch: "{{ 'yes' if kolla_enable_central_logging | bool or kolla_enable_freezer | bool or kolla_enable_osprofiler | bool or kolla_enable_skydive | bool else 'no' }}"
kolla_enable_etcd: "no"
kolla_enable_freezer: "no"
kolla_enable_glance: "yes"
kolla_enable_gnocchi: "no"

3
ansible/group_vars/controllers/kolla
View File

@ -1,3 +0,0 @@
---
# List of Kolla container image sets for controllers.
container_image_sets: "{{ controller_container_image_sets }}"

3
ansible/group_vars/seed/kolla
View File

@ -1,3 +0,0 @@
---
# List of Kolla container image sets for the seed.
container_image_sets: "{{ seed_container_image_sets }}"

3
ansible/kolla-build.yml
View File

@ -1,6 +1,7 @@
---
- name: Ensure Kolla is installed and configured
hosts: seed:controllers
hosts: container-image-builders
roles:
- role: kolla
- role: kolla-build
kolla_build_extra_config_path: "{{ kayobe_config_path }}/kolla/kolla-build.conf"

6
ansible/kolla.yml
View File

@ -1,6 +0,0 @@
---
- name: Ensure Kolla is configured
hosts: config-mgmt
roles:
- role: kolla
- role: kolla-build

6
ansible/roles/kolla-build/defaults/main.yml
View File

@ -2,6 +2,9 @@
# Directory where Kolla config files will be installed.
kolla_build_config_path:
# Path to extra kolla configuration files.
kolla_build_extra_config_path:
# Valid options are [ centos, fedora, oraclelinux, ubuntu ]
kolla_base_distro:
@ -34,3 +37,6 @@ kolla_build_blocks: {}
# most commonly packages. The operation should be one of override, append or
# remove. The value should be a list.
kolla_build_customizations: {}
# Free-form extra configuration for kolla-build.
kolla_extra_build: {}

14
ansible/roles/kolla-build/tasks/main.yml
View File

@ -1,4 +1,18 @@
---
- name: Check whether a Kolla build extra configuration file exists
local_action:
module: stat
path: "{{ kolla_build_extra_config_path }}"
get_checksum: False
get_md5: False
mime: False
register: stat_result
- name: Set a fact containing extra configuration
set_fact:
kolla_build_extra_config: "{{ lookup('template', kolla_build_extra_config_path) }}"
when: stat_result.stat.exists
- name: Ensure the Kolla build configuration files exist
template:
src: "{{ item.src }}"

8
ansible/roles/kolla-build/templates/kolla-build.conf.j2
View File

@ -33,3 +33,11 @@ reference = {{ source_def.reference }}
{% endif %}
{% endfor %}
{% if kolla_build_extra_config is defined %}
#######################
# Extra configuration
#######################
{{ kolla_build_extra_config }}
{% endif %}

16
etc/kayobe/inventory/groups
View File

@ -1,9 +1,22 @@
# Kayobe groups inventory file. This file should generally not be modified.
# If declares the top-level groups and sub-groups.
###############################################################################
# Seed groups.
[seed]
# Empty group to provide declaration of seed group.
[seed-hypervisor]
# Empty group to provide declaration of seed-hypervisor group.
[container-image-builders:children]
# Build container images on the seed by default.
seed
###############################################################################
# Overcloud groups.
[controllers]
# Empty group to provide declaration of controllers group.
@ -24,6 +37,9 @@ network
monitoring
compute
###############################################################################
# Docker groups.
[docker:children]
# Hosts in this group will have Docker installed.
seed

6
etc/kayobe/seed-vm.yml
View File

@ -26,6 +26,12 @@
# Base image for the seed VM root volume.
#seed_vm_root_image:
# Capacity of the seed VM data volume.
#seed_vm_data_capacity:
# Format of the seed VM data volume.
#seed_vm_data_format:
###############################################################################
# Dummy variable to allow Ansible to accept this file.
workaround_ansible_issue_8743: yes

16
kayobe/cli/commands.py
View File

@ -414,12 +414,16 @@ class SeedContainerImageBuild(KayobeAnsibleMixin, VaultMixin, Command):
def take_action(self, parsed_args):
self.app.LOG.debug("Building seed container images")
playbooks = _build_playbook_list(
"kolla-build", "container-image-build")
"container-image-builders-check", "kolla-build",
"container-image-build")
extra_vars = {"push_images": parsed_args.push}
if parsed_args.regex:
regexes = "'%s'" % " ".join(parsed_args.regex)
extra_vars["container_image_regexes"] = regexes
self.run_kayobe_playbooks(parsed_args, playbooks, limit="seed",
else:
extra_vars["container_image_sets"] = (
"{{ seed_container_image_sets }}")
self.run_kayobe_playbooks(parsed_args, playbooks,
extra_vars=extra_vars)
@ -940,12 +944,16 @@ class OvercloudContainerImageBuild(KayobeAnsibleMixin, VaultMixin, Command):
def take_action(self, parsed_args):
self.app.LOG.debug("Building overcloud container images")
playbooks = _build_playbook_list(
"kolla-build", "container-image-build")
"container-image-builders-check", "kolla-build",
"container-image-build")
extra_vars = {"push_images": parsed_args.push}
if parsed_args.regex:
regexes = "'%s'" % " ".join(parsed_args.regex)
extra_vars["container_image_regexes"] = regexes
self.run_kayobe_playbooks(parsed_args, playbooks, limit="controllers",
else:
extra_vars["container_image_sets"] = (
"{{ overcloud_container_image_sets }}")
self.run_kayobe_playbooks(parsed_args, playbooks,
extra_vars=extra_vars)

98
kayobe/tests/unit/cli/test_commands.py
View File

@ -81,3 +81,101 @@ class TestCase(unittest.TestCase):
mock.call(mock.ANY, ["ansible/network-connectivity.yml"]),
]
self.assertEqual(expected_calls, mock_run.call_args_list)
@mock.patch.object(commands.KayobeAnsibleMixin,
"run_kayobe_playbooks")
def test_seed_container_image_build(self, mock_run):
command = commands.SeedContainerImageBuild(TestApp(), [])
parser = command.get_parser("test")
parsed_args = parser.parse_args([])
result = command.run(parsed_args)
self.assertEqual(0, result)
expected_calls = [
mock.call(
mock.ANY,
[
"ansible/container-image-builders-check.yml",
"ansible/kolla-build.yml",
"ansible/container-image-build.yml"
],
extra_vars={
"container_image_sets": (
"{{ seed_container_image_sets }}"),
"push_images": False,
}
),
]
self.assertEqual(expected_calls, mock_run.call_args_list)
@mock.patch.object(commands.KayobeAnsibleMixin,
"run_kayobe_playbooks")
def test_seed_container_image_build_with_regex(self, mock_run):
command = commands.SeedContainerImageBuild(TestApp(), [])
parser = command.get_parser("test")
parsed_args = parser.parse_args(["--push", "^regex1$", "^regex2$"])
result = command.run(parsed_args)
self.assertEqual(0, result)
expected_calls = [
mock.call(
mock.ANY,
[
"ansible/container-image-builders-check.yml",
"ansible/kolla-build.yml",
"ansible/container-image-build.yml"
],
extra_vars={
"container_image_regexes": "'^regex1$ ^regex2$'",
"push_images": True,
}
),
]
self.assertEqual(expected_calls, mock_run.call_args_list)
@mock.patch.object(commands.KayobeAnsibleMixin,
"run_kayobe_playbooks")
def test_overcloud_container_image_build(self, mock_run):
command = commands.OvercloudContainerImageBuild(TestApp(), [])
parser = command.get_parser("test")
parsed_args = parser.parse_args([])
result = command.run(parsed_args)
self.assertEqual(0, result)
expected_calls = [
mock.call(
mock.ANY,
[
"ansible/container-image-builders-check.yml",
"ansible/kolla-build.yml",
"ansible/container-image-build.yml"
],
extra_vars={
"container_image_sets": (
"{{ overcloud_container_image_sets }}"),
"push_images": False,
}
),
]
self.assertEqual(expected_calls, mock_run.call_args_list)
@mock.patch.object(commands.KayobeAnsibleMixin,
"run_kayobe_playbooks")
def test_overcloud_container_image_build_with_regex(self, mock_run):
command = commands.OvercloudContainerImageBuild(TestApp(), [])
parser = command.get_parser("test")
parsed_args = parser.parse_args(["--push", "^regex1$", "^regex2$"])
result = command.run(parsed_args)
self.assertEqual(0, result)
expected_calls = [
mock.call(
mock.ANY,
[
"ansible/container-image-builders-check.yml",
"ansible/kolla-build.yml",
"ansible/container-image-build.yml"
],
extra_vars={
"container_image_regexes": "'^regex1$ ^regex2$'",
"push_images": True,
}
),
]
self.assertEqual(expected_calls, mock_run.call_args_list)