tripleo_container_manage: reduce downtime for minor updates

If a container has a new config (e.g. new image), it'll now be removed
right before being re-created.

Before, we were first removing that container among potential orphan
containers, and the container would be re-created later but the downtime
in the middle can be several minutes; since we batch the container
creation.

Now, we separate the cleanup of orphan containers and the ones with new
configs.

The workflow is the following:

1) Remove orphan containers (not in the config anymore, missing Labels,
   etc).

Then for each batch of containers:

2) Remove the containers where the config_data has changed.

3) Create the containers that are in the config.

This patch should reduce the downtime of containers that are updated.

Change-Id: I821d674dead4a21b7ac30b47b31b8dd34e0ecc8b
This commit is contained in:
Emilien Macchi 2020-02-13 13:14:04 -05:00
parent 99b8a1c4b8
commit 0e24247d2c
5 changed files with 139 additions and 6 deletions

View File

@ -88,7 +88,7 @@ class FilterModule(object):
return return_dict
def needs_delete(self, container_infos, config, config_id,
clean_orphans=True):
clean_orphans=True, check_config=True):
"""Returns a list of containers which need to be removed.
This filter will check which containers need to be removed for these
@ -99,6 +99,7 @@ class FilterModule(object):
:param config: dict
:param config_id: string
:param clean_orphans: bool
:param check_config: bool to whether or not check if config changed
:returns: list
"""
to_delete = []
@ -177,7 +178,7 @@ class FilterModule(object):
except ValueError: # c_data is not data
c_data = dict()
if cmp(c_data, config_data) != 0:
if cmp(c_data, config_data) != 0 and check_config:
to_delete += [c_name]
# Cleanup installed containers that aren't in config anymore.

View File

@ -20,8 +20,10 @@
when:
- tripleo_container_manage_cli == 'podman'
- name: "Delete containers managed by Podman for {{ tripleo_container_manage_config }}"
- name: "Delete orphan containers managed by Podman for {{ tripleo_container_manage_config }}"
when:
- tripleo_container_manage_cli == 'podman'
include_tasks: podman/delete.yml
loop: "{{ podman_containers.containers | needs_delete(config=all_containers_hash, config_id=tripleo_container_manage_config_id) }}"
loop: >-
{{ podman_containers.containers | needs_delete(config=all_containers_hash,
config_id=tripleo_container_manage_config_id, check_config=False) }}

View File

@ -101,8 +101,8 @@
include_tasks: puppet_config.yml
when:
- tripleo_container_manage_check_puppet_config|bool
- name: "Delete containers from {{ tripleo_container_manage_config }}"
include_tasks: delete.yml
- name: "Delete orphan containers from {{ tripleo_container_manage_config }}"
include_tasks: delete_orphan.yml
- name: "Create containers from {{ tripleo_container_manage_config }}"
include_tasks: create.yml
- name: "Show all container commands for {{ tripleo_container_manage_config }}"

View File

@ -14,6 +14,14 @@
# License for the specific language governing permissions and limitations
# under the License.
- name: "Tear-down containers that need to be re-created (new-config detected)"
when:
- tripleo_container_manage_cli == 'podman'
include_tasks: podman/delete.yml
loop: >
{{ podman_containers.containers | needs_delete(config=batched_container_data|haskey(attribute='action', reverse=True)|singledict,
config_id=tripleo_container_manage_config_id, clean_orphans=False) }}
- name: "Async container create/run"
async: "{{ (not ansible_check_mode | bool) | ternary('600', omit) }}"
poll: "{{ (not ansible_check_mode | bool) | ternary('0', omit) }}"

View File

@ -407,6 +407,128 @@ class TestHelperFilters(tests_base.TestCase):
config_id='tripleo_step1')
self.assertEqual(result, expected_list)
def test_needs_delete_no_config_check(self):
data = [
{
'Name': 'mysql',
'Config': {
'Labels': {
'config_id': 'tripleo_step1'
}
}
},
{
'Name': 'rabbitmq',
'Config': {
'Labels': {
'managed_by': 'tripleo_ansible',
'config_id': 'tripleo_step1',
'container_name': 'rabbitmq',
'name': 'rabbitmq'
}
}
},
{
'Name': 'swift',
'Config': {
'Labels': {
'managed_by': 'tripleo',
'config_id': 'tripleo_step1',
'container_name': 'swift',
'name': 'swift',
'config_data': {'foo': 'bar'}
}
}
},
{
'Name': 'heat',
'Config': {
'Labels': {
'managed_by': 'tripleo-Undercloud',
'config_id': 'tripleo_step1',
'container_name': 'heat',
'name': 'heat',
'config_data': "{'start_order': 0}"
}
}
},
{
'Name': 'test1',
'Config': {
'Labels': {
'managed_by': 'tripleo-other',
'config_id': 'tripleo_step1',
'container_name': 'test1',
'name': 'test1',
'config_data': {'start_order': 0}
}
}
},
{
'Name': 'haproxy',
'Config': {
'Labels': {
'managed_by': 'paunch',
'config_id': 'tripleo_step1',
'config_data': ""
}
}
},
{
'Name': 'tripleo',
'Config': {
'Labels': {
'foo': 'bar'
}
}
},
{
'Name': 'none_tripleo',
'Config': {
'Labels': None
}
},
{
'Name': 'old_tripleo',
'Config': {
'Labels': {
'managed_by': 'tripleo_ansible',
'config_id': 'tripleo_step1',
'config_data': ""
}
}
},
]
config = {
# we don't want that container to be touched: no restart
'mysql': '',
# container has no Config, therefore no Labels: restart needed
# but will be skipped because check_config is False
'rabbitmq': '',
# container has no config_data: restart needed
# but will be skipped because check_config is False
'haproxy': '',
# container isn't part of config_id: no restart
'tripleo': '',
# container isn't in container_infos but not part of config_id:
# no restart.
'doesnt_exist': '',
# config_data didn't change: no restart
'swift': {'foo': 'bar'},
# config_data changed: restart needed
# but will be skipped because check_config is False
'heat': {'start_order': 1},
# config_data changed: restart needed
# but will be skipped because check_config is False
'test1': {'start_order': 2},
}
expected_list = ['rabbitmq', 'old_tripleo']
result = self.filters.needs_delete(container_infos=data,
config=config,
config_id='tripleo_step1',
check_config=False)
self.assertEqual(result, expected_list)
def test_needs_delete_single_config(self):
data = [
{