tripleo_container_manage: reduce downtime for minor updates
If a container has a new config (e.g. new image), it'll now be removed
right before being re-created.
Before, we were first removing that container among potential orphan
containers, and the container would be re-created later but the downtime
in the middle can be several minutes; since we batch the container
creation.
Now, we separate the cleanup of orphan containers and the ones with new
configs.
The workflow is the following:
1) Remove orphan containers (not in the config anymore, missing Labels,
etc).
Then for each batch of containers:
2) Remove the containers where the config_data has changed.
3) Create the containers that are in the config.
This patch should reduce the downtime of containers that are updated.
Change-Id: I821d674dead4a21b7ac30b47b31b8dd34e0ecc8b
(cherry picked from commit 0e24247d2c
)
This commit is contained in:
parent
327e8fa386
commit
8fb15c6d06
@ -87,7 +87,7 @@ class FilterModule(object):
|
|||||||
return return_dict
|
return return_dict
|
||||||
|
|
||||||
def needs_delete(self, container_infos, config, config_id,
|
def needs_delete(self, container_infos, config, config_id,
|
||||||
clean_orphans=True):
|
clean_orphans=True, check_config=True):
|
||||||
"""Returns a list of containers which need to be removed.
|
"""Returns a list of containers which need to be removed.
|
||||||
|
|
||||||
This filter will check which containers need to be removed for these
|
This filter will check which containers need to be removed for these
|
||||||
@ -98,6 +98,7 @@ class FilterModule(object):
|
|||||||
:param config: dict
|
:param config: dict
|
||||||
:param config_id: string
|
:param config_id: string
|
||||||
:param clean_orphans: bool
|
:param clean_orphans: bool
|
||||||
|
:param check_config: bool to whether or not check if config changed
|
||||||
:returns: list
|
:returns: list
|
||||||
"""
|
"""
|
||||||
to_delete = []
|
to_delete = []
|
||||||
@ -176,7 +177,7 @@ class FilterModule(object):
|
|||||||
except ValueError: # c_data is not data
|
except ValueError: # c_data is not data
|
||||||
c_data = dict()
|
c_data = dict()
|
||||||
|
|
||||||
if cmp(c_data, config_data) != 0:
|
if cmp(c_data, config_data) != 0 and check_config:
|
||||||
to_delete += [c_name]
|
to_delete += [c_name]
|
||||||
|
|
||||||
# Cleanup installed containers that aren't in config anymore.
|
# Cleanup installed containers that aren't in config anymore.
|
||||||
|
@ -20,8 +20,10 @@
|
|||||||
when:
|
when:
|
||||||
- tripleo_container_manage_cli == 'podman'
|
- tripleo_container_manage_cli == 'podman'
|
||||||
|
|
||||||
- name: "Delete containers managed by Podman for {{ tripleo_container_manage_config }}"
|
- name: "Delete orphan containers managed by Podman for {{ tripleo_container_manage_config }}"
|
||||||
when:
|
when:
|
||||||
- tripleo_container_manage_cli == 'podman'
|
- tripleo_container_manage_cli == 'podman'
|
||||||
include_tasks: podman/delete.yml
|
include_tasks: podman/delete.yml
|
||||||
loop: "{{ podman_containers.containers | needs_delete(config=all_containers_hash, config_id=tripleo_container_manage_config_id) }}"
|
loop: >-
|
||||||
|
{{ podman_containers.containers | needs_delete(config=all_containers_hash,
|
||||||
|
config_id=tripleo_container_manage_config_id, check_config=False) }}
|
@ -101,8 +101,8 @@
|
|||||||
include_tasks: puppet_config.yml
|
include_tasks: puppet_config.yml
|
||||||
when:
|
when:
|
||||||
- tripleo_container_manage_check_puppet_config|bool
|
- tripleo_container_manage_check_puppet_config|bool
|
||||||
- name: "Delete containers from {{ tripleo_container_manage_config }}"
|
- name: "Delete orphan containers from {{ tripleo_container_manage_config }}"
|
||||||
include_tasks: delete.yml
|
include_tasks: delete_orphan.yml
|
||||||
- name: "Create containers from {{ tripleo_container_manage_config }}"
|
- name: "Create containers from {{ tripleo_container_manage_config }}"
|
||||||
include_tasks: create.yml
|
include_tasks: create.yml
|
||||||
- name: "Show all container commands for {{ tripleo_container_manage_config }}"
|
- name: "Show all container commands for {{ tripleo_container_manage_config }}"
|
||||||
|
@ -14,6 +14,14 @@
|
|||||||
# License for the specific language governing permissions and limitations
|
# License for the specific language governing permissions and limitations
|
||||||
# under the License.
|
# under the License.
|
||||||
|
|
||||||
|
- name: "Tear-down containers that need to be re-created (new-config detected)"
|
||||||
|
when:
|
||||||
|
- tripleo_container_manage_cli == 'podman'
|
||||||
|
include_tasks: podman/delete.yml
|
||||||
|
loop: >
|
||||||
|
{{ podman_containers.containers | needs_delete(config=batched_container_data|haskey(attribute='action', reverse=True)|singledict,
|
||||||
|
config_id=tripleo_container_manage_config_id, clean_orphans=False) }}
|
||||||
|
|
||||||
- name: "Async container create/run"
|
- name: "Async container create/run"
|
||||||
async: "{{ (not ansible_check_mode | bool) | ternary('600', omit) }}"
|
async: "{{ (not ansible_check_mode | bool) | ternary('600', omit) }}"
|
||||||
poll: "{{ (not ansible_check_mode | bool) | ternary('0', omit) }}"
|
poll: "{{ (not ansible_check_mode | bool) | ternary('0', omit) }}"
|
||||||
|
@ -407,6 +407,128 @@ class TestHelperFilters(tests_base.TestCase):
|
|||||||
config_id='tripleo_step1')
|
config_id='tripleo_step1')
|
||||||
self.assertEqual(result, expected_list)
|
self.assertEqual(result, expected_list)
|
||||||
|
|
||||||
|
def test_needs_delete_no_config_check(self):
|
||||||
|
data = [
|
||||||
|
{
|
||||||
|
'Name': 'mysql',
|
||||||
|
'Config': {
|
||||||
|
'Labels': {
|
||||||
|
'config_id': 'tripleo_step1'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'Name': 'rabbitmq',
|
||||||
|
'Config': {
|
||||||
|
'Labels': {
|
||||||
|
'managed_by': 'tripleo_ansible',
|
||||||
|
'config_id': 'tripleo_step1',
|
||||||
|
'container_name': 'rabbitmq',
|
||||||
|
'name': 'rabbitmq'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'Name': 'swift',
|
||||||
|
'Config': {
|
||||||
|
'Labels': {
|
||||||
|
'managed_by': 'tripleo',
|
||||||
|
'config_id': 'tripleo_step1',
|
||||||
|
'container_name': 'swift',
|
||||||
|
'name': 'swift',
|
||||||
|
'config_data': {'foo': 'bar'}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'Name': 'heat',
|
||||||
|
'Config': {
|
||||||
|
'Labels': {
|
||||||
|
'managed_by': 'tripleo-Undercloud',
|
||||||
|
'config_id': 'tripleo_step1',
|
||||||
|
'container_name': 'heat',
|
||||||
|
'name': 'heat',
|
||||||
|
'config_data': "{'start_order': 0}"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'Name': 'test1',
|
||||||
|
'Config': {
|
||||||
|
'Labels': {
|
||||||
|
'managed_by': 'tripleo-other',
|
||||||
|
'config_id': 'tripleo_step1',
|
||||||
|
'container_name': 'test1',
|
||||||
|
'name': 'test1',
|
||||||
|
'config_data': {'start_order': 0}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'Name': 'haproxy',
|
||||||
|
'Config': {
|
||||||
|
'Labels': {
|
||||||
|
'managed_by': 'paunch',
|
||||||
|
'config_id': 'tripleo_step1',
|
||||||
|
'config_data': ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'Name': 'tripleo',
|
||||||
|
'Config': {
|
||||||
|
'Labels': {
|
||||||
|
'foo': 'bar'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'Name': 'none_tripleo',
|
||||||
|
'Config': {
|
||||||
|
'Labels': None
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'Name': 'old_tripleo',
|
||||||
|
'Config': {
|
||||||
|
'Labels': {
|
||||||
|
'managed_by': 'tripleo_ansible',
|
||||||
|
'config_id': 'tripleo_step1',
|
||||||
|
'config_data': ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
]
|
||||||
|
config = {
|
||||||
|
# we don't want that container to be touched: no restart
|
||||||
|
'mysql': '',
|
||||||
|
# container has no Config, therefore no Labels: restart needed
|
||||||
|
# but will be skipped because check_config is False
|
||||||
|
'rabbitmq': '',
|
||||||
|
# container has no config_data: restart needed
|
||||||
|
# but will be skipped because check_config is False
|
||||||
|
'haproxy': '',
|
||||||
|
# container isn't part of config_id: no restart
|
||||||
|
'tripleo': '',
|
||||||
|
# container isn't in container_infos but not part of config_id:
|
||||||
|
# no restart.
|
||||||
|
'doesnt_exist': '',
|
||||||
|
# config_data didn't change: no restart
|
||||||
|
'swift': {'foo': 'bar'},
|
||||||
|
# config_data changed: restart needed
|
||||||
|
# but will be skipped because check_config is False
|
||||||
|
'heat': {'start_order': 1},
|
||||||
|
# config_data changed: restart needed
|
||||||
|
# but will be skipped because check_config is False
|
||||||
|
'test1': {'start_order': 2},
|
||||||
|
}
|
||||||
|
expected_list = ['rabbitmq', 'old_tripleo']
|
||||||
|
result = self.filters.needs_delete(container_infos=data,
|
||||||
|
config=config,
|
||||||
|
config_id='tripleo_step1',
|
||||||
|
check_config=False)
|
||||||
|
self.assertEqual(result, expected_list)
|
||||||
|
|
||||||
def test_needs_delete_single_config(self):
|
def test_needs_delete_single_config(self):
|
||||||
data = [
|
data = [
|
||||||
{
|
{
|
||||||
|
Loading…
Reference in New Issue
Block a user