wait: Remove test pods from wait
When waiting on resources that share labels with existing test pods, an upgrade can fail due to a wait operation on the existing test pods. This change skips wait operations on test resources by filtering them using Helm hooks. Change-Id: I465d3429216457ea8d088064cafa74b2b0d9b8cb
This commit is contained in:
parent
32baf9851e
commit
1a28e6b72f
|
@ -27,6 +27,8 @@ DEFAULT_CHART_TIMEOUT = 900
|
||||||
|
|
||||||
# Tiller
|
# Tiller
|
||||||
DEFAULT_TILLER_TIMEOUT = 300
|
DEFAULT_TILLER_TIMEOUT = 300
|
||||||
|
HELM_HOOK_ANNOTATION = 'helm.sh/hook'
|
||||||
|
HELM_TEST_HOOKS = ['test-success', 'test-failure']
|
||||||
STATUS_UNKNOWN = 'UNKNOWN'
|
STATUS_UNKNOWN = 'UNKNOWN'
|
||||||
STATUS_DEPLOYED = 'DEPLOYED'
|
STATUS_DEPLOYED = 'DEPLOYED'
|
||||||
STATUS_DELETED = 'DELETED'
|
STATUS_DELETED = 'DELETED'
|
||||||
|
|
|
@ -148,6 +148,16 @@ class ResourceWait(ABC):
|
||||||
'''
|
'''
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
def include_resource(self, resource):
|
||||||
|
'''
|
||||||
|
Test to include or exclude a resource in a wait operation. This method
|
||||||
|
can be used to exclude resources that should not be included in wait
|
||||||
|
operations (e.g. test pods).
|
||||||
|
:param resource: resource to test
|
||||||
|
:returns: boolean representing test result
|
||||||
|
'''
|
||||||
|
return True
|
||||||
|
|
||||||
def handle_resource(self, resource):
|
def handle_resource(self, resource):
|
||||||
resource_name = resource.metadata.name
|
resource_name = resource.metadata.name
|
||||||
|
|
||||||
|
@ -259,7 +269,9 @@ class ResourceWait(ABC):
|
||||||
|
|
||||||
resource_list = self.get_resources(**kwargs)
|
resource_list = self.get_resources(**kwargs)
|
||||||
for resource in resource_list.items:
|
for resource in resource_list.items:
|
||||||
ready[resource.metadata.name] = self.handle_resource(resource)
|
# Only include resources that should be included in wait ops
|
||||||
|
if self.include_resource(resource):
|
||||||
|
ready[resource.metadata.name] = self.handle_resource(resource)
|
||||||
if not resource_list.items:
|
if not resource_list.items:
|
||||||
if self.skip_if_none_found:
|
if self.skip_if_none_found:
|
||||||
msg = 'Skipping wait, no %s resources found.'
|
msg = 'Skipping wait, no %s resources found.'
|
||||||
|
@ -279,6 +291,11 @@ class ResourceWait(ABC):
|
||||||
resource = event['object']
|
resource = event['object']
|
||||||
resource_name = resource.metadata.name
|
resource_name = resource.metadata.name
|
||||||
resource_version = resource.metadata.resource_version
|
resource_version = resource.metadata.resource_version
|
||||||
|
|
||||||
|
# Skip resources that should be excluded from wait operations
|
||||||
|
if not self.include_resource(resource):
|
||||||
|
continue
|
||||||
|
|
||||||
msg = ('Watch event: type=%s, name=%s, namespace=%s,'
|
msg = ('Watch event: type=%s, name=%s, namespace=%s,'
|
||||||
'resource_version=%s')
|
'resource_version=%s')
|
||||||
LOG.debug(msg, event_type, resource_name,
|
LOG.debug(msg, event_type, resource_name,
|
||||||
|
@ -330,6 +347,28 @@ class PodWait(ResourceWait):
|
||||||
resource_type, chart_wait, labels,
|
resource_type, chart_wait, labels,
|
||||||
chart_wait.k8s.client.list_namespaced_pod, **kwargs)
|
chart_wait.k8s.client.list_namespaced_pod, **kwargs)
|
||||||
|
|
||||||
|
def include_resource(self, resource):
|
||||||
|
pod = resource
|
||||||
|
annotations = pod.metadata.annotations
|
||||||
|
|
||||||
|
# Retrieve pod's Helm test hooks
|
||||||
|
test_hooks = None
|
||||||
|
if annotations:
|
||||||
|
hook_string = annotations.get(const.HELM_HOOK_ANNOTATION)
|
||||||
|
if hook_string:
|
||||||
|
hooks = hook_string.split(',')
|
||||||
|
test_hooks = [h for h in hooks if h in const.HELM_TEST_HOOKS]
|
||||||
|
|
||||||
|
# NOTE(drewwalters96): Test pods may cause wait operations to fail
|
||||||
|
# when old resources remain from previous upgrades/tests. Indicate that
|
||||||
|
# test pods should not be included in wait operations.
|
||||||
|
if test_hooks:
|
||||||
|
LOG.debug('Pod %s will be skipped during wait operations.',
|
||||||
|
pod.metadata.name)
|
||||||
|
return False
|
||||||
|
else:
|
||||||
|
return True
|
||||||
|
|
||||||
def is_resource_ready(self, resource):
|
def is_resource_ready(self, resource):
|
||||||
pod = resource
|
pod = resource
|
||||||
name = pod.metadata.name
|
name = pod.metadata.name
|
||||||
|
|
|
@ -185,3 +185,49 @@ class ChartWaitTestCase(base.ArmadaTestCase):
|
||||||
self.assertEqual(2, len(unit.waits))
|
self.assertEqual(2, len(unit.waits))
|
||||||
for w in unit.waits:
|
for w in unit.waits:
|
||||||
w.wait.assert_called_once()
|
w.wait.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
class PodWaitTestCase(base.ArmadaTestCase):
|
||||||
|
|
||||||
|
def get_unit(self, labels):
|
||||||
|
return wait.PodWait(
|
||||||
|
resource_type='pod', chart_wait=mock.MagicMock(), labels=labels)
|
||||||
|
|
||||||
|
def test_include_resource(self):
|
||||||
|
|
||||||
|
def mock_resource(annotations):
|
||||||
|
resource = mock.Mock()
|
||||||
|
resource.metadata.annotations = annotations
|
||||||
|
return resource
|
||||||
|
|
||||||
|
test_resources = [
|
||||||
|
mock_resource({
|
||||||
|
'key': 'value',
|
||||||
|
'helm.sh/hook': 'test-success'
|
||||||
|
}),
|
||||||
|
mock_resource({
|
||||||
|
'helm.sh/hook': 'test-failure'
|
||||||
|
}),
|
||||||
|
mock_resource({
|
||||||
|
'helm.sh/hook': 'test-success,pre-install'
|
||||||
|
})
|
||||||
|
]
|
||||||
|
non_test_resources = [
|
||||||
|
mock_resource({
|
||||||
|
'helm.sh/hook': 'pre-install'
|
||||||
|
}),
|
||||||
|
mock_resource({
|
||||||
|
'key': 'value'
|
||||||
|
}),
|
||||||
|
mock_resource({})
|
||||||
|
]
|
||||||
|
|
||||||
|
unit = self.get_unit({})
|
||||||
|
|
||||||
|
# Validate test resources excluded
|
||||||
|
for resource in test_resources:
|
||||||
|
self.assertFalse(unit.include_resource(resource))
|
||||||
|
|
||||||
|
# Validate other resources included
|
||||||
|
for resource in non_test_resources:
|
||||||
|
self.assertTrue(unit.include_resource(resource))
|
||||||
|
|
Loading…
Reference in New Issue