Browse Source

Automate deletion of test pods

When running helm tests for a chart release multiple times in a site,
if the previous test pod is not deleted, then the test pod creation
can fail due to a name conflict. Armada/helm support immediate test pod
cleanup, but using this means that upon test failure, the test pod logs will
not be available for debugging purposes. Due to this, the recommended approach
for deleting test pods in Armada has been using `upgrade.pre.delete` actions.
So chart authors can accomplish test pod deletion using this
feature, however, it often takes awhile, usually not until they test upgrading
the chart for chart authors to realize that this is necessary and to get it
implemented.

This patchset automates deletion of test pods directly before running tests by
using the `wait.labels` field in the chart doc when they exist to find all pods
in the release and then using their annotations to determine if they are test
pods and deleting them if so.

A later patchset is planned to implement defaulting of the wait labels when
they are not defined.

Change-Id: I2092f448acb88b5ade3b31b397f9c874c0061668
changes/13/629313/6
Sean Eagan 6 months ago
parent
commit
c31a961bf1

+ 3
- 4
armada/api/controller/test.py View File

@@ -39,7 +39,7 @@ class TestReleasesReleaseNameController(api.BaseResource):
39 39
         with self.get_tiller(req, resp) as tiller:
40 40
             cleanup = req.get_param_as_bool('cleanup')
41 41
 
42
-            test_handler = Test(release, tiller, cleanup=cleanup)
42
+            test_handler = Test({}, release, tiller, cleanup=cleanup)
43 43
             success = test_handler.test_release_for_success()
44 44
 
45 45
         if success:
@@ -136,15 +136,14 @@ class TestReleasesManifestController(api.BaseResource):
136 136
                     cleanup = req.get_param_as_bool('cleanup')
137 137
                     enable_all = req.get_param_as_bool('enable_all')
138 138
                     cg_test_charts = group.get('test_charts')
139
-                    test_values = chart.get('test', {})
140 139
 
141 140
                     test_handler = Test(
141
+                        chart,
142 142
                         release_name,
143 143
                         tiller,
144 144
                         cg_test_charts=cg_test_charts,
145 145
                         cleanup=cleanup,
146
-                        enable_all=enable_all,
147
-                        test_values=test_values)
146
+                        enable_all=enable_all)
148 147
 
149 148
                     if test_handler.test_enabled:
150 149
                         success = test_handler.test_release_for_success()

+ 6
- 5
armada/cli/test.py View File

@@ -124,7 +124,10 @@ class TestChartManifest(CliAction):
124 124
 
125 125
         if self.release:
126 126
             if not self.ctx.obj.get('api', False):
127
-                test_handler = Test(self.release, tiller, cleanup=self.cleanup)
127
+                test_handler = Test({},
128
+                                    self.release,
129
+                                    tiller,
130
+                                    cleanup=self.cleanup)
128 131
                 test_handler.test_release_for_success()
129 132
             else:
130 133
                 client = self.ctx.obj.get('CLIENT')
@@ -156,14 +159,12 @@ class TestChartManifest(CliAction):
156 159
                         release_name = release_prefixer(
157 160
                             prefix, chart.get('release'))
158 161
                         if release_name in known_release_names:
159
-                            test_values = chart.get('test', {})
160
-
161 162
                             test_handler = Test(
163
+                                chart,
162 164
                                 release_name,
163 165
                                 tiller,
164 166
                                 cleanup=self.cleanup,
165
-                                enable_all=self.enable_all,
166
-                                test_values=test_values)
167
+                                enable_all=self.enable_all)
167 168
 
168 169
                             if test_handler.test_enabled:
169 170
                                 test_handler.test_release_for_success()

+ 0
- 2
armada/const.py View File

@@ -27,8 +27,6 @@ DEFAULT_CHART_TIMEOUT = 900
27 27
 
28 28
 # Tiller
29 29
 DEFAULT_TILLER_TIMEOUT = 300
30
-HELM_HOOK_ANNOTATION = 'helm.sh/hook'
31
-HELM_TEST_HOOKS = ['test-success', 'test-failure']
32 30
 STATUS_UNKNOWN = 'UNKNOWN'
33 31
 STATUS_DEPLOYED = 'DEPLOYED'
34 32
 STATUS_DELETED = 'DELETED'

+ 2
- 3
armada/handlers/chart_deploy.py View File

@@ -232,12 +232,11 @@ class ChartDeploy(object):
232 232
         just_deployed = ('install' in result) or ('upgrade' in result)
233 233
         last_test_passed = old_release and r.get_last_test_result(old_release)
234 234
 
235
-        test_values = chart.get('test')
236 235
         test_handler = Test(
236
+            chart,
237 237
             release_name,
238 238
             self.tiller,
239
-            cg_test_charts=cg_test_all_charts,
240
-            test_values=test_values)
239
+            cg_test_charts=cg_test_all_charts)
241 240
 
242 241
         run_test = test_handler.test_enabled and (just_deployed or
243 242
                                                   not last_test_passed)

+ 52
- 13
armada/handlers/test.py View File

@@ -16,10 +16,9 @@ from oslo_log import log as logging
16 16
 
17 17
 from armada import const
18 18
 
19
-TESTRUN_STATUS_UNKNOWN = 0
20
-TESTRUN_STATUS_SUCCESS = 1
21
-TESTRUN_STATUS_FAILURE = 2
22
-TESTRUN_STATUS_RUNNING = 3
19
+from armada.handlers.wait import get_wait_labels
20
+from armada.utils.release import label_selectors
21
+from armada.utils.helm import get_test_suite_run_success, is_test_pod
23 22
 
24 23
 LOG = logging.getLogger(__name__)
25 24
 
@@ -27,33 +26,37 @@ LOG = logging.getLogger(__name__)
27 26
 class Test(object):
28 27
 
29 28
     def __init__(self,
29
+                 chart,
30 30
                  release_name,
31 31
                  tiller,
32 32
                  cg_test_charts=None,
33 33
                  cleanup=None,
34
-                 enable_all=False,
35
-                 test_values=None):
34
+                 enable_all=False):
36 35
         """Initialize a test handler to run Helm tests corresponding to a
37 36
         release.
38 37
 
38
+        :param chart: The armada chart document
39 39
         :param release_name: Name of a Helm release
40 40
         :param tiller: Tiller object
41 41
         :param cg_test_charts: Chart group `test_charts` key
42 42
         :param cleanup: Triggers cleanup; overrides `test.options.cleanup`
43 43
         :param enable_all: Run tests regardless of the value of `test.enabled`
44
-        :param test_values: Test values retrieved from a chart's `test` key
45 44
 
45
+        :type chart: dict
46 46
         :type release_name: str
47 47
         :type tiller: Tiller object
48 48
         :type cg_test_charts: bool
49 49
         :type cleanup: bool
50 50
         :type enable_all: bool
51
-        :type test_values: dict or bool (deprecated)
52 51
         """
53 52
 
53
+        self.chart = chart
54 54
         self.release_name = release_name
55 55
         self.tiller = tiller
56 56
         self.cleanup = cleanup
57
+        self.k8s_timeout = const.DEFAULT_K8S_TIMEOUT
58
+
59
+        test_values = self.chart.get('test', None)
57 60
 
58 61
         # NOTE(drewwalters96): Support the chart_group `test_charts` key until
59 62
         # its deprecation period ends. The `test.enabled`, `enable_all` flag,
@@ -106,10 +109,16 @@ class Test(object):
106 109
         """
107 110
         LOG.info('RUNNING: %s tests', self.release_name)
108 111
 
112
+        try:
113
+            self.delete_test_pods()
114
+        except Exception:
115
+            LOG.exception("Exception when deleting test pods for release: %s",
116
+                          self.release_name)
117
+
109 118
         test_suite_run = self.tiller.test_release(
110 119
             self.release_name, timeout=timeout, cleanup=self.cleanup)
111 120
 
112
-        success = self.get_test_suite_run_success(test_suite_run)
121
+        success = get_test_suite_run_success(test_suite_run)
113 122
         if success:
114 123
             LOG.info('PASSED: %s', self.release_name)
115 124
         else:
@@ -117,7 +126,37 @@ class Test(object):
117 126
 
118 127
         return success
119 128
 
120
-    @classmethod
121
-    def get_test_suite_run_success(self, test_suite_run):
122
-        return all(
123
-            r.status == TESTRUN_STATUS_SUCCESS for r in test_suite_run.results)
129
+    def delete_test_pods(self):
130
+        """Deletes any existing test pods for the release, as identified by the
131
+        wait labels for the chart, to avoid test pod name conflicts when
132
+        creating the new test pod as well as just for general cleanup since
133
+        the new test pod should supercede it.
134
+        """
135
+        labels = get_wait_labels(self.chart)
136
+
137
+        # Guard against labels being left empty, so we don't delete other
138
+        # chart's test pods.
139
+        if labels:
140
+            label_selector = label_selectors(labels)
141
+
142
+            namespace = self.chart['namespace']
143
+
144
+            list_args = {
145
+                'namespace': namespace,
146
+                'label_selector': label_selector,
147
+                'timeout_seconds': self.k8s_timeout
148
+            }
149
+
150
+            pod_list = self.tiller.k8s.client.list_namespaced_pod(**list_args)
151
+            test_pods = (pod for pod in pod_list.items if is_test_pod(pod))
152
+
153
+            if test_pods:
154
+                LOG.info(
155
+                    'Found existing test pods for release with '
156
+                    'namespace=%s, labels=(%s)', namespace, label_selector)
157
+
158
+            for test_pod in test_pods:
159
+                pod_name = test_pod.metadata.name
160
+                LOG.info('Deleting existing test pod: %s', pod_name)
161
+                self.tiller.k8s.delete_pod_action(
162
+                    pod_name, namespace, timeout=self.k8s_timeout)

+ 2
- 2
armada/handlers/tiller.py View File

@@ -32,7 +32,7 @@ from oslo_log import log as logging
32 32
 from armada import const
33 33
 from armada.exceptions import tiller_exceptions as ex
34 34
 from armada.handlers.k8s import K8s
35
-from armada.handlers import test
35
+from armada.utils import helm
36 36
 from armada.utils.release import label_selectors, get_release_status
37 37
 
38 38
 TILLER_VERSION = b'2.12.1'
@@ -502,7 +502,7 @@ class Tiller(object):
502 502
 
503 503
             failed = 0
504 504
             for test_message in test_message_stream:
505
-                if test_message.status == test.TESTRUN_STATUS_FAILURE:
505
+                if test_message.status == helm.TESTRUN_STATUS_FAILURE:
506 506
                     failed += 1
507 507
                 LOG.info(test_message.msg)
508 508
             if failed:

+ 12
- 15
armada/handlers/wait.py View File

@@ -21,6 +21,7 @@ import time
21 21
 from oslo_log import log as logging
22 22
 
23 23
 from armada import const
24
+from armada.utils.helm import is_test_pod
24 25
 from armada.utils.release import label_selectors
25 26
 from armada.exceptions import k8s_exceptions
26 27
 from armada.exceptions import manifest_exceptions
@@ -32,6 +33,11 @@ LOG = logging.getLogger(__name__)
32 33
 ROLLING_UPDATE_STRATEGY_TYPE = 'RollingUpdate'
33 34
 
34 35
 
36
+def get_wait_labels(chart):
37
+    wait_config = chart.get('wait', {})
38
+    return wait_config.get('labels', {})
39
+
40
+
35 41
 # TODO: Validate this object up front in armada validate flow.
36 42
 class ChartWait():
37 43
 
@@ -46,7 +52,7 @@ class ChartWait():
46 52
         self.k8s_wait_attempt_sleep = max(k8s_wait_attempt_sleep, 1)
47 53
 
48 54
         resources = self.wait_config.get('resources')
49
-        labels = self.wait_config.get('labels', {})
55
+        labels = get_wait_labels(self.chart)
50 56
 
51 57
         if resources is not None:
52 58
             waits = []
@@ -349,25 +355,16 @@ class PodWait(ResourceWait):
349 355
 
350 356
     def include_resource(self, resource):
351 357
         pod = resource
352
-        annotations = pod.metadata.annotations
353
-
354
-        # Retrieve pod's Helm test hooks
355
-        test_hooks = None
356
-        if annotations:
357
-            hook_string = annotations.get(const.HELM_HOOK_ANNOTATION)
358
-            if hook_string:
359
-                hooks = hook_string.split(',')
360
-                test_hooks = [h for h in hooks if h in const.HELM_TEST_HOOKS]
358
+        include = not is_test_pod(pod)
361 359
 
362 360
         # NOTE(drewwalters96): Test pods may cause wait operations to fail
363 361
         # when old resources remain from previous upgrades/tests. Indicate that
364 362
         # test pods should not be included in wait operations.
365
-        if test_hooks:
366
-            LOG.debug('Pod %s will be skipped during wait operations.',
363
+        if not include:
364
+            LOG.debug('Test pod %s will be skipped during wait operations.',
367 365
                       pod.metadata.name)
368
-            return False
369
-        else:
370
-            return True
366
+
367
+        return include
371 368
 
372 369
     def is_resource_ready(self, resource):
373 370
         pod = resource

+ 3
- 4
armada/tests/unit/handlers/test_armada.py View File

@@ -17,7 +17,7 @@ import yaml
17 17
 
18 18
 from armada import const
19 19
 from armada.handlers import armada
20
-from armada.handlers.test import TESTRUN_STATUS_SUCCESS, TESTRUN_STATUS_FAILURE
20
+from armada.utils.helm import TESTRUN_STATUS_SUCCESS, TESTRUN_STATUS_FAILURE
21 21
 from armada.tests.unit import base
22 22
 from armada.tests.test_utils import AttrDict, makeMockThreadSafe
23 23
 from armada.utils.release import release_prefixer, get_release_status
@@ -459,13 +459,12 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase):
459 459
                                         wait=native_wait_enabled,
460 460
                                         timeout=mock.ANY))
461 461
 
462
-                test_chart_override = chart.get('test')
463 462
                 expected_test_constructor_calls.append(
464 463
                     mock.call(
464
+                        chart,
465 465
                         release_name,
466 466
                         m_tiller,
467
-                        cg_test_charts=cg_test_all_charts,
468
-                        test_values=test_chart_override))
467
+                        cg_test_charts=cg_test_all_charts))
469 468
 
470 469
             any_order = not chart_group['sequenced']
471 470
             # Verify that at least 1 release is either installed or updated.

+ 66
- 49
armada/tests/unit/handlers/test_test.py View File

@@ -18,6 +18,7 @@ from armada.handlers import test
18 18
 from armada.handlers import tiller
19 19
 from armada.tests.unit import base
20 20
 from armada.tests.test_utils import AttrDict
21
+from armada.utils import helm
21 22
 
22 23
 
23 24
 class TestHandlerTestCase(base.ArmadaTestCase):
@@ -33,7 +34,7 @@ class TestHandlerTestCase(base.ArmadaTestCase):
33 34
             tiller_obj.test_release.return_value = AttrDict(
34 35
                 **{'results': results})
35 36
 
36
-            test_handler = test.Test(release, tiller_obj)
37
+            test_handler = test.Test({}, release, tiller_obj)
37 38
             success = test_handler.test_release_for_success()
38 39
 
39 40
             self.assertEqual(expected_success, success)
@@ -45,24 +46,24 @@ class TestHandlerTestCase(base.ArmadaTestCase):
45 46
 
46 47
     def test_unknown(self):
47 48
         self._test_test_release_for_success(False, [
48
-            AttrDict(**{'status': test.TESTRUN_STATUS_SUCCESS}),
49
-            AttrDict(**{'status': test.TESTRUN_STATUS_UNKNOWN})
49
+            AttrDict(**{'status': helm.TESTRUN_STATUS_SUCCESS}),
50
+            AttrDict(**{'status': helm.TESTRUN_STATUS_UNKNOWN})
50 51
         ])
51 52
 
52 53
     def test_success(self):
53 54
         self._test_test_release_for_success(
54
-            True, [AttrDict(**{'status': test.TESTRUN_STATUS_SUCCESS})])
55
+            True, [AttrDict(**{'status': helm.TESTRUN_STATUS_SUCCESS})])
55 56
 
56 57
     def test_failure(self):
57 58
         self._test_test_release_for_success(False, [
58
-            AttrDict(**{'status': test.TESTRUN_STATUS_SUCCESS}),
59
-            AttrDict(**{'status': test.TESTRUN_STATUS_FAILURE})
59
+            AttrDict(**{'status': helm.TESTRUN_STATUS_SUCCESS}),
60
+            AttrDict(**{'status': helm.TESTRUN_STATUS_FAILURE})
60 61
         ])
61 62
 
62 63
     def test_running(self):
63 64
         self._test_test_release_for_success(False, [
64
-            AttrDict(**{'status': test.TESTRUN_STATUS_SUCCESS}),
65
-            AttrDict(**{'status': test.TESTRUN_STATUS_RUNNING})
65
+            AttrDict(**{'status': helm.TESTRUN_STATUS_SUCCESS}),
66
+            AttrDict(**{'status': helm.TESTRUN_STATUS_RUNNING})
66 67
         ])
67 68
 
68 69
     def test_cg_disabled(self):
@@ -70,7 +71,10 @@ class TestHandlerTestCase(base.ArmadaTestCase):
70 71
         tests.
71 72
         """
72 73
         test_handler = test.Test(
73
-            release_name='release', tiller=mock.Mock(), cg_test_charts=False)
74
+            chart={},
75
+            release_name='release',
76
+            tiller=mock.Mock(),
77
+            cg_test_charts=False)
74 78
 
75 79
         assert test_handler.test_enabled is False
76 80
 
@@ -79,10 +83,10 @@ class TestHandlerTestCase(base.ArmadaTestCase):
79 83
         tests and the deprecated, boolean `test` key is enabled.
80 84
         """
81 85
         test_handler = test.Test(
86
+            chart={'test': True},
82 87
             release_name='release',
83 88
             tiller=mock.Mock(),
84
-            cg_test_charts=False,
85
-            test_values=True)
89
+            cg_test_charts=False)
86 90
 
87 91
         assert test_handler.test_enabled is True
88 92
 
@@ -90,13 +94,13 @@ class TestHandlerTestCase(base.ArmadaTestCase):
90 94
         """Test that tests are enabled when a chart group disables all
91 95
         tests and the `test.enabled` key is False.
92 96
         """
93
-        test_values = {'enabled': True}
94
-
95 97
         test_handler = test.Test(
98
+            chart={'test': {
99
+                'enabled': True
100
+            }},
96 101
             release_name='release',
97 102
             tiller=mock.Mock(),
98
-            cg_test_charts=False,
99
-            test_values=test_values)
103
+            cg_test_charts=False)
100 104
 
101 105
         assert test_handler.test_enabled is True
102 106
 
@@ -105,10 +109,10 @@ class TestHandlerTestCase(base.ArmadaTestCase):
105 109
         tests and the deprecated, boolean `test` key is disabled.
106 110
         """
107 111
         test_handler = test.Test(
112
+            chart={'test': False},
108 113
             release_name='release',
109 114
             tiller=mock.Mock(),
110
-            cg_test_charts=True,
111
-            test_values=False)
115
+            cg_test_charts=True)
112 116
 
113 117
         assert test_handler.test_enabled is False
114 118
 
@@ -116,13 +120,13 @@ class TestHandlerTestCase(base.ArmadaTestCase):
116 120
         """Test that tests are disabled when a chart group enables all
117 121
         tests and the deprecated, boolean `test` key is disabled.
118 122
         """
119
-        test_values = {'enabled': False}
120
-
121 123
         test_handler = test.Test(
124
+            chart={'test': {
125
+                'enabled': False
126
+            }},
122 127
             release_name='release',
123 128
             tiller=mock.Mock(),
124
-            cg_test_charts=True,
125
-            test_values=test_values)
129
+            cg_test_charts=True)
126 130
 
127 131
         assert test_handler.test_enabled is False
128 132
 
@@ -131,6 +135,7 @@ class TestHandlerTestCase(base.ArmadaTestCase):
131 135
         True and the chart group `test_enabled` key is disabled.
132 136
         """
133 137
         test_handler = test.Test(
138
+            chart={},
134 139
             release_name='release',
135 140
             tiller=mock.Mock(),
136 141
             cg_test_charts=False,
@@ -143,10 +148,10 @@ class TestHandlerTestCase(base.ArmadaTestCase):
143 148
         True and the deprecated, boolean `test` key is disabled.
144 149
         """
145 150
         test_handler = test.Test(
151
+            chart={'test': True},
146 152
             release_name='release',
147 153
             tiller=mock.Mock(),
148
-            enable_all=True,
149
-            test_values=False)
154
+            enable_all=True)
150 155
 
151 156
         assert test_handler.test_enabled is True
152 157
 
@@ -154,13 +159,13 @@ class TestHandlerTestCase(base.ArmadaTestCase):
154 159
         """Test that tests are enabled when the `enable_all` parameter is
155 160
         True and the `test.enabled` key is False.
156 161
         """
157
-        test_values = {'enabled': False}
158
-
159 162
         test_handler = test.Test(
163
+            chart={'test': {
164
+                'enabled': False
165
+            }},
160 166
             release_name='release',
161 167
             tiller=mock.Mock(),
162
-            enable_all=True,
163
-            test_values=test_values)
168
+            enable_all=True)
164 169
 
165 170
         assert test_handler.test_enabled is True
166 171
 
@@ -169,16 +174,16 @@ class TestHandlerTestCase(base.ArmadaTestCase):
169 174
         for a chart's test key.
170 175
         """
171 176
         test_handler = test.Test(
172
-            release_name='release', tiller=mock.Mock(), test_values=True)
177
+            chart={'test': False}, release_name='release', tiller=mock.Mock())
173 178
 
174
-        assert test_handler.test_enabled
179
+        assert not test_handler.test_enabled
175 180
 
176 181
     def test_deprecated_test_key_true(self):
177 182
         """Test that cleanup is enabled by default when tests are enabled using
178 183
         the deprecated, boolean value for a chart's `test` key.
179 184
         """
180 185
         test_handler = test.Test(
181
-            release_name='release', tiller=mock.Mock(), test_values=True)
186
+            chart={'test': True}, release_name='release', tiller=mock.Mock())
182 187
 
183 188
         assert test_handler.test_enabled is True
184 189
         assert test_handler.cleanup is True
@@ -187,11 +192,12 @@ class TestHandlerTestCase(base.ArmadaTestCase):
187 192
         """Test that tests are disabled by a chart's values using the
188 193
         `test.enabled` path.
189 194
         """
190
-        test_values = {'enabled': False}
191 195
         test_handler = test.Test(
196
+            chart={'test': {
197
+                'enabled': False
198
+            }},
192 199
             release_name='release',
193
-            tiller=mock.Mock(),
194
-            test_values=test_values)
200
+            tiller=mock.Mock())
195 201
 
196 202
         assert test_handler.test_enabled is False
197 203
 
@@ -199,11 +205,12 @@ class TestHandlerTestCase(base.ArmadaTestCase):
199 205
         """Test that cleanup is disabled (by default) when tests are enabled by
200 206
         a chart's values using the `test.enabled` path.
201 207
         """
202
-        test_values = {'enabled': True}
203 208
         test_handler = test.Test(
209
+            chart={'test': {
210
+                'enabled': True
211
+            }},
204 212
             release_name='release',
205
-            tiller=mock.Mock(),
206
-            test_values=test_values)
213
+            tiller=mock.Mock())
207 214
 
208 215
         assert test_handler.test_enabled is True
209 216
         assert test_handler.cleanup is False
@@ -212,12 +219,15 @@ class TestHandlerTestCase(base.ArmadaTestCase):
212 219
         """Test that the test handler uses the values provided by a chart's
213 220
         `test` key.
214 221
         """
215
-        test_values = {'enabled': True, 'options': {'cleanup': True}}
216
-
217 222
         test_handler = test.Test(
223
+            chart={'test': {
224
+                'enabled': True,
225
+                'options': {
226
+                    'cleanup': True
227
+                }
228
+            }},
218 229
             release_name='release',
219
-            tiller=mock.Mock(),
220
-            test_values=test_values)
230
+            tiller=mock.Mock())
221 231
 
222 232
         assert test_handler.test_enabled is True
223 233
         assert test_handler.cleanup is True
@@ -226,12 +236,15 @@ class TestHandlerTestCase(base.ArmadaTestCase):
226 236
         """Test that the test handler uses the values provided by a chart's
227 237
         `test` key.
228 238
         """
229
-        test_values = {'enabled': True, 'options': {'cleanup': False}}
230
-
231 239
         test_handler = test.Test(
240
+            chart={'test': {
241
+                'enabled': True,
242
+                'options': {
243
+                    'cleanup': False
244
+                }
245
+            }},
232 246
             release_name='release',
233
-            tiller=mock.Mock(),
234
-            test_values=test_values)
247
+            tiller=mock.Mock())
235 248
 
236 249
         assert test_handler.test_enabled is True
237 250
         assert test_handler.cleanup is False
@@ -240,7 +253,8 @@ class TestHandlerTestCase(base.ArmadaTestCase):
240 253
         """Test that the default values are enforced when no chart `test`
241 254
         values are provided (i.e. tests are enabled and cleanup is disabled).
242 255
         """
243
-        test_handler = test.Test(release_name='release', tiller=mock.Mock())
256
+        test_handler = test.Test(
257
+            chart={}, release_name='release', tiller=mock.Mock())
244 258
 
245 259
         assert test_handler.test_enabled is True
246 260
         assert test_handler.cleanup is False
@@ -249,13 +263,16 @@ class TestHandlerTestCase(base.ArmadaTestCase):
249 263
         """Test that a cleanup value passed to the Test handler (i.e. from the
250 264
         API/CLI) takes precedence over a chart's `test.cleanup` value.
251 265
         """
252
-        test_values = {'enabled': True, 'options': {'cleanup': False}}
253
-
254 266
         test_handler = test.Test(
267
+            chart={'test': {
268
+                'enabled': True,
269
+                'options': {
270
+                    'cleanup': False
271
+                }
272
+            }},
255 273
             release_name='release',
256 274
             tiller=mock.Mock(),
257
-            cleanup=True,
258
-            test_values=test_values)
275
+            cleanup=True)
259 276
 
260 277
         assert test_handler.test_enabled is True
261 278
         assert test_handler.cleanup is True

+ 6
- 6
armada/tests/unit/handlers/test_tiller.py View File

@@ -17,7 +17,7 @@ from mock import MagicMock
17 17
 
18 18
 from armada.exceptions import tiller_exceptions as ex
19 19
 from armada.handlers import tiller
20
-from armada.handlers import test
20
+from armada.utils import helm
21 21
 from armada.tests.unit import base
22 22
 from armada.tests.test_utils import AttrDict
23 23
 
@@ -552,7 +552,7 @@ class TillerTestCase(base.ArmadaTestCase):
552 552
         self._test_test_release([
553 553
             AttrDict(**{
554 554
                 'msg': 'No Tests Found',
555
-                'status': test.TESTRUN_STATUS_UNKNOWN
555
+                'status': helm.TESTRUN_STATUS_UNKNOWN
556 556
             })
557 557
         ])
558 558
 
@@ -560,11 +560,11 @@ class TillerTestCase(base.ArmadaTestCase):
560 560
         self._test_test_release([
561 561
             AttrDict(**{
562 562
                 'msg': 'RUNNING: ...',
563
-                'status': test.TESTRUN_STATUS_RUNNING
563
+                'status': helm.TESTRUN_STATUS_RUNNING
564 564
             }),
565 565
             AttrDict(**{
566 566
                 'msg': 'SUCCESS: ...',
567
-                'status': test.TESTRUN_STATUS_SUCCESS
567
+                'status': helm.TESTRUN_STATUS_SUCCESS
568 568
             })
569 569
         ])
570 570
 
@@ -572,11 +572,11 @@ class TillerTestCase(base.ArmadaTestCase):
572 572
         self._test_test_release([
573 573
             AttrDict(**{
574 574
                 'msg': 'RUNNING: ...',
575
-                'status': test.TESTRUN_STATUS_RUNNING
575
+                'status': helm.TESTRUN_STATUS_RUNNING
576 576
             }),
577 577
             AttrDict(**{
578 578
                 'msg': 'FAILURE: ...',
579
-                'status': test.TESTRUN_STATUS_FAILURE
579
+                'status': helm.TESTRUN_STATUS_FAILURE
580 580
             })
581 581
         ])
582 582
 

+ 40
- 0
armada/utils/helm.py View File

@@ -0,0 +1,40 @@
1
+# Copyright 2019 The Armada Authors.
2
+#
3
+# Licensed under the Apache License, Version 2.0 (the "License");
4
+# you may not use this file except in compliance with the License.
5
+# You may obtain a copy of the License at
6
+#
7
+#     http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+# Unless required by applicable law or agreed to in writing, software
10
+# distributed under the License is distributed on an "AS IS" BASIS,
11
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+# See the License for the specific language governing permissions and
13
+# limitations under the License.
14
+
15
+TESTRUN_STATUS_UNKNOWN = 0
16
+TESTRUN_STATUS_SUCCESS = 1
17
+TESTRUN_STATUS_FAILURE = 2
18
+TESTRUN_STATUS_RUNNING = 3
19
+
20
+HELM_HOOK_ANNOTATION = 'helm.sh/hook'
21
+HELM_TEST_HOOKS = ['test-success', 'test-failure']
22
+
23
+
24
+def is_test_pod(pod):
25
+    annotations = pod.metadata.annotations
26
+
27
+    # Retrieve pod's Helm test hooks
28
+    test_hooks = None
29
+    if annotations:
30
+        hook_string = annotations.get(HELM_HOOK_ANNOTATION)
31
+        if hook_string:
32
+            hooks = hook_string.split(',')
33
+            test_hooks = [h for h in hooks if h in HELM_TEST_HOOKS]
34
+
35
+    return bool(test_hooks)
36
+
37
+
38
+def get_test_suite_run_success(test_suite_run):
39
+    return all(
40
+        r.status == TESTRUN_STATUS_SUCCESS for r in test_suite_run.results)

+ 2
- 2
armada/utils/release.py View File

@@ -12,7 +12,7 @@
12 12
 # See the License for the specific language governing permissions and
13 13
 # limitations under the License.
14 14
 
15
-from armada.handlers.test import Test
15
+from armada.utils.helm import get_test_suite_run_success
16 16
 
17 17
 import time
18 18
 
@@ -54,7 +54,7 @@ def get_last_test_result(release):
54 54
     status = release.info.status
55 55
     if not status.HasField('last_test_suite_run'):
56 56
         return None
57
-    return Test.get_test_suite_run_success(status.last_test_suite_run)
57
+    return get_test_suite_run_success(status.last_test_suite_run)
58 58
 
59 59
 
60 60
 def get_last_deployment_age(release):

+ 8
- 3
doc/source/operations/guide-build-armada-yaml.rst View File

@@ -203,10 +203,15 @@ Test options to pass through directly to helm.
203 203
 
204 204
 .. note::
205 205
 
206
-    The preferred way to achieve test cleanup is to add a pre-upgrade delete
207
-    action on the test pod, which allows for debugging the test pod up until the
208
-    next upgrade.
206
+    If cleanup is ``true`` this prevents being able to debug a test in the event of failure.
209 207
 
208
+    Historically, the preferred way to achieve test cleanup has been to add a pre-upgrade delete
209
+    action on the test pod.
210
+
211
+    This still works, however it is usually no longer necessary as Armada now automatically
212
+    cleans up any test pods which match the ``wait.labels`` of the chart, immediately before
213
+    running tests. Similar suggestions have been made for how ``helm test --cleanup`` itself
214
+    ought to work (https://github.com/helm/helm/issues/3279).
210 215
 
211 216
 Upgrade - Pre
212 217
 ^^^^^^^^^^^^^

Loading…
Cancel
Save