Browse Source

test: Refactor test handler

While authoring [0], it was discovered that Armada has duplicate logic
for deciding if Helm test cleanup should be enabled as well as the tests
themselves. Because of this, changes to test logic (e.g. adding pre-test
actions), requires changing all traces of the repeated logic, which can
lead to inconsistent behavior if not properly addressed. This change
moves all test decision logic to a singular Test handler, implemented by
the `Test` class. This change does NOT change the expected behavior of
testing during upgrades; however, tests initiated from the API and CLI
will not execute when testing a manifest if they are disabled in a
chart, unless using the `--enable-all` flag.

[0] https://review.openstack.org/617834

Change-Id: I1530d7637b0eb6a83f048895053a5db80d033046
changes/55/618355/4
Drew Walters 7 months ago
parent
commit
adfe3ae505

+ 25
- 30
armada/api/controller/test.py View File

@@ -21,8 +21,8 @@ from oslo_config import cfg
21 21
 from armada import api
22 22
 from armada.common import policy
23 23
 from armada import const
24
-from armada.handlers.test import test_release_for_success
25 24
 from armada.handlers.manifest import Manifest
25
+from armada.handlers.test import Test
26 26
 from armada.utils.release import release_prefixer
27 27
 from armada.utils import validate
28 28
 
@@ -36,14 +36,11 @@ class TestReleasesReleaseNameController(api.BaseResource):
36 36
 
37 37
     @policy.enforce('armada:test_release')
38 38
     def on_get(self, req, resp, release):
39
-        self.logger.info('RUNNING: %s', release)
40 39
         with self.get_tiller(req, resp) as tiller:
41
-
42 40
             cleanup = req.get_param_as_bool('cleanup')
43
-            if cleanup is None:
44
-                cleanup = False
45
-            success = test_release_for_success(
46
-                tiller, release, cleanup=cleanup)
41
+
42
+            test_handler = Test(release, tiller, cleanup=cleanup)
43
+            success = test_handler.test_release_for_success()
47 44
 
48 45
         if success:
49 46
             msg = {
@@ -56,8 +53,6 @@ class TestReleasesReleaseNameController(api.BaseResource):
56 53
                 'message': 'MESSAGE: Test Fail'
57 54
             }
58 55
 
59
-        self.logger.info(msg)
60
-
61 56
         resp.body = json.dumps(msg)
62 57
         resp.status = falcon.HTTP_200
63 58
         resp.content_type = 'application/json'
@@ -135,29 +130,29 @@ class TestReleasesManifestController(api.BaseResource):
135 130
                 const.KEYWORD_GROUPS):
136 131
             for ch in group.get(const.KEYWORD_CHARTS):
137 132
                 chart = ch['chart']
133
+
138 134
                 release_name = release_prefixer(prefix, chart.get('release'))
139
-                cleanup = req.get_param_as_bool('cleanup')
140
-                if cleanup is None:
141
-                    test_chart_override = chart.get('test', {})
142
-                    if isinstance(test_chart_override, bool):
143
-                        self.logger.warn(
144
-                            'Boolean value for chart `test` key is deprecated '
145
-                            'and will be removed. Use `test.enabled` instead.')
146
-                        # Use old default value.
147
-                        cleanup = True
148
-                    else:
149
-                        cleanup = test_chart_override.get('options', {}).get(
150
-                            'cleanup', False)
151 135
                 if release_name in known_releases:
152
-                    self.logger.info('RUNNING: %s tests', release_name)
153
-                    success = test_release_for_success(
154
-                        tiller, release_name, cleanup=cleanup)
155
-                    if success:
156
-                        self.logger.info("PASSED: %s", release_name)
157
-                        message['test']['passed'].append(release_name)
158
-                    else:
159
-                        self.logger.info("FAILED: %s", release_name)
160
-                        message['test']['failed'].append(release_name)
136
+                    cleanup = req.get_param_as_bool('cleanup')
137
+                    enable_all = req.get_param_as_bool('enable_all')
138
+                    cg_test_charts = group.get('test_charts')
139
+                    test_values = chart.get('test', {})
140
+
141
+                    test_handler = Test(
142
+                        release_name,
143
+                        tiller,
144
+                        cg_test_charts=cg_test_charts,
145
+                        cleanup=cleanup,
146
+                        enable_all=enable_all,
147
+                        test_values=test_values)
148
+
149
+                    if test_handler.test_enabled:
150
+                        success = test_handler.test_release_for_success()
151
+
152
+                        if success:
153
+                            message['test']['passed'].append(release_name)
154
+                        else:
155
+                            message['test']['failed'].append(release_name)
161 156
                 else:
162 157
                     self.logger.info('Release %s not found - SKIPPING',
163 158
                                      release_name)

+ 25
- 34
armada/cli/test.py View File

@@ -19,8 +19,8 @@ from oslo_config import cfg
19 19
 
20 20
 from armada.cli import CliAction
21 21
 from armada import const
22
-from armada.handlers.test import test_release_for_success
23 22
 from armada.handlers.manifest import Manifest
23
+from armada.handlers.test import Test
24 24
 from armada.handlers.tiller import Tiller
25 25
 from armada.utils.release import release_prefixer
26 26
 
@@ -79,20 +79,26 @@ SHORT_DESC = "Command tests releases."
79 79
     help=("Delete test pods upon completion."),
80 80
     is_flag=True,
81 81
     default=None)
82
+@click.option(
83
+    '--enable-all',
84
+    help=("Run all tests for all releases regardless of any disabled chart "
85
+          "tests."),
86
+    is_flag=True,
87
+    default=False)
82 88
 @click.option('--debug', help="Enable debug logging.", is_flag=True)
83 89
 @click.pass_context
84 90
 def test_charts(ctx, file, release, tiller_host, tiller_port, tiller_namespace,
85
-                target_manifest, cleanup, debug):
91
+                target_manifest, cleanup, enable_all, debug):
86 92
     CONF.debug = debug
87 93
     TestChartManifest(ctx, file, release, tiller_host, tiller_port,
88
-                      tiller_namespace, target_manifest,
89
-                      cleanup).safe_invoke()
94
+                      tiller_namespace, target_manifest, cleanup,
95
+                      enable_all).safe_invoke()
90 96
 
91 97
 
92 98
 class TestChartManifest(CliAction):
93 99
 
94 100
     def __init__(self, ctx, file, release, tiller_host, tiller_port,
95
-                 tiller_namespace, target_manifest, cleanup):
101
+                 tiller_namespace, target_manifest, cleanup, enable_all):
96 102
 
97 103
         super(TestChartManifest, self).__init__()
98 104
         self.ctx = ctx
@@ -103,6 +109,7 @@ class TestChartManifest(CliAction):
103 109
         self.tiller_namespace = tiller_namespace
104 110
         self.target_manifest = target_manifest
105 111
         self.cleanup = cleanup
112
+        self.enable_all = enable_all
106 113
 
107 114
     def invoke(self):
108 115
         with Tiller(
@@ -117,13 +124,8 @@ class TestChartManifest(CliAction):
117 124
 
118 125
         if self.release:
119 126
             if not self.ctx.obj.get('api', False):
120
-                self.logger.info("RUNNING: %s tests", self.release)
121
-                success = test_release_for_success(
122
-                    tiller, self.release, cleanup=self.cleanup)
123
-                if success:
124
-                    self.logger.info("PASSED: %s", self.release)
125
-                else:
126
-                    self.logger.info("FAILED: %s", self.release)
127
+                test_handler = Test(self.release, tiller, cleanup=self.cleanup)
128
+                test_handler.test_release_for_success()
127 129
             else:
128 130
                 client = self.ctx.obj.get('CLIENT')
129 131
                 query = {
@@ -150,32 +152,21 @@ class TestChartManifest(CliAction):
150 152
                         const.KEYWORD_GROUPS):
151 153
                     for ch in group.get(const.KEYWORD_CHARTS):
152 154
                         chart = ch['chart']
155
+
153 156
                         release_name = release_prefixer(
154 157
                             prefix, chart.get('release'))
155
-
156 158
                         if release_name in known_release_names:
157
-                            cleanup = self.cleanup
158
-                            if cleanup is None:
159
-                                test_chart_override = chart.get('test', {})
160
-                                if isinstance(test_chart_override, bool):
161
-                                    self.logger.warn(
162
-                                        'Boolean value for chart `test` key is'
163
-                                        ' deprecated and support for this will'
164
-                                        ' be removed. Use `test.enabled` '
165
-                                        'instead.')
166
-                                    # Use old default value.
167
-                                    cleanup = True
168
-                                else:
169
-                                    cleanup = test_chart_override.get(
170
-                                        'options', {}).get('cleanup', False)
171
-                            self.logger.info('RUNNING: %s tests', release_name)
172
-                            success = test_release_for_success(
173
-                                tiller, release_name, cleanup=cleanup)
174
-                            if success:
175
-                                self.logger.info("PASSED: %s", release_name)
176
-                            else:
177
-                                self.logger.info("FAILED: %s", release_name)
159
+                            test_values = chart.get('test', {})
160
+
161
+                            test_handler = Test(
162
+                                release_name,
163
+                                tiller,
164
+                                cleanup=self.cleanup,
165
+                                enable_all=self.enable_all,
166
+                                test_values=test_values)
178 167
 
168
+                            if test_handler.test_enabled:
169
+                                test_handler.test_release_for_success()
179 170
                         else:
180 171
                             self.logger.info('Release %s not found - SKIPPING',
181 172
                                              release_name)

+ 0
- 8
armada/handlers/armada.py View File

@@ -200,14 +200,6 @@ class Armada(object):
200 200
 
201 201
             # TODO(MarshM): Deprecate the `test_charts` key
202 202
             cg_test_all_charts = chartgroup.get('test_charts')
203
-            if isinstance(cg_test_all_charts, bool):
204
-                LOG.warn('The ChartGroup `test_charts` key is deprecated, '
205
-                         'and support for this will be removed. See the '
206
-                         'Chart `test` key for more information.')
207
-            else:
208
-                # This key defaults to True. Individual charts must
209
-                # explicitly disable helm tests if they choose
210
-                cg_test_all_charts = True
211 203
 
212 204
             cg_charts = chartgroup.get(const.KEYWORD_CHARTS, [])
213 205
             charts = map(lambda x: x.get('chart', {}), cg_charts)

+ 14
- 27
armada/handlers/chart_deploy.py View File

@@ -19,8 +19,8 @@ import yaml
19 19
 from armada import const
20 20
 from armada.exceptions import armada_exceptions
21 21
 from armada.handlers.chartbuilder import ChartBuilder
22
-from armada.handlers.test import test_release_for_success
23 22
 from armada.handlers.release_diff import ReleaseDiff
23
+from armada.handlers.test import Test
24 24
 from armada.handlers.wait import ChartWait
25 25
 from armada.exceptions import tiller_exceptions
26 26
 import armada.utils.release as r
@@ -202,34 +202,25 @@ class ChartDeploy(object):
202 202
         chart_wait.wait(timer)
203 203
 
204 204
         # Test
205
-        test_chart_override = chart.get('test')
206
-        # Use old default value when not using newer `test` key
207
-        test_cleanup = True
208
-        if test_chart_override is None:
209
-            test_enabled = cg_test_all_charts
210
-        elif isinstance(test_chart_override, bool):
211
-            LOG.warn('Boolean value for chart `test` key is'
212
-                     ' deprecated and support for this will'
213
-                     ' be removed. Use `test.enabled` '
214
-                     'instead.')
215
-            test_enabled = test_chart_override
216
-        else:
217
-            # NOTE: helm tests are enabled by default
218
-            test_enabled = test_chart_override.get('enabled', True)
219
-            test_cleanup = test_chart_override.get('options', {}).get(
220
-                'cleanup', False)
221
-
222 205
         just_deployed = ('install' in result) or ('upgrade' in result)
223 206
         last_test_passed = old_release and r.get_last_test_result(old_release)
224
-        run_test = test_enabled and (just_deployed or not last_test_passed)
225 207
 
208
+        test_values = chart.get('test')
209
+        test_handler = Test(
210
+            release_name,
211
+            self.tiller,
212
+            cg_test_charts=cg_test_all_charts,
213
+            test_values=test_values)
214
+
215
+        run_test = test_handler.test_enabled and (just_deployed or
216
+                                                  not last_test_passed)
226 217
         if run_test:
227 218
             timer = int(round(deadline - time.time()))
228
-            self._test_chart(release_name, timer, test_cleanup)
219
+            self._test_chart(release_name, timer, test_handler)
229 220
 
230 221
         return result
231 222
 
232
-    def _test_chart(self, release_name, timeout, cleanup):
223
+    def _test_chart(self, release_name, timeout, test_handler):
233 224
         if self.dry_run:
234 225
             LOG.info(
235 226
                 'Skipping test during `dry-run`, would have tested '
@@ -242,12 +233,8 @@ class ChartDeploy(object):
242 233
             LOG.error(reason)
243 234
             raise armada_exceptions.ArmadaTimeoutException(reason)
244 235
 
245
-        success = test_release_for_success(
246
-            self.tiller, release_name, timeout=timeout, cleanup=cleanup)
247
-        if success:
248
-            LOG.info("Test passed for release: %s", release_name)
249
-        else:
250
-            LOG.info("Test failed for release: %s", release_name)
236
+        success = test_handler.test_release_for_success(timeout=timeout)
237
+        if not success:
251 238
             raise tiller_exceptions.TestFailedException(release_name)
252 239
 
253 240
     def get_diff(self, old_chart, old_values, new_chart, values):

+ 99
- 10
armada/handlers/test.py View File

@@ -12,6 +12,8 @@
12 12
 # See the License for the specific language governing permissions and
13 13
 # limitations under the License.
14 14
 
15
+from oslo_log import log as logging
16
+
15 17
 from armada import const
16 18
 
17 19
 TESTRUN_STATUS_UNKNOWN = 0
@@ -19,16 +21,103 @@ TESTRUN_STATUS_SUCCESS = 1
19 21
 TESTRUN_STATUS_FAILURE = 2
20 22
 TESTRUN_STATUS_RUNNING = 3
21 23
 
24
+LOG = logging.getLogger(__name__)
25
+
26
+
27
+class Test(object):
28
+
29
+    def __init__(self,
30
+                 release_name,
31
+                 tiller,
32
+                 cg_test_charts=None,
33
+                 cleanup=None,
34
+                 enable_all=False,
35
+                 test_values=None):
36
+        """Initialize a test handler to run Helm tests corresponding to a
37
+        release.
38
+
39
+        :param release_name: Name of a Helm release
40
+        :param tiller: Tiller object
41
+        :param cg_test_charts: Chart group `test_charts` key
42
+        :param cleanup: Triggers cleanup; overrides `test.options.cleanup`
43
+        :param enable_all: Run tests regardless of the value of `test.enabled`
44
+        :param test_values: Test values retrieved from a chart's `test` key
45
+
46
+        :type release_name: str
47
+        :type tiller: Tiller object
48
+        :type cg_test_charts: bool
49
+        :type cleanup: bool
50
+        :type enable_all: bool
51
+        :type test_values: dict or bool (deprecated)
52
+        """
53
+
54
+        self.release_name = release_name
55
+        self.tiller = tiller
56
+        self.cleanup = cleanup
57
+
58
+        # NOTE(drewwalters96): Support the chart_group `test_charts` key until
59
+        # its deprecation period ends. The `test.enabled`, `enable_all` flag,
60
+        # and deprecated, boolean `test` key override this value if provided.
61
+        if cg_test_charts is not None:
62
+            LOG.warn('Chart group key `test_charts` is deprecated and will be '
63
+                     'removed. Use `test.enabled` instead.')
64
+            self.test_enabled = cg_test_charts
65
+        else:
66
+            self.test_enabled = True
67
+
68
+        # NOTE: Support old, boolean `test` key until deprecation period ends.
69
+        if (type(test_values) == bool):
70
+            LOG.warn('Boolean value for chart `test` key is deprecated and '
71
+                     'will be removed. Use `test.enabled` instead.')
72
+
73
+            self.test_enabled = test_values
74
+
75
+            # NOTE: Use old, default cleanup value (i.e. True) if none is
76
+            # provided.
77
+            if self.cleanup is None:
78
+                self.cleanup = True
79
+        elif test_values:
80
+            test_enabled_opt = test_values.get('enabled')
81
+            if test_enabled_opt is not None:
82
+                self.test_enabled = test_enabled_opt
83
+
84
+            # NOTE(drewwalters96): `self.cleanup`, the cleanup value provided
85
+            # by the API/CLI, takes precedence over the chart value
86
+            # `test.cleanup`.
87
+            if self.cleanup is None:
88
+                test_options = test_values.get('options', {})
89
+                self.cleanup = test_options.get('cleanup', False)
90
+        else:
91
+            # Default cleanup value
92
+            if self.cleanup is None:
93
+                self.cleanup = False
94
+
95
+        if enable_all:
96
+            self.test_enabled = True
97
+
98
+    def test_release_for_success(self, timeout=const.DEFAULT_TILLER_TIMEOUT):
99
+        """Run the Helm tests corresponding to a release for success (i.e. exit
100
+        code 0).
101
+
102
+        :param timeout: Timeout value for a release's tests completion
103
+        :type timeout: int
104
+
105
+        :rtype: Helm test suite run result
106
+        """
107
+        LOG.info('RUNNING: %s tests', self.release_name)
108
+
109
+        test_suite_run = self.tiller.test_release(
110
+            self.release_name, timeout=timeout, cleanup=self.cleanup)
22 111
 
23
-def test_release_for_success(tiller,
24
-                             release,
25
-                             timeout=const.DEFAULT_TILLER_TIMEOUT,
26
-                             cleanup=False):
27
-    test_suite_run = tiller.test_release(
28
-        release, timeout=timeout, cleanup=cleanup)
29
-    return get_test_suite_run_success(test_suite_run)
112
+        success = self.get_test_suite_run_success(test_suite_run)
113
+        if success:
114
+            LOG.info('PASSED: %s', self.release_name)
115
+        else:
116
+            LOG.info('FAILED: %s', self.release_name)
30 117
 
118
+        return success
31 119
 
32
-def get_test_suite_run_success(test_suite_run):
33
-    return all(
34
-        r.status == TESTRUN_STATUS_SUCCESS for r in test_suite_run.results)
120
+    @classmethod
121
+    def get_test_suite_run_success(self, test_suite_run):
122
+        return all(
123
+            r.status == TESTRUN_STATUS_SUCCESS for r in test_suite_run.results)

+ 7
- 9
armada/tests/unit/api/test_test_controller.py View File

@@ -59,7 +59,7 @@ class TestReleasesManifestControllerTest(base.BaseControllerTest):
59 59
 
60 60
 class TestReleasesReleaseNameControllerTest(base.BaseControllerTest):
61 61
 
62
-    @mock.patch.object(test, 'test_release_for_success')
62
+    @mock.patch.object(test.Test, 'test_release_for_success')
63 63
     @mock.patch.object(api, 'Tiller')
64 64
     def test_test_controller_test_pass(self, mock_tiller,
65 65
                                        mock_test_release_for_success):
@@ -73,14 +73,13 @@ class TestReleasesReleaseNameControllerTest(base.BaseControllerTest):
73 73
 
74 74
         release = 'fake-release'
75 75
         resp = self.app.simulate_get('/api/v1.0/test/{}'.format(release))
76
-        mock_test_release_for_success.assert_has_calls(
77
-            [mock.call(mock_tiller.return_value, release, cleanup=False)])
76
+        mock_test_release_for_success.assert_called_once()
78 77
         self.assertEqual(200, resp.status_code)
79 78
         self.assertEqual('MESSAGE: Test Pass',
80 79
                          json.loads(resp.text)['message'])
81 80
         m_tiller.__exit__.assert_called()
82 81
 
83
-    @mock.patch.object(test, 'test_release_for_success')
82
+    @mock.patch.object(test.Test, 'test_release_for_success')
84 83
     @mock.patch.object(api, 'Tiller')
85 84
     def test_test_controller_test_fail(self, mock_tiller,
86 85
                                        mock_test_release_for_success):
@@ -98,7 +97,7 @@ class TestReleasesReleaseNameControllerTest(base.BaseControllerTest):
98 97
                          json.loads(resp.text)['message'])
99 98
         m_tiller.__exit__.assert_called()
100 99
 
101
-    @mock.patch.object(test, 'test_release_for_success')
100
+    @mock.patch.object(test.Test, 'test_release_for_success')
102 101
     @mock.patch.object(api, 'Tiller')
103 102
     def test_test_controller_cleanup(self, mock_tiller,
104 103
                                      mock_test_release_for_success):
@@ -112,8 +111,7 @@ class TestReleasesReleaseNameControllerTest(base.BaseControllerTest):
112 111
         release = 'fake-release'
113 112
         resp = self.app.simulate_get(
114 113
             '/api/v1.0/test/{}'.format(release), query_string='cleanup=true')
115
-        mock_test_release_for_success.assert_has_calls(
116
-            [mock.call(m_tiller, release, cleanup=True)])
114
+        mock_test_release_for_success.assert_called_once()
117 115
         self.assertEqual(200, resp.status_code)
118 116
         self.assertEqual('MESSAGE: Test Pass',
119 117
                          json.loads(resp.text)['message'])
@@ -125,7 +123,7 @@ class TestReleasesManifestControllerNegativeTest(base.BaseControllerTest):
125 123
 
126 124
     @mock.patch.object(test, 'Manifest')
127 125
     @mock.patch.object(api, 'Tiller')
128
-    @mock.patch.object(test, 'test_release_for_success')
126
+    @mock.patch.object(test.Test, 'test_release_for_success')
129 127
     def test_test_controller_tiller_exc_returns_500(
130 128
             self, mock_test_release_for_success, mock_tiller, _):
131 129
         rules = {'armada:test_manifest': '@'}
@@ -227,7 +225,7 @@ class TestReleasesManifestControllerNegativeTest(base.BaseControllerTest):
227 225
 class TestReleasesReleaseNameControllerNegativeTest(base.BaseControllerTest):
228 226
 
229 227
     @mock.patch.object(api, 'Tiller')
230
-    @mock.patch.object(test, 'test_release_for_success')
228
+    @mock.patch.object(test.Test, 'test_release_for_success')
231 229
     def test_test_controller_tiller_exc_returns_500(
232 230
             self, mock_test_release_for_success, mock_tiller):
233 231
         rules = {'armada:test_release': '@'}

+ 19
- 35
armada/tests/unit/handlers/test_armada.py View File

@@ -17,7 +17,6 @@ import yaml
17 17
 
18 18
 from armada import const
19 19
 from armada.handlers import armada
20
-from armada.handlers import chart_deploy
21 20
 from armada.handlers.test import TESTRUN_STATUS_SUCCESS, TESTRUN_STATUS_FAILURE
22 21
 from armada.tests.unit import base
23 22
 from armada.tests.test_utils import AttrDict, makeMockThreadSafe
@@ -337,9 +336,9 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase):
337 336
         @mock.patch.object(armada.Armada, 'post_flight_ops')
338 337
         @mock.patch.object(armada.Armada, 'pre_flight_ops')
339 338
         @mock.patch('armada.handlers.chart_deploy.ChartBuilder')
340
-        @mock.patch.object(chart_deploy, 'test_release_for_success')
341
-        def _do_test(mock_test_release_for_success, mock_chartbuilder,
342
-                     mock_pre_flight, mock_post_flight):
339
+        @mock.patch('armada.handlers.chart_deploy.Test')
340
+        def _do_test(mock_test, mock_chartbuilder, mock_pre_flight,
341
+                     mock_post_flight):
343 342
             # Instantiate Armada object.
344 343
             yaml_documents = list(yaml.safe_load_all(TEST_YAML))
345 344
 
@@ -351,8 +350,9 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase):
351 350
 
352 351
             chart_group = armada_obj.manifest['armada']['chart_groups'][0]
353 352
             charts = chart_group['chart_group']
354
-            cg_test_all_charts = chart_group.get('test_charts', True)
353
+            cg_test_all_charts = chart_group.get('test_charts')
355 354
 
355
+            mock_test_release = mock_test.return_value.test_release_for_success
356 356
             if test_failure_to_run:
357 357
 
358 358
                 def fail(tiller, release, timeout=None, cleanup=False):
@@ -361,9 +361,9 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase):
361 361
                     raise tiller_exceptions.ReleaseException(
362 362
                         release, status, 'Test')
363 363
 
364
-                mock_test_release_for_success.side_effect = fail
364
+                mock_test_release.side_effect = fail
365 365
             else:
366
-                mock_test_release_for_success.return_value = test_success
366
+                mock_test_release.return_value = test_success
367 367
 
368 368
             # Stub out irrelevant methods called by `armada.sync()`.
369 369
             mock_chartbuilder.get_source_path.return_value = None
@@ -377,7 +377,7 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase):
377 377
             expected_install_release_calls = []
378 378
             expected_update_release_calls = []
379 379
             expected_uninstall_release_calls = []
380
-            expected_test_release_for_success_calls = []
380
+            expected_test_constructor_calls = []
381 381
 
382 382
             for c in charts:
383 383
                 chart = c['chart']
@@ -389,7 +389,6 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase):
389 389
                 native_wait_enabled = (chart['wait'].get('native', {}).get(
390 390
                     'enabled', True))
391 391
 
392
-                expected_apply = True
393 392
                 if release_name not in [x.name for x in known_releases]:
394 393
                     expected_install_release_calls.append(
395 394
                         mock.call(
@@ -435,9 +434,7 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase):
435 434
                                         break
436 435
 
437 436
                         if status == const.STATUS_DEPLOYED:
438
-                            if not diff:
439
-                                expected_apply = False
440
-                            else:
437
+                            if diff:
441 438
                                 upgrade = chart.get('upgrade', {})
442 439
                                 disable_hooks = upgrade.get('no_hooks', False)
443 440
                                 force = upgrade.get('force', False)
@@ -462,25 +459,12 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase):
462 459
                                         timeout=mock.ANY))
463 460
 
464 461
                 test_chart_override = chart.get('test')
465
-                # Use old default value when not using newer `test` key
466
-                test_cleanup = True
467
-                if test_chart_override is None:
468
-                    test_this_chart = cg_test_all_charts
469
-                elif isinstance(test_chart_override, bool):
470
-                    test_this_chart = test_chart_override
471
-                else:
472
-                    test_this_chart = test_chart_override.get('enabled', True)
473
-                    test_cleanup = test_chart_override.get('options', {}).get(
474
-                        'cleanup', False)
475
-
476
-                if test_this_chart and (expected_apply or
477
-                                        not expected_last_test_result):
478
-                    expected_test_release_for_success_calls.append(
479
-                        mock.call(
480
-                            m_tiller,
481
-                            release_name,
482
-                            timeout=mock.ANY,
483
-                            cleanup=test_cleanup))
462
+                expected_test_constructor_calls.append(
463
+                    mock.call(
464
+                        release_name,
465
+                        m_tiller,
466
+                        cg_test_charts=cg_test_all_charts,
467
+                        test_values=test_chart_override))
484 468
 
485 469
             any_order = not chart_group['sequenced']
486 470
             # Verify that at least 1 release is either installed or updated.
@@ -511,10 +495,10 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase):
511 495
             # Verify that the expected number of deployed releases are
512 496
             # tested with expected arguments.
513 497
             self.assertEqual(
514
-                len(expected_test_release_for_success_calls),
515
-                mock_test_release_for_success.call_count)
516
-            mock_test_release_for_success.assert_has_calls(
517
-                expected_test_release_for_success_calls, any_order=any_order)
498
+                len(expected_test_constructor_calls), mock_test.call_count)
499
+
500
+            mock_test.assert_has_calls(
501
+                expected_test_constructor_calls, any_order=True)
518 502
 
519 503
         _do_test()
520 504
 

+ 199
- 2
armada/tests/unit/handlers/test_test.py View File

@@ -14,8 +14,8 @@
14 14
 
15 15
 import mock
16 16
 
17
-from armada.handlers import tiller
18 17
 from armada.handlers import test
18
+from armada.handlers import tiller
19 19
 from armada.tests.unit import base
20 20
 from armada.tests.test_utils import AttrDict
21 21
 
@@ -32,7 +32,9 @@ class TestHandlerTestCase(base.ArmadaTestCase):
32 32
             tiller_obj.test_release = mock.Mock()
33 33
             tiller_obj.test_release.return_value = AttrDict(
34 34
                 **{'results': results})
35
-            success = test.test_release_for_success(tiller_obj, release)
35
+
36
+            test_handler = test.Test(release, tiller_obj)
37
+            success = test_handler.test_release_for_success()
36 38
 
37 39
             self.assertEqual(expected_success, success)
38 40
 
@@ -62,3 +64,198 @@ class TestHandlerTestCase(base.ArmadaTestCase):
62 64
             AttrDict(**{'status': test.TESTRUN_STATUS_SUCCESS}),
63 65
             AttrDict(**{'status': test.TESTRUN_STATUS_RUNNING})
64 66
         ])
67
+
68
+    def test_cg_disabled(self):
69
+        """Test that tests are disabled when a chart group disables all
70
+        tests.
71
+        """
72
+        test_handler = test.Test(
73
+            release_name='release', tiller=mock.Mock(), cg_test_charts=False)
74
+
75
+        assert test_handler.test_enabled is False
76
+
77
+    def test_cg_disabled_test_key_enabled(self):
78
+        """Test that tests are enabled when a chart group disables all
79
+        tests and the deprecated, boolean `test` key is enabled.
80
+        """
81
+        test_handler = test.Test(
82
+            release_name='release',
83
+            tiller=mock.Mock(),
84
+            cg_test_charts=False,
85
+            test_values=True)
86
+
87
+        assert test_handler.test_enabled is True
88
+
89
+    def test_cg_disabled_test_values_enabled(self):
90
+        """Test that tests are enabled when a chart group disables all
91
+        tests and the `test.enabled` key is False.
92
+        """
93
+        test_values = {'enabled': True}
94
+
95
+        test_handler = test.Test(
96
+            release_name='release',
97
+            tiller=mock.Mock(),
98
+            cg_test_charts=False,
99
+            test_values=test_values)
100
+
101
+        assert test_handler.test_enabled is True
102
+
103
+    def test_cg_enabled_test_key_disabled(self):
104
+        """Test that tests are disabled when a chart group enables all
105
+        tests and the deprecated, boolean `test` key is disabled.
106
+        """
107
+        test_handler = test.Test(
108
+            release_name='release',
109
+            tiller=mock.Mock(),
110
+            cg_test_charts=True,
111
+            test_values=False)
112
+
113
+        assert test_handler.test_enabled is False
114
+
115
+    def test_cg_enabled_test_values_disabled(self):
116
+        """Test that tests are disabled when a chart group enables all
117
+        tests and the deprecated, boolean `test` key is disabled.
118
+        """
119
+        test_values = {'enabled': False}
120
+
121
+        test_handler = test.Test(
122
+            release_name='release',
123
+            tiller=mock.Mock(),
124
+            cg_test_charts=True,
125
+            test_values=test_values)
126
+
127
+        assert test_handler.test_enabled is False
128
+
129
+    def test_enable_all_cg_disabled(self):
130
+        """Test that tests are enabled when the `enable_all` parameter is
131
+        True and the chart group `test_enabled` key is disabled.
132
+        """
133
+        test_handler = test.Test(
134
+            release_name='release',
135
+            tiller=mock.Mock(),
136
+            cg_test_charts=False,
137
+            enable_all=True)
138
+
139
+        assert test_handler.test_enabled is True
140
+
141
+    def test_enable_all_test_key_disabled(self):
142
+        """Test that tests are enabled when the `enable_all` parameter is
143
+        True and the deprecated, boolean `test` key is disabled.
144
+        """
145
+        test_handler = test.Test(
146
+            release_name='release',
147
+            tiller=mock.Mock(),
148
+            enable_all=True,
149
+            test_values=False)
150
+
151
+        assert test_handler.test_enabled is True
152
+
153
+    def test_enable_all_test_values_disabled(self):
154
+        """Test that tests are enabled when the `enable_all` parameter is
155
+        True and the `test.enabled` key is False.
156
+        """
157
+        test_values = {'enabled': False}
158
+
159
+        test_handler = test.Test(
160
+            release_name='release',
161
+            tiller=mock.Mock(),
162
+            enable_all=True,
163
+            test_values=test_values)
164
+
165
+        assert test_handler.test_enabled is True
166
+
167
+    def test_deprecated_test_key_false(self):
168
+        """Test that tests can be disabled using the deprecated, boolean value
169
+        for a chart's test key.
170
+        """
171
+        test_handler = test.Test(
172
+            release_name='release', tiller=mock.Mock(), test_values=True)
173
+
174
+        assert test_handler.test_enabled
175
+
176
+    def test_deprecated_test_key_true(self):
177
+        """Test that cleanup is enabled by default when tests are enabled using
178
+        the deprecated, boolean value for a chart's `test` key.
179
+        """
180
+        test_handler = test.Test(
181
+            release_name='release', tiller=mock.Mock(), test_values=True)
182
+
183
+        assert test_handler.test_enabled is True
184
+        assert test_handler.cleanup is True
185
+
186
+    def test_tests_disabled(self):
187
+        """Test that tests are disabled by a chart's values using the
188
+        `test.enabled` path.
189
+        """
190
+        test_values = {'enabled': False}
191
+        test_handler = test.Test(
192
+            release_name='release',
193
+            tiller=mock.Mock(),
194
+            test_values=test_values)
195
+
196
+        assert test_handler.test_enabled is False
197
+
198
+    def test_tests_enabled(self):
199
+        """Test that cleanup is disabled (by default) when tests are enabled by
200
+        a chart's values using the `test.enabled` path.
201
+        """
202
+        test_values = {'enabled': True}
203
+        test_handler = test.Test(
204
+            release_name='release',
205
+            tiller=mock.Mock(),
206
+            test_values=test_values)
207
+
208
+        assert test_handler.test_enabled is True
209
+        assert test_handler.cleanup is False
210
+
211
+    def test_tests_enabled_cleanup_enabled(self):
212
+        """Test that the test handler uses the values provided by a chart's
213
+        `test` key.
214
+        """
215
+        test_values = {'enabled': True, 'options': {'cleanup': True}}
216
+
217
+        test_handler = test.Test(
218
+            release_name='release',
219
+            tiller=mock.Mock(),
220
+            test_values=test_values)
221
+
222
+        assert test_handler.test_enabled is True
223
+        assert test_handler.cleanup is True
224
+
225
+    def test_tests_enabled_cleanup_disabled(self):
226
+        """Test that the test handler uses the values provided by a chart's
227
+        `test` key.
228
+        """
229
+        test_values = {'enabled': True, 'options': {'cleanup': False}}
230
+
231
+        test_handler = test.Test(
232
+            release_name='release',
233
+            tiller=mock.Mock(),
234
+            test_values=test_values)
235
+
236
+        assert test_handler.test_enabled is True
237
+        assert test_handler.cleanup is False
238
+
239
+    def test_no_test_values(self):
240
+        """Test that the default values are enforced when no chart `test`
241
+        values are provided (i.e. tests are enabled and cleanup is disabled).
242
+        """
243
+        test_handler = test.Test(release_name='release', tiller=mock.Mock())
244
+
245
+        assert test_handler.test_enabled is True
246
+        assert test_handler.cleanup is False
247
+
248
+    def test_override_cleanup(self):
249
+        """Test that a cleanup value passed to the Test handler (i.e. from the
250
+        API/CLI) takes precedence over a chart's `test.cleanup` value.
251
+        """
252
+        test_values = {'enabled': True, 'options': {'cleanup': False}}
253
+
254
+        test_handler = test.Test(
255
+            release_name='release',
256
+            tiller=mock.Mock(),
257
+            cleanup=True,
258
+            test_values=test_values)
259
+
260
+        assert test_handler.test_enabled is True
261
+        assert test_handler.cleanup is True

+ 2
- 2
armada/utils/release.py View File

@@ -12,7 +12,7 @@
12 12
 # See the License for the specific language governing permissions and
13 13
 # limitations under the License.
14 14
 
15
-from armada.handlers.test import get_test_suite_run_success
15
+from armada.handlers.test import Test
16 16
 
17 17
 
18 18
 def release_prefixer(prefix, release):
@@ -52,4 +52,4 @@ def get_last_test_result(release):
52 52
     status = release.info.status
53 53
     if not status.HasField('last_test_suite_run'):
54 54
         return None
55
-    return get_test_suite_run_success(status.last_test_suite_run)
55
+    return Test.get_test_suite_run_success(status.last_test_suite_run)

+ 2
- 0
doc/source/commands/test.rst View File

@@ -24,6 +24,8 @@ Commands
24 24
           $ armada test --release blog-1
25 25
 
26 26
     Options:
27
+      --cleanup                     Delete test pods after test completion
28
+      --enable-all                  Run disabled chart tests
27 29
       --file TEXT                   armada manifest
28 30
       --release TEXT                helm release
29 31
       --tiller-host TEXT            Tiller Host IP

+ 8
- 0
swagger/swaggerV3-api.yaml View File

@@ -155,6 +155,7 @@ paths:
155 155
         - $ref: "#/components/parameters/tiller-port"
156 156
         - $ref: "#/components/parameters/tiller-namespace"
157 157
         - $ref: "#/components/parameters/target-manifest"
158
+        - $ref: "#/components/parameters/enable-all"
158 159
       requestBody:
159 160
         $ref: "#/components/requestBodies/manifest-body"
160 161
       responses:
@@ -337,6 +338,13 @@ components:
337 338
       description: Flag to allow for chart cleanup
338 339
       schema:
339 340
         type: boolean
341
+    enable-all:
342
+      in: query
343
+      name: enable_all
344
+      required: false
345
+      description: Flag to test disabled tests
346
+      schema:
347
+        type: boolean
340 348
     dry-run:
341 349
       in: query
342 350
       name: dry_run

Loading…
Cancel
Save