actions: Add Shipyard action to test site

This commit introduces an action, `test_site`, that invokes Helm
tests for all deployed releases using the
`ArmadaTestReleasesOperator` introduced in [1]. This action supports
the ability to invoke Helm tests for a specific release using the
`release` parameter and cleanup resources if the `cleanup` parameter
is set to `true`.

[1] https://review.openstack.org/#/c/603236/

Depends-On: https://review.openstack.org/#/c/603236/
Change-Id: Ib5f38fe4b8a6516ee2afae62774ec84f1d2eb1ad

doc/source/_static/shipyard.policy.yaml.sample

@@ -86,3 +86,8 @@
 # POST  /api/v1.0/actions
 #"workflow_orchestrator:action_relabel_nodes": "rule:admin_required"
 
+# Create a workflow action to invoke Helm tests on all releases or a
+# targeted release
+# POST  /api/v1.0/actions
+#"workflow_orchestrator:action_test_site": "rule:admin_required"
+

doc/source/action-commands.rst

@@ -237,21 +237,42 @@ relabel_nodes
 Using parameters to indicate which server(s), triggers an update to the
 Kubernetes node labels for those servers.
 
-Future actions
+.. _test_site:
-~~~~~~~~~~~~~~
 
-These actions are anticipated for development
+test_site
+~~~~~~~~~
 
-test region
+Triggers the execution of the Helm tests corresponding to all deployed releases
-  Invoke site validation testing - perhaps a baseline is an invocation of all
+in all namespaces. Steps, conceptually:
-  components' exposed tests or extended health checks. This test would be used
-  as a preflight-style test to ensure all components are in a working state.
 
-test component
+#. Preflight checks
-  Invoke a particular platform component to test it. This test would be
+    Ensures all Airship components are in a responsive state.
-  used to interrogate a particular platform component to ensure it is in a
+#. Armada test
-  working state, and that its own downstream dependencies are also
+    Invokes Armada to execute Helm tests for all releases.
-  operational
+
+Using test_site
+```````````````
+
+The ``test_site`` action accepts two optional parameters:
+
+#. cleanup: A boolean value that instructs Armada to delete test pods after
+   test execution. Default value is ``false``. Failure to set this value to
+   ``True`` may require manual intervention to re-execute tests, as test pods
+   will not be deleted.
+#. release: The name of a release to test. When provided, tests are only
+   executed for the specified release.
+
+An example of invoking Helm tests with cleanup enabled::
+
+  shipyard create action test_site --param="cleanup=true"
+
+An example of invoking Helm tests for a single release::
+
+  shipyard create action test_site --param="release=keystone"
+
+.. _update_labels:
 
 update labels
-  Triggers an update to the Kubernetes node labels for specified server(s)
+~~~~~~~~~~~~~
+
+Triggers an update to the Kubernetes node labels for specified server(s)

src/bin/shipyard_airflow/etc/shipyard/policy.yaml.sample

@@ -86,3 +86,8 @@
 # POST  /api/v1.0/actions
 #"workflow_orchestrator:action_relabel_nodes": "rule:admin_required"
 
+# Create a workflow action to invoke Helm tests on all releases or a
+# targeted release
+# POST  /api/v1.0/actions
+#"workflow_orchestrator:action_test_site": "rule:admin_required"
+

src/bin/shipyard_airflow/shipyard_airflow/control/action/action_validators.py

@@ -27,6 +27,8 @@ from shipyard_airflow.control.validators.validate_intermediate_commit import \
     ValidateIntermediateCommit
 from shipyard_airflow.control.validators.validate_target_nodes import \
     ValidateTargetNodes
+from shipyard_airflow.control.validators.validate_test_cleanup import \
+    ValidateTestCleanup
 
 LOG = logging.getLogger(__name__)
 
@@ -90,3 +92,12 @@ def validate_target_nodes(action, **kwargs):
     """
     validator = ValidateTargetNodes(action=action)
     validator.validate()
+
+
+def validate_test_cleanup(action, **kwargs):
+    """Validates the cleanup parameter
+
+    Ensures the cleanup parameter is a boolean value.
+    """
+    validator = ValidateTestCleanup(action=action)
+    validator.validate()

src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_api.py

@@ -87,6 +87,13 @@ def _action_mappings():
                 action_validators.validate_committed_revision,
                 action_validators.validate_deployment_action_basic,
             ]
+        },
+        'test_site': {
+            'dag': 'test_site',
+            'rbac_policy': policy.ACTION_TEST_SITE,
+            'validators': [
+                action_validators.validate_test_cleanup,
+            ]
         }
     }
 

src/bin/shipyard_airflow/shipyard_airflow/control/validators/validate_test_cleanup.py

@@ -0,0 +1,45 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import falcon
+
+from shipyard_airflow.errors import ApiError
+
+
+class ValidateTestCleanup:
+    """Validate that a valid cleanup value is specified for release testing"""
+    def __init__(self, action):
+        self.action = action
+
+    def validate(self):
+        """Retrieve cleanup parameter and verify it is a boolean value"""
+        # Retrieve optional parameters
+        parameters = self.action.get('parameters')
+        if not parameters:
+            return
+
+        # Verify cleanup param (optional) is a boolean value
+        cleanup = parameters.get('cleanup')
+        if not cleanup:
+            return
+        elif str.lower(cleanup) in ['true', 'false']:
+            return
+
+        raise ApiError(
+            title='Invalid cleanup value',
+            description=(
+                'Cleanup must be a boolean value.'
+            ),
+            status=falcon.HTTP_400,
+            retry=False
+        )

src/bin/shipyard_airflow/shipyard_airflow/policy.py

@@ -46,6 +46,7 @@ ACTION_UPDATE_SITE = 'workflow_orchestrator:action_update_site'
 ACTION_UPDATE_SOFTWARE = 'workflow_orchestrator:action_update_software'
 ACTION_REDEPLOY_SERVER = 'workflow_orchestrator:action_redeploy_server'
 ACTION_RELABEL_NODES = 'workflow_orchestrator:action_relabel_nodes'
+ACTION_TEST_SITE = 'workflow_orchestrator:action_test_site'
 
 
 class ShipyardPolicy(object):
@@ -262,6 +263,16 @@ class ShipyardPolicy(object):
                 'method': 'POST'
             }]
         ),
+        policy.DocumentedRuleDefault(
+            ACTION_TEST_SITE,
+            RULE_ADMIN_REQUIRED,
+            'Create a workflow action to invoke Helm tests on all releases ' \
+            'or a targeted release',
+            [{
+                'path': '/api/v1.0/actions',
+                'method': 'POST'
+            }]
+        ),
     ]
 
     # Regions Policy

src/bin/shipyard_airflow/tests/unit/control/test_action_validators.py

@@ -28,7 +28,8 @@ from shipyard_airflow.control.action.action_validators import (
     validate_deployment_action_basic,
     validate_deployment_action_full,
     validate_intermediate_commits,
-    validate_target_nodes
+    validate_target_nodes,
+    validate_test_cleanup
 )
 from shipyard_airflow.errors import ApiError
 from tests.unit.common.deployment_group.node_lookup_stubs import node_lookup
@@ -271,6 +272,22 @@ class TestActionValidator:
             )
         assert apie.value.title == 'Invalid target_nodes parameter'
 
+    def test_validate_test_cleanup(self, **args):
+        """Test that the validate_test_cleanup validator enforces an optional,
+        boolean value.
+        """
+        # No cleanup param provided
+        validate_test_cleanup(self._action(None))
+
+        # Valid cleanup params
+        validate_test_cleanup(self._action({'cleanup': 'True'}))
+        validate_test_cleanup(self._action({'cleanup': 'false'}))
+
+        # Bad cleanup params
+        with pytest.raises(ApiError):
+            validate_test_cleanup(self._action({'cleanup': 'string'}))
+            validate_test_cleanup(self._action({'cleanup': '10000'}))
+
     def test_validate_committed_revision(self, *args):
         """Test the committed revision validator"""
         validate_committed_revision(self._action(None))

src/bin/shipyard_airflow/tests/unit/control/test_actions_api.py

@@ -530,6 +530,9 @@ def test_create_targeted_action_no_committed(basic_val, *args):
 @mock.patch('shipyard_airflow.control.action.action_validators'
             '.validate_target_nodes',
             side_effect=Exception('purposeful'))
+@mock.patch('shipyard_airflow.control.action.action_validators'
+            '.validate_test_cleanup',
+            side_effect=Exception('purposeful'))
 @mock.patch('shipyard_airflow.policy.check_auth')
 def test_auth_alignment(auth, *args):
     action_resource = _gen_action_resource_stubbed()

src/bin/shipyard_client/shipyard_client/cli/input_checks.py

@@ -19,7 +19,7 @@ from arrow.parser import ParserError
 def check_action_command(ctx, action_command):
     """Verifies the action command is valid"""
     valid_commands = ['deploy_site', 'update_site', 'update_software',
-                      'redeploy_server', 'relabel_nodes']
+                      'redeploy_server', 'relabel_nodes', 'test_site']
     if action_command not in valid_commands:
         ctx.fail('Invalid action command.  The action commands available are '
                  ' {}'.format(', '.join(valid_commands)))