Add udpate_software action to Shipyard

Provides an action that can be invoked by a user to deploy only the
software portion of a design.

Change-Id: I880bdc245064364dfdd6a482a3cf2d2a293f6c0d

docs/source/API-action-commands.rst

@@ -54,6 +54,22 @@ update_site
 Applies a new committed configuration to the environment. The steps of
 update_site mirror those of :ref:`deploy_site`.
 
+.. _update_software:
+
+update_software
+~~~~~~~~~~~~~~~
+Triggers an update of the software in a site, using the latest committed
+configuration documents. Steps, conceptually:
+
+#. Concurrency check
+    Prevents concurrent site modifications by conflicting
+    actions/workflows.
+#. Validate design
+    Asks each involved Airship component to validate the design. This ensures
+    that the previously committed design is valid at the present time.
+#. Armada build
+    Orchestrates Armada to configure software on the nodes as designed.
+
 Actions under development
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 

docs/source/API.rst

@@ -93,11 +93,11 @@ be several versions of documents in a site that are accessible via this API:
   default. (This behavior can be overridden by query parameters issued by the
   user of Shipyard)
 - The "Last Site Action" version represents the version of documents associated
-  with the last successful or failed site action. Site actions include 'deploy_site'
+  with the last successful or failed site action.
-  and 'update_site'.
 - The "Successful Site Action" version represents the version of documents
-  associated with the last successful site action. Site actions include 'deploy_site'
+  associated with the last successful site action.
-  and 'update_site'.
+- Site actions include ``deploy_site``, ``update_site``, and
+  ``update_software``.
 
 All versions of documents rely upon Deckhand for storage. Shipyard uses the
 tagging features of Deckhand to find the appropriate Committed Documents,

docs/source/CLI.rst

@@ -582,9 +582,9 @@ get configdocs
 
 Retrieve documents loaded into Shipyard. The possible options include last
 committed, last site action, last successful site action and retrieval from
-the Shipyard Buffer. Site actions include deploy_site and update_site. Note
+the Shipyard Buffer. Site actions include ``deploy_site``, ``update_site`` and
-that we can only select one of the options when we retrieve the documents
+`update_software`. Note that only one option may be selected when retrieving
-for a particular collection.
+the documents for a particular collection.
 
 The command will compare the differences between the revisions specified if
 the collection option is not specified. Note that we can only compare between

src/bin/shipyard_airflow/shipyard_airflow/control/action/actions_api.py

@@ -50,6 +50,10 @@ def _action_mappings():
             'dag': 'update_site',
             'validators': [action_validators.validate_site_action]
         },
+        'update_software': {
+            'dag': 'update_software',
+            'validators': [action_validators.validate_site_action]
+        },
         'redeploy_server': {
             'dag': 'redeploy_server',
             'validators': []

src/bin/shipyard_airflow/shipyard_airflow/dags/common_step_factory.py

@@ -105,6 +105,7 @@ class CommonStepFactory(object):
             dag=self.dag)
 
     def get_validate_site_design(self,
+                                 targets=None,
                                  task_id=dn.VALIDATE_SITE_DESIGN_DAG_NAME):
         """Generate the validate site design step
 
@@ -115,7 +116,8 @@ class CommonStepFactory(object):
             subdag=validate_site_design(
                 self.parent_dag_name,
                 task_id,
-                args=self.default_args),
+                args=self.default_args
+                targets=targets),
             task_id=task_id,
             on_failure_callback=step_failure_handler,
             dag=self.dag)

src/bin/shipyard_airflow/shipyard_airflow/dags/update_software.py

@@ -0,0 +1,72 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from datetime import timedelta
+
+import airflow
+from airflow import DAG
+
+from common_step_factory import CommonStepFactory
+from validate_site_design import SOFTWARE
+
+"""update_software
+
+The top-level orchestration DAG for updating only the software components
+using the Undercloud platform.
+"""
+PARENT_DAG_NAME = 'update_software'
+
+default_args = {
+    'owner': 'airflow',
+    'depends_on_past': False,
+    'start_date': airflow.utils.dates.days_ago(1),
+    'email': [''],
+    'email_on_failure': False,
+    'email_on_retry': False,
+    'provide_context': True,
+    'retries': 0,
+    'retry_delay': timedelta(seconds=30),
+}
+
+dag = DAG(PARENT_DAG_NAME, default_args=default_args, schedule_interval=None)
+
+step_factory = CommonStepFactory(parent_dag_name=PARENT_DAG_NAME,
+                                 dag=dag,
+                                 default_args=default_args)
+
+action_xcom = step_factory.get_action_xcom()
+concurrency_check = step_factory.get_concurrency_check()
+deployment_configuration = step_factory.get_deployment_configuration()
+validate_site_design = step_factory.get_validate_site_design(targets=SOFTWARE)
+armada_build = step_factory.get_armada_build()
+decide_airflow_upgrade = step_factory.get_decide_airflow_upgrade()
+upgrade_airflow = step_factory.get_upgrade_airflow()
+skip_upgrade_airflow = step_factory.get_skip_upgrade_airflow()
+create_action_tag = step_factory.get_create_action_tag()
+
+# DAG Wiring
+deployment_configuration.set_upstream(action_xcom)
+validate_site_design.set_upstream([
+    concurrency_check,
+    deployment_configuration
+])
+armada_build.set_upstream(validate_site_design)
+decide_airflow_upgrade.set_upstream(armada_build)
+decide_airflow_upgrade.set_downstream([
+    upgrade_airflow,
+    skip_upgrade_airflow
+])
+create_action_tag.set_upstream([
+    upgrade_airflow,
+    skip_upgrade_airflow
+])

src/bin/shipyard_airflow/shipyard_airflow/dags/validate_site_design.py

@@ -21,7 +21,10 @@ from airflow.operators import PromenadeValidateSiteDesignOperator
 from config_path import config_path
 
 
-def validate_site_design(parent_dag_name, child_dag_name, args):
+BAREMETAL = 'baremetal'
+SOFTWARE = 'software'
+
+def validate_site_design(parent_dag_name, child_dag_name, args, targets=None):
     """Subdag to delegate design verification to the UCP components
 
     There is no wiring of steps - they all execute in parallel
@@ -30,32 +33,44 @@ def validate_site_design(parent_dag_name, child_dag_name, args):
         '{}.{}'.format(parent_dag_name, child_dag_name),
         default_args=args)
 
-    deckhand_validate_docs = DeckhandValidateSiteDesignOperator(
+    if targets is None:
+        targets = [BAREMETAL, SOFTWARE]
+
+    # Always add Deckhand validations
+    DeckhandValidateSiteDesignOperator(
         task_id='deckhand_validate_site_design',
         shipyard_conf=config_path,
         main_dag_name=parent_dag_name,
         retries=1,
-        dag=dag)
+        dag=dag
+    )
 
-    drydock_validate_docs = DrydockValidateDesignOperator(
+    if BAREMETAL in targets:
-        task_id='drydock_validate_site_design',
+        # Add Drydock and Promenade validations
-        shipyard_conf=config_path,
+        DrydockValidateDesignOperator(
-        main_dag_name=parent_dag_name,
+            task_id='drydock_validate_site_design',
-        retries=1,
+            shipyard_conf=config_path,
-        dag=dag)
+            main_dag_name=parent_dag_name,
+            retries=1,
+            dag=dag
+        )
 
-    armada_validate_docs = ArmadaValidateDesignOperator(
+        PromenadeValidateSiteDesignOperator(
-        task_id='armada_validate_site_design',
+            task_id='promenade_validate_site_design',
-        shipyard_conf=config_path,
+            shipyard_conf=config_path,
-        main_dag_name=parent_dag_name,
+            main_dag_name=parent_dag_name,
-        retries=1,
+            retries=1,
-        dag=dag)
+            dag=dag
+        )
 
-    promenade_validate_docs = PromenadeValidateSiteDesignOperator(
+    if SOFTWARE in targets:
-        task_id='promenade_validate_site_design',
+        # Add Armada validations
-        shipyard_conf=config_path,
+        ArmadaValidateDesignOperator(
-        main_dag_name=parent_dag_name,
+            task_id='armada_validate_site_design',
-        retries=1,
+            shipyard_conf=config_path,
-        dag=dag)
+            main_dag_name=parent_dag_name,
+            retries=1,
+            dag=dag
+        )
 
     return dag

src/bin/shipyard_airflow/shipyard_airflow/plugins/concurrency_check_operator.py

@@ -28,7 +28,8 @@ DAG_RUN_SELECT_RUNNING_SQL = ("select dag_id, execution_date "
 AIRFLOW_DB = 'airflows_own_db'
 
 # each set in this list of sets indicates DAGs that shouldn't execute together
-CONFLICTING_DAG_SETS = [set(['deploy_site', 'update_site', 'redeploy_server'])]
+CONFLICTING_DAG_SETS = [set(['deploy_site', 'update_site', 'update_software',
+                             'redeploy_server'])]
 
 
 def find_conflicting_dag_set(dag_name, conflicting_dag_sets=None):

src/bin/shipyard_airflow/shipyard_airflow/plugins/deckhand_create_site_action_tag.py

@@ -111,7 +111,7 @@ class DeckhandCreateSiteActionTagOperator(DeckhandBaseOperator):
         task = ['armada_build']
         task_result = {}
 
-        if self.main_dag_name == 'update_site':
+        if self.main_dag_name in ['update_site', 'update_software']:
             # NOTE: We will check the final state of the 'armada_build' task
             # as a 'success' means that all tasks preceding it would either
             # be in 'skipped' or 'success' state. A failure of 'armada_build'
@@ -119,8 +119,8 @@ class DeckhandCreateSiteActionTagOperator(DeckhandBaseOperator):
             # to determine the success/failure of the 'deploy_site' workflow
             # with the final state of the 'armada_build' task.
             #
-            # NOTE: The 'update_site' workflow contains additional steps for
+            # NOTE: The 'update_site' and 'update_software' workflows contain
-            # upgrading of worker pods.
+            # additional steps for upgrading of worker pods.
             for k in ['skip_upgrade_airflow', 'upgrade_airflow']:
                 task.append(k)
 

src/bin/shipyard_airflow/shipyard_airflow/plugins/ucp_preflight_check_operator.py

@@ -108,6 +108,8 @@ class UcpHealthCheckOperator(BaseOperator):
         """
         # If Drydock health check fails and continue-on-fail, continue
         # and create xcom key 'drydock_continue_on_fail'
+        # Note that 'update_software' does not interact with Drydock, and
+        # therefore does not use the continue-on-fail option.
         if (component == service_endpoint.DRYDOCK and
                 self.action_info['parameters'].get(
                     'continue-on-fail', 'false').lower() == 'true' and

src/bin/shipyard_client/shipyard_client/cli/get/commands.py

@@ -222,7 +222,7 @@ def get_version(ctx, buffer, committed, last_site_action,
             'successful or failed site action\n'
             '--successful-site-action for the documents associated with the '
             'last successful site action\n'
-            'Site actions include deploy_site and update_site.')
+            'Site actions are deploy_site, update_site, and update_software')
 
     elif len(optional_site_parameters) == 1:
         return optional_site_parameters[0]

src/bin/shipyard_client/shipyard_client/cli/help/output.py

@@ -37,9 +37,12 @@ The workflow actions that may be invoked using Shipyard
 deploy_site: Triggers the initial deployment of a site using the latest
              committed configuration documents.
 
-update_site: Triggers the initial deployment of a site, using the latest
+update_site: Triggers the update to a deployment of a site, using the latest
              committed configuration documents.
 
+update_software: Starts an update that only exercises the software portion of
+                 the commited configuration documents.
+
 redeploy_server: Using parameters to indicate which server(s), triggers a
                  redeployment of servers to the last committed design and
                  secrets.

src/bin/shipyard_client/shipyard_client/cli/input_checks.py

@@ -18,9 +18,11 @@ from arrow.parser import ParserError
 
 def check_action_command(ctx, action_command):
     """Verifies the action command is valid"""
-    if action_command not in ['deploy_site', 'update_site', 'redeploy_server']:
+    valid_commands = ['deploy_site', 'update_site', 'update_software',
+                      'redeploy_server']
+    if action_command not in valid_commands:
         ctx.fail('Invalid action command.  The action commands available are '
-                 'deploy_site, update_site, and redeploy_server.')
+                 ' {}'.format(', '.join(valid_commands)))
 
 
 def check_control_action(ctx, action):

src/bin/shipyard_client/tests/unit/cli/test_input_checks.py

@@ -176,6 +176,7 @@ def test_check_action_commands():
     ctx = Mock(side_effect=Exception("failed"))
     input_checks.check_action_command(ctx, 'deploy_site')
     input_checks.check_action_command(ctx, 'update_site')
+    input_checks.check_action_command(ctx, 'update_software')
     input_checks.check_action_command(ctx, 'redeploy_server')
     ctx.fail.assert_not_called()
 

tools/execute_shipyard_action.sh

@@ -13,8 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# This is a common script that is used by the deploy_site, update_site
+# This is a common script that is used by the deploy_site, update_site,
-# and redeploy_server scripts
+# update_software and redeploy_server scripts
 
 set -ex
 

tools/set_env

@@ -13,13 +13,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# NOTE: If user is executing deploy_site, update_site or redeploy_server
+# NOTE: If a user is executing deploy_site, update_site, update_software or
-# workflow from outside the cluster, e.g. from a remote jump server, then
+# redeploy_server workflow from outside the cluster, e.g. from a remote jump
-# he/she will need to ensure that the DNS server is able to resolve the
+# server, then the user will need to ensure that the DNS server is able to
-# FQDN of the Shipyard and Keystone public URL (both will be pointing to
+# resolve the FQDN of the Shipyard and Keystone public URL (both will be
-# the IP of the Ingress Controller). If the DNS resolution is not available,
+# pointing to the IP of the Ingress Controller). If the DNS resolution is not
-# the user will need to ensure that the /etc/hosts file is properly updated
+# available, the user will need to ensure that the /etc/hosts file is properly
-# before setting up the environment variables and running the worflow.
+# updated before setting up the environment variables and running the worflow.
 
 # Define Variable
 namespace="${namespace:-ucp}"

tools/shipyard.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+# User can run the script as they would execute the Shipyard CLI.
+# For instance, to run the 'shipyard get actions' command, user can execute
+# the following command after setting up the required environment variables:
+#
+# $ ./tools/shipyard.sh get actions
+#
+
+# NOTE: If user is executing the script from outside the cluster, e.g. from
+# a remote jump server, then he/she will need to ensure that the DNS server
+# is able to resolve the FQDN of the Shipyard and Keystone public URL (both
+# will be pointing to the IP of the Ingress Controller). If the DNS resolution
+# is not available, the user will need to ensure that the /etc/hosts file is
+# properly updated before running the script.
+
+# Commands requiring files as input utilize the pwd mounted into the container
+# as the /target directory, e.g.:
+#
+# $ ./tools/shipyard.sh create configdocs design --filename=/target/afile.yaml
+
+# Get the path of the directory where the script is located
+# Source Base Docker Command
+SHIPYARD_HOSTPATH=${SHIPYARD_HOSTPATH:-"/target"}
+NAMESPACE="${NAMESPACE:-ucp}"
+SHIPYARD_IMAGE="${SHIPYARD_IMAGE:-quay.io/airshipit/shipyard:master}"
+
+# Define Base Docker Command
+base_docker_command=$(cat << EndOfCommand
+sudo docker run -t --rm --net=host
+-e http_proxy=${HTTP_PROXY}
+-e https_proxy=${HTTPS_PROXY}
+-e OS_AUTH_URL=${OS_AUTH_URL:-http://keystone.${NAMESPACE}.svc.cluster.local:80/v3}
+-e OS_USERNAME=${OS_USERNAME:-shipyard}
+-e OS_USER_DOMAIN_NAME=${OS_USER_DOMAIN_NAME:-default}
+-e OS_PASSWORD=${OS_PASSWORD:-password}
+-e OS_PROJECT_DOMAIN_NAME=${OS_PROJECT_DOMAIN_NAME:-default}
+-e OS_PROJECT_NAME=${OS_PROJECT_NAME:-service}
+EndOfCommand
+)
+
+# Execute Shipyard CLI
+${base_docker_command} -v "$(pwd)":"${SHIPYARD_HOSTPATH}" "${SHIPYARD_IMAGE}" $@

tools/update_software.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# We can excute the script in the following manner:
+#
+# $ ./deploy_site.sh
+#
+
+# Source environment variables
+source set_env
+
+# Execute shipyard action for update_software
+bash execute_shipyard_action.sh 'update_software'