Update DryDock Operator

1) Make use of keystone client to retrieve the keystone token
   instead of 'subprocess.Popen' as the latter can be a source
   of security risk. Create a separate Module for keystone
   token-get task.
2) Make use of decorator instead as we want to re-use the
   'shipyard_service_token' function to authenticate against the
   other components in the UCP, e.g. Armada, DeckHand
3) We will allow for 1 retry in getting the Keystone Token with a
   backoff interval of 10 seconds in case there is a temporary
   glitch in the network or transient problems with the keystone-api
   pod
4) Make use of the 'keystone_authtoken' section in the shipyard.conf
   instead to retrieve the required Keystone session information
5) Remove DryDock dags that are out-dated and no longer valid
6) Remove hard-coded DryDock token input.  Use the keystone token
   instead.
7) Add/Edit Banner

Change-Id: I6ac38845f9df2ae61d1d4af523197f8dd9275be6

shipyard_airflow/dags/drydock_deploy_site.py

@@ -29,7 +29,6 @@ config.read(config_path)
 # Define Variables
 drydock_target_host = config.get('drydock', 'host')
 drydock_port = config.get('drydock', 'port')
-drydock_token = config.get('drydock', 'token')
 drydock_conf = config.get('drydock', 'site_yaml')
 promenade_conf = config.get('drydock', 'prom_yaml')
 parent_dag = 'deploy_site'
@@ -48,7 +47,6 @@ def create_drydock_client(parent_dag_name, child_dag_name, args):
         task_id='create_drydock_client',
         host=drydock_target_host,
         port=drydock_port,
-        token=drydock_token,
         shipyard_conf=config_path,
         action='create_drydock_client',
         main_dag_name=parent_dag,

shipyard_airflow/dags/drydock_operator_child.py

@@ -1,107 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""
-### DryDock Operator Child Dag
-"""
-import configparser
-from airflow import DAG
-from airflow.operators import DryDockOperator
-
-
-def sub_dag(parent_dag_name, child_dag_name, args, schedule_interval):
-    dag = DAG(
-        '%s.%s' % (parent_dag_name, child_dag_name),
-        default_args=args,
-        start_date=args['start_date'],
-        max_active_runs=1,
-    )
-
-    # Location of shiyard.conf
-    config_path = '/usr/local/airflow/plugins/shipyard.conf'
-
-    # Read and parse shiyard.conf
-    config = configparser.ConfigParser()
-    config.read(config_path)
-
-    # Define Variables
-    drydock_target_host = config.get('drydock', 'host')
-    drydock_port = config.get('drydock', 'port')
-    drydock_token = config.get('drydock', 'token')
-    drydock_conf = config.get('drydock', 'site_yaml')
-    promenade_conf = config.get('drydock', 'prom_yaml')
-
-    # Create Drydock Client
-    t1 = DryDockOperator(
-        task_id='create_drydock_client',
-        host=drydock_target_host,
-        port=drydock_port,
-        token=drydock_token,
-        shipyard_conf=config_path,
-        action='create_drydock_client',
-        dag=dag)
-
-    # Get Design ID
-    t2 = DryDockOperator(
-        task_id='drydock_get_design_id',
-        action='get_design_id',
-        dag=dag)
-
-    # DryDock Load Parts
-    t3 = DryDockOperator(
-        task_id='drydock_load_parts',
-        drydock_conf=drydock_conf,
-        action='drydock_load_parts',
-        dag=dag)
-
-    # Promenade Load Parts
-    t4 = DryDockOperator(
-        task_id='promenade_load_parts',
-        promenade_conf=promenade_conf,
-        action='promenade_load_parts',
-        dag=dag)
-
-    # Verify Site
-    t5 = DryDockOperator(
-        task_id='drydock_verify_site',
-        action='verify_site',
-        dag=dag)
-
-    # Prepare Site
-    t6 = DryDockOperator(
-        task_id='drydock_prepare_site',
-        action='prepare_site',
-        dag=dag)
-
-    # Prepare Node
-    t7 = DryDockOperator(
-        task_id='drydock_prepare_node',
-        action='prepare_node',
-        dag=dag)
-
-    # Deploy Node
-    t8 = DryDockOperator(
-        task_id='drydock_deploy_node',
-        action='deploy_node',
-        dag=dag)
-
-    # Define dependencies
-    t2.set_upstream(t1)
-    t3.set_upstream(t2)
-    t4.set_upstream(t3)
-    t5.set_upstream(t4)
-    t6.set_upstream(t5)
-    t7.set_upstream(t6)
-    t8.set_upstream(t7)
-
-    return dag

shipyard_airflow/dags/drydock_operator_parent.py

@@ -1,66 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""
-### DryDock Operator Parent Dag
-"""
-import airflow
-from airflow import DAG
-from datetime import timedelta
-from airflow.operators.python_operator import PythonOperator
-from airflow.operators.subdag_operator import SubDagOperator
-from drydock_operator_child import sub_dag
-
-parent_dag_name = 'drydock_operator_parent'
-child_dag_name = 'drydock_operator_child'
-
-args = {
-    'owner': 'airflow',
-    'depends_on_past': False,
-    'start_date': airflow.utils.dates.days_ago(1),
-    'retries': 0,
-    'retry_delay': timedelta(minutes=1),
-    'provide_context': True
-}
-
-main_dag = DAG(
-    dag_id=parent_dag_name,
-    default_args=args,
-    schedule_interval=None,
-    start_date=airflow.utils.dates.days_ago(1),
-    max_active_runs=1
-)
-
-# Define push function to store the content of 'action' that is
-# defined via 'dag_run' in XCOM so that it can be used by the
-# DryDock Operators
-
-
-def push(**kwargs):
-    # Pushes action XCom
-    kwargs['ti'].xcom_push(key='action',
-                           value=kwargs['dag_run'].conf['action'])
-
-
-action_xcom = PythonOperator(
-    task_id='action_xcom', dag=main_dag, python_callable=push)
-
-subdag = SubDagOperator(
-    subdag=sub_dag(parent_dag_name, child_dag_name, args,
-                   main_dag.schedule_interval),
-    task_id=child_dag_name,
-    default_args=args,
-    dag=main_dag)
-
-# Set dependencies
-subdag.set_upstream(action_xcom)

shipyard_airflow/plugins/drydock_operators.py

@@ -1,10 +1,10 @@
-# -*- coding: utf-8 -*-
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-# http://www.apache.org/licenses/LICENSE-2.0
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
@@ -13,11 +13,7 @@
 # limitations under the License.
 
 import logging
-import subprocess
-import os
 import time
-import re
-import configparser
 
 from airflow.exceptions import AirflowException
 from airflow.models import BaseOperator
@@ -27,13 +23,14 @@ from airflow.utils.decorators import apply_defaults
 import drydock_provisioner.drydock_client.client as client
 import drydock_provisioner.drydock_client.session as session
 
+from service_token import shipyard_service_token
+
 
 class DryDockOperator(BaseOperator):
     """
     DryDock Client
     :host: Target Host
     :port: DryDock Port
-    :token: DryDock Token
     :shipyard_conf: Location of shipyard.conf
     :drydock_conf: Location of drydock YAML
     :promenade_conf: Location of promenade YAML
@@ -47,7 +44,6 @@ class DryDockOperator(BaseOperator):
     def __init__(self,
                  host=None,
                  port=None,
-                 token=None,
                  action=None,
                  design_id=None,
                  shipyard_conf=None,
@@ -62,7 +58,6 @@ class DryDockOperator(BaseOperator):
         super(DryDockOperator, self).__init__(*args, **kwargs)
         self.host = host
         self.port = port
-        self.token = token
         self.shipyard_conf = shipyard_conf
         self.drydock_conf = drydock_conf
         self.promenade_conf = promenade_conf
@@ -236,70 +231,20 @@ class DryDockOperator(BaseOperator):
         else:
             logging.info('No Action to Perform')
 
-    def keystone_token_get(self, conf_path):
+    @shipyard_service_token
-
-        # Read and parse shiyard.conf
-        config = configparser.ConfigParser()
-        config.read(conf_path)
-
-        # Construct Envrionment variables
-        for attr in ('OS_AUTH_URL', 'OS_PROJECT_NAME', 'OS_USER_DOMAIN_NAME',
-                     'OS_USERNAME', 'OS_PASSWORD', 'OS_REGION_NAME',
-                     'OS_IDENTITY_API_VERSION'):
-            os.environ[attr] = config.get('keystone', attr)
-
-        # Execute 'openstack token issue' command
-        logging.info("Get Keystone Token")
-        keystone_output = subprocess.Popen(["openstack", "token", "issue"],
-                                           stdout=subprocess.PIPE,
-                                           stderr=subprocess.STDOUT)
-
-        # Get Keystone Token from output
-        line = ''
-        for line in iter(keystone_output.stdout.readline, b''):
-            line = line.strip()
-            if re.search(r'\bid\b', str(line, 'utf-8')):
-                token = str(line, 'utf-8').split(' |')[1].split(' ')[1]
-
-        # Wait for child process to terminate
-        # Set and return returncode attribute.
-        keystone_output.wait()
-        logging.info(
-            "Command exited with "
-            "return code {0}".format(keystone_output.returncode))
-
-        # Raise Execptions if 'openstack token issue' fails to execute
-        if keystone_output.returncode:
-            raise AirflowException("Unable to get Keystone Token!")
-            return 'keystone_token_error'
-        else:
-            logging.info(token)
-            return token
-
     def drydock_session_client(self, context):
 
-        # Retrieve Keystone Token
-        keystone_token = self.keystone_token_get(self.shipyard_conf)
-
-        # Raise Exception and Exit if we are not able to get Keystone
-        # Token, else continue
-        if keystone_token == 'keystone_token_error':
-            raise AirflowException("Unable to get Keystone Token!")
-        else:
-            pass
-
         # Build a DrydockSession with credentials and target host
-        # information.  Note that hard-coded token will be replaced
+        # information.
-        # by keystone_token in near future
         logging.info("Build DryDock Session")
         dd_session = session.DrydockSession(self.host, port=self.port,
-                                            token=self.token)
+                                            token=context['svc_token'])
 
         # Raise Exception if we are not able to get a drydock session
         if dd_session:
-            pass
+            logging.info("Successfully Built DryDock Session")
         else:
-            raise AirflowException("Unable to get a drydock session")
+            raise AirflowException("Unable to get a Drydock Session")
 
         # Use session to build a DrydockClient to make one or more API calls
         # The DrydockSession will care for TCP connection pooling
@@ -309,9 +254,9 @@ class DryDockOperator(BaseOperator):
 
         # Raise Exception if we are not able to build drydock client
         if dd_client:
-            pass
+            logging.info("Successfully Built DryDock client")
         else:
-            raise AirflowException("Unable to build drydock client")
+            raise AirflowException("Unable to Build Drydock Client")
 
         # Drydock client for XCOM Usage
         return dd_client

shipyard_airflow/plugins/service_token.py

@@ -0,0 +1,77 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import configparser
+import logging
+import time
+
+from airflow.exceptions import AirflowException
+
+from functools import wraps
+
+from keystoneauth1.identity import v3 as keystone_v3
+from keystoneauth1 import session as keystone_session
+from keystoneclient.v3 import client as keystone_client
+
+
+def shipyard_service_token(func):
+    @wraps(func)
+    def keystone_token_get(self, context):
+        # Read and parse shiyard.conf
+        config = configparser.ConfigParser()
+        config.read(self.shipyard_conf)
+
+        # Initialize variables
+        retry = 0
+        token = None
+        keystone_auth = {}
+
+        # We will allow 1 retry in getting the Keystone Token with a
+        # backoff interval of 10 seconds in case there is a temporary
+        # glitch in the network or transient problems with the keystone-api
+        # pod
+        while retry <= 1:
+            # Construct Session Argument
+            for attr in ('auth_url', 'password', 'project_domain_name',
+                         'project_name', 'username', 'user_domain_name'):
+                keystone_auth[attr] = config.get('keystone_authtoken', attr)
+
+            # Set up keystone session
+            auth = keystone_v3.Password(**keystone_auth)
+            sess = keystone_session.Session(auth=auth)
+            keystone = keystone_client.Client(session=sess)
+
+            # Retrieve Keystone Token
+            logging.info("Get Keystone Token")
+            token = keystone.get_raw_token_from_identity_service(
+                **keystone_auth)['auth_token']
+
+            # Retry if we fail to get the keystone token
+            if token:
+                logging.info("Successfully Retrieved Keystone Token")
+                context['svc_token'] = token
+                break
+            else:
+                logging.info("Unable to get Keystone Token on first attempt")
+                logging.info("Retrying after 10 seconds...")
+                time.sleep(10)
+                retry += 1
+
+        # Raise Execptions if we fail to get a proper response
+        if not token:
+            raise AirflowException("Unable to get Keystone Token!")
+        else:
+            return func(self, context)
+
+    return keystone_token_get