airship
/
shipyard
Code Issues Proposed changes

Update DryDock Operator 

1) Make use of keystone client to retrieve the keystone token
   instead of 'subprocess.Popen' as the latter can be a source
   of security risk. Create a separate Module for keystone
   token-get task.
2) Make use of decorator instead as we want to re-use the
   'shipyard_service_token' function to authenticate against the
   other components in the UCP, e.g. Armada, DeckHand
3) We will allow for 1 retry in getting the Keystone Token with a
   backoff interval of 10 seconds in case there is a temporary
   glitch in the network or transient problems with the keystone-api
   pod
4) Make use of the 'keystone_authtoken' section in the shipyard.conf
   instead to retrieve the required Keystone session information
5) Remove DryDock dags that are out-dated and no longer valid
6) Remove hard-coded DryDock token input.  Use the keystone token
   instead.
7) Add/Edit Banner

Change-Id: I6ac38845f9df2ae61d1d4af523197f8dd9275be6
This commit is contained in:
Anthony Lin 2017-09-28 22:57:45 +00:00
parent 6ae83ed258
commit b4b68c2a54
5 changed files with 88 additions and 241 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
shipyard_airflow/dags/drydock_deploy_site.py
View File

@ -29,7 +29,6 @@ config.read(config_path)
# Define Variables
drydock_target_host = config.get('drydock', 'host')
drydock_port = config.get('drydock', 'port')
drydock_token = config.get('drydock', 'token')
drydock_conf = config.get('drydock', 'site_yaml')
promenade_conf = config.get('drydock', 'prom_yaml')
parent_dag = 'deploy_site'
@ -48,7 +47,6 @@ def create_drydock_client(parent_dag_name, child_dag_name, args):
task_id='create_drydock_client',
host=drydock_target_host,
port=drydock_port,
token=drydock_token,
shipyard_conf=config_path,
action='create_drydock_client',
main_dag_name=parent_dag,

107
shipyard_airflow/dags/drydock_operator_child.py
View File

@ -1,107 +0,0 @@
# -*- coding: utf-8 -*-
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
### DryDock Operator Child Dag
"""
import configparser
from airflow import DAG
from airflow.operators import DryDockOperator
def sub_dag(parent_dag_name, child_dag_name, args, schedule_interval):
dag = DAG(
'%s.%s' % (parent_dag_name, child_dag_name),
default_args=args,
start_date=args['start_date'],
max_active_runs=1,
)
# Location of shiyard.conf
config_path = '/usr/local/airflow/plugins/shipyard.conf'
# Read and parse shiyard.conf
config = configparser.ConfigParser()
config.read(config_path)
# Define Variables
drydock_target_host = config.get('drydock', 'host')
drydock_port = config.get('drydock', 'port')
drydock_token = config.get('drydock', 'token')
drydock_conf = config.get('drydock', 'site_yaml')
promenade_conf = config.get('drydock', 'prom_yaml')
# Create Drydock Client
t1 = DryDockOperator(
task_id='create_drydock_client',
host=drydock_target_host,
port=drydock_port,
token=drydock_token,
shipyard_conf=config_path,
action='create_drydock_client',
dag=dag)
# Get Design ID
t2 = DryDockOperator(
task_id='drydock_get_design_id',
action='get_design_id',
dag=dag)
# DryDock Load Parts
t3 = DryDockOperator(
task_id='drydock_load_parts',
drydock_conf=drydock_conf,
action='drydock_load_parts',
dag=dag)
# Promenade Load Parts
t4 = DryDockOperator(
task_id='promenade_load_parts',
promenade_conf=promenade_conf,
action='promenade_load_parts',
dag=dag)
# Verify Site
t5 = DryDockOperator(
task_id='drydock_verify_site',
action='verify_site',
dag=dag)
# Prepare Site
t6 = DryDockOperator(
task_id='drydock_prepare_site',
action='prepare_site',
dag=dag)
# Prepare Node
t7 = DryDockOperator(
task_id='drydock_prepare_node',
action='prepare_node',
dag=dag)
# Deploy Node
t8 = DryDockOperator(
task_id='drydock_deploy_node',
action='deploy_node',
dag=dag)
# Define dependencies
t2.set_upstream(t1)
t3.set_upstream(t2)
t4.set_upstream(t3)
t5.set_upstream(t4)
t6.set_upstream(t5)
t7.set_upstream(t6)
t8.set_upstream(t7)
return dag

66
shipyard_airflow/dags/drydock_operator_parent.py
View File

@ -1,66 +0,0 @@
# -*- coding: utf-8 -*-
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
### DryDock Operator Parent Dag
"""
import airflow
from airflow import DAG
from datetime import timedelta
from airflow.operators.python_operator import PythonOperator
from airflow.operators.subdag_operator import SubDagOperator
from drydock_operator_child import sub_dag
parent_dag_name = 'drydock_operator_parent'
child_dag_name = 'drydock_operator_child'
args = {
'owner': 'airflow',
'depends_on_past': False,
'start_date': airflow.utils.dates.days_ago(1),
'retries': 0,
'retry_delay': timedelta(minutes=1),
'provide_context': True
}
main_dag = DAG(
dag_id=parent_dag_name,
default_args=args,
schedule_interval=None,
start_date=airflow.utils.dates.days_ago(1),
max_active_runs=1
)
# Define push function to store the content of 'action' that is
# defined via 'dag_run' in XCOM so that it can be used by the
# DryDock Operators
def push(**kwargs):
# Pushes action XCom
kwargs['ti'].xcom_push(key='action',
value=kwargs['dag_run'].conf['action'])
action_xcom = PythonOperator(
task_id='action_xcom', dag=main_dag, python_callable=push)
subdag = SubDagOperator(
subdag=sub_dag(parent_dag_name, child_dag_name, args,
main_dag.schedule_interval),
task_id=child_dag_name,
default_args=args,
dag=main_dag)
# Set dependencies
subdag.set_upstream(action_xcom)

77
shipyard_airflow/plugins/drydock_operators.py
View File

@ -1,10 +1,10 @@
# -*- coding: utf-8 -*-
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@ -13,11 +13,7 @@
# limitations under the License.
import logging
import subprocess
import os
import time
import re
import configparser
from airflow.exceptions import AirflowException
from airflow.models import BaseOperator
@ -27,13 +23,14 @@ from airflow.utils.decorators import apply_defaults
import drydock_provisioner.drydock_client.client as client
import drydock_provisioner.drydock_client.session as session
from service_token import shipyard_service_token
class DryDockOperator(BaseOperator):
"""
DryDock Client
:host: Target Host
:port: DryDock Port
:token: DryDock Token
:shipyard_conf: Location of shipyard.conf
:drydock_conf: Location of drydock YAML
:promenade_conf: Location of promenade YAML
@ -47,7 +44,6 @@ class DryDockOperator(BaseOperator):
def __init__(self,
host=None,
port=None,
token=None,
action=None,
design_id=None,
shipyard_conf=None,
@ -62,7 +58,6 @@ class DryDockOperator(BaseOperator):
super(DryDockOperator, self).__init__(*args, **kwargs)
self.host = host
self.port = port
self.token = token
self.shipyard_conf = shipyard_conf
self.drydock_conf = drydock_conf
self.promenade_conf = promenade_conf
@ -236,70 +231,20 @@ class DryDockOperator(BaseOperator):
else:
logging.info('No Action to Perform')
def keystone_token_get(self, conf_path):
# Read and parse shiyard.conf
config = configparser.ConfigParser()
config.read(conf_path)
# Construct Envrionment variables
for attr in ('OS_AUTH_URL', 'OS_PROJECT_NAME', 'OS_USER_DOMAIN_NAME',
'OS_USERNAME', 'OS_PASSWORD', 'OS_REGION_NAME',
'OS_IDENTITY_API_VERSION'):
os.environ[attr] = config.get('keystone', attr)
# Execute 'openstack token issue' command
logging.info("Get Keystone Token")
keystone_output = subprocess.Popen(["openstack", "token", "issue"],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
# Get Keystone Token from output
line = ''
for line in iter(keystone_output.stdout.readline, b''):
line = line.strip()
if re.search(r'\bid\b', str(line, 'utf-8')):
token = str(line, 'utf-8').split(' |')[1].split(' ')[1]
# Wait for child process to terminate
# Set and return returncode attribute.
keystone_output.wait()
logging.info(
"Command exited with "
"return code {0}".format(keystone_output.returncode))
# Raise Execptions if 'openstack token issue' fails to execute
if keystone_output.returncode:
raise AirflowException("Unable to get Keystone Token!")
return 'keystone_token_error'
else:
logging.info(token)
return token
@shipyard_service_token
def drydock_session_client(self, context):
# Retrieve Keystone Token
keystone_token = self.keystone_token_get(self.shipyard_conf)
# Raise Exception and Exit if we are not able to get Keystone
# Token, else continue
if keystone_token == 'keystone_token_error':
raise AirflowException("Unable to get Keystone Token!")
else:
pass
# Build a DrydockSession with credentials and target host
# information. Note that hard-coded token will be replaced
# by keystone_token in near future
# information.
logging.info("Build DryDock Session")
dd_session = session.DrydockSession(self.host, port=self.port,
token=self.token)
token=context['svc_token'])
# Raise Exception if we are not able to get a drydock session
if dd_session:
pass
logging.info("Successfully Built DryDock Session")
else:
raise AirflowException("Unable to get a drydock session")
raise AirflowException("Unable to get a Drydock Session")
# Use session to build a DrydockClient to make one or more API calls
# The DrydockSession will care for TCP connection pooling
@ -309,9 +254,9 @@ class DryDockOperator(BaseOperator):
# Raise Exception if we are not able to build drydock client
if dd_client:
pass
logging.info("Successfully Built DryDock client")
else:
raise AirflowException("Unable to build drydock client")
raise AirflowException("Unable to Build Drydock Client")
# Drydock client for XCOM Usage
return dd_client

77
shipyard_airflow/plugins/service_token.py Normal file
View File

@ -0,0 +1,77 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import configparser
import logging
import time
from airflow.exceptions import AirflowException
from functools import wraps
from keystoneauth1.identity import v3 as keystone_v3
from keystoneauth1 import session as keystone_session
from keystoneclient.v3 import client as keystone_client
def shipyard_service_token(func):
@wraps(func)
def keystone_token_get(self, context):
# Read and parse shiyard.conf
config = configparser.ConfigParser()
config.read(self.shipyard_conf)
# Initialize variables
retry = 0
token = None
keystone_auth = {}
# We will allow 1 retry in getting the Keystone Token with a
# backoff interval of 10 seconds in case there is a temporary
# glitch in the network or transient problems with the keystone-api
# pod
while retry <= 1:
# Construct Session Argument
for attr in ('auth_url', 'password', 'project_domain_name',
'project_name', 'username', 'user_domain_name'):
keystone_auth[attr] = config.get('keystone_authtoken', attr)
# Set up keystone session
auth = keystone_v3.Password(**keystone_auth)
sess = keystone_session.Session(auth=auth)
keystone = keystone_client.Client(session=sess)
# Retrieve Keystone Token
logging.info("Get Keystone Token")
token = keystone.get_raw_token_from_identity_service(
**keystone_auth)['auth_token']
# Retry if we fail to get the keystone token
if token:
logging.info("Successfully Retrieved Keystone Token")
context['svc_token'] = token
break
else:
logging.info("Unable to get Keystone Token on first attempt")
logging.info("Retrying after 10 seconds...")
time.sleep(10)
retry += 1
# Raise Execptions if we fail to get a proper response
if not token:
raise AirflowException("Unable to get Keystone Token!")
else:
return func(self, context)
return keystone_token_get