Update DryDock Operator
1) Make use of keystone client to retrieve the keystone token instead of 'subprocess.Popen' as the latter can be a source of security risk. Create a separate Module for keystone token-get task. 2) Make use of decorator instead as we want to re-use the 'shipyard_service_token' function to authenticate against the other components in the UCP, e.g. Armada, DeckHand 3) We will allow for 1 retry in getting the Keystone Token with a backoff interval of 10 seconds in case there is a temporary glitch in the network or transient problems with the keystone-api pod 4) Make use of the 'keystone_authtoken' section in the shipyard.conf instead to retrieve the required Keystone session information 5) Remove DryDock dags that are out-dated and no longer valid 6) Remove hard-coded DryDock token input. Use the keystone token instead. 7) Add/Edit Banner Change-Id: I6ac38845f9df2ae61d1d4af523197f8dd9275be6
This commit is contained in:
parent
6ae83ed258
commit
b4b68c2a54
|
@ -29,7 +29,6 @@ config.read(config_path)
|
|||
# Define Variables
|
||||
drydock_target_host = config.get('drydock', 'host')
|
||||
drydock_port = config.get('drydock', 'port')
|
||||
drydock_token = config.get('drydock', 'token')
|
||||
drydock_conf = config.get('drydock', 'site_yaml')
|
||||
promenade_conf = config.get('drydock', 'prom_yaml')
|
||||
parent_dag = 'deploy_site'
|
||||
|
@ -48,7 +47,6 @@ def create_drydock_client(parent_dag_name, child_dag_name, args):
|
|||
task_id='create_drydock_client',
|
||||
host=drydock_target_host,
|
||||
port=drydock_port,
|
||||
token=drydock_token,
|
||||
shipyard_conf=config_path,
|
||||
action='create_drydock_client',
|
||||
main_dag_name=parent_dag,
|
||||
|
|
|
@ -1,107 +0,0 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
### DryDock Operator Child Dag
|
||||
"""
|
||||
import configparser
|
||||
from airflow import DAG
|
||||
from airflow.operators import DryDockOperator
|
||||
|
||||
|
||||
def sub_dag(parent_dag_name, child_dag_name, args, schedule_interval):
|
||||
dag = DAG(
|
||||
'%s.%s' % (parent_dag_name, child_dag_name),
|
||||
default_args=args,
|
||||
start_date=args['start_date'],
|
||||
max_active_runs=1,
|
||||
)
|
||||
|
||||
# Location of shiyard.conf
|
||||
config_path = '/usr/local/airflow/plugins/shipyard.conf'
|
||||
|
||||
# Read and parse shiyard.conf
|
||||
config = configparser.ConfigParser()
|
||||
config.read(config_path)
|
||||
|
||||
# Define Variables
|
||||
drydock_target_host = config.get('drydock', 'host')
|
||||
drydock_port = config.get('drydock', 'port')
|
||||
drydock_token = config.get('drydock', 'token')
|
||||
drydock_conf = config.get('drydock', 'site_yaml')
|
||||
promenade_conf = config.get('drydock', 'prom_yaml')
|
||||
|
||||
# Create Drydock Client
|
||||
t1 = DryDockOperator(
|
||||
task_id='create_drydock_client',
|
||||
host=drydock_target_host,
|
||||
port=drydock_port,
|
||||
token=drydock_token,
|
||||
shipyard_conf=config_path,
|
||||
action='create_drydock_client',
|
||||
dag=dag)
|
||||
|
||||
# Get Design ID
|
||||
t2 = DryDockOperator(
|
||||
task_id='drydock_get_design_id',
|
||||
action='get_design_id',
|
||||
dag=dag)
|
||||
|
||||
# DryDock Load Parts
|
||||
t3 = DryDockOperator(
|
||||
task_id='drydock_load_parts',
|
||||
drydock_conf=drydock_conf,
|
||||
action='drydock_load_parts',
|
||||
dag=dag)
|
||||
|
||||
# Promenade Load Parts
|
||||
t4 = DryDockOperator(
|
||||
task_id='promenade_load_parts',
|
||||
promenade_conf=promenade_conf,
|
||||
action='promenade_load_parts',
|
||||
dag=dag)
|
||||
|
||||
# Verify Site
|
||||
t5 = DryDockOperator(
|
||||
task_id='drydock_verify_site',
|
||||
action='verify_site',
|
||||
dag=dag)
|
||||
|
||||
# Prepare Site
|
||||
t6 = DryDockOperator(
|
||||
task_id='drydock_prepare_site',
|
||||
action='prepare_site',
|
||||
dag=dag)
|
||||
|
||||
# Prepare Node
|
||||
t7 = DryDockOperator(
|
||||
task_id='drydock_prepare_node',
|
||||
action='prepare_node',
|
||||
dag=dag)
|
||||
|
||||
# Deploy Node
|
||||
t8 = DryDockOperator(
|
||||
task_id='drydock_deploy_node',
|
||||
action='deploy_node',
|
||||
dag=dag)
|
||||
|
||||
# Define dependencies
|
||||
t2.set_upstream(t1)
|
||||
t3.set_upstream(t2)
|
||||
t4.set_upstream(t3)
|
||||
t5.set_upstream(t4)
|
||||
t6.set_upstream(t5)
|
||||
t7.set_upstream(t6)
|
||||
t8.set_upstream(t7)
|
||||
|
||||
return dag
|
|
@ -1,66 +0,0 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
### DryDock Operator Parent Dag
|
||||
"""
|
||||
import airflow
|
||||
from airflow import DAG
|
||||
from datetime import timedelta
|
||||
from airflow.operators.python_operator import PythonOperator
|
||||
from airflow.operators.subdag_operator import SubDagOperator
|
||||
from drydock_operator_child import sub_dag
|
||||
|
||||
parent_dag_name = 'drydock_operator_parent'
|
||||
child_dag_name = 'drydock_operator_child'
|
||||
|
||||
args = {
|
||||
'owner': 'airflow',
|
||||
'depends_on_past': False,
|
||||
'start_date': airflow.utils.dates.days_ago(1),
|
||||
'retries': 0,
|
||||
'retry_delay': timedelta(minutes=1),
|
||||
'provide_context': True
|
||||
}
|
||||
|
||||
main_dag = DAG(
|
||||
dag_id=parent_dag_name,
|
||||
default_args=args,
|
||||
schedule_interval=None,
|
||||
start_date=airflow.utils.dates.days_ago(1),
|
||||
max_active_runs=1
|
||||
)
|
||||
|
||||
# Define push function to store the content of 'action' that is
|
||||
# defined via 'dag_run' in XCOM so that it can be used by the
|
||||
# DryDock Operators
|
||||
|
||||
|
||||
def push(**kwargs):
|
||||
# Pushes action XCom
|
||||
kwargs['ti'].xcom_push(key='action',
|
||||
value=kwargs['dag_run'].conf['action'])
|
||||
|
||||
|
||||
action_xcom = PythonOperator(
|
||||
task_id='action_xcom', dag=main_dag, python_callable=push)
|
||||
|
||||
subdag = SubDagOperator(
|
||||
subdag=sub_dag(parent_dag_name, child_dag_name, args,
|
||||
main_dag.schedule_interval),
|
||||
task_id=child_dag_name,
|
||||
default_args=args,
|
||||
dag=main_dag)
|
||||
|
||||
# Set dependencies
|
||||
subdag.set_upstream(action_xcom)
|
|
@ -1,4 +1,4 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
@ -13,11 +13,7 @@
|
|||
# limitations under the License.
|
||||
|
||||
import logging
|
||||
import subprocess
|
||||
import os
|
||||
import time
|
||||
import re
|
||||
import configparser
|
||||
|
||||
from airflow.exceptions import AirflowException
|
||||
from airflow.models import BaseOperator
|
||||
|
@ -27,13 +23,14 @@ from airflow.utils.decorators import apply_defaults
|
|||
import drydock_provisioner.drydock_client.client as client
|
||||
import drydock_provisioner.drydock_client.session as session
|
||||
|
||||
from service_token import shipyard_service_token
|
||||
|
||||
|
||||
class DryDockOperator(BaseOperator):
|
||||
"""
|
||||
DryDock Client
|
||||
:host: Target Host
|
||||
:port: DryDock Port
|
||||
:token: DryDock Token
|
||||
:shipyard_conf: Location of shipyard.conf
|
||||
:drydock_conf: Location of drydock YAML
|
||||
:promenade_conf: Location of promenade YAML
|
||||
|
@ -47,7 +44,6 @@ class DryDockOperator(BaseOperator):
|
|||
def __init__(self,
|
||||
host=None,
|
||||
port=None,
|
||||
token=None,
|
||||
action=None,
|
||||
design_id=None,
|
||||
shipyard_conf=None,
|
||||
|
@ -62,7 +58,6 @@ class DryDockOperator(BaseOperator):
|
|||
super(DryDockOperator, self).__init__(*args, **kwargs)
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.token = token
|
||||
self.shipyard_conf = shipyard_conf
|
||||
self.drydock_conf = drydock_conf
|
||||
self.promenade_conf = promenade_conf
|
||||
|
@ -236,70 +231,20 @@ class DryDockOperator(BaseOperator):
|
|||
else:
|
||||
logging.info('No Action to Perform')
|
||||
|
||||
def keystone_token_get(self, conf_path):
|
||||
|
||||
# Read and parse shiyard.conf
|
||||
config = configparser.ConfigParser()
|
||||
config.read(conf_path)
|
||||
|
||||
# Construct Envrionment variables
|
||||
for attr in ('OS_AUTH_URL', 'OS_PROJECT_NAME', 'OS_USER_DOMAIN_NAME',
|
||||
'OS_USERNAME', 'OS_PASSWORD', 'OS_REGION_NAME',
|
||||
'OS_IDENTITY_API_VERSION'):
|
||||
os.environ[attr] = config.get('keystone', attr)
|
||||
|
||||
# Execute 'openstack token issue' command
|
||||
logging.info("Get Keystone Token")
|
||||
keystone_output = subprocess.Popen(["openstack", "token", "issue"],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT)
|
||||
|
||||
# Get Keystone Token from output
|
||||
line = ''
|
||||
for line in iter(keystone_output.stdout.readline, b''):
|
||||
line = line.strip()
|
||||
if re.search(r'\bid\b', str(line, 'utf-8')):
|
||||
token = str(line, 'utf-8').split(' |')[1].split(' ')[1]
|
||||
|
||||
# Wait for child process to terminate
|
||||
# Set and return returncode attribute.
|
||||
keystone_output.wait()
|
||||
logging.info(
|
||||
"Command exited with "
|
||||
"return code {0}".format(keystone_output.returncode))
|
||||
|
||||
# Raise Execptions if 'openstack token issue' fails to execute
|
||||
if keystone_output.returncode:
|
||||
raise AirflowException("Unable to get Keystone Token!")
|
||||
return 'keystone_token_error'
|
||||
else:
|
||||
logging.info(token)
|
||||
return token
|
||||
|
||||
@shipyard_service_token
|
||||
def drydock_session_client(self, context):
|
||||
|
||||
# Retrieve Keystone Token
|
||||
keystone_token = self.keystone_token_get(self.shipyard_conf)
|
||||
|
||||
# Raise Exception and Exit if we are not able to get Keystone
|
||||
# Token, else continue
|
||||
if keystone_token == 'keystone_token_error':
|
||||
raise AirflowException("Unable to get Keystone Token!")
|
||||
else:
|
||||
pass
|
||||
|
||||
# Build a DrydockSession with credentials and target host
|
||||
# information. Note that hard-coded token will be replaced
|
||||
# by keystone_token in near future
|
||||
# information.
|
||||
logging.info("Build DryDock Session")
|
||||
dd_session = session.DrydockSession(self.host, port=self.port,
|
||||
token=self.token)
|
||||
token=context['svc_token'])
|
||||
|
||||
# Raise Exception if we are not able to get a drydock session
|
||||
if dd_session:
|
||||
pass
|
||||
logging.info("Successfully Built DryDock Session")
|
||||
else:
|
||||
raise AirflowException("Unable to get a drydock session")
|
||||
raise AirflowException("Unable to get a Drydock Session")
|
||||
|
||||
# Use session to build a DrydockClient to make one or more API calls
|
||||
# The DrydockSession will care for TCP connection pooling
|
||||
|
@ -309,9 +254,9 @@ class DryDockOperator(BaseOperator):
|
|||
|
||||
# Raise Exception if we are not able to build drydock client
|
||||
if dd_client:
|
||||
pass
|
||||
logging.info("Successfully Built DryDock client")
|
||||
else:
|
||||
raise AirflowException("Unable to build drydock client")
|
||||
raise AirflowException("Unable to Build Drydock Client")
|
||||
|
||||
# Drydock client for XCOM Usage
|
||||
return dd_client
|
||||
|
|
|
@ -0,0 +1,77 @@
|
|||
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import configparser
|
||||
import logging
|
||||
import time
|
||||
|
||||
from airflow.exceptions import AirflowException
|
||||
|
||||
from functools import wraps
|
||||
|
||||
from keystoneauth1.identity import v3 as keystone_v3
|
||||
from keystoneauth1 import session as keystone_session
|
||||
from keystoneclient.v3 import client as keystone_client
|
||||
|
||||
|
||||
def shipyard_service_token(func):
|
||||
@wraps(func)
|
||||
def keystone_token_get(self, context):
|
||||
# Read and parse shiyard.conf
|
||||
config = configparser.ConfigParser()
|
||||
config.read(self.shipyard_conf)
|
||||
|
||||
# Initialize variables
|
||||
retry = 0
|
||||
token = None
|
||||
keystone_auth = {}
|
||||
|
||||
# We will allow 1 retry in getting the Keystone Token with a
|
||||
# backoff interval of 10 seconds in case there is a temporary
|
||||
# glitch in the network or transient problems with the keystone-api
|
||||
# pod
|
||||
while retry <= 1:
|
||||
# Construct Session Argument
|
||||
for attr in ('auth_url', 'password', 'project_domain_name',
|
||||
'project_name', 'username', 'user_domain_name'):
|
||||
keystone_auth[attr] = config.get('keystone_authtoken', attr)
|
||||
|
||||
# Set up keystone session
|
||||
auth = keystone_v3.Password(**keystone_auth)
|
||||
sess = keystone_session.Session(auth=auth)
|
||||
keystone = keystone_client.Client(session=sess)
|
||||
|
||||
# Retrieve Keystone Token
|
||||
logging.info("Get Keystone Token")
|
||||
token = keystone.get_raw_token_from_identity_service(
|
||||
**keystone_auth)['auth_token']
|
||||
|
||||
# Retry if we fail to get the keystone token
|
||||
if token:
|
||||
logging.info("Successfully Retrieved Keystone Token")
|
||||
context['svc_token'] = token
|
||||
break
|
||||
else:
|
||||
logging.info("Unable to get Keystone Token on first attempt")
|
||||
logging.info("Retrying after 10 seconds...")
|
||||
time.sleep(10)
|
||||
retry += 1
|
||||
|
||||
# Raise Execptions if we fail to get a proper response
|
||||
if not token:
|
||||
raise AirflowException("Unable to get Keystone Token!")
|
||||
else:
|
||||
return func(self, context)
|
||||
|
||||
return keystone_token_get
|
Loading…
Reference in New Issue