From 1d522e3f410155a623ec17950455f5ae87bd00a5 Mon Sep 17 00:00:00 2001 From: Anthony Lin Date: Tue, 3 Oct 2017 14:13:24 +0000 Subject: [PATCH] Add Keystone Operators It is noted that the keystoneclient might go away and hence we should avoid using it. We will make use of the recommended 'keystoneauth1.session' instead. This Patch Set is meant to do the following 1) Add service_session(common function) to retrieve keystone session 2) Add service_endpoint to retrieve keystone endpoint for the requested service type 3) Update service_token to avoid the usage of keystoneclient Change-Id: I69dee3360365a1138bb7482add5e1477b4360111 --- shipyard_airflow/plugins/service_endpoint.py | 57 +++++++++++++++++ shipyard_airflow/plugins/service_session.py | 65 ++++++++++++++++++++ shipyard_airflow/plugins/service_token.py | 43 +++++++------ 3 files changed, 143 insertions(+), 22 deletions(-) create mode 100644 shipyard_airflow/plugins/service_endpoint.py create mode 100644 shipyard_airflow/plugins/service_session.py diff --git a/shipyard_airflow/plugins/service_endpoint.py b/shipyard_airflow/plugins/service_endpoint.py new file mode 100644 index 00000000..6a77eab5 --- /dev/null +++ b/shipyard_airflow/plugins/service_endpoint.py @@ -0,0 +1,57 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import time + +from airflow.exceptions import AirflowException + +from service_session import ucp_keystone_session + + +def ucp_service_endpoint(self, context): + + # Initialize variables + retry = 0 + int_endpoint = None + + # Retrieve Keystone Session + sess = ucp_keystone_session(self, context) + + # We will allow 1 retry in getting the Keystone Endpoint with a + # backoff interval of 10 seconds in case there is a temporary + # glitch in the network or transient problems with the keystone-api + # pod + while retry <= 1: + # Retrieve Keystone Endpoint + # We will make use of internal endpoint + logging.info("Get Keystone Endpoint") + int_endpoint = sess.get_endpoint(interface='internal', + service_type=context['svc_type']) + + # Retry if we fail to get keystone endpoint + if int_endpoint: + logging.info("Successfully Retrieved Keystone Endpoint") + break + else: + logging.info("Unable to get Keystone endpoint on first attempt") + logging.info("Retrying after 10 seconds...") + time.sleep(10) + retry += 1 + + # Raise Execptions if we fail to get the keystone endpoint + if not int_endpoint: + raise AirflowException("Unable to get Keystone Endpoint!") + else: + return int_endpoint diff --git a/shipyard_airflow/plugins/service_session.py b/shipyard_airflow/plugins/service_session.py new file mode 100644 index 00000000..1e6372b7 --- /dev/null +++ b/shipyard_airflow/plugins/service_session.py @@ -0,0 +1,65 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import configparser +import logging +import time + +from airflow.exceptions import AirflowException + +from keystoneauth1.identity import v3 as keystone_v3 +from keystoneauth1 import session as keystone_session + + +def ucp_keystone_session(self, context): + + # Read and parse shiyard.conf + config = configparser.ConfigParser() + config.read(self.shipyard_conf) + + # Initialize variables + retry = 0 + sess = None + keystone_auth = {} + + # We will allow 1 retry in getting the Keystone Session with a + # backoff interval of 10 seconds in case there is a temporary + # glitch in the network or transient problems with the keystone-api + # pod + while retry <= 1: + # Construct Session Argument + for attr in ('auth_url', 'password', 'project_domain_name', + 'project_name', 'username', 'user_domain_name'): + keystone_auth[attr] = config.get('keystone_authtoken', attr) + + # Set up keystone session + logging.info("Get Keystone Session") + auth = keystone_v3.Password(**keystone_auth) + sess = keystone_session.Session(auth=auth) + + # Retry if we fail to get keystone session + if sess: + logging.info("Successfully Retrieved Keystone Session") + break + else: + logging.info("Unable to get Keystone Session on first attempt") + logging.info("Retrying after 10 seconds...") + time.sleep(10) + retry += 1 + + # Raise Execptions if we fail to get the keystone session + if not sess: + raise AirflowException("Unable to get Keystone Session!") + else: + return sess diff --git a/shipyard_airflow/plugins/service_token.py b/shipyard_airflow/plugins/service_token.py index ac4b9f78..a96246d6 100644 --- a/shipyard_airflow/plugins/service_token.py +++ b/shipyard_airflow/plugins/service_token.py @@ -12,50 +12,49 @@ # See the License for the specific language governing permissions and # limitations under the License. -import configparser +from functools import wraps import logging import time from airflow.exceptions import AirflowException -from functools import wraps - -from keystoneauth1.identity import v3 as keystone_v3 -from keystoneauth1 import session as keystone_session -from keystoneclient.v3 import client as keystone_client +from service_session import ucp_keystone_session def shipyard_service_token(func): @wraps(func) def keystone_token_get(self, context): - # Read and parse shiyard.conf - config = configparser.ConfigParser() - config.read(self.shipyard_conf) + """This function retrieves Keystone token for UCP Services + :param context: Information on the current workflow + + Example:: + + from service_token import shipyard_service_token + + @shipyard_service_token + def on_get(self, context): + svc_token=context['svc_token'] + + # Use the token to perform tasks such as setting + # up a DrydockSession which requires keystone + # token for authentication + """ # Initialize variables retry = 0 token = None - keystone_auth = {} + + # Retrieve Keystone Session + sess = ucp_keystone_session(self, context) # We will allow 1 retry in getting the Keystone Token with a # backoff interval of 10 seconds in case there is a temporary # glitch in the network or transient problems with the keystone-api # pod while retry <= 1: - # Construct Session Argument - for attr in ('auth_url', 'password', 'project_domain_name', - 'project_name', 'username', 'user_domain_name'): - keystone_auth[attr] = config.get('keystone_authtoken', attr) - - # Set up keystone session - auth = keystone_v3.Password(**keystone_auth) - sess = keystone_session.Session(auth=auth) - keystone = keystone_client.Client(session=sess) - # Retrieve Keystone Token logging.info("Get Keystone Token") - token = keystone.get_raw_token_from_identity_service( - **keystone_auth)['auth_token'] + token = sess.get_auth_headers().get('X-Auth-Token') # Retry if we fail to get the keystone token if token: