diff --git a/src/bin/shipyard_airflow/requirements.txt b/src/bin/shipyard_airflow/requirements.txt index b18696b3..13de559f 100644 --- a/src/bin/shipyard_airflow/requirements.txt +++ b/src/bin/shipyard_airflow/requirements.txt @@ -41,4 +41,4 @@ Werkzeug==0.14.1 # Dependencies for other UCP components git+https://git.openstack.org/openstack/airship-deckhand@0aae9dc1776bd97ab8f4152a51baedb24eb61396#egg=deckhand git+https://git.openstack.org/openstack/airship-drydock.git@d93d6d5a0a370ced536180612d1ade708e29cd47#egg=drydock_provisioner&subdirectory=python -git+https://git.openstack.org/openstack/airship-armada.git@269083c0c063efa50bd5e88374dbd89abb3921c3#egg=armada +git+https://git.openstack.org/openstack/airship-armada.git@37249f3492b137ff6beb3d3c7e5e9dc1732ef8fb#egg=armada diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_base_operator.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_base_operator.py index aa96446d..4640a656 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_base_operator.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_base_operator.py @@ -23,13 +23,11 @@ import armada.common.session as session from armada.exceptions import api_exceptions as errors try: - from get_k8s_pod_port_ip import get_pod_port_ip import service_endpoint from service_token import shipyard_service_token from ucp_base_operator import UcpBaseOperator from xcom_pusher import XcomPusher except ImportError: - from shipyard_airflow.plugins.get_k8s_pod_port_ip import get_pod_port_ip from shipyard_airflow.plugins import service_endpoint from shipyard_airflow.plugins.service_token import shipyard_service_token from shipyard_airflow.plugins.ucp_base_operator import UcpBaseOperator @@ -90,11 +88,6 @@ class ArmadaBaseOperator(UcpBaseOperator): self.svc_token ) - # Retrieve Tiller Information - # TODO(@drewwalters96): This should be explicit. Refactor in - # conjunction with `get_pod_port_ip` decorator. - self.get_tiller_info(pods_ip_port={}) - @staticmethod def _init_armada_client(armada_svc_endpoint, svc_token): @@ -144,14 +137,6 @@ class ArmadaBaseOperator(UcpBaseOperator): self.get_k8s_logs() raise AirflowException(client_error) - @get_pod_port_ip('tiller', namespace='kube-system') - def get_tiller_info(self, pods_ip_port={}): - - # Assign value to the 'query' dictionary so that we can pass - # it via the Armada Client - self.query['tiller_host'] = pods_ip_port['tiller']['ip'] - self.query['tiller_port'] = pods_ip_port['tiller']['port'] - class ArmadaBaseOperatorPlugin(AirflowPlugin): diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_get_status.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_get_status.py index 07b41d61..d9a8686a 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_get_status.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_get_status.py @@ -38,9 +38,6 @@ class ArmadaGetStatusOperator(ArmadaBaseOperator): def do_execute(self): - # Retrieve Tiller Information - self.get_tiller_info(pods_ip_port={}) - # Retrieve read timeout timeout = self.dc['armada.get_status_timeout'] diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_post_apply.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_post_apply.py index 81e868e1..b78f9d4f 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_post_apply.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_post_apply.py @@ -47,9 +47,6 @@ class ArmadaPostApplyOperator(ArmadaBaseOperator): self.dc = self.xcom_puller.get_deployment_configuration() self.target_manifest = self.dc['armada.manifest'] - # Retrieve Tiller Information - self.get_tiller_info(pods_ip_port={}) - # Update query dict with information of target_manifest self.query['target_manifest'] = self.target_manifest diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/get_k8s_pod_port_ip.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/get_k8s_pod_port_ip.py deleted file mode 100644 index d1c9e99a..00000000 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/get_k8s_pod_port_ip.py +++ /dev/null @@ -1,121 +0,0 @@ -# Copyright 2018 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -from functools import wraps - -from airflow.exceptions import AirflowException -from kubernetes import client, config - - -def get_pod_port_ip(*pods, namespace): - def get_k8s_pod_port_ip(func): - @wraps(func) - def k8s_pod_port_ip_get(self, pods_ip_port): - """This function retrieves Kubernetes Pod Port and IP - information. It can be used to retrieve information of - single pod deployment and/or statefulsets. For instance, - it can be used to retrieve the tiller pod IP and port - information for usage in the Armada Operator. - - :param pods_ip_port: IP and port information of the pods - - Example:: - - from get_k8s_pod_port_ip import get_pod_port_ip - - @get_pod_port_ip('tiller', namespace='kube-system') - def get_pod_info(self, pods_ip_port={}): - - tiller_ip = pods_ip_port['tiller']['ip'] - tiller_port = pods_ip_port['tiller']['port'] - - """ - # Initialize variable - k8s_pods = {} - - # The function allows us to query information on multiple - # pods - for pod_name in pods: - # Initialize variables - pod_attr = {} - pod_attr[pod_name] = {} - - # Initialize/Reset counter - count = 0 - - # Make use of kubernetes client to retrieve pod IP - # and port information - # Note that we should use 'in_cluster_config' - # Note that we will only search for pods in the namespace - # that was specified in the request - config.load_incluster_config() - v1 = client.CoreV1Api() - ret = v1.list_namespaced_pod(namespace=namespace, - watch=False) - - # Loop through items to extract port and IP information - # of the pod - for i in ret.items: - if pod_name in i.metadata.name: - # Get pod IP - logging.info("Retrieving %s IP", pod_name) - pod_attr[pod_name]['ip'] = i.status.pod_ip - logging.info("%s IP is %s", pod_name, - pod_attr[pod_name]['ip']) - - # Get pod port - logging.info("Retrieving %s Port", pod_name) - - # It is possible for a pod to have an IP with no - # port. For instance maas-rack takes on genesis - # node IP and has no port associated with it. We - # will assign the value 'None' to the port value - # in such cases. - try: - specs_dict = i.spec.containers[0].__dict__ - ports_dict = specs_dict['_ports'][0].__dict__ - pod_attr[pod_name]['port'] = ( - ports_dict['_container_port']) - logging.info("%s Port is %s", pod_name, - pod_attr[pod_name]['port']) - except: - pod_attr[pod_name]['port'] = 'None' - logging.warning("%s Port is None", pod_name) - - # Update k8s_pods with new entry - k8s_pods.update(pod_attr) - - # It is possible for different pods to have the same - # partial names. This means that we can end up with - # inconsistent results depending on how the pods were - # ordered in the results for 'list_namespaced_pod'. - # Hence an exception should be raised when the function - # returns results for 2 or more pods. - if count > 0: - raise AirflowException( - "Pod search string is not unique!") - - # Step counter - count += 1 - - # Raise Execptions if the pod does not exits in the - # Kubernetes cluster - if not pod_attr[pod_name]: - raise AirflowException("Unable to locate", pod_name) - - return func(self, pods_ip_port=k8s_pods) - - return k8s_pod_port_ip_get - return get_k8s_pod_port_ip diff --git a/src/bin/shipyard_airflow/tests/unit/plugins/test_armada_test_releases_operator.py b/src/bin/shipyard_airflow/tests/unit/plugins/test_armada_test_releases_operator.py index e68e5814..76dfbbfb 100644 --- a/src/bin/shipyard_airflow/tests/unit/plugins/test_armada_test_releases_operator.py +++ b/src/bin/shipyard_airflow/tests/unit/plugins/test_armada_test_releases_operator.py @@ -45,8 +45,7 @@ class TestArmadaTestReleasesOperator: @mock.patch.object(ArmadaBaseOperator, 'armada_client', create=True) @mock.patch.object(ArmadaBaseOperator, 'get_releases', return_value=RELEASES) - @mock.patch.object(ArmadaBaseOperator, 'get_tiller_info') - def test_do_execute(self, mock_tiller_info, mock_releases, mock_client, + def test_do_execute(self, mock_releases, mock_client, mock_logs): op = ArmadaTestReleasesOperator(main_dag_name='main', shipyard_conf=CONF_FILE, @@ -69,8 +68,7 @@ class TestArmadaTestReleasesOperator: @mock.patch('shipyard_airflow.plugins.armada_test_releases.LOG.info') @mock.patch.object(ArmadaBaseOperator, 'armada_client', create=True) - @mock.patch.object(ArmadaBaseOperator, 'get_tiller_info') - def test_do_execute_with_params(self, mock_tiller, mock_client, mock_logs): + def test_do_execute_with_params(self, mock_client, mock_logs): op = ArmadaTestReleasesOperator(main_dag_name='main', shipyard_conf=CONF_FILE, task_id='t1') @@ -91,9 +89,8 @@ class TestArmadaTestReleasesOperator: @mock.patch.object(ArmadaBaseOperator, 'armada_client', create=True) @mock.patch.object(ArmadaBaseOperator, 'get_releases', return_value=RELEASES) - @mock.patch.object(ArmadaBaseOperator, 'get_tiller_info') @mock.patch.object(UcpBaseOperator, 'get_k8s_logs') - def test_do_execute_fail(self, mock_k8s_logs, mock_tiller_info, + def test_do_execute_fail(self, mock_k8s_logs, mock_releases, mock_client): mock_client.get_test_release.return_value = None