From afd2788729b0a0cf7d2d000d6c15cc5fd42bfe02 Mon Sep 17 00:00:00 2001 From: Nishant kumar Date: Thu, 24 Jan 2019 21:42:25 +0000 Subject: [PATCH] Remove Shipyard queries for Tiller information Currently Shipyard is attempting to retrieve Tiller information to pass to Armada. Once the dependent change is merged, Tiller will be moved into the same pod as Armada, and this information will be retrieved from the Armada configuration file. Therefore, Shipyard should no longer be retrieving this information for Armada. Co-Authored-By: Michael Beaver Depends-On: https://review.openstack.org/#/c/632788/ Change-Id: I689e396ef9f184525f09f819f13c1490344481f2 --- src/bin/shipyard_airflow/requirements.txt | 2 +- .../plugins/armada_base_operator.py | 15 --- .../plugins/armada_get_status.py | 3 - .../plugins/armada_post_apply.py | 3 - .../plugins/get_k8s_pod_port_ip.py | 121 ------------------ .../test_armada_test_releases_operator.py | 9 +- 6 files changed, 4 insertions(+), 149 deletions(-) delete mode 100644 src/bin/shipyard_airflow/shipyard_airflow/plugins/get_k8s_pod_port_ip.py diff --git a/src/bin/shipyard_airflow/requirements.txt b/src/bin/shipyard_airflow/requirements.txt index b18696b3..13de559f 100644 --- a/src/bin/shipyard_airflow/requirements.txt +++ b/src/bin/shipyard_airflow/requirements.txt @@ -41,4 +41,4 @@ Werkzeug==0.14.1 # Dependencies for other UCP components git+https://git.openstack.org/openstack/airship-deckhand@0aae9dc1776bd97ab8f4152a51baedb24eb61396#egg=deckhand git+https://git.openstack.org/openstack/airship-drydock.git@d93d6d5a0a370ced536180612d1ade708e29cd47#egg=drydock_provisioner&subdirectory=python -git+https://git.openstack.org/openstack/airship-armada.git@269083c0c063efa50bd5e88374dbd89abb3921c3#egg=armada +git+https://git.openstack.org/openstack/airship-armada.git@37249f3492b137ff6beb3d3c7e5e9dc1732ef8fb#egg=armada diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_base_operator.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_base_operator.py index aa96446d..4640a656 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_base_operator.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_base_operator.py @@ -23,13 +23,11 @@ import armada.common.session as session from armada.exceptions import api_exceptions as errors try: - from get_k8s_pod_port_ip import get_pod_port_ip import service_endpoint from service_token import shipyard_service_token from ucp_base_operator import UcpBaseOperator from xcom_pusher import XcomPusher except ImportError: - from shipyard_airflow.plugins.get_k8s_pod_port_ip import get_pod_port_ip from shipyard_airflow.plugins import service_endpoint from shipyard_airflow.plugins.service_token import shipyard_service_token from shipyard_airflow.plugins.ucp_base_operator import UcpBaseOperator @@ -90,11 +88,6 @@ class ArmadaBaseOperator(UcpBaseOperator): self.svc_token ) - # Retrieve Tiller Information - # TODO(@drewwalters96): This should be explicit. Refactor in - # conjunction with `get_pod_port_ip` decorator. - self.get_tiller_info(pods_ip_port={}) - @staticmethod def _init_armada_client(armada_svc_endpoint, svc_token): @@ -144,14 +137,6 @@ class ArmadaBaseOperator(UcpBaseOperator): self.get_k8s_logs() raise AirflowException(client_error) - @get_pod_port_ip('tiller', namespace='kube-system') - def get_tiller_info(self, pods_ip_port={}): - - # Assign value to the 'query' dictionary so that we can pass - # it via the Armada Client - self.query['tiller_host'] = pods_ip_port['tiller']['ip'] - self.query['tiller_port'] = pods_ip_port['tiller']['port'] - class ArmadaBaseOperatorPlugin(AirflowPlugin): diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_get_status.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_get_status.py index 07b41d61..d9a8686a 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_get_status.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_get_status.py @@ -38,9 +38,6 @@ class ArmadaGetStatusOperator(ArmadaBaseOperator): def do_execute(self): - # Retrieve Tiller Information - self.get_tiller_info(pods_ip_port={}) - # Retrieve read timeout timeout = self.dc['armada.get_status_timeout'] diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_post_apply.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_post_apply.py index 81e868e1..b78f9d4f 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_post_apply.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_post_apply.py @@ -47,9 +47,6 @@ class ArmadaPostApplyOperator(ArmadaBaseOperator): self.dc = self.xcom_puller.get_deployment_configuration() self.target_manifest = self.dc['armada.manifest'] - # Retrieve Tiller Information - self.get_tiller_info(pods_ip_port={}) - # Update query dict with information of target_manifest self.query['target_manifest'] = self.target_manifest diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/get_k8s_pod_port_ip.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/get_k8s_pod_port_ip.py deleted file mode 100644 index d1c9e99a..00000000 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/get_k8s_pod_port_ip.py +++ /dev/null @@ -1,121 +0,0 @@ -# Copyright 2018 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -from functools import wraps - -from airflow.exceptions import AirflowException -from kubernetes import client, config - - -def get_pod_port_ip(*pods, namespace): - def get_k8s_pod_port_ip(func): - @wraps(func) - def k8s_pod_port_ip_get(self, pods_ip_port): - """This function retrieves Kubernetes Pod Port and IP - information. It can be used to retrieve information of - single pod deployment and/or statefulsets. For instance, - it can be used to retrieve the tiller pod IP and port - information for usage in the Armada Operator. - - :param pods_ip_port: IP and port information of the pods - - Example:: - - from get_k8s_pod_port_ip import get_pod_port_ip - - @get_pod_port_ip('tiller', namespace='kube-system') - def get_pod_info(self, pods_ip_port={}): - - tiller_ip = pods_ip_port['tiller']['ip'] - tiller_port = pods_ip_port['tiller']['port'] - - """ - # Initialize variable - k8s_pods = {} - - # The function allows us to query information on multiple - # pods - for pod_name in pods: - # Initialize variables - pod_attr = {} - pod_attr[pod_name] = {} - - # Initialize/Reset counter - count = 0 - - # Make use of kubernetes client to retrieve pod IP - # and port information - # Note that we should use 'in_cluster_config' - # Note that we will only search for pods in the namespace - # that was specified in the request - config.load_incluster_config() - v1 = client.CoreV1Api() - ret = v1.list_namespaced_pod(namespace=namespace, - watch=False) - - # Loop through items to extract port and IP information - # of the pod - for i in ret.items: - if pod_name in i.metadata.name: - # Get pod IP - logging.info("Retrieving %s IP", pod_name) - pod_attr[pod_name]['ip'] = i.status.pod_ip - logging.info("%s IP is %s", pod_name, - pod_attr[pod_name]['ip']) - - # Get pod port - logging.info("Retrieving %s Port", pod_name) - - # It is possible for a pod to have an IP with no - # port. For instance maas-rack takes on genesis - # node IP and has no port associated with it. We - # will assign the value 'None' to the port value - # in such cases. - try: - specs_dict = i.spec.containers[0].__dict__ - ports_dict = specs_dict['_ports'][0].__dict__ - pod_attr[pod_name]['port'] = ( - ports_dict['_container_port']) - logging.info("%s Port is %s", pod_name, - pod_attr[pod_name]['port']) - except: - pod_attr[pod_name]['port'] = 'None' - logging.warning("%s Port is None", pod_name) - - # Update k8s_pods with new entry - k8s_pods.update(pod_attr) - - # It is possible for different pods to have the same - # partial names. This means that we can end up with - # inconsistent results depending on how the pods were - # ordered in the results for 'list_namespaced_pod'. - # Hence an exception should be raised when the function - # returns results for 2 or more pods. - if count > 0: - raise AirflowException( - "Pod search string is not unique!") - - # Step counter - count += 1 - - # Raise Execptions if the pod does not exits in the - # Kubernetes cluster - if not pod_attr[pod_name]: - raise AirflowException("Unable to locate", pod_name) - - return func(self, pods_ip_port=k8s_pods) - - return k8s_pod_port_ip_get - return get_k8s_pod_port_ip diff --git a/src/bin/shipyard_airflow/tests/unit/plugins/test_armada_test_releases_operator.py b/src/bin/shipyard_airflow/tests/unit/plugins/test_armada_test_releases_operator.py index e68e5814..76dfbbfb 100644 --- a/src/bin/shipyard_airflow/tests/unit/plugins/test_armada_test_releases_operator.py +++ b/src/bin/shipyard_airflow/tests/unit/plugins/test_armada_test_releases_operator.py @@ -45,8 +45,7 @@ class TestArmadaTestReleasesOperator: @mock.patch.object(ArmadaBaseOperator, 'armada_client', create=True) @mock.patch.object(ArmadaBaseOperator, 'get_releases', return_value=RELEASES) - @mock.patch.object(ArmadaBaseOperator, 'get_tiller_info') - def test_do_execute(self, mock_tiller_info, mock_releases, mock_client, + def test_do_execute(self, mock_releases, mock_client, mock_logs): op = ArmadaTestReleasesOperator(main_dag_name='main', shipyard_conf=CONF_FILE, @@ -69,8 +68,7 @@ class TestArmadaTestReleasesOperator: @mock.patch('shipyard_airflow.plugins.armada_test_releases.LOG.info') @mock.patch.object(ArmadaBaseOperator, 'armada_client', create=True) - @mock.patch.object(ArmadaBaseOperator, 'get_tiller_info') - def test_do_execute_with_params(self, mock_tiller, mock_client, mock_logs): + def test_do_execute_with_params(self, mock_client, mock_logs): op = ArmadaTestReleasesOperator(main_dag_name='main', shipyard_conf=CONF_FILE, task_id='t1') @@ -91,9 +89,8 @@ class TestArmadaTestReleasesOperator: @mock.patch.object(ArmadaBaseOperator, 'armada_client', create=True) @mock.patch.object(ArmadaBaseOperator, 'get_releases', return_value=RELEASES) - @mock.patch.object(ArmadaBaseOperator, 'get_tiller_info') @mock.patch.object(UcpBaseOperator, 'get_k8s_logs') - def test_do_execute_fail(self, mock_k8s_logs, mock_tiller_info, + def test_do_execute_fail(self, mock_k8s_logs, mock_releases, mock_client): mock_client.get_test_release.return_value = None