diff --git a/armada/handlers/k8s.py b/armada/handlers/k8s.py index 96786421..ae106d97 100644 --- a/armada/handlers/k8s.py +++ b/armada/handlers/k8s.py @@ -1,4 +1,4 @@ -# Copyright 2017 The Armada Authors. +# Copyright 2015 The Armada Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,9 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from kubernetes import client, config +import re +from kubernetes import client, config, watch from kubernetes.client.rest import ApiException + from oslo_config import cfg from oslo_log import log as logging @@ -92,3 +94,30 @@ class K8s(object): return self.client \ .delete_namespaced_pod(name, namespace, body) + + def wait_for_pod_redeployment(self, old_pod_name, namespace): + base_pod_pattern = re.compile('^(.+)-[a-zA-Z0-9]+$') + if not base_pod_pattern.match(old_pod_name): + LOG.error( + 'Could not identify new pod after purging %s', old_pod_name) + return + pod_base_name = base_pod_pattern.match(old_pod_name).group(1) + + new_pod_name = '' + w = watch.Watch() + for event in w.stream(self.client.list_namespaced_pod, namespace): + event_name = event['object'].metadata.name + event_match = base_pod_pattern.match(event_name) + if not event_match or not event_match.group(1) == pod_base_name: + continue + + pod_conditions = event['object'].status.conditions + # wait for new pod deployment + if event['type'] == 'ADDED' and not pod_conditions: + new_pod_name = event_name + elif new_pod_name: + for condition in pod_conditions: + if (condition.type == 'Ready' and + condition.status == 'True'): + LOG.info('New pod %s deployed', new_pod_name) + w.stop() diff --git a/armada/handlers/tiller.py b/armada/handlers/tiller.py index 11af275a..7111c471 100644 --- a/armada/handlers/tiller.py +++ b/armada/handlers/tiller.py @@ -207,6 +207,7 @@ class Tiller(object): LOG.info("Deleting %s in namespace: %s", pod_name, namespace) self.k8s.delete_namespace_pod(pod_name, namespace) + self.k8s.wait_for_pod_redeployment(pod_name, namespace) else: LOG.error("Unable to execute name: %s type: %s ", resource_name, resource_type)