Handle case where nova server is in DELETED state

The nova api can return instance records for an instance that has been
deleted. When it does this the status should be "DELETED". This means we
should check for the instance to have no more record or if the record is
present check that the status if DELETED.

Change-Id: I7ad753a3c73f3d2cd78f4a380f78279af9206ada

nodepool/driver/fake/provider.py

@@ -206,7 +206,9 @@ class FakeOpenStackCloud(object):
         for maybe in instance_list:
             if maybe.name == name_or_id or maybe.id == name_or_id:
                 instance = maybe
-        if instance:
+        if instance and not instance.status == 'DELETED':
+            # We don't remove DELETED instances as we want to test
+            # they are handled properly.
             instance_list.remove(instance)
         self.log.debug("Deleted from %s" % (repr(instance_list),))
 

nodepool/driver/openstack/provider.py

@@ -380,7 +380,8 @@ class OpenStackProvider(Provider):
         for count in iterate_timeout(
                 timeout, exceptions.ServerDeleteException,
                 "server %s deletion" % server_id):
-            if not self.getServer(server_id):
+            server = self.getServer(server_id)
+            if not server or server.status == "DELETED":
                 return
 
     def createImage(self, server, image_name, meta):

nodepool/tests/unit/test_launcher.py

@@ -810,6 +810,45 @@ class TestLauncher(tests.DBTestCase):
         self.assertEqual('fake-provider', new_nodes[0].provider)
         self.assertNotEqual(nodes[0], new_nodes[0])
 
+    def test_node_delete_DELETED_success(self):
+        """Test we treat a node in DELETING state as deleted"""
+        configfile = self.setup_config('node.yaml')
+        pool = self.useNodepool(configfile, watermark_sleep=1)
+        self.useBuilder(configfile)
+        pool.start()
+        self.waitForImage('fake-provider', 'fake-image')
+        nodes = self.waitForNodes('fake-label')
+        self.assertEqual(len(nodes), 1)
+        self.assertEqual(zk.READY, nodes[0].state)
+        self.assertEqual('fake-provider', nodes[0].provider)
+
+        # Get fake cloud record and set status to DELETING
+        manager = pool.getProviderManager('fake-provider')
+        for instance in manager.listNodes():
+            if instance.id == nodes[0].external_id:
+                instance.status = 'DELETED'
+                break
+
+        nodes[0].state = zk.DELETING
+        self.zk.storeNode(nodes[0])
+
+        # Wait for this one to be deleted
+        self.waitForNodeDeletion(nodes[0])
+
+        api_record_remains = False
+        for instance in manager.listNodes():
+            if instance.id == nodes[0].external_id:
+                api_record_remains = True
+                break
+        self.assertTrue(api_record_remains)
+
+        # Wait for a new one to take it's place
+        new_nodes = self.waitForNodes('fake-label')
+        self.assertEqual(len(new_nodes), 1)
+        self.assertEqual(zk.READY, new_nodes[0].state)
+        self.assertEqual('fake-provider', new_nodes[0].provider)
+        self.assertNotEqual(nodes[0], new_nodes[0])
+
     def test_node_launch_retries(self):
         configfile = self.setup_config('node_launch_retry.yaml')
         pool = self.useNodepool(configfile, watermark_sleep=1)