Fix image delete logic

The logic around when to delete an image was just completely wrong.

Also, rackspace sometimes returns a deleted image when we request
it, while hp returns a 404.  Deal with both of those situations.

Change-Id: I4b6d620a750bd39a1d3b89e6eb51baf37694f8a7

nodepool/nodepool.py

@@ -853,22 +853,23 @@ class NodePool(threading.Thread):
             if node.state in [nodedb.READY, nodedb.HOLD]:
                 continue
             delete = False
+            now = time.time()
             if (node.state == nodedb.DELETE):
                 self.log.warning("Deleting node id: %s which is in delete "
                                  "state" % node.id)
                 delete = True
             elif (node.state == nodedb.TEST and
-                  time.time() - node.state_time > TEST_CLEANUP):
+                  (now - node.state_time) > TEST_CLEANUP):
                 self.log.warning("Deleting node id: %s which has been in %s "
                                  "state for %s hours" %
                                  (node.id, node.state,
-                                  node.state_time / (60 * 60)))
+                                  (now - node.state_time) / (60 * 60)))
                 delete = True
-            elif time.time() - node.state_time > NODE_CLEANUP:
+            elif (now - node.state_time) > NODE_CLEANUP:
                 self.log.warning("Deleting node id: %s which has been in %s "
                                  "state for %s hours" %
                                  (node.id, node.state,
-                                  node.state_time / (60 * 60)))
+                                  (now - node.state_time) / (60 * 60)))
                 delete = True
             if delete:
                 try:
@@ -881,6 +882,7 @@ class NodePool(threading.Thread):
             # Normally, reap images that have sat in their current state
             # for 24 hours, unless the image is the current snapshot
             delete = False
+            now = time.time()
             if image.provider_name not in self.config.providers:
                 delete = True
                 self.log.info("Deleting image id: %s which has no current "
@@ -894,10 +896,11 @@ class NodePool(threading.Thread):
                 current = session.getCurrentSnapshotImage(image.provider_name,
                                                           image.image_name)
                 if (current and image != current and
-                    (time.time() - current.state_time) > KEEP_OLD_IMAGE):
-                    self.log.info("Deleting image id: %s because the current "
-                                  "image is %s hours old" %
-                                  (image.id, current.state_time / (60 * 60)))
+                    (now - image.state_time) > KEEP_OLD_IMAGE):
+                    self.log.info("Deleting non-current image id: %s because "
+                                  "the image is %s hours old" %
+                                  (image.id,
+                                   (now - image.state_time) / (60 * 60)))
                     delete = True
             if delete:
                 try:

nodepool/provider_manager.py

@@ -152,6 +152,9 @@ class GetImageTask(Task):
             image = client.images.get(**self.args)
         except novaclient.exceptions.NotFound:
             raise NotFound()
+        # HP returns 404, rackspace can return a 'DELETED' image.
+        if image.status == 'DELETED':
+            raise NotFound()
         d = dict(id=image.id, status=image.status)
         if hasattr(image, 'progress'):
             d['progress'] = image.progress