Azure: handle leaked image upload resources

We could conceivable leak disks used for image uploads as well as
the images themselves.  Handle that in the cleanup method.

Change-Id: I6e72682995c50a57684cd5a0407a85049cdbda16

nodepool/driver/azurestate/adapter.py

@@ -300,30 +300,36 @@ class AzureAdapter(statemachine.Adapter):
     def getDeleteStateMachine(self, external_id):
         return AzureDeleteStateMachine(self, external_id)
 
-    def cleanupLeakedResources(self, known_nodes, metadata):
+    def cleanupLeakedResources(self, known_nodes, known_uploads, metadata):
         for vm in self._listVirtualMachines():
-            node_id = self._metadataMatches(vm, metadata)
+            node_id, upload_id = self._metadataMatches(vm, metadata)
             if (node_id and node_id not in known_nodes):
                 self.log.info(f"Deleting leaked vm: {vm['name']}")
                 self.azul.virtual_machines.delete(
                     self.resource_group, vm['name'])
         for nic in self._listNetworkInterfaces():
-            node_id = self._metadataMatches(nic, metadata)
+            node_id, upload_id = self._metadataMatches(nic, metadata)
             if (node_id and node_id not in known_nodes):
                 self.log.info(f"Deleting leaked nic: {nic['name']}")
                 self.azul.network_interfaces.delete(
                     self.resource_group, nic['name'])
         for pip in self._listPublicIPAddresses():
-            node_id = self._metadataMatches(pip, metadata)
+            node_id, upload_id = self._metadataMatches(pip, metadata)
             if (node_id and node_id not in known_nodes):
                 self.log.info(f"Deleting leaked pip: {pip['name']}")
                 self.azul.public_ip_addresses.delete(
                     self.resource_group, pip['name'])
         for disk in self._listDisks():
-            node_id = self._metadataMatches(disk, metadata)
-            if (node_id and node_id not in known_nodes):
+            node_id, upload_id = self._metadataMatches(disk, metadata)
+            if ((node_id and node_id not in known_nodes) or
+                (upload_id and upload_id not in known_uploads)):
                 self.log.info(f"Deleting leaked disk: {disk['name']}")
                 self.azul.disks.delete(self.resource_group, disk['name'])
+        for image in self._listImages():
+            node_id, upload_id = self._metadataMatches(image, metadata)
+            if (upload_id and upload_id not in known_uploads):
+                self.log.info(f"Deleting leaked image: {image['name']}")
+                self.azul.images.delete(self.resource_group, image['name'])
 
     def listInstances(self):
         for vm in self._listVirtualMachines():
@@ -444,7 +450,8 @@ class AzureAdapter(statemachine.Adapter):
         for k, v in metadata.items():
             if obj['tags'].get(k) != v:
                 return None
-        return obj['tags']['nodepool_node_id']
+        return (obj['tags'].get('nodepool_node_id'),
+                obj['tags'].get('nodepool_upload_id'))
 
     @staticmethod
     def _succeeded(obj):
@@ -659,3 +666,7 @@ class AzureAdapter(statemachine.Adapter):
         # the last thing we delete anyway, let's just fire and forget.
         self.azul.disks.delete(self.resource_group, name)
         return None
+
+    @cachetools.func.ttl_cache(maxsize=1, ttl=10)
+    def _listImages(self):
+        return self.azul.images.list(self.resource_group)

nodepool/driver/statemachine.py

@@ -579,13 +579,23 @@ class StateMachineProvider(Provider, QuotaSupport):
                 continue
             known_nodes.add(node.id)
 
+        known_uploads = set()
+        uploads = self._zk.getProviderUploads(self.provider.name)
+        for image in uploads.values():
+            for build in image.values():
+                for upload in build:
+                    known_uploads.add(upload.id)
+
         metadata = {'nodepool_provider_name': self.provider.name}
-        self.adapter.cleanupLeakedResources(known_nodes, metadata)
+        self.adapter.cleanupLeakedResources(known_nodes,
+                                            known_uploads, metadata)
 
     # Image handling
 
     def uploadImage(self, image_name, filename, image_type=None, meta=None,
                     md5=None, sha256=None):
+        meta = meta.copy()
+        meta['nodepool_provider_name'] = self.provider.name
         return self.adapter.uploadImage(image_name, filename,
                                         image_format=image_type,
                                         metadata=meta, md5=md5,
@@ -762,7 +772,7 @@ class Adapter:
         """
         raise NotImplementedError()
 
-    def cleanupLeakedResources(self, known_nodes, metadata):
+    def cleanupLeakedResources(self, known_nodes, known_uploads, metadata):
         """Delete any leaked resources
 
         Use the supplied metadata to delete any leaked resources.
@@ -774,10 +784,13 @@ class Adapter:
         be deleted.
 
         Look up the `nodepool_node_id` metadata attribute on cloud
-        instances to compare to known_nodes.
+        instances to compare to known_nodes, and `nodepool_upload_id`
+        for known_uploads.
 
         :param set known_nodes: A set of string values representing
             known node ids.
+        :param set known_uploads: A set of string values representing
+            known image upload ids.
         :param dict metadata: Metadata values to compare with cloud
             instances.