Offload openstack delete api calls to an executor

Use a thread pool exeuctor to run openstack API server delete
calls since they can take an appreciable amount of time to
return.

This also increases the number of worker threads to 16 to account
for the extra executor usage.

It also sets the number of keyscan workers to 16 (which is probably
an increase; it was undefined before).

This allows for more parallel operation at the cost of a little more
threading overhead.

Change-Id: I55984b21677eb1291d11a0e80945015f68f67b5e

nodepool/driver/openstack/adapter.py

@@ -69,6 +69,7 @@ class OpenStackResource(statemachine.Resource):
 
 class OpenStackDeleteStateMachine(statemachine.StateMachine):
     FLOATING_IP_DELETING = 'deleting floating ip'
+    SERVER_DELETE_SUBMIT = 'submit delete server'
     SERVER_DELETE = 'delete server'
     SERVER_DELETING = 'deleting server'
     COMPLETE = 'complete'
@@ -91,7 +92,7 @@ class OpenStackDeleteStateMachine(statemachine.StateMachine):
                     self.adapter._deleteFloatingIp(fip)
                     self.state = self.FLOATING_IP_DELETING
             if not self.floating_ips:
-                self.state = self.SERVER_DELETE
+                self.state = self.SERVER_DELETE_SUBMIT
 
         if self.state == self.FLOATING_IP_DELETING:
             fips = []
@@ -105,11 +106,17 @@ class OpenStackDeleteStateMachine(statemachine.StateMachine):
             if self.floating_ips:
                 return
             else:
-                self.state = self.SERVER_DELETE
+                self.state = self.SERVER_DELETE_SUBMIT
+
+        if self.state == self.SERVER_DELETE_SUBMIT:
+            self.delete_future = self.adapter._submitApi(
+                self.adapter._deleteServer,
+                self.external_id)
+            self.state = self.SERVER_DELETE
 
         if self.state == self.SERVER_DELETE:
-            self.adapter._deleteServer(self.external_id)
-            self.state = self.SERVER_DELETING
+            if self.adapter._completeApi(self.delete_future):
+                self.state = self.SERVER_DELETING
 
         if self.state == self.SERVER_DELETING:
             self.server = self.adapter._refreshServerDelete(self.server)
@@ -363,7 +370,9 @@ class OpenStackAdapter(statemachine.Adapter):
             f"nodepool.OpenStackAdapter.{provider_config.name}")
         self.provider = provider_config
 
-        workers = 8
+        # The default http connection pool size is 10; match it for
+        # efficiency.
+        workers = 10
         self.log.info("Create executor with max workers=%s", workers)
         self.api_executor = ThreadPoolExecutor(
             thread_name_prefix=f'openstack-api-{provider_config.name}',
@@ -733,6 +742,7 @@ class OpenStackAdapter(statemachine.Adapter):
     def _deleteServer(self, external_id):
         with Timer(self.log, 'API call delete_server'):
             self._client.delete_server(external_id)
+            return True
 
     def _getFlavorFromServer(self, server):
         # In earlier versions of nova or the sdk, flavor has just an id.

nodepool/driver/statemachine.py

@@ -539,8 +539,14 @@ class StateMachineProvider(Provider, QuotaSupport):
         super().start(zk_conn)
         self.running = True
         self._zk = zk_conn
+
+        # Matching the workers in openstack/adapter.py
+        # TODO: unify thread pool handling across drivers
+        workers = 10
+        self.log.info("Create keyscan executor with max workers=%s", workers)
         self.keyscan_worker = ThreadPoolExecutor(
-            thread_name_prefix=f'keyscan-{self.provider.name}')
+            thread_name_prefix=f'keyscan-{self.provider.name}',
+            max_workers=workers)
         self.state_machine_thread = threading.Thread(
             target=self._runStateMachines,
             daemon=True)