Browse Source

Workaround MAAS race condition

- If two threads concurrently update the power parameters
  of VMs on the same libvirt host, it seems to leave MAAS
  with broken state. Add locking in Drydock so that Drydock
  does not do this concurrently.

Change-Id: I85575f4ba48152b4dc79c646871f33b69f845ab9
Scott Hussey 4 months ago
parent
commit
54ea0e1374
1 changed files with 36 additions and 31 deletions
  1. 36
    31
      python/drydock_provisioner/drivers/node/maasdriver/models/machine.py

+ 36
- 31
python/drydock_provisioner/drivers/node/maasdriver/models/machine.py View File

@@ -15,6 +15,8 @@
15 15
 import logging
16 16
 import base64
17 17
 
18
+from threading import Lock, Condition
19
+
18 20
 import drydock_provisioner.error as errors
19 21
 import drydock_provisioner.drivers.node.maasdriver.models.base as model_base
20 22
 import drydock_provisioner.drivers.node.maasdriver.models.interface as maas_interface
@@ -26,6 +28,8 @@ from bson import BSON
26 28
 
27 29
 LOG = logging.getLogger(__name__)
28 30
 
31
+power_lock = Lock()
32
+power_cv = Condition(lock=power_lock)
29 33
 
30 34
 class Machine(model_base.ResourceBase):
31 35
 
@@ -388,32 +392,33 @@ class Machine(model_base.ResourceBase):
388 392
         :param kwargs: Each kwargs key will be prepended with 'power_parameters_' and
389 393
                        added to the list of updates for the node.
390 394
         """
391
-        if not power_type:
392
-            raise errors.DriverError(
393
-                "Cannot set power parameters. Must specify a power type.")
395
+        with power_cv:
396
+            if not power_type:
397
+                raise errors.DriverError(
398
+                    "Cannot set power parameters. Must specify a power type.")
394 399
 
395
-        url = self.interpolate_url()
400
+            url = self.interpolate_url()
396 401
 
397
-        if kwargs:
398
-            power_params = dict()
402
+            if kwargs:
403
+                power_params = dict()
399 404
 
400
-            self.logger.debug("Setting node power type to %s." % power_type)
401
-            self.power_type = power_type
402
-            power_params['power_type'] = power_type
405
+                self.logger.debug("Setting node power type to %s." % power_type)
406
+                self.power_type = power_type
407
+                power_params['power_type'] = power_type
403 408
 
404
-            for k, v in kwargs.items():
405
-                power_params['power_parameters_' + k] = v
409
+                for k, v in kwargs.items():
410
+                    power_params['power_parameters_' + k] = v
406 411
 
407
-            self.logger.debug("Updating node %s power parameters: %s" %
408
-                              (self.hostname, str(power_params)))
409
-            resp = self.api_client.put(url, files=power_params)
412
+                self.logger.debug("Updating node %s power parameters: %s" %
413
+                                  (self.hostname, str(power_params)))
414
+                resp = self.api_client.put(url, files=power_params)
410 415
 
411
-            if resp.status_code == 200:
412
-                return True
416
+                if resp.status_code == 200:
417
+                    return True
413 418
 
414
-            raise errors.DriverError(
415
-                "Failed updating power parameters MAAS url %s - return code %s\n%s"
416
-                % (url, resp.status_code.resp.text))
419
+                raise errors.DriverError(
420
+                    "Failed updating power parameters MAAS url %s - return code %s\n%s"
421
+                    % (url, resp.status_code.resp.text))
417 422
 
418 423
     def reset_power_parameters(self):
419 424
         """Reset power type and parameters for this node to manual.
@@ -422,21 +427,21 @@ class Machine(model_base.ResourceBase):
422 427
 
423 428
         Only available after the node has been added to MAAS.
424 429
         """
430
+        with power_cv:
431
+            url = self.interpolate_url()
425 432
 
426
-        url = self.interpolate_url()
427
-
428
-        self.logger.debug("Resetting node power type for machine {}".format(
429
-            self.resource_id))
430
-        self.power_type = 'manual'
431
-        power_params = {'power_type': 'manual'}
432
-        resp = self.api_client.put(url, files=power_params)
433
+            self.logger.debug("Resetting node power type for machine {}".format(
434
+                self.resource_id))
435
+            self.power_type = 'manual'
436
+            power_params = {'power_type': 'manual'}
437
+            resp = self.api_client.put(url, files=power_params)
433 438
 
434
-        if resp.status_code == 200:
435
-            return True
439
+            if resp.status_code == 200:
440
+                return True
436 441
 
437
-        raise errors.DriverError(
438
-            "Failed updating power parameters MAAS url {} - return code {}\n{}"
439
-            .format(url, resp.status_code.resp.text))
442
+            raise errors.DriverError(
443
+                "Failed updating power parameters MAAS url {} - return code {}\n{}"
444
+                .format(url, resp.status_code.resp.text))
440 445
 
441 446
     def update_identity(self, n, domain="local"):
442 447
         """Update this node's identity based on the Node object ``n``

Loading…
Cancel
Save