airship/drydock
Code Issues Proposed changes

Workaround MAAS race condition

Browse Source 
- If two threads concurrently update the power parameters
  of VMs on the same libvirt host, it seems to leave MAAS
  with broken state. Add locking in Drydock so that Drydock
  does not do this concurrently.

Change-Id: I85575f4ba48152b4dc79c646871f33b69f845ab9
This commit is contained in:
Scott Hussey 2018-12-17 11:29:52 -06:00
parent 485f919822
commit 54ea0e1374
1 changed files with 36 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
python/drydock_provisioner/drivers/node/maasdriver/models/machine.py
View File

@ -15,6 +15,8 @@
import logging import logging
import base64 import base64
from threading import Lock, Condition
import drydock_provisioner.error as errors import drydock_provisioner.error as errors
import drydock_provisioner.drivers.node.maasdriver.models.base as model_base import drydock_provisioner.drivers.node.maasdriver.models.base as model_base
import drydock_provisioner.drivers.node.maasdriver.models.interface as maas_interface import drydock_provisioner.drivers.node.maasdriver.models.interface as maas_interface
@ -26,6 +28,8 @@ from bson import BSON
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
power_lock = Lock()
power_cv = Condition(lock=power_lock)
class Machine(model_base.ResourceBase): class Machine(model_base.ResourceBase):
@ -388,6 +392,7 @@ class Machine(model_base.ResourceBase):
:param kwargs: Each kwargs key will be prepended with 'power_parameters_' and :param kwargs: Each kwargs key will be prepended with 'power_parameters_' and
added to the list of updates for the node. added to the list of updates for the node.
""" """
with power_cv:
if not power_type: if not power_type:
raise errors.DriverError( raise errors.DriverError(
"Cannot set power parameters. Must specify a power type.") "Cannot set power parameters. Must specify a power type.")
@ -422,7 +427,7 @@ class Machine(model_base.ResourceBase):
Only available after the node has been added to MAAS. Only available after the node has been added to MAAS.
""" """
with power_cv:
url = self.interpolate_url() url = self.interpolate_url()
self.logger.debug("Resetting node power type for machine {}".format( self.logger.debug("Resetting node power type for machine {}".format(