Fixing availability-zone not take effect error

when add/remove a host to one aggregate or update aggregate
metadata incluing availability_zone, "OS-EXT-AZ:availability_zone"
property of some instances in the host can not show correctly.

The cause is that when getting availability_zone of one instance,
it will try to get the value from cache first, but unfortunately
the cache does not update or reset in time, and it will keep one
hour if we do not change it.

This patch will add update or reset after adding/removing a host
to one aggregate or updating aggregate metadata including
availability_zone.

Change-Id: I5dd07f876471b5faf8fb1016e25a861124b7cb6f
Closes-bug: #1240374

nova/availability_zones.py

@@ -74,10 +74,8 @@ def set_availability_zones(context, services):
             else:
                 az = CONF.default_availability_zone
                 # update the cache
-                cache = _get_cache()
-                cache_key = _make_cache_key(service['host'])
-                cache.delete(cache_key)
-                cache.set(cache_key, az, AZ_CACHE_SECONDS)
+                update_host_availability_zone_cache(context,
+                                                    service['host'], az)
         service['availability_zone'] = az
     return services
 
@@ -96,6 +94,15 @@ def get_host_availability_zone(context, host, conductor_api=None):
     return az
 
 
+def update_host_availability_zone_cache(context, host, availability_zone=None):
+    if not availability_zone:
+        availability_zone = get_host_availability_zone(context, host)
+    cache = _get_cache()
+    cache_key = _make_cache_key(host)
+    cache.delete(cache_key)
+    cache.set(cache_key, availability_zone, AZ_CACHE_SECONDS)
+
+
 def get_availability_zones(context, get_only_available=False):
     """Return available and unavailable zones on demand.
 

nova/compute/api.py

@@ -3196,7 +3196,6 @@ class AggregateAPI(base.Base):
         if values:
             aggregate.metadata = values
         aggregate.save()
-
         # If updated values include availability_zones, then the cache
         # which stored availability_zones and host need to be reset
         if values.get('availability_zone'):
@@ -3208,6 +3207,10 @@ class AggregateAPI(base.Base):
         """Updates the aggregate metadata."""
         aggregate = aggregate_obj.Aggregate.get_by_id(context, aggregate_id)
         aggregate.update_metadata(metadata)
+        # If updated metadata include availability_zones, then the cache
+        # which stored availability_zones and host need to be reset
+        if metadata and metadata.get('availability_zone'):
+            availability_zones.reset_cache()
         return aggregate
 
     @wrap_exception()
@@ -3245,6 +3248,14 @@ class AggregateAPI(base.Base):
                     action=action_name, aggregate_id=aggregate_id,
                     reason=msg)
 
+    def _update_az_cache_for_host(self, context, host_name, aggregate_meta):
+        # Update the availability_zone cache to avoid getting wrong
+        # availability_zone in cache retention time when add/remove
+        # host to/from aggregate.
+        if aggregate_meta and aggregate_meta.get('availability_zone'):
+            availability_zones.update_host_availability_zone_cache(context,
+                                                                   host_name)
+
     @wrap_exception()
     def add_host_to_aggregate(self, context, aggregate_id, host_name):
         """Adds the host to an aggregate."""
@@ -3264,6 +3275,7 @@ class AggregateAPI(base.Base):
                 self._check_az_for_host(aggregate_meta, host_az, aggregate_id)
         aggregate = aggregate_obj.Aggregate.get_by_id(context, aggregate_id)
         aggregate.add_host(context, host_name)
+        self._update_az_cache_for_host(context, host_name, aggregate.metadata)
         #NOTE(jogo): Send message to host to support resource pools
         self.compute_rpcapi.add_aggregate_host(context,
                 aggregate=aggregate, host_param=host_name, host=host_name)
@@ -3285,6 +3297,7 @@ class AggregateAPI(base.Base):
         service_obj.Service.get_by_compute_host(context, host_name)
         aggregate = aggregate_obj.Aggregate.get_by_id(context, aggregate_id)
         aggregate.delete_host(host_name)
+        self._update_az_cache_for_host(context, host_name, aggregate.metadata)
         self.compute_rpcapi.remove_aggregate_host(context,
                 aggregate=aggregate, host_param=host_name, host=host_name)
         compute_utils.notify_about_aggregate_update(context,

nova/tests/compute/test_compute.py

@@ -8549,10 +8549,13 @@ class ComputeAPIAggrTestCase(BaseTestCase):
         aggr = self.api.create_aggregate(self.context, 'fake_aggregate',
                                          'fake_zone')
         metadata = {'foo_key1': 'foo_value1',
-                    'foo_key2': 'foo_value2', }
+                    'foo_key2': 'foo_value2',
+                    'availability_zone': 'fake_zone'}
         fake_notifier.NOTIFICATIONS = []
+        availability_zones._get_cache().add('fake_key', 'fake_value')
         aggr = self.api.update_aggregate_metadata(self.context, aggr['id'],
                                                   metadata)
+        self.assertIsNone(availability_zones._get_cache().get('fake_key'))
         self.assertEqual(len(fake_notifier.NOTIFICATIONS), 2)
         msg = fake_notifier.NOTIFICATIONS[0]
         self.assertEqual(msg.event_type,
@@ -8618,6 +8621,12 @@ class ComputeAPIAggrTestCase(BaseTestCase):
         self.stubs.Set(self.api.compute_rpcapi, 'add_aggregate_host',
                        fake_add_aggregate_host)
 
+        self.mox.StubOutWithMock(availability_zones,
+                                 'update_host_availability_zone_cache')
+        availability_zones.update_host_availability_zone_cache(self.context,
+                                                               fake_host)
+        self.mox.ReplayAll()
+
         fake_notifier.NOTIFICATIONS = []
         aggr = self.api.add_host_to_aggregate(self.context,
                                               aggr['id'], fake_host)
@@ -8724,6 +8733,12 @@ class ComputeAPIAggrTestCase(BaseTestCase):
         self.stubs.Set(self.api.compute_rpcapi, 'remove_aggregate_host',
                        fake_remove_aggregate_host)
 
+        self.mox.StubOutWithMock(availability_zones,
+                                 'update_host_availability_zone_cache')
+        availability_zones.update_host_availability_zone_cache(self.context,
+                                                               host_to_remove)
+        self.mox.ReplayAll()
+
         fake_notifier.NOTIFICATIONS = []
         expected = self.api.remove_host_from_aggregate(self.context,
                                                        aggr['id'],

nova/tests/test_availability_zones.py

@@ -87,6 +87,20 @@ class AvailabilityZoneTestCases(test.TestCase):
         az.reset_cache()
         self.assertIsNone(az._get_cache().get('cache'))
 
+    def test_update_host_availability_zone_cache(self):
+        """Test availability zone cache could be update."""
+        service = self._create_service_with_topic('compute', self.host)
+
+        # Create a new aggregate with an AZ and add the host to the AZ
+        az_name = 'az1'
+        cache_key = az._make_cache_key(self.host)
+        agg_az1 = self._create_az('agg-az1', az_name)
+        self._add_to_aggregate(service, agg_az1)
+        az.update_host_availability_zone_cache(self.context, self.host)
+        self.assertEqual(az._get_cache().get(cache_key), 'az1')
+        az.update_host_availability_zone_cache(self.context, self.host, 'az2')
+        self.assertEqual(az._get_cache().get(cache_key), 'az2')
+
     def test_set_availability_zone_compute_service(self):
         """Test for compute service get right availability zone."""
         service = self._create_service_with_topic('compute', self.host)