Add availability_zone attribute to gnocchi instance resources.

Since moving to libvirt nova-compute poller we no longer have
the availability_zone attribute of an instance available.

We can capture instance.create.end events and create gnocchi resources
based on these which do have the availability_zone availabile.

Currently we are only handling delete events for resources, this
patch adds the handling of create events for this purpose.

Change-Id: Ieddbddd4ddb6af11d5158f5c90c87ae4f847bb96

ceilometer/gnocchi_client.py

@@ -224,6 +224,15 @@ resources_update_operations = [
          {"op": "add", "path": "/attributes/instance_id",
           "value": {"type": "uuid", "required": False}},
      ]},
+    {"desc": "add availability_zone to instance",
+     "type": "update_attribute_type",
+     "resource_type": "instance",
+     "data": [{
+         "op": "add",
+         "path": "/attributes/availability_zone",
+         "value": {"type": "string", "min_length": 0, "max_length": 255,
+                   "required": False}
+     }]},
 ]
 
 # NOTE(sileht): We use LooseVersion because pbr can generate invalid

ceilometer/pipeline/data/event_definitions.yaml

@@ -50,6 +50,11 @@
     deleted_at:
       type: datetime
       fields: payload.deleted_at
+- event_type: compute.instance.create.end
+  traits:
+    <<: *instance_traits
+    availability_zone:
+      fields: payload.availability_zone
 - event_type: compute.instance.update
   traits:
     <<: *instance_traits

ceilometer/publisher/data/gnocchi_resources.yaml

@@ -110,8 +110,16 @@ resources:
       flavor_name: resource_metadata.(instance_type|(flavor.name)|flavor_name)
       server_group: resource_metadata.user_metadata.server_group
     event_delete: compute.instance.delete.start
+    event_create: compute.instance.create.end
     event_attributes:
       id: instance_id
+      display_name: display_name
+      host: host
+      availability_zone: availability_zone
+      flavor_id: instance_type_id
+      flavor_name: instance_type
+      user_id: user_id
+      project_id: project_id
     event_associated_resources:
       instance_network_interface: '{"=": {"instance_id": "%s"}}'
       instance_disk: '{"=": {"instance_id": "%s"}}'

ceilometer/publisher/gnocchi.py

@@ -492,6 +492,8 @@ class GnocchiPublisher(publisher.ConfigPublisherBase):
             rd, operation = rd
             if operation == EVENT_DELETE:
                 self._delete_event(rd, event)
+            if operation == EVENT_CREATE:
+                self._create_event(rd, event)
 
     def _delete_event(self, rd, event):
         ended_at = timeutils.utcnow().isoformat()
@@ -510,6 +512,19 @@ class GnocchiPublisher(publisher.ConfigPublisherBase):
         for resource in to_end:
             self._set_ended_at(resource, ended_at)
 
+    def _create_event(self, rd, event):
+        resource = rd.event_attributes(event)
+        resource_type = resource.pop('type')
+
+        try:
+            self._create_resource(resource_type, resource)
+        except gnocchi_exc.ResourceAlreadyExists:
+            LOG.debug("Create event received on existing resource (%s), "
+                      "ignore it.", resource['id'])
+        except Exception:
+            LOG.error("Failed to create resource %s", resource,
+                      exc_info=True)
+
     def _search_resource(self, resource_type, query):
         try:
             return self._gnocchi.resource.search(

ceilometer/tests/unit/publisher/test_gnocchi.py

@@ -44,6 +44,7 @@ INSTANCE_DELETE_START = models.Event(
             models.Trait(
                 'user_id', 1, u'1e3ce043029547f1a61c1996d1a531a2'),
             models.Trait('service', 1, u'compute'),
+            models.Trait('availability_zone', 1, u'zone1'),
             models.Trait('disk_gb', 2, 0),
             models.Trait('instance_type', 1, u'm1.tiny'),
             models.Trait('tenant_id', 1, u'7c150a59fe714e6f9263774af9688f0e'),
@@ -64,6 +65,33 @@ INSTANCE_DELETE_START = models.Event(
     message_id=u'a15b94ee-cb8e-4c71-9abe-14aa80055fb4',
 )
 
+INSTANCE_CREATE_END = models.Event(
+    event_type=u'compute.instance.create.end',
+    traits=[models.Trait('state', 1, u'active'),
+            models.Trait(
+                'user_id', 1, u'1e3ce043029547f1a61c1996d1a531a2'),
+            models.Trait('service', 1, u'compute'),
+            models.Trait('availability_zone', 1, u'zone1'),
+            models.Trait('disk_gb', 2, 0),
+            models.Trait('instance_type', 1, u'm1.tiny'),
+            models.Trait('tenant_id', 1, u'7c150a59fe714e6f9263774af9688f0e'),
+            models.Trait('root_gb', 2, 0),
+            models.Trait('ephemeral_gb', 2, 0),
+            models.Trait('instance_type_id', 2, u'2'),
+            models.Trait('vcpus', 2, 1),
+            models.Trait('memory_mb', 2, 512),
+            models.Trait(
+                'instance_id', 1, u'9f9d01b9-4a58-4271-9e27-398b21ab20d1'),
+            models.Trait('host', 1, u'vagrant-precise'),
+            models.Trait(
+                'request_id', 1, u'req-fb3c4546-a2e5-49b7-9fd2-a63bd658bc39'),
+            models.Trait('project_id', 1, u'7c150a59fe714e6f9263774af9688f0e'),
+            models.Trait('launched_at', 4, '2012-05-08T20:23:47')],
+    raw={},
+    generated='2012-05-08T20:24:14.824743',
+    message_id=u'202f745e-4913-11e9-affe-9797342bd3a8',
+)
+
 IMAGE_DELETE_START = models.Event(
     event_type=u'image.delete',
     traits=[models.Trait(u'status', 1, u'deleted'),
@@ -465,7 +493,7 @@ class PublisherWorkflowTest(base.BaseTestCase,
         self.ks_client = ks_client
 
     @mock.patch('gnocchiclient.v1.client.Client')
-    def test_event_workflow(self, fakeclient_cls):
+    def test_delete_event_workflow(self, fakeclient_cls):
         url = netutils.urlsplit("gnocchi://")
         self.publisher = gnocchi.GnocchiPublisher(self.conf.conf, url)
 
@@ -521,6 +549,33 @@ class PublisherWorkflowTest(base.BaseTestCase,
         for call in expected_calls:
             self.assertIn(call, fakeclient.mock_calls)
 
+    @mock.patch('gnocchiclient.v1.client.Client')
+    def test_create_event_workflow(self, fakeclient_cls):
+        url = netutils.urlsplit("gnocchi://")
+        self.publisher = gnocchi.GnocchiPublisher(self.conf.conf, url)
+
+        fakeclient = fakeclient_cls.return_value
+
+        now = timeutils.utcnow()
+        self.useFixture(utils_fixture.TimeFixture(now))
+
+        expected_calls = [
+            mock.call.resource.create(
+                'instance',
+                {'id': '9f9d01b9-4a58-4271-9e27-398b21ab20d1',
+                 'user_id': '1e3ce043029547f1a61c1996d1a531a2',
+                 'project_id': '7c150a59fe714e6f9263774af9688f0e',
+                 'availability_zone': 'zone1',
+                 'flavor_name': 'm1.tiny',
+                 'flavor_id': '2',
+                 'host': 'vagrant-precise'}),
+        ]
+
+        self.publisher.publish_events([INSTANCE_CREATE_END])
+        self.assertEqual(1, len(fakeclient.mock_calls))
+        for call in expected_calls:
+            self.assertIn(call, fakeclient.mock_calls)
+
     @mock.patch('ceilometer.publisher.gnocchi.LOG')
     @mock.patch('gnocchiclient.v1.client.Client')
     def test_workflow(self, fakeclient_cls, logger):

releasenotes/notes/add-availability_zone-gnocchi-instance-15170e4966a89d63.yaml

@@ -0,0 +1,11 @@
+---
+features:
+  - |
+    Add availability_zone attribute to gnocchi instance resources.
+    Populates this attribute by consuming instance.create.end events.
+upgrade:
+  - |
+    To take advantage of this new feature you will need to update your
+    gnocchi_resources.yaml file. See the example file for an example.
+    You will need to ensure all required attributes of an instance
+    are specified in the event_attributes.