Add support for Cinder volumes

Change-Id: I33401c86051293d5246856451571e4827be833ed

atmosphere/models.py

@@ -46,6 +46,8 @@ def get_model_type_from_event(event):
     """get_model_type_from_event"""
     if event.startswith('compute.instance'):
         return Instance, InstanceSpec
+    if event.startswith('volume.'):
+        return Volume, VolumeSpec
     if event.startswith('aggregate.'):
         raise exceptions.IgnoredEvent
     if event.startswith('compute_task.'):
@@ -66,8 +68,6 @@ def get_model_type_from_event(event):
         raise exceptions.IgnoredEvent
     if event.startswith('service.'):
         raise exceptions.IgnoredEvent
-    if event == 'volume.usage':
-        raise exceptions.IgnoredEvent
 
     raise exceptions.UnsupportedEventType
 
@@ -198,7 +198,11 @@ class Resource(db.Model, GetOrCreateMixin):
 
         # If we're deleted, then we close the current period.
         if resource.__class__.is_event_delete(event):
-            period.ended_at = event['traits']['deleted_at']
+            # NOTE(mnaser): Some resources don't have `deleted_at`, so we
+            #               resort to the timestamp of the event instead.
+            period.ended_at = event['traits'].get(
+                'deleted_at', event['generated']
+            )
         elif period.spec != spec:
             period.ended_at = event['generated']
 
@@ -265,6 +269,26 @@ class Instance(Resource):
         return 'deleted_at' in event['traits']
 
 
+class Volume(Resource):
+    """Volume"""
+
+    STATE_ALLOW_LIST = ('available', 'deleted')
+
+    __mapper_args__ = {
+        'polymorphic_identity': 'OS::Cinder::Volume'
+    }
+
+    @classmethod
+    def is_event_ignored(cls, event):
+        """is_event_ignored"""
+        return event['traits']['state'] not in cls.STATE_ALLOW_LIST
+
+    @classmethod
+    def is_event_delete(cls, event):
+        """is_event_delete"""
+        return event['traits']['state'] == 'deleted'
+
+
 class BigIntegerDateTime(TypeDecorator):
     """BigIntegerDateTime"""
 
@@ -368,3 +392,30 @@ class InstanceSpec(Spec):
             'instance_type': self.instance_type,
             'state': self.state,
             }
+
+
+class VolumeSpec(Spec):
+    """VolumeSpec"""
+
+    id = db.Column(db.Integer, db.ForeignKey('spec.id'), primary_key=True)
+    volume_type = db.Column(db.String(255))
+    volume_size = db.Column(db.String(255))
+    state = db.Column(db.String(255))
+
+    __table_args__ = (
+        db.UniqueConstraint('volume_type', 'volume_size', 'state'),
+    )
+
+    __mapper_args__ = {
+        'polymorphic_identity': 'OS::Cinder::Volume',
+    }
+
+    @property
+    def serialize(self):
+        """Return object data in easily serializable format"""
+
+        return {
+            'volume_type': self.volume_type,
+            'volume_size': self.volume_size,
+            'state': self.state,
+            }

atmosphere/tests/unit/api/test_ingress.py

@@ -44,7 +44,7 @@ class TestEvent:
         assert response.status_code == 400
 
     def test_with_one_event_provided(self, client):
-        event = fake.get_event()
+        event = fake.get_instance_event()
         response = client.post('/v1/event', json=[event])
 
         assert response.status_code == 204
@@ -53,8 +53,8 @@ class TestEvent:
         assert models.Spec.query.count() == 1
 
     def test_with_multiple_events_provided(self, client):
-        event_1 = fake.get_event(resource_id='fake-resource-1')
+        event_1 = fake.get_instance_event(resource_id='fake-resource-1')
-        event_2 = fake.get_event(resource_id='fake-resource-2')
+        event_2 = fake.get_instance_event(resource_id='fake-resource-2')
 
         response = client.post('/v1/event', json=[event_1, event_2])
 
@@ -64,7 +64,7 @@ class TestEvent:
         assert models.Spec.query.count() == 1
 
     def test_with_old_event_provided(self, client):
-        event_new = fake.get_event()
+        event_new = fake.get_instance_event()
         event_new['generated'] = '2020-06-07T01:42:54.736337'
         response = client.post('/v1/event', json=[event_new])
 
@@ -73,7 +73,7 @@ class TestEvent:
         assert models.Period.query.count() == 1
         assert models.Spec.query.count() == 1
 
-        event_old = fake.get_event()
+        event_old = fake.get_instance_event()
         event_old['generated'] = '2020-06-07T01:40:54.736337'
         response = client.post('/v1/event', json=[event_old])
 
@@ -83,7 +83,7 @@ class TestEvent:
         assert models.Spec.query.count() == 1
 
     def test_with_invalid_event_provided(self, client):
-        event = fake.get_event(event_type='foo.bar.exists')
+        event = fake.get_instance_event(event_type='foo.bar.exists')
         response = client.post('/v1/event', json=[event])
 
         assert response.status_code == 400
@@ -92,7 +92,7 @@ class TestEvent:
         assert models.Spec.query.count() == 0
 
     def test_with_ignored_event_provided(self, client, ignored_event):
-        event = fake.get_event(event_type=ignored_event)
+        event = fake.get_instance_event(event_type=ignored_event)
         response = client.post('/v1/event', json=[event])
 
         assert response.status_code == 202

atmosphere/tests/unit/conftest.py

@@ -30,8 +30,7 @@ from atmosphere.api import ingress
     'metrics.update',
     'scheduler.select_destinations.end',
     'server_group.add_member',
-    'service.create',
+    'service.create'
-    'volume.usage',
 ])
 def ignored_event(request):
     yield request.param

atmosphere/tests/unit/fake.py

@@ -20,7 +20,7 @@ from atmosphere import models
 from atmosphere import utils
 
 
-def get_event(resource_id='fake-uuid', event_type='compute.instance.exists'):
+def get_instance_event(resource_id='fake-uuid', event_type='compute.instance.exists'):
     return dict({
         'generated': '2020-06-07T01:42:54.736337',
         'event_type': event_type,
@@ -36,8 +36,32 @@ def get_event(resource_id='fake-uuid', event_type='compute.instance.exists'):
     })
 
 
-def get_normalized_event():
+def get_volume_event(resource_id='fake-uuid', event_type='volume.exists'):
-    event = get_event()
+    return dict({
+        'generated': '2020-06-07T01:42:54.736337',
+        'event_type': event_type,
+        'traits': [
+            ["service", 1, "volume.ironic-devstack"],
+            ["request_id", 1, "req-66a05c73-964e-4d12-a0c3-7d3b0d5801ce"],
+            ["project_id", 1, "a1da863b589642558b7a87f09840a565"],
+            ["user_id", 1, "a6db2097a75a4cf3b3b4336108017aae"],
+            ["tenant_id", 1, "a1da863b589642558b7a87f09840a565"],
+            ["resource_id", 1, "3c1e0499-7621-496c-bab4-59a7053f8b59"],
+            ["volume_type", 1, "7d233c12-d346-4948-8901-7afd5c5dd590"],
+            ["volume_size", 2, 1],
+            ["state", 1, "available"],
+            ["created_at", 4, "2021-03-26T00:36:28"],
+        ]
+    })
+
+
+def get_normalized_instance_event():
+    event = get_instance_event()
+    return utils.normalize_event(event)
+
+
+def get_normalized_volume_event():
+    event = get_volume_event()
     return utils.normalize_event(event)
 
 
@@ -53,6 +77,16 @@ def get_instance_spec(**kwargs):
     return models.InstanceSpec(**kwargs)
 
 
+def get_volume_spec(**kwargs):
+    if not kwargs:
+        kwargs = {
+            'volume_type': '7d233c12-d346-4948-8901-7afd5c5dd590',
+            'volume_size': 3,
+            'state': 'available'
+        }
+    return models.VolumeSpec(**kwargs)
+
+
 def get_resource_with_periods(number):
     resource = get_resource()
 

atmosphere/tests/unit/test_models.py

@@ -46,7 +46,7 @@ def _db(app):
 
 class GetOrCreateTestMixin:
     def test_with_existing_object(self):
-        event = fake.get_normalized_event()
+        event = fake.get_normalized_instance_event()
         assert self.MODEL.query_from_event(event).count() == 0
 
         old_object = self.MODEL.get_or_create(event)
@@ -58,14 +58,14 @@ class GetOrCreateTestMixin:
         assert old_object == new_object
 
     def test_with_no_existing_object(self):
-        event = fake.get_normalized_event()
+        event = fake.get_normalized_instance_event()
         assert self.MODEL.query_from_event(event).count() == 0
 
         new_object = self.MODEL.get_or_create(event)
         assert self.MODEL.query_from_event(event).count() == 1
 
     def test_with_object_created_during_creation(self):
-        event = fake.get_normalized_event()
+        event = fake.get_normalized_instance_event()
         assert self.MODEL.query_from_event(event).count() == 0
 
         def before_session_begin(*args, **kwargs):
@@ -89,7 +89,7 @@ class TestResource(GetOrCreateTestMixin):
         assert len(data) == 0
 
     def test_get_all_by_time_range_by_project(self):
-        event = fake.get_normalized_event()
+        event = fake.get_normalized_instance_event()
         resource = models.Resource.get_or_create(event)
 
         start = event['traits']['created_at'] - relativedelta(hours=+1)
@@ -105,7 +105,7 @@ class TestResource(GetOrCreateTestMixin):
         assert data[0].periods[0].seconds == 3600
 
     def test_get_all_by_time_range_with_resource_ended_before_start(self):
-        event = fake.get_normalized_event()
+        event = fake.get_normalized_instance_event()
         event['traits']['deleted_at'] = event['traits']['created_at'] + \
             relativedelta(hours=+1)
 
@@ -118,7 +118,7 @@ class TestResource(GetOrCreateTestMixin):
         assert len(data) == 0
 
     def test_get_all_by_time_range_with_resource_started_after_end(self):
-        event = fake.get_normalized_event()
+        event = fake.get_normalized_instance_event()
         resource = models.Resource.get_or_create(event)
 
         ended = event['traits']['created_at'] - relativedelta(hours=+1)
@@ -128,7 +128,7 @@ class TestResource(GetOrCreateTestMixin):
         assert len(data) == 0
 
     def test_get_all_by_time_range_with_active_resource_after_start(self):
-        event = fake.get_normalized_event()
+        event = fake.get_normalized_instance_event()
         resource = models.Resource.get_or_create(event)
 
         start = event['traits']['created_at'] - relativedelta(hours=+1)
@@ -139,7 +139,7 @@ class TestResource(GetOrCreateTestMixin):
         assert data[0].periods[0].seconds == 3600
 
     def test_get_all_by_time_range_with_active_resource_before_start(self):
-        event = fake.get_normalized_event()
+        event = fake.get_normalized_instance_event()
         resource = models.Resource.get_or_create(event)
 
         start = event['traits']['created_at'] + relativedelta(minutes=+30)
@@ -150,7 +150,7 @@ class TestResource(GetOrCreateTestMixin):
         assert data[0].periods[0].seconds == 1800
 
     def test_get_all_by_time_range_with_active_resource_after_end(self):
-        event = fake.get_normalized_event()
+        event = fake.get_normalized_instance_event()
         event['traits']['deleted_at'] = event['traits']['created_at'] + \
             relativedelta(hours=+1)
 
@@ -163,7 +163,7 @@ class TestResource(GetOrCreateTestMixin):
         assert len(data) == 0
 
     def test_get_all_by_time_range_with_resource_inside_range(self):
-        event = fake.get_normalized_event()
+        event = fake.get_normalized_instance_event()
         event['traits']['deleted_at'] = event['traits']['created_at'] + \
             relativedelta(minutes=+15)
 
@@ -177,7 +177,7 @@ class TestResource(GetOrCreateTestMixin):
         assert data[0].periods[0].seconds == 900
 
     def test_get_all_by_time_range_with_resource_with_multiple_periods(self):
-        event = fake.get_normalized_event()
+        event = fake.get_normalized_instance_event()
         event['traits']['created_at'] = event['traits']['created_at'] + \
             relativedelta(microseconds=0)
         models.Resource.get_or_create(event)
@@ -196,7 +196,7 @@ class TestResource(GetOrCreateTestMixin):
         assert data[0].periods[1].seconds == 2700
 
     def test_get_all_by_time_range_with_resource_with_one_active_period(self):
-        event = fake.get_normalized_event()
+        event = fake.get_normalized_instance_event()
         event['traits']['created_at'] = event['traits']['created_at'] + \
             relativedelta(microseconds=0)
         models.Resource.get_or_create(event)
@@ -215,7 +215,7 @@ class TestResource(GetOrCreateTestMixin):
         assert data[0].periods[0].seconds == 2700
 
     def test_from_event(self):
-        event = fake.get_normalized_event()
+        event = fake.get_normalized_instance_event()
         resource = models.Resource.from_event(event)
 
         assert resource.uuid == event['traits']['resource_id']
@@ -226,7 +226,7 @@ class TestResource(GetOrCreateTestMixin):
     def test_query_from_event(self, mock_query_property_getter):
         mock_filter_by = mock_query_property_getter.return_value.filter_by
 
-        event = fake.get_normalized_event()
+        event = fake.get_normalized_instance_event()
         query = models.Resource.query_from_event(event)
 
         mock_filter_by.assert_called_with(
@@ -235,7 +235,7 @@ class TestResource(GetOrCreateTestMixin):
         )
 
     def test_get_or_create_with_old_event(self):
-        event = fake.get_normalized_event()
+        event = fake.get_normalized_instance_event()
         new_object = models.Resource.get_or_create(event)
 
         old_event = event.copy()
@@ -246,7 +246,7 @@ class TestResource(GetOrCreateTestMixin):
             models.Resource.get_or_create(old_event)
 
     def test_get_or_create_refresh_updated_at(self):
-        event = fake.get_normalized_event()
+        event = fake.get_normalized_instance_event()
         old_object = models.Resource.get_or_create(event)
 
         new_event = event.copy()
@@ -259,14 +259,14 @@ class TestResource(GetOrCreateTestMixin):
         assert models.Resource.query_from_event(event).count() == 1
 
     def test_get_or_create_using_created_at(self):
-        event = fake.get_normalized_event()
+        event = fake.get_normalized_instance_event()
         resource = models.Resource.get_or_create(event)
 
         assert resource.get_open_period().started_at == \
             event['traits']['created_at']
 
     def test_get_or_create_using_deleted_event_only(self):
-        event = fake.get_normalized_event()
+        event = fake.get_normalized_instance_event()
         event['traits']['deleted_at'] = event['traits']['created_at'] + \
             relativedelta(hours=+1)
 
@@ -278,7 +278,7 @@ class TestResource(GetOrCreateTestMixin):
         assert resource.periods[0].seconds == 3600
 
     def test_get_or_create_using_multiple_deleted_events(self):
-        event = fake.get_normalized_event()
+        event = fake.get_normalized_instance_event()
         event['traits']['deleted_at'] = event['traits']['created_at'] + \
             relativedelta(hours=+1)
 
@@ -287,7 +287,7 @@ class TestResource(GetOrCreateTestMixin):
             models.Resource.get_or_create(event)
 
     def test_get_or_create_using_deleted_event(self):
-        event = fake.get_normalized_event()
+        event = fake.get_normalized_instance_event()
         old_resource = models.Resource.get_or_create(event)
 
         assert old_resource.get_open_period() is not None
@@ -305,7 +305,7 @@ class TestResource(GetOrCreateTestMixin):
         assert new_resource.periods[0].seconds == 3600
 
     def test_get_or_create_using_updated_spec(self):
-        event = fake.get_normalized_event()
+        event = fake.get_normalized_instance_event()
         old_resource = models.Resource.get_or_create(event)
 
         assert old_resource.get_open_period() is not None
@@ -323,7 +323,7 @@ class TestResource(GetOrCreateTestMixin):
         assert new_resource.get_open_period().started_at == event['generated']
 
     def test_get_or_create_using_same_spec(self):
-        event = fake.get_normalized_event()
+        event = fake.get_normalized_instance_event()
         old_resource = models.Resource.get_or_create(event)
 
         assert old_resource.get_open_period() is not None
@@ -429,33 +429,33 @@ class TestResource(GetOrCreateTestMixin):
 @pytest.mark.usefixtures("db_session")
 class TestInstance:
     def test_is_event_delete(self):
-        event = fake.get_normalized_event()
+        event = fake.get_normalized_instance_event()
         assert models.Instance.is_event_delete(event) == False
 
     def test_is_event_delete_for_actual_delete(self):
-        event = fake.get_normalized_event()
+        event = fake.get_normalized_instance_event()
         event['traits']['deleted_at'] = event['generated']
         assert models.Instance.is_event_delete(event) == True
 
     def test_is_event_ignored(self):
-        event = fake.get_normalized_event()
+        event = fake.get_normalized_instance_event()
         assert models.Instance.is_event_ignored(event) == False
 
     def test_is_event_ignored_for_pending_delete(self):
-        event = fake.get_normalized_event()
+        event = fake.get_normalized_instance_event()
         event['event_type'] = 'compute.instance.delete.start'
         event['traits']['state'] = 'deleted'
         assert models.Instance.is_event_ignored(event) == True
 
     def test_is_event_ignored_for_deleted(self):
-        event = fake.get_normalized_event()
+        event = fake.get_normalized_instance_event()
         event['event_type'] = 'compute.instance.delete.start'
         event['traits']['state'] = 'deleted'
         event['traits']['deleted_at'] = event['generated']
         assert models.Instance.is_event_ignored(event) == False
 
-    def test_get_or_create_has_no_deleted_period(self):
+    def _test_get_or_create_has_no_deleted_period(self, event, delete_event):
-        event = fake.get_normalized_event()
+        event = fake.get_normalized_instance_event()
         resource = models.Resource.get_or_create(event)
 
         assert resource.get_open_period() is not None
@@ -479,6 +479,59 @@ class TestInstance:
         assert len(resource.periods) == 1
 
 
+@pytest.mark.usefixtures("db_session")
+class TestVolume:
+    def test_is_event_delete(self):
+        event = fake.get_normalized_volume_event()
+        assert models.Volume.is_event_delete(event) == False
+
+    def test_is_event_delete_for_actual_delete(self):
+        event = fake.get_normalized_volume_event()
+        event['traits']['state'] = 'deleted'
+        assert models.Volume.is_event_delete(event) == True
+
+    def test_is_event_ignored(self):
+        event = fake.get_normalized_volume_event()
+        assert models.Volume.is_event_ignored(event) == False
+
+    def test_is_event_ignored_for_pending_delete(self):
+        event = fake.get_normalized_instance_event()
+        event['event_type'] = 'volume.delete.start'
+        event['traits']['state'] = 'deleting'
+        assert models.Volume.is_event_ignored(event) == True
+
+    def test_is_event_ignored_for_pending_create(self):
+        event = fake.get_normalized_instance_event()
+        event['event_type'] = 'volume.delete.start'
+        event['traits']['state'] = 'creating'
+        assert models.Volume.is_event_ignored(event) == True
+
+    def _test_get_or_create_has_no_deleted_period(self, event, delete_event):
+        event = fake.get_normalized_instance_event()
+        resource = models.Resource.get_or_create(event)
+
+        assert resource.get_open_period() is not None
+        assert len(resource.periods) == 1
+
+        event['event_type'] = 'volume.delete.start'
+        event['traits']['state'] = 'deleting'
+        event['generated'] += relativedelta(hours=+1)
+
+        with pytest.raises(exceptions.IgnoredEvent) as e:
+            models.Resource.get_or_create(event)
+
+        assert resource.get_open_period() is not None
+        assert len(resource.periods) == 1
+
+        event['event_type'] = 'volume.delete.end'
+        event['traits']['state'] = 'deleted'
+        event['generated'] += relativedelta(seconds=+2)
+        resource = models.Resource.get_or_create(event)
+
+        assert resource.get_open_period() is None
+        assert len(resource.periods) == 1
+
+
 @pytest.mark.usefixtures("db_session")
 class TestPeriod:
     def test_serialize_without_start(self):
@@ -541,7 +594,7 @@ class TestSpec(GetOrCreateTestMixin):
     MODEL = models.Spec
 
     def test_from_event(self):
-        event = fake.get_normalized_event()
+        event = fake.get_normalized_instance_event()
         spec = models.Spec.from_event(event)
 
         assert spec.instance_type == 'v1-standard-1'
@@ -551,7 +604,7 @@ class TestSpec(GetOrCreateTestMixin):
     def test_query_from_event(self, mock_query_property_getter):
         mock_filter_by = mock_query_property_getter.return_value.filter_by
 
-        event = fake.get_normalized_event()
+        event = fake.get_normalized_instance_event()
         query = models.Spec.query_from_event(event)
 
         mock_filter_by.assert_called_with(
@@ -569,3 +622,14 @@ class TestInstanceSpec:
             'instance_type': spec.instance_type,
             'state': spec.state,
         }
+
+@pytest.mark.usefixtures("db_session")
+class TestVolumeSpec:
+    def test_serialize(self):
+        spec = fake.get_volume_spec()
+
+        assert spec.serialize == {
+            'volume_type': spec.volume_type,
+            'volume_size': spec.volume_size,
+            'state': spec.state,
+        }

atmosphere/tests/unit/test_utils.py

@@ -24,8 +24,8 @@ from atmosphere import utils
 
 class TestNormalizeEvent:
     def test_normalize_event(self):
-        event = fake.get_event()
+        event = fake.get_instance_event()
-        event_expected = fake.get_event()
+        event_expected = fake.get_instance_event()
         event_expected.update({
             "generated": datetime.datetime(2020, 6, 7, 1, 42, 54, 736337),
             "traits": {
@@ -47,6 +47,10 @@ class TestModelTypeDetection:
         assert models.get_model_type_from_event('compute.instance.exists') == \
             (models.Instance, models.InstanceSpec)
 
+    def test_volume(self):
+        assert models.get_model_type_from_event('volume.create.start') == \
+            (models.Volume, models.VolumeSpec)
+
     def test_ignored_resource(self, ignored_event):
         with pytest.raises(exceptions.IgnoredEvent) as e:
             models.get_model_type_from_event(ignored_event)

contrib/event_definitions.yaml

@@ -31,7 +31,4 @@
     created_at:
       type: datetime
       fields: payload.created_at
-    deleted_at:
-      type: datetime
-      fields: payload.deleted_at
 ...