vexxhost/atmosphere
Code Issues Proposed changes

Add support for Cinder volumes

Browse Source 
Change-Id: I33401c86051293d5246856451571e4827be833ed
This commit is contained in:
Mohammed Naser 2021-03-25 22:05:35 -04:00
parent 582b9aa4fd
commit 75ab643e19
7 changed files with 200 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
atmosphere/models.py
View File

@ -46,6 +46,8 @@ def get_model_type_from_event(event):
"""get_model_type_from_event"""
if event.startswith('compute.instance'):
return Instance, InstanceSpec
if event.startswith('volume.'):
return Volume, VolumeSpec
if event.startswith('aggregate.'):
raise exceptions.IgnoredEvent
if event.startswith('compute_task.'):
@ -66,8 +68,6 @@ def get_model_type_from_event(event):
raise exceptions.IgnoredEvent
if event.startswith('service.'):
raise exceptions.IgnoredEvent
if event == 'volume.usage':
raise exceptions.IgnoredEvent
raise exceptions.UnsupportedEventType
@ -198,7 +198,11 @@ class Resource(db.Model, GetOrCreateMixin):
# If we're deleted, then we close the current period.
if resource.__class__.is_event_delete(event):
period.ended_at = event['traits']['deleted_at']
# NOTE(mnaser): Some resources don't have `deleted_at`, so we
# resort to the timestamp of the event instead.
period.ended_at = event['traits'].get(
'deleted_at', event['generated']
)
elif period.spec != spec:
period.ended_at = event['generated']
@ -265,6 +269,26 @@ class Instance(Resource):
return 'deleted_at' in event['traits']
class Volume(Resource):
"""Volume"""
STATE_ALLOW_LIST = ('available', 'deleted')
__mapper_args__ = {
'polymorphic_identity': 'OS::Cinder::Volume'
}
@classmethod
def is_event_ignored(cls, event):
"""is_event_ignored"""
return event['traits']['state'] not in cls.STATE_ALLOW_LIST
@classmethod
def is_event_delete(cls, event):
"""is_event_delete"""
return event['traits']['state'] == 'deleted'
class BigIntegerDateTime(TypeDecorator):
"""BigIntegerDateTime"""
@ -368,3 +392,30 @@ class InstanceSpec(Spec):
'instance_type': self.instance_type,
'state': self.state,
}
class VolumeSpec(Spec):
"""VolumeSpec"""
id = db.Column(db.Integer, db.ForeignKey('spec.id'), primary_key=True)
volume_type = db.Column(db.String(255))
volume_size = db.Column(db.String(255))
state = db.Column(db.String(255))
__table_args__ = (
db.UniqueConstraint('volume_type', 'volume_size', 'state'),
)
__mapper_args__ = {
'polymorphic_identity': 'OS::Cinder::Volume',
}
@property
def serialize(self):
"""Return object data in easily serializable format"""
return {
'volume_type': self.volume_type,
'volume_size': self.volume_size,
'state': self.state,
}

14
atmosphere/tests/unit/api/test_ingress.py
View File

@ -44,7 +44,7 @@ class TestEvent:
assert response.status_code == 400
def test_with_one_event_provided(self, client):
event = fake.get_event()
event = fake.get_instance_event()
response = client.post('/v1/event', json=[event])
assert response.status_code == 204
@ -53,8 +53,8 @@ class TestEvent:
assert models.Spec.query.count() == 1
def test_with_multiple_events_provided(self, client):
event_1 = fake.get_event(resource_id='fake-resource-1')
event_2 = fake.get_event(resource_id='fake-resource-2')
event_1 = fake.get_instance_event(resource_id='fake-resource-1')
event_2 = fake.get_instance_event(resource_id='fake-resource-2')
response = client.post('/v1/event', json=[event_1, event_2])
@ -64,7 +64,7 @@ class TestEvent:
assert models.Spec.query.count() == 1
def test_with_old_event_provided(self, client):
event_new = fake.get_event()
event_new = fake.get_instance_event()
event_new['generated'] = '2020-06-07T01:42:54.736337'
response = client.post('/v1/event', json=[event_new])
@ -73,7 +73,7 @@ class TestEvent:
assert models.Period.query.count() == 1
assert models.Spec.query.count() == 1
event_old = fake.get_event()
event_old = fake.get_instance_event()
event_old['generated'] = '2020-06-07T01:40:54.736337'
response = client.post('/v1/event', json=[event_old])
@ -83,7 +83,7 @@ class TestEvent:
assert models.Spec.query.count() == 1
def test_with_invalid_event_provided(self, client):
event = fake.get_event(event_type='foo.bar.exists')
event = fake.get_instance_event(event_type='foo.bar.exists')
response = client.post('/v1/event', json=[event])
assert response.status_code == 400
@ -92,7 +92,7 @@ class TestEvent:
assert models.Spec.query.count() == 0
def test_with_ignored_event_provided(self, client, ignored_event):
event = fake.get_event(event_type=ignored_event)
event = fake.get_instance_event(event_type=ignored_event)
response = client.post('/v1/event', json=[event])
assert response.status_code == 202

3
atmosphere/tests/unit/conftest.py
View File

@ -30,8 +30,7 @@ from atmosphere.api import ingress
'metrics.update',
'scheduler.select_destinations.end',
'server_group.add_member',
'service.create',
'volume.usage',
'service.create'
])
def ignored_event(request):
yield request.param

40
atmosphere/tests/unit/fake.py
View File

@ -20,7 +20,7 @@ from atmosphere import models
from atmosphere import utils
def get_event(resource_id='fake-uuid', event_type='compute.instance.exists'):
def get_instance_event(resource_id='fake-uuid', event_type='compute.instance.exists'):
return dict({
'generated': '2020-06-07T01:42:54.736337',
'event_type': event_type,
@ -36,8 +36,32 @@ def get_event(resource_id='fake-uuid', event_type='compute.instance.exists'):
})
def get_normalized_event():
event = get_event()
def get_volume_event(resource_id='fake-uuid', event_type='volume.exists'):
return dict({
'generated': '2020-06-07T01:42:54.736337',
'event_type': event_type,
'traits': [
["service", 1, "volume.ironic-devstack"],
["request_id", 1, "req-66a05c73-964e-4d12-a0c3-7d3b0d5801ce"],
["project_id", 1, "a1da863b589642558b7a87f09840a565"],
["user_id", 1, "a6db2097a75a4cf3b3b4336108017aae"],
["tenant_id", 1, "a1da863b589642558b7a87f09840a565"],
["resource_id", 1, "3c1e0499-7621-496c-bab4-59a7053f8b59"],
["volume_type", 1, "7d233c12-d346-4948-8901-7afd5c5dd590"],
["volume_size", 2, 1],
["state", 1, "available"],
["created_at", 4, "2021-03-26T00:36:28"],
]
})
def get_normalized_instance_event():
event = get_instance_event()
return utils.normalize_event(event)
def get_normalized_volume_event():
event = get_volume_event()
return utils.normalize_event(event)
@ -53,6 +77,16 @@ def get_instance_spec(**kwargs):
return models.InstanceSpec(**kwargs)
def get_volume_spec(**kwargs):
if not kwargs:
kwargs = {
'volume_type': '7d233c12-d346-4948-8901-7afd5c5dd590',
'volume_size': 3,
'state': 'available'
}
return models.VolumeSpec(**kwargs)
def get_resource_with_periods(number):
resource = get_resource()

126
atmosphere/tests/unit/test_models.py
View File

@ -46,7 +46,7 @@ def _db(app):
class GetOrCreateTestMixin:
def test_with_existing_object(self):
event = fake.get_normalized_event()
event = fake.get_normalized_instance_event()
assert self.MODEL.query_from_event(event).count() == 0
old_object = self.MODEL.get_or_create(event)
@ -58,14 +58,14 @@ class GetOrCreateTestMixin:
assert old_object == new_object
def test_with_no_existing_object(self):
event = fake.get_normalized_event()
event = fake.get_normalized_instance_event()
assert self.MODEL.query_from_event(event).count() == 0
new_object = self.MODEL.get_or_create(event)
assert self.MODEL.query_from_event(event).count() == 1
def test_with_object_created_during_creation(self):
event = fake.get_normalized_event()
event = fake.get_normalized_instance_event()
assert self.MODEL.query_from_event(event).count() == 0
def before_session_begin(*args, **kwargs):
@ -89,7 +89,7 @@ class TestResource(GetOrCreateTestMixin):
assert len(data) == 0
def test_get_all_by_time_range_by_project(self):
event = fake.get_normalized_event()
event = fake.get_normalized_instance_event()
resource = models.Resource.get_or_create(event)
start = event['traits']['created_at'] - relativedelta(hours=+1)
@ -105,7 +105,7 @@ class TestResource(GetOrCreateTestMixin):
assert data[0].periods[0].seconds == 3600
def test_get_all_by_time_range_with_resource_ended_before_start(self):
event = fake.get_normalized_event()
event = fake.get_normalized_instance_event()
event['traits']['deleted_at'] = event['traits']['created_at'] + \
relativedelta(hours=+1)
@ -118,7 +118,7 @@ class TestResource(GetOrCreateTestMixin):
assert len(data) == 0
def test_get_all_by_time_range_with_resource_started_after_end(self):
event = fake.get_normalized_event()
event = fake.get_normalized_instance_event()
resource = models.Resource.get_or_create(event)
ended = event['traits']['created_at'] - relativedelta(hours=+1)
@ -128,7 +128,7 @@ class TestResource(GetOrCreateTestMixin):
assert len(data) == 0
def test_get_all_by_time_range_with_active_resource_after_start(self):
event = fake.get_normalized_event()
event = fake.get_normalized_instance_event()
resource = models.Resource.get_or_create(event)
start = event['traits']['created_at'] - relativedelta(hours=+1)
@ -139,7 +139,7 @@ class TestResource(GetOrCreateTestMixin):
assert data[0].periods[0].seconds == 3600
def test_get_all_by_time_range_with_active_resource_before_start(self):
event = fake.get_normalized_event()
event = fake.get_normalized_instance_event()
resource = models.Resource.get_or_create(event)
start = event['traits']['created_at'] + relativedelta(minutes=+30)
@ -150,7 +150,7 @@ class TestResource(GetOrCreateTestMixin):
assert data[0].periods[0].seconds == 1800
def test_get_all_by_time_range_with_active_resource_after_end(self):
event = fake.get_normalized_event()
event = fake.get_normalized_instance_event()
event['traits']['deleted_at'] = event['traits']['created_at'] + \
relativedelta(hours=+1)
@ -163,7 +163,7 @@ class TestResource(GetOrCreateTestMixin):
assert len(data) == 0
def test_get_all_by_time_range_with_resource_inside_range(self):
event = fake.get_normalized_event()
event = fake.get_normalized_instance_event()
event['traits']['deleted_at'] = event['traits']['created_at'] + \
relativedelta(minutes=+15)
@ -177,7 +177,7 @@ class TestResource(GetOrCreateTestMixin):
assert data[0].periods[0].seconds == 900
def test_get_all_by_time_range_with_resource_with_multiple_periods(self):
event = fake.get_normalized_event()
event = fake.get_normalized_instance_event()
event['traits']['created_at'] = event['traits']['created_at'] + \
relativedelta(microseconds=0)
models.Resource.get_or_create(event)
@ -196,7 +196,7 @@ class TestResource(GetOrCreateTestMixin):
assert data[0].periods[1].seconds == 2700
def test_get_all_by_time_range_with_resource_with_one_active_period(self):
event = fake.get_normalized_event()
event = fake.get_normalized_instance_event()
event['traits']['created_at'] = event['traits']['created_at'] + \
relativedelta(microseconds=0)
models.Resource.get_or_create(event)
@ -215,7 +215,7 @@ class TestResource(GetOrCreateTestMixin):
assert data[0].periods[0].seconds == 2700
def test_from_event(self):
event = fake.get_normalized_event()
event = fake.get_normalized_instance_event()
resource = models.Resource.from_event(event)
assert resource.uuid == event['traits']['resource_id']
@ -226,7 +226,7 @@ class TestResource(GetOrCreateTestMixin):
def test_query_from_event(self, mock_query_property_getter):
mock_filter_by = mock_query_property_getter.return_value.filter_by
event = fake.get_normalized_event()
event = fake.get_normalized_instance_event()
query = models.Resource.query_from_event(event)
mock_filter_by.assert_called_with(
@ -235,7 +235,7 @@ class TestResource(GetOrCreateTestMixin):
)
def test_get_or_create_with_old_event(self):
event = fake.get_normalized_event()
event = fake.get_normalized_instance_event()
new_object = models.Resource.get_or_create(event)
old_event = event.copy()
@ -246,7 +246,7 @@ class TestResource(GetOrCreateTestMixin):
models.Resource.get_or_create(old_event)
def test_get_or_create_refresh_updated_at(self):
event = fake.get_normalized_event()
event = fake.get_normalized_instance_event()
old_object = models.Resource.get_or_create(event)
new_event = event.copy()
@ -259,14 +259,14 @@ class TestResource(GetOrCreateTestMixin):
assert models.Resource.query_from_event(event).count() == 1
def test_get_or_create_using_created_at(self):
event = fake.get_normalized_event()
event = fake.get_normalized_instance_event()
resource = models.Resource.get_or_create(event)
assert resource.get_open_period().started_at == \
event['traits']['created_at']
def test_get_or_create_using_deleted_event_only(self):
event = fake.get_normalized_event()
event = fake.get_normalized_instance_event()
event['traits']['deleted_at'] = event['traits']['created_at'] + \
relativedelta(hours=+1)
@ -278,7 +278,7 @@ class TestResource(GetOrCreateTestMixin):
assert resource.periods[0].seconds == 3600
def test_get_or_create_using_multiple_deleted_events(self):
event = fake.get_normalized_event()
event = fake.get_normalized_instance_event()
event['traits']['deleted_at'] = event['traits']['created_at'] + \
relativedelta(hours=+1)
@ -287,7 +287,7 @@ class TestResource(GetOrCreateTestMixin):
models.Resource.get_or_create(event)
def test_get_or_create_using_deleted_event(self):
event = fake.get_normalized_event()
event = fake.get_normalized_instance_event()
old_resource = models.Resource.get_or_create(event)
assert old_resource.get_open_period() is not None
@ -305,7 +305,7 @@ class TestResource(GetOrCreateTestMixin):
assert new_resource.periods[0].seconds == 3600
def test_get_or_create_using_updated_spec(self):
event = fake.get_normalized_event()
event = fake.get_normalized_instance_event()
old_resource = models.Resource.get_or_create(event)
assert old_resource.get_open_period() is not None
@ -323,7 +323,7 @@ class TestResource(GetOrCreateTestMixin):
assert new_resource.get_open_period().started_at == event['generated']
def test_get_or_create_using_same_spec(self):
event = fake.get_normalized_event()
event = fake.get_normalized_instance_event()
old_resource = models.Resource.get_or_create(event)
assert old_resource.get_open_period() is not None
@ -429,33 +429,33 @@ class TestResource(GetOrCreateTestMixin):
@pytest.mark.usefixtures("db_session")
class TestInstance:
def test_is_event_delete(self):
event = fake.get_normalized_event()
event = fake.get_normalized_instance_event()
assert models.Instance.is_event_delete(event) == False
def test_is_event_delete_for_actual_delete(self):
event = fake.get_normalized_event()
event = fake.get_normalized_instance_event()
event['traits']['deleted_at'] = event['generated']
assert models.Instance.is_event_delete(event) == True
def test_is_event_ignored(self):
event = fake.get_normalized_event()
event = fake.get_normalized_instance_event()
assert models.Instance.is_event_ignored(event) == False
def test_is_event_ignored_for_pending_delete(self):
event = fake.get_normalized_event()
event = fake.get_normalized_instance_event()
event['event_type'] = 'compute.instance.delete.start'
event['traits']['state'] = 'deleted'
assert models.Instance.is_event_ignored(event) == True
def test_is_event_ignored_for_deleted(self):
event = fake.get_normalized_event()
event = fake.get_normalized_instance_event()
event['event_type'] = 'compute.instance.delete.start'
event['traits']['state'] = 'deleted'
event['traits']['deleted_at'] = event['generated']
assert models.Instance.is_event_ignored(event) == False
def test_get_or_create_has_no_deleted_period(self):
event = fake.get_normalized_event()
def _test_get_or_create_has_no_deleted_period(self, event, delete_event):
event = fake.get_normalized_instance_event()
resource = models.Resource.get_or_create(event)
assert resource.get_open_period() is not None
@ -479,6 +479,59 @@ class TestInstance:
assert len(resource.periods) == 1
@pytest.mark.usefixtures("db_session")
class TestVolume:
def test_is_event_delete(self):
event = fake.get_normalized_volume_event()
assert models.Volume.is_event_delete(event) == False
def test_is_event_delete_for_actual_delete(self):
event = fake.get_normalized_volume_event()
event['traits']['state'] = 'deleted'
assert models.Volume.is_event_delete(event) == True
def test_is_event_ignored(self):
event = fake.get_normalized_volume_event()
assert models.Volume.is_event_ignored(event) == False
def test_is_event_ignored_for_pending_delete(self):
event = fake.get_normalized_instance_event()
event['event_type'] = 'volume.delete.start'
event['traits']['state'] = 'deleting'
assert models.Volume.is_event_ignored(event) == True
def test_is_event_ignored_for_pending_create(self):
event = fake.get_normalized_instance_event()
event['event_type'] = 'volume.delete.start'
event['traits']['state'] = 'creating'
assert models.Volume.is_event_ignored(event) == True
def _test_get_or_create_has_no_deleted_period(self, event, delete_event):
event = fake.get_normalized_instance_event()
resource = models.Resource.get_or_create(event)
assert resource.get_open_period() is not None
assert len(resource.periods) == 1
event['event_type'] = 'volume.delete.start'
event['traits']['state'] = 'deleting'
event['generated'] += relativedelta(hours=+1)
with pytest.raises(exceptions.IgnoredEvent) as e:
models.Resource.get_or_create(event)
assert resource.get_open_period() is not None
assert len(resource.periods) == 1
event['event_type'] = 'volume.delete.end'
event['traits']['state'] = 'deleted'
event['generated'] += relativedelta(seconds=+2)
resource = models.Resource.get_or_create(event)
assert resource.get_open_period() is None
assert len(resource.periods) == 1
@pytest.mark.usefixtures("db_session")
class TestPeriod:
def test_serialize_without_start(self):
@ -541,7 +594,7 @@ class TestSpec(GetOrCreateTestMixin):
MODEL = models.Spec
def test_from_event(self):
event = fake.get_normalized_event()
event = fake.get_normalized_instance_event()
spec = models.Spec.from_event(event)
assert spec.instance_type == 'v1-standard-1'
@ -551,7 +604,7 @@ class TestSpec(GetOrCreateTestMixin):
def test_query_from_event(self, mock_query_property_getter):
mock_filter_by = mock_query_property_getter.return_value.filter_by
event = fake.get_normalized_event()
event = fake.get_normalized_instance_event()
query = models.Spec.query_from_event(event)
mock_filter_by.assert_called_with(
@ -569,3 +622,14 @@ class TestInstanceSpec:
'instance_type': spec.instance_type,
'state': spec.state,
}
@pytest.mark.usefixtures("db_session")
class TestVolumeSpec:
def test_serialize(self):
spec = fake.get_volume_spec()
assert spec.serialize == {
'volume_type': spec.volume_type,
'volume_size': spec.volume_size,
'state': spec.state,
}

8
atmosphere/tests/unit/test_utils.py
View File

@ -24,8 +24,8 @@ from atmosphere import utils
class TestNormalizeEvent:
def test_normalize_event(self):
event = fake.get_event()
event_expected = fake.get_event()
event = fake.get_instance_event()
event_expected = fake.get_instance_event()
event_expected.update({
"generated": datetime.datetime(2020, 6, 7, 1, 42, 54, 736337),
"traits": {
@ -47,6 +47,10 @@ class TestModelTypeDetection:
assert models.get_model_type_from_event('compute.instance.exists') == \
(models.Instance, models.InstanceSpec)
def test_volume(self):
assert models.get_model_type_from_event('volume.create.start') == \
(models.Volume, models.VolumeSpec)
def test_ignored_resource(self, ignored_event):
with pytest.raises(exceptions.IgnoredEvent) as e:
models.get_model_type_from_event(ignored_event)

3
contrib/event_definitions.yaml
View File

@ -31,7 +31,4 @@
created_at:
type: datetime
fields: payload.created_at
deleted_at:
type: datetime
fields: payload.deleted_at
...