From 052f17eb209081a048f7c7970b707fa8f1c41da8 Mon Sep 17 00:00:00 2001 From: sbauza Date: Wed, 2 Oct 2013 14:37:49 +0200 Subject: [PATCH] DB API and models for supporting Compute Hosts Reservations As per our design doc for Hosts reservations [1], Climate needs to be added three new models called ComputeHost, ComputeHostExtraCapabilities and ComputeHostReservation * ComputeHost will store all info concerning hosts managed by Climate and will keep tracability on if they are free for a lease or not * ComputeHostExtraCapabilities will allow admin user to add extra info for each host to add to Climate like GPU or SSD which are not yet visible thru Nova-Compute * ComputeHostReservation will store all reservation details when creating a lease, needed by Manager when starting the leases Implements bp:host-db-api [1] : https://docs.google.com/drawings/d/1-_DOB65LsSGR4JUaPN525bWAg1yhvEv3KNcHfmuLvYw/edit Change-Id: Ib8f289064a9621372908be1d44db92f3719b24e1 --- climate/db/api.py | 117 +++++++- climate/db/sqlalchemy/api.py | 250 ++++++++++++++++++ climate/db/sqlalchemy/models.py | 69 +++++ .../db/sqlalchemy/test_sqlalchemy_api.py | 228 ++++++++++++++++ 4 files changed, 662 insertions(+), 2 deletions(-) diff --git a/climate/db/api.py b/climate/db/api.py index b18069bc..2045b481 100644 --- a/climate/db/api.py +++ b/climate/db/api.py @@ -112,9 +112,15 @@ def reservation_create(reservation_values): @to_dict -def reservation_get_all_by_lease(lease_id): +def reservation_get_all_by_lease_id(lease_id): """Return all reservations belongs to specific lease.""" - return IMPL.reservation_get_all_by_lease(lease_id) + return IMPL.reservation_get_all_by_lease_id(lease_id) + + +@to_dict +def reservation_get_all_by_values(**kwargs): + """Returns all entries filtered by col=value.""" + return IMPL.reservation_get_all_by_values(**kwargs) @to_dict @@ -221,3 +227,110 @@ def event_destroy(event_id): def event_update(event_id, event_values): """Update event or raise if not exists.""" IMPL.event_update(event_id, event_values) + + +# Host reservations + +def host_reservation_create(host_reservation_values): + """Create a host reservation from the values.""" + return IMPL.host_reservation_create(host_reservation_values) + + +@to_dict +def host_reservation_get_by_reservation_id(reservation_id): + """Return host reservation belonging to specific reservation.""" + return IMPL.host_reservation_get_by_reservation_id(reservation_id) + + +@to_dict +def host_reservation_get(host_reservation_id): + """Return specific host reservation.""" + return IMPL.host_reservation_get(host_reservation_id) + + +@to_dict +def host_reservation_get_all(): + """Return all hosts reservations.""" + return IMPL.host_reservation_get_all() + + +def host_reservation_destroy(host_reservation_id): + """Delete specific host reservation.""" + IMPL.host_reservation_destroy(host_reservation_id) + + +def host_reservation_update(host_reservation_id, + host_reservation_values): + """Update host reservation.""" + IMPL.host_reservation_update(host_reservation_id, + host_reservation_values) + + +# Compute Hosts + +def host_create(values): + """Create a Compute host from the values.""" + return IMPL.host_create(values) + + +@to_dict +def host_get(host_id): + """Return a specific Compute host.""" + return IMPL.host_get(host_id) + + +@to_dict +def host_list(): + """Return a list of events.""" + return IMPL.host_list() + + +@to_dict +def host_get_all_by_filters(filters): + """Returns Compute hosts filtered by name of the field.""" + return IMPL.host_get_all_by_filters(filters) + + +@to_dict +def host_get_all_by_queries(queries): + """Returns hosts filtered by an array of queries.""" + return IMPL.host_get_all_by_queries(queries) + + +def host_destroy(host_id): + """Delete specific Compute host.""" + IMPL.host_destroy(host_id) + + +def host_update(host_id, values): + """Update Compute host.""" + IMPL.host_update(host_id, values) + + +# ComputeHostExtraCapabilities + +def host_extra_capability_create(values): + """Create a Host ExtraCapability from the values.""" + return IMPL.host_extra_capability_create(values) + + +@to_dict +def host_extra_capability_get(host_extra_capability_id): + """Return a specific Host Extracapability.""" + return IMPL.host_extra_capability_get(host_extra_capability_id) + + +@to_dict +def host_extra_capability_get_all_per_host(host_id): + """Return all extra_capabilities belonging to a specific Compute host.""" + return IMPL.host_extra_capability_get_all_per_host(host_id) + + +def host_extra_capability_destroy(host_extra_capability_id): + """Delete specific host ExtraCapability.""" + IMPL.host_extra_capability_destroy(host_extra_capability_id) + + +def host_extra_capability_update(host_extra_capability_id, values): + """Update specific host ExtraCapability.""" + IMPL.host_extra_capability_update(host_extra_capability_id, values) diff --git a/climate/db/sqlalchemy/api.py b/climate/db/sqlalchemy/api.py index 2ca0d382..04f3982a 100644 --- a/climate/db/sqlalchemy/api.py +++ b/climate/db/sqlalchemy/api.py @@ -153,6 +153,17 @@ def reservation_get_all_by_lease_id(lease_id): return reservations.all() +def reservation_get_all_by_values(**kwargs): + """Returns all entries filtered by col=value.""" + + reservation_query = model_query(models.Reservation, get_session()) + for name, value in kwargs.items(): + column = getattr(models.Reservation, name, None) + if column: + reservation_query = reservation_query.filter(column == value) + return reservation_query.all() + + def reservation_create(values): values = values.copy() reservation = models.Reservation() @@ -357,3 +368,242 @@ def event_destroy(event_id): raise RuntimeError("Event not found!") session.delete(event) + + +#ComputeHostReservation +def _host_reservation_get(session, host_reservation_id): + query = model_query(models.ComputeHostReservation, session) + return query.filter_by(id=host_reservation_id).first() + + +def host_reservation_get(host_reservation_id): + return _host_reservation_get(get_session(), + host_reservation_id) + + +def host_reservation_get_all(): + query = model_query(models.ComputeHostReservation, get_session()) + return query.all() + + +def _host_reservation_get_by_reservation_id(session, reservation_id): + query = model_query(models.ComputeHostReservation, session) + return query.filter_by(reservation_id=reservation_id).first() + + +def host_reservation_get_by_reservation_id(reservation_id): + return _host_reservation_get_by_reservation_id(get_session(), + reservation_id) + + +def host_reservation_create(values): + values = values.copy() + host_reservation = models.ComputeHostReservation() + host_reservation.update(values) + + session = get_session() + with session.begin(): + try: + host_reservation.save(session=session) + except db_exc.DBDuplicateEntry as e: + # raise exception about duplicated columns (e.columns) + raise RuntimeError("DBDuplicateEntry: %s" % e.columns) + + return host_reservation_get(host_reservation.id) + + +def host_reservation_update(host_reservation_id, values): + session = get_session() + + with session.begin(): + host_reservation = _host_reservation_get(session, + host_reservation_id) + host_reservation.update(values) + host_reservation.save(session=session) + + return host_reservation_get(host_reservation_id) + + +def host_reservation_destroy(host_reservation_id): + session = get_session() + with session.begin(): + host_reservation = _host_reservation_get(session, + host_reservation_id) + + if not host_reservation: + # raise not found error + raise RuntimeError("Host Reservation not found!") + + session.delete(host_reservation) + + +#ComputeHost +def _host_get(session, host_id): + query = model_query(models.ComputeHost, session) + return query.filter_by(id=host_id).first() + + +def _host_get_all(session): + query = model_query(models.ComputeHost, session) + return query + + +def host_get(host_id): + return _host_get(get_session(), host_id) + + +def host_list(): + return model_query(models.ComputeHost, get_session()).all() + + +def host_get_all_by_filters(filters): + """Returns hosts filtered by name of the field.""" + + hosts_query = _host_get_all(get_session()) + + if 'status' in filters: + hosts_query = hosts_query.\ + filter(models.ComputeHost.status == filters['status']) + + return hosts_query.all() + + +def host_get_all_by_queries(queries): + """Returns hosts filtered by an array of queries. + + :param queries: array of queries "key op value" where op can be + http://docs.sqlalchemy.org/en/rel_0_7/core/expression_api.html + #sqlalchemy.sql.operators.ColumnOperators + """ + + hosts_query = model_query(models.ComputeHost, get_session()) + + oper = dict({'<': 'lt', '>': 'gt', '<=': 'le', '>=': 'ge', '==': 'eq', + '!=': 'ne'}) + for query in queries: + try: + key, op, value = query.split(' ', 3) + except ValueError: + raise RuntimeError('Invalid filter: %s' % query) + column = getattr(models.ComputeHost, key, None) + if not column: + raise RuntimeError('Invalid filter column: %s' % key) + if op == 'in': + filt = column.in_(value.split(',')) + else: + if op in oper: + op = oper[op] + try: + attr = filter(lambda e: hasattr(column, e % op), + ['%s', '%s_', '__%s__'])[0] % op + except IndexError: + raise RuntimeError('Invalid filter operator: %s' % op) + if value == 'null': + value = None + filt = getattr(column, attr)(value) + hosts_query = hosts_query.filter(filt) + + return hosts_query.all() + + +def host_create(values): + values = values.copy() + host = models.ComputeHost() + host.update(values) + + session = get_session() + with session.begin(): + try: + host.save(session=session) + except db_exc.DBDuplicateEntry as e: + # raise exception about duplicated columns (e.columns) + raise RuntimeError("DBDuplicateEntry: %s" % e.columns) + + return host_get(host.id) + + +def host_update(host_id, values): + session = get_session() + + with session.begin(): + host = _host_get(session, host_id) + host.update(values) + host.save(session=session) + + return host_get(host_id) + + +def host_destroy(host_id): + session = get_session() + with session.begin(): + host = _host_get(session, host_id) + + if not host: + # raise not found error + raise RuntimeError("Host not found!") + + session.delete(host) + + +#ComputeHostExtraCapability +def _host_extra_capability_get(session, host_extra_capability_id): + query = model_query(models.ComputeHostExtraCapability, session) + return query.filter_by(id=host_extra_capability_id).first() + + +def host_extra_capability_get(host_extra_capability_id): + return _host_extra_capability_get(get_session(), + host_extra_capability_id) + + +def _host_extra_capability_get_all_per_host(session, host_id): + query = model_query(models.ComputeHostExtraCapability, session) + return query.filter_by(computehost_id=host_id).all() + + +def host_extra_capability_get_all_per_host(host_id): + return _host_extra_capability_get_all_per_host(get_session(), + host_id) + + +def host_extra_capability_create(values): + values = values.copy() + host_extra_capability = models.ComputeHostExtraCapability() + host_extra_capability.update(values) + + session = get_session() + with session.begin(): + try: + host_extra_capability.save(session=session) + except db_exc.DBDuplicateEntry as e: + # raise exception about duplicated columns (e.columns) + raise RuntimeError("DBDuplicateEntry: %s" % e.columns) + + return host_extra_capability_get(host_extra_capability.id) + + +def host_extra_capability_update(host_extra_capability_id, values): + session = get_session() + + with session.begin(): + host_extra_capability = \ + _host_extra_capability_get(session, + host_extra_capability_id) + host_extra_capability.update(values) + host_extra_capability.save(session=session) + + return host_extra_capability_get(host_extra_capability_id) + + +def host_extra_capability_destroy(host_extra_capability_id): + session = get_session() + with session.begin(): + host_extra_capability = \ + _host_extra_capability_get(session, + host_extra_capability_id) + + if not host_extra_capability: + # raise not found error + raise RuntimeError("Host Extracapability not found!") + + session.delete(host_extra_capability) diff --git a/climate/db/sqlalchemy/models.py b/climate/db/sqlalchemy/models.py index aec4fe3c..b631fe1c 100644 --- a/climate/db/sqlalchemy/models.py +++ b/climate/db/sqlalchemy/models.py @@ -14,6 +14,7 @@ # limitations under the License. import sqlalchemy as sa +from sqlalchemy.dialects.mysql import MEDIUMTEXT from sqlalchemy.orm import relationship from climate.db.sqlalchemy import model_base as mb @@ -26,6 +27,10 @@ def _generate_unicode_uuid(): return unicode(uuidutils.generate_uuid()) +def MediumText(): + return sa.Text().with_variant(MEDIUMTEXT(), 'mysql') + + def _id_column(): return sa.Column(sa.String(36), primary_key=True, @@ -72,6 +77,11 @@ class Reservation(mb.ClimateBase): resource_id = sa.Column(sa.String(36)) resource_type = sa.Column(sa.String(66)) status = sa.Column(sa.String(13)) + computehost_reservations = relationship('ComputeHostReservation', + uselist=False, + cascade="all,delete", + backref='reservation', + lazy='joined') def to_dict(self): return super(Reservation, self).to_dict() @@ -90,3 +100,62 @@ class Event(mb.ClimateBase): def to_dict(self): return super(Event, self).to_dict() + + +class ComputeHostReservation(mb.ClimateBase): + """Specifies resources asked by reservation from + Compute Host Reservation API. + """ + + __tablename__ = 'computehost_reservations' + + id = _id_column() + reservation_id = sa.Column(sa.String(36), sa.ForeignKey('reservations.id')) + resource_properties = sa.Column(MediumText()) + count_range = sa.Column(sa.String(36)) + hypervisor_properties = sa.Column(MediumText()) + status = sa.Column(sa.String(13)) + + def to_dict(self): + return super(ComputeHostReservation, self).to_dict() + + +class ComputeHost(mb.ClimateBase): + """Specifies resources asked by reservation from + Compute Host Reservation API. + """ + + __tablename__ = 'computehosts' + + id = _id_column() + vcpu = sa.Column(sa.Integer, nullable=False) + cpu_info = sa.Column(MediumText(), nullable=False) + hypervisor_type = sa.Column(MediumText(), nullable=False) + hypervisor_version = sa.Column(sa.Integer, nullable=False) + hypervisor_hostname = sa.Column(sa.String(255), nullable=True) + memory_mb = sa.Column(sa.Integer, nullable=False) + local_gb = sa.Column(sa.Integer, nullable=False) + status = sa.Column(sa.String(13)) + computehost_extra_capabilities = relationship('ComputeHostExtraCapability', + cascade="all,delete", + backref='computehost', + lazy='joined') + + def to_dict(self): + return super(ComputeHost, self).to_dict() + + +class ComputeHostExtraCapability(mb.ClimateBase): + """Allows to define extra capabilities per administrator request for each + Compute Host added. + """ + + __tablename__ = 'computehost_extra_capabilities' + + id = _id_column() + computehost_id = sa.Column(sa.String(36), sa.ForeignKey('computehosts.id')) + capability_name = sa.Column(sa.String(64), nullable=False) + capability_value = sa.Column(MediumText(), nullable=False) + + def to_dict(self): + return super(ComputeHostExtraCapability, self).to_dict() diff --git a/climate/tests/db/sqlalchemy/test_sqlalchemy_api.py b/climate/tests/db/sqlalchemy/test_sqlalchemy_api.py index 80ca02e7..7498abce 100644 --- a/climate/tests/db/sqlalchemy/test_sqlalchemy_api.py +++ b/climate/tests/db/sqlalchemy/test_sqlalchemy_api.py @@ -92,6 +92,43 @@ def _create_physical_lease(values=_get_fake_phys_lease_values(), return db_api.lease_create(values) +def _get_fake_host_reservation_values(id=_get_fake_random_uuid(), + reservation_id=_get_fake_random_uuid()): + return {'id': id, + 'reservation_id': reservation_id, + 'resource_properties': "fake", + 'hypervisor_properties': "fake"} + + +def _get_fake_cpu_info(): + return str({'vendor': 'Intel', + 'model': 'Westmere', + 'arch': 'x86_64', + 'features': ['rdtscp', 'pdpe1gb', 'hypervisor', 'vmx', 'ss', + 'vme'], + 'topology': {'cores': 1, 'threads': 1, 'sockets': 2}}) + + +def _get_fake_host_values(id=_get_fake_random_uuid(), mem=8192, disk=10): + return {'id': id, + 'vcpu': 1, + 'cpu_info': _get_fake_cpu_info(), + 'hypervisor_type': 'QEMU', + 'hypervisor_version': 1000, + 'memory_mb': mem, + 'local_gb': disk, + 'status': 'free' + } + + +def _get_fake_host_extra_capabilities(id=_get_fake_random_uuid(), + computehost_id=_get_fake_random_uuid()): + return {'id': id, + 'computehost_id': computehost_id, + 'capability_name': 'vgpu', + 'capability_value': '2'} + + class SQLAlchemyDBApiTestCase(tests.DBTestCase): """Test case for SQLAlchemy DB API.""" @@ -182,3 +219,194 @@ class SQLAlchemyDBApiTestCase(tests.DBTestCase): values={'start_date': _get_datetime('2014-02-01 00:00')}) self.assertEqual(_get_datetime('2014-02-01 00:00'), result['start_date']) + + # Reservations + + def test_create_reservation(self): + """Create a reservation and verify that all tables have been + populated. + """ + + result = db_api.reservation_create(_get_fake_phys_reservation_values()) + self.assertEqual(result['lease_id'], + _get_fake_phys_reservation_values() + ['lease_id']) + + def test_reservation_get_all_by_values(self): + """Create two reservations and verify that we can find reservation per + resource_id or resource_type. + """ + db_api.reservation_create(_get_fake_phys_reservation_values()) + db_api.reservation_create(_get_fake_virt_reservation_values()) + self.assertEqual(2, len(db_api.reservation_get_all_by_values())) + self.assertEqual(1, len(db_api.reservation_get_all_by_values( + resource_id='5678'))) + self.assertEqual(1, len(db_api.reservation_get_all_by_values( + resource_type='physical:host'))) + + # Host reservations + + def test_create_host_reservation(self): + """Create a host reservation and verify that all tables + have been populated. + """ + + result = db_api.host_reservation_create( + _get_fake_host_reservation_values(id='1')) + self.assertEqual(result['id'], + _get_fake_host_reservation_values(id='1') + ['id']) + # Making sure we still raise a DuplicateDBEntry + self.assertRaises(RuntimeError, db_api.host_reservation_create, + _get_fake_host_reservation_values(id='1')) + + def test_delete_host_reservation(self): + """Check all deletion cases for host reservation, + including cascade deletion from reservations table. + """ + + self.assertRaises(RuntimeError, + db_api.host_reservation_destroy, 'fake_id') + + result = db_api.host_reservation_create( + _get_fake_host_reservation_values()) + db_api.host_reservation_destroy(result['id']) + self.assertIsNone(db_api.host_reservation_get(result['id'])) + reserv = db_api.reservation_create(_get_fake_phys_reservation_values()) + result = db_api.host_reservation_create( + _get_fake_host_reservation_values(reservation_id=reserv['id'])) + db_api.reservation_destroy(reserv['id']) + self.assertIsNone(db_api.host_reservation_get(result['id'])) + + def test_host_reservation_get_all(self): + """Check that we return 2 hosts.""" + + db_api.host_reservation_create(_get_fake_host_reservation_values(id=1)) + db_api.host_reservation_create(_get_fake_host_reservation_values(id=2)) + hosts_reservations = db_api.host_reservation_get_all() + self.assertEqual(['1', '2'], [x['id'] for x in hosts_reservations]) + + def test_host_reservation_get_by_reservation_id(self): + """Check that we return 2 hosts.""" + + db_api.host_reservation_create( + _get_fake_host_reservation_values(id=1, reservation_id=1)) + db_api.host_reservation_create( + _get_fake_host_reservation_values(id=2, reservation_id=2)) + res = db_api.host_reservation_get_by_reservation_id(2) + self.assertEqual('2', res['id']) + + def test_update_host_reservation(self): + db_api.host_reservation_create(_get_fake_host_reservation_values(id=1)) + db_api.host_reservation_update(1, {'resource_properties': 'updated'}) + res = db_api.host_reservation_get(1) + self.assertEqual('updated', res['resource_properties']) + + def test_create_host(self): + """Create a host and verify that all tables + have been populated. + """ + result = db_api.host_create(_get_fake_host_values(id='1')) + self.assertEqual(result['id'], _get_fake_host_values(id='1')['id']) + # Making sure we still raise a DuplicateDBEntry + self.assertRaises(RuntimeError, db_api.host_create, + _get_fake_host_values(id='1')) + + def test_search_for_hosts_by_ram(self): + """Create two hosts and check that we can find a host per its RAM info. + """ + db_api.host_create(_get_fake_host_values(id=1, mem=2048)) + db_api.host_create(_get_fake_host_values(id=2, mem=4096)) + self.assertEqual(2, len( + db_api.host_get_all_by_queries(['memory_mb >= 2048']))) + self.assertEqual(0, len( + db_api.host_get_all_by_queries(['memory_mb lt 2048']))) + + def test_search_for_hosts_by_cpu_info(self): + """Create one host and search within cpu_info.""" + + db_api.host_create(_get_fake_host_values()) + self.assertEqual(1, len( + db_api.host_get_all_by_queries(['cpu_info like %Westmere%']))) + + def test_search_for_hosts_by_composed_queries(self): + """Create one host and test composed queries.""" + + db_api.host_create(_get_fake_host_values(mem=8192)) + self.assertEqual(1, len( + db_api.host_get_all_by_queries(['memory_mb > 2048', + 'cpu_info like %Westmere%']))) + self.assertEqual(0, len( + db_api.host_get_all_by_queries(['memory_mb < 2048', + 'cpu_info like %Westmere%']))) + self.assertRaises(RuntimeError, + db_api.host_get_all_by_queries, ['memory_mb <']) + self.assertRaises(RuntimeError, + db_api.host_get_all_by_queries, ['apples < 2048']) + self.assertRaises(RuntimeError, + db_api.host_get_all_by_queries, + ['memory_mb wrongop 2048']) + self.assertEqual(1, len( + db_api.host_get_all_by_queries(['memory_mb in 4096,8192']))) + self.assertEqual(1, len( + db_api.host_get_all_by_queries(['memory_mb != null']))) + + def test_list_hosts(self): + db_api.host_create(_get_fake_host_values(id=1)) + db_api.host_create(_get_fake_host_values(id=2)) + self.assertEqual(2, len(db_api.host_list())) + + def test_get_hosts_per_filter(self): + db_api.host_create(_get_fake_host_values(id=1)) + db_api.host_create(_get_fake_host_values(id=2)) + filters = {'status': 'free'} + self.assertEqual(2, len( + db_api.host_get_all_by_filters(filters))) + + def test_update_host(self): + db_api.host_create(_get_fake_host_values(id=1)) + db_api.host_update(1, {'status': 'updated'}) + self.assertEqual('updated', db_api.host_get(1)['status']) + + def test_delete_host(self): + db_api.host_create(_get_fake_host_values(id=1)) + db_api.host_destroy(1) + self.assertEqual(None, db_api.host_get(1)) + self.assertRaises(RuntimeError, db_api.host_destroy, 2) + + def test_create_host_extra_capability(self): + result = db_api.host_extra_capability_create( + _get_fake_host_extra_capabilities(id=1)) + self.assertEqual(result['id'], _get_fake_host_values(id='1')['id']) + # Making sure we still raise a DuplicateDBEntry + self.assertRaises(RuntimeError, db_api.host_extra_capability_create, + _get_fake_host_extra_capabilities(id='1')) + + def test_get_host_extra_capability_per_id(self): + db_api.host_extra_capability_create( + _get_fake_host_extra_capabilities(id='1')) + result = db_api.host_extra_capability_get('1') + self.assertEqual('1', result['id']) + + def test_host_extra_capability_get_all_per_host(self): + db_api.host_extra_capability_create( + _get_fake_host_extra_capabilities(id='1', computehost_id='1')) + db_api.host_extra_capability_create( + _get_fake_host_extra_capabilities(id='2', computehost_id='1')) + res = db_api.host_extra_capability_get_all_per_host('1') + self.assertEqual(2, len(res)) + + def test_update_host_extra_capability(self): + db_api.host_extra_capability_create( + _get_fake_host_extra_capabilities(id='1')) + db_api.host_extra_capability_update('1', {'capability_value': '2'}) + res = db_api.host_extra_capability_get('1') + self.assertEqual('2', res['capability_value']) + + def test_delete_host_extra_capability(self): + db_api.host_extra_capability_create( + _get_fake_host_extra_capabilities(id='1')) + db_api.host_extra_capability_destroy('1') + self.assertEqual(None, db_api.host_extra_capability_get('1')) + self.assertRaises(RuntimeError, + db_api.host_extra_capability_destroy, '1')