DB API and models for supporting Compute Hosts Reservations

As per our design doc for Hosts reservations [1], Climate needs to be
added three new models called ComputeHost, ComputeHostExtraCapabilities
and ComputeHostReservation

 * ComputeHost will store all info concerning hosts managed by Climate
   and will keep tracability on if they are free for a lease or not
 * ComputeHostExtraCapabilities will allow admin user to add extra info
   for each host to add to Climate like GPU or SSD which are not yet
   visible thru Nova-Compute
 * ComputeHostReservation will store all reservation details when
   creating a lease, needed by Manager when starting the leases

Implements bp:host-db-api

[1] : https://docs.google.com/drawings/d/1-_DOB65LsSGR4JUaPN525bWAg1yhvEv3KNcHfmuLvYw/edit

Change-Id: Ib8f289064a9621372908be1d44db92f3719b24e1
This commit is contained in:
sbauza 2013-10-02 14:37:49 +02:00 committed by sbauza
parent 23595d493d
commit 052f17eb20
4 changed files with 662 additions and 2 deletions

View File

@ -112,9 +112,15 @@ def reservation_create(reservation_values):
@to_dict
def reservation_get_all_by_lease(lease_id):
def reservation_get_all_by_lease_id(lease_id):
"""Return all reservations belongs to specific lease."""
return IMPL.reservation_get_all_by_lease(lease_id)
return IMPL.reservation_get_all_by_lease_id(lease_id)
@to_dict
def reservation_get_all_by_values(**kwargs):
"""Returns all entries filtered by col=value."""
return IMPL.reservation_get_all_by_values(**kwargs)
@to_dict
@ -221,3 +227,110 @@ def event_destroy(event_id):
def event_update(event_id, event_values):
"""Update event or raise if not exists."""
IMPL.event_update(event_id, event_values)
# Host reservations
def host_reservation_create(host_reservation_values):
"""Create a host reservation from the values."""
return IMPL.host_reservation_create(host_reservation_values)
@to_dict
def host_reservation_get_by_reservation_id(reservation_id):
"""Return host reservation belonging to specific reservation."""
return IMPL.host_reservation_get_by_reservation_id(reservation_id)
@to_dict
def host_reservation_get(host_reservation_id):
"""Return specific host reservation."""
return IMPL.host_reservation_get(host_reservation_id)
@to_dict
def host_reservation_get_all():
"""Return all hosts reservations."""
return IMPL.host_reservation_get_all()
def host_reservation_destroy(host_reservation_id):
"""Delete specific host reservation."""
IMPL.host_reservation_destroy(host_reservation_id)
def host_reservation_update(host_reservation_id,
host_reservation_values):
"""Update host reservation."""
IMPL.host_reservation_update(host_reservation_id,
host_reservation_values)
# Compute Hosts
def host_create(values):
"""Create a Compute host from the values."""
return IMPL.host_create(values)
@to_dict
def host_get(host_id):
"""Return a specific Compute host."""
return IMPL.host_get(host_id)
@to_dict
def host_list():
"""Return a list of events."""
return IMPL.host_list()
@to_dict
def host_get_all_by_filters(filters):
"""Returns Compute hosts filtered by name of the field."""
return IMPL.host_get_all_by_filters(filters)
@to_dict
def host_get_all_by_queries(queries):
"""Returns hosts filtered by an array of queries."""
return IMPL.host_get_all_by_queries(queries)
def host_destroy(host_id):
"""Delete specific Compute host."""
IMPL.host_destroy(host_id)
def host_update(host_id, values):
"""Update Compute host."""
IMPL.host_update(host_id, values)
# ComputeHostExtraCapabilities
def host_extra_capability_create(values):
"""Create a Host ExtraCapability from the values."""
return IMPL.host_extra_capability_create(values)
@to_dict
def host_extra_capability_get(host_extra_capability_id):
"""Return a specific Host Extracapability."""
return IMPL.host_extra_capability_get(host_extra_capability_id)
@to_dict
def host_extra_capability_get_all_per_host(host_id):
"""Return all extra_capabilities belonging to a specific Compute host."""
return IMPL.host_extra_capability_get_all_per_host(host_id)
def host_extra_capability_destroy(host_extra_capability_id):
"""Delete specific host ExtraCapability."""
IMPL.host_extra_capability_destroy(host_extra_capability_id)
def host_extra_capability_update(host_extra_capability_id, values):
"""Update specific host ExtraCapability."""
IMPL.host_extra_capability_update(host_extra_capability_id, values)

View File

@ -153,6 +153,17 @@ def reservation_get_all_by_lease_id(lease_id):
return reservations.all()
def reservation_get_all_by_values(**kwargs):
"""Returns all entries filtered by col=value."""
reservation_query = model_query(models.Reservation, get_session())
for name, value in kwargs.items():
column = getattr(models.Reservation, name, None)
if column:
reservation_query = reservation_query.filter(column == value)
return reservation_query.all()
def reservation_create(values):
values = values.copy()
reservation = models.Reservation()
@ -357,3 +368,242 @@ def event_destroy(event_id):
raise RuntimeError("Event not found!")
session.delete(event)
#ComputeHostReservation
def _host_reservation_get(session, host_reservation_id):
query = model_query(models.ComputeHostReservation, session)
return query.filter_by(id=host_reservation_id).first()
def host_reservation_get(host_reservation_id):
return _host_reservation_get(get_session(),
host_reservation_id)
def host_reservation_get_all():
query = model_query(models.ComputeHostReservation, get_session())
return query.all()
def _host_reservation_get_by_reservation_id(session, reservation_id):
query = model_query(models.ComputeHostReservation, session)
return query.filter_by(reservation_id=reservation_id).first()
def host_reservation_get_by_reservation_id(reservation_id):
return _host_reservation_get_by_reservation_id(get_session(),
reservation_id)
def host_reservation_create(values):
values = values.copy()
host_reservation = models.ComputeHostReservation()
host_reservation.update(values)
session = get_session()
with session.begin():
try:
host_reservation.save(session=session)
except db_exc.DBDuplicateEntry as e:
# raise exception about duplicated columns (e.columns)
raise RuntimeError("DBDuplicateEntry: %s" % e.columns)
return host_reservation_get(host_reservation.id)
def host_reservation_update(host_reservation_id, values):
session = get_session()
with session.begin():
host_reservation = _host_reservation_get(session,
host_reservation_id)
host_reservation.update(values)
host_reservation.save(session=session)
return host_reservation_get(host_reservation_id)
def host_reservation_destroy(host_reservation_id):
session = get_session()
with session.begin():
host_reservation = _host_reservation_get(session,
host_reservation_id)
if not host_reservation:
# raise not found error
raise RuntimeError("Host Reservation not found!")
session.delete(host_reservation)
#ComputeHost
def _host_get(session, host_id):
query = model_query(models.ComputeHost, session)
return query.filter_by(id=host_id).first()
def _host_get_all(session):
query = model_query(models.ComputeHost, session)
return query
def host_get(host_id):
return _host_get(get_session(), host_id)
def host_list():
return model_query(models.ComputeHost, get_session()).all()
def host_get_all_by_filters(filters):
"""Returns hosts filtered by name of the field."""
hosts_query = _host_get_all(get_session())
if 'status' in filters:
hosts_query = hosts_query.\
filter(models.ComputeHost.status == filters['status'])
return hosts_query.all()
def host_get_all_by_queries(queries):
"""Returns hosts filtered by an array of queries.
:param queries: array of queries "key op value" where op can be
http://docs.sqlalchemy.org/en/rel_0_7/core/expression_api.html
#sqlalchemy.sql.operators.ColumnOperators
"""
hosts_query = model_query(models.ComputeHost, get_session())
oper = dict({'<': 'lt', '>': 'gt', '<=': 'le', '>=': 'ge', '==': 'eq',
'!=': 'ne'})
for query in queries:
try:
key, op, value = query.split(' ', 3)
except ValueError:
raise RuntimeError('Invalid filter: %s' % query)
column = getattr(models.ComputeHost, key, None)
if not column:
raise RuntimeError('Invalid filter column: %s' % key)
if op == 'in':
filt = column.in_(value.split(','))
else:
if op in oper:
op = oper[op]
try:
attr = filter(lambda e: hasattr(column, e % op),
['%s', '%s_', '__%s__'])[0] % op
except IndexError:
raise RuntimeError('Invalid filter operator: %s' % op)
if value == 'null':
value = None
filt = getattr(column, attr)(value)
hosts_query = hosts_query.filter(filt)
return hosts_query.all()
def host_create(values):
values = values.copy()
host = models.ComputeHost()
host.update(values)
session = get_session()
with session.begin():
try:
host.save(session=session)
except db_exc.DBDuplicateEntry as e:
# raise exception about duplicated columns (e.columns)
raise RuntimeError("DBDuplicateEntry: %s" % e.columns)
return host_get(host.id)
def host_update(host_id, values):
session = get_session()
with session.begin():
host = _host_get(session, host_id)
host.update(values)
host.save(session=session)
return host_get(host_id)
def host_destroy(host_id):
session = get_session()
with session.begin():
host = _host_get(session, host_id)
if not host:
# raise not found error
raise RuntimeError("Host not found!")
session.delete(host)
#ComputeHostExtraCapability
def _host_extra_capability_get(session, host_extra_capability_id):
query = model_query(models.ComputeHostExtraCapability, session)
return query.filter_by(id=host_extra_capability_id).first()
def host_extra_capability_get(host_extra_capability_id):
return _host_extra_capability_get(get_session(),
host_extra_capability_id)
def _host_extra_capability_get_all_per_host(session, host_id):
query = model_query(models.ComputeHostExtraCapability, session)
return query.filter_by(computehost_id=host_id).all()
def host_extra_capability_get_all_per_host(host_id):
return _host_extra_capability_get_all_per_host(get_session(),
host_id)
def host_extra_capability_create(values):
values = values.copy()
host_extra_capability = models.ComputeHostExtraCapability()
host_extra_capability.update(values)
session = get_session()
with session.begin():
try:
host_extra_capability.save(session=session)
except db_exc.DBDuplicateEntry as e:
# raise exception about duplicated columns (e.columns)
raise RuntimeError("DBDuplicateEntry: %s" % e.columns)
return host_extra_capability_get(host_extra_capability.id)
def host_extra_capability_update(host_extra_capability_id, values):
session = get_session()
with session.begin():
host_extra_capability = \
_host_extra_capability_get(session,
host_extra_capability_id)
host_extra_capability.update(values)
host_extra_capability.save(session=session)
return host_extra_capability_get(host_extra_capability_id)
def host_extra_capability_destroy(host_extra_capability_id):
session = get_session()
with session.begin():
host_extra_capability = \
_host_extra_capability_get(session,
host_extra_capability_id)
if not host_extra_capability:
# raise not found error
raise RuntimeError("Host Extracapability not found!")
session.delete(host_extra_capability)

View File

@ -14,6 +14,7 @@
# limitations under the License.
import sqlalchemy as sa
from sqlalchemy.dialects.mysql import MEDIUMTEXT
from sqlalchemy.orm import relationship
from climate.db.sqlalchemy import model_base as mb
@ -26,6 +27,10 @@ def _generate_unicode_uuid():
return unicode(uuidutils.generate_uuid())
def MediumText():
return sa.Text().with_variant(MEDIUMTEXT(), 'mysql')
def _id_column():
return sa.Column(sa.String(36),
primary_key=True,
@ -72,6 +77,11 @@ class Reservation(mb.ClimateBase):
resource_id = sa.Column(sa.String(36))
resource_type = sa.Column(sa.String(66))
status = sa.Column(sa.String(13))
computehost_reservations = relationship('ComputeHostReservation',
uselist=False,
cascade="all,delete",
backref='reservation',
lazy='joined')
def to_dict(self):
return super(Reservation, self).to_dict()
@ -90,3 +100,62 @@ class Event(mb.ClimateBase):
def to_dict(self):
return super(Event, self).to_dict()
class ComputeHostReservation(mb.ClimateBase):
"""Specifies resources asked by reservation from
Compute Host Reservation API.
"""
__tablename__ = 'computehost_reservations'
id = _id_column()
reservation_id = sa.Column(sa.String(36), sa.ForeignKey('reservations.id'))
resource_properties = sa.Column(MediumText())
count_range = sa.Column(sa.String(36))
hypervisor_properties = sa.Column(MediumText())
status = sa.Column(sa.String(13))
def to_dict(self):
return super(ComputeHostReservation, self).to_dict()
class ComputeHost(mb.ClimateBase):
"""Specifies resources asked by reservation from
Compute Host Reservation API.
"""
__tablename__ = 'computehosts'
id = _id_column()
vcpu = sa.Column(sa.Integer, nullable=False)
cpu_info = sa.Column(MediumText(), nullable=False)
hypervisor_type = sa.Column(MediumText(), nullable=False)
hypervisor_version = sa.Column(sa.Integer, nullable=False)
hypervisor_hostname = sa.Column(sa.String(255), nullable=True)
memory_mb = sa.Column(sa.Integer, nullable=False)
local_gb = sa.Column(sa.Integer, nullable=False)
status = sa.Column(sa.String(13))
computehost_extra_capabilities = relationship('ComputeHostExtraCapability',
cascade="all,delete",
backref='computehost',
lazy='joined')
def to_dict(self):
return super(ComputeHost, self).to_dict()
class ComputeHostExtraCapability(mb.ClimateBase):
"""Allows to define extra capabilities per administrator request for each
Compute Host added.
"""
__tablename__ = 'computehost_extra_capabilities'
id = _id_column()
computehost_id = sa.Column(sa.String(36), sa.ForeignKey('computehosts.id'))
capability_name = sa.Column(sa.String(64), nullable=False)
capability_value = sa.Column(MediumText(), nullable=False)
def to_dict(self):
return super(ComputeHostExtraCapability, self).to_dict()

View File

@ -92,6 +92,43 @@ def _create_physical_lease(values=_get_fake_phys_lease_values(),
return db_api.lease_create(values)
def _get_fake_host_reservation_values(id=_get_fake_random_uuid(),
reservation_id=_get_fake_random_uuid()):
return {'id': id,
'reservation_id': reservation_id,
'resource_properties': "fake",
'hypervisor_properties': "fake"}
def _get_fake_cpu_info():
return str({'vendor': 'Intel',
'model': 'Westmere',
'arch': 'x86_64',
'features': ['rdtscp', 'pdpe1gb', 'hypervisor', 'vmx', 'ss',
'vme'],
'topology': {'cores': 1, 'threads': 1, 'sockets': 2}})
def _get_fake_host_values(id=_get_fake_random_uuid(), mem=8192, disk=10):
return {'id': id,
'vcpu': 1,
'cpu_info': _get_fake_cpu_info(),
'hypervisor_type': 'QEMU',
'hypervisor_version': 1000,
'memory_mb': mem,
'local_gb': disk,
'status': 'free'
}
def _get_fake_host_extra_capabilities(id=_get_fake_random_uuid(),
computehost_id=_get_fake_random_uuid()):
return {'id': id,
'computehost_id': computehost_id,
'capability_name': 'vgpu',
'capability_value': '2'}
class SQLAlchemyDBApiTestCase(tests.DBTestCase):
"""Test case for SQLAlchemy DB API."""
@ -182,3 +219,194 @@ class SQLAlchemyDBApiTestCase(tests.DBTestCase):
values={'start_date': _get_datetime('2014-02-01 00:00')})
self.assertEqual(_get_datetime('2014-02-01 00:00'),
result['start_date'])
# Reservations
def test_create_reservation(self):
"""Create a reservation and verify that all tables have been
populated.
"""
result = db_api.reservation_create(_get_fake_phys_reservation_values())
self.assertEqual(result['lease_id'],
_get_fake_phys_reservation_values()
['lease_id'])
def test_reservation_get_all_by_values(self):
"""Create two reservations and verify that we can find reservation per
resource_id or resource_type.
"""
db_api.reservation_create(_get_fake_phys_reservation_values())
db_api.reservation_create(_get_fake_virt_reservation_values())
self.assertEqual(2, len(db_api.reservation_get_all_by_values()))
self.assertEqual(1, len(db_api.reservation_get_all_by_values(
resource_id='5678')))
self.assertEqual(1, len(db_api.reservation_get_all_by_values(
resource_type='physical:host')))
# Host reservations
def test_create_host_reservation(self):
"""Create a host reservation and verify that all tables
have been populated.
"""
result = db_api.host_reservation_create(
_get_fake_host_reservation_values(id='1'))
self.assertEqual(result['id'],
_get_fake_host_reservation_values(id='1')
['id'])
# Making sure we still raise a DuplicateDBEntry
self.assertRaises(RuntimeError, db_api.host_reservation_create,
_get_fake_host_reservation_values(id='1'))
def test_delete_host_reservation(self):
"""Check all deletion cases for host reservation,
including cascade deletion from reservations table.
"""
self.assertRaises(RuntimeError,
db_api.host_reservation_destroy, 'fake_id')
result = db_api.host_reservation_create(
_get_fake_host_reservation_values())
db_api.host_reservation_destroy(result['id'])
self.assertIsNone(db_api.host_reservation_get(result['id']))
reserv = db_api.reservation_create(_get_fake_phys_reservation_values())
result = db_api.host_reservation_create(
_get_fake_host_reservation_values(reservation_id=reserv['id']))
db_api.reservation_destroy(reserv['id'])
self.assertIsNone(db_api.host_reservation_get(result['id']))
def test_host_reservation_get_all(self):
"""Check that we return 2 hosts."""
db_api.host_reservation_create(_get_fake_host_reservation_values(id=1))
db_api.host_reservation_create(_get_fake_host_reservation_values(id=2))
hosts_reservations = db_api.host_reservation_get_all()
self.assertEqual(['1', '2'], [x['id'] for x in hosts_reservations])
def test_host_reservation_get_by_reservation_id(self):
"""Check that we return 2 hosts."""
db_api.host_reservation_create(
_get_fake_host_reservation_values(id=1, reservation_id=1))
db_api.host_reservation_create(
_get_fake_host_reservation_values(id=2, reservation_id=2))
res = db_api.host_reservation_get_by_reservation_id(2)
self.assertEqual('2', res['id'])
def test_update_host_reservation(self):
db_api.host_reservation_create(_get_fake_host_reservation_values(id=1))
db_api.host_reservation_update(1, {'resource_properties': 'updated'})
res = db_api.host_reservation_get(1)
self.assertEqual('updated', res['resource_properties'])
def test_create_host(self):
"""Create a host and verify that all tables
have been populated.
"""
result = db_api.host_create(_get_fake_host_values(id='1'))
self.assertEqual(result['id'], _get_fake_host_values(id='1')['id'])
# Making sure we still raise a DuplicateDBEntry
self.assertRaises(RuntimeError, db_api.host_create,
_get_fake_host_values(id='1'))
def test_search_for_hosts_by_ram(self):
"""Create two hosts and check that we can find a host per its RAM info.
"""
db_api.host_create(_get_fake_host_values(id=1, mem=2048))
db_api.host_create(_get_fake_host_values(id=2, mem=4096))
self.assertEqual(2, len(
db_api.host_get_all_by_queries(['memory_mb >= 2048'])))
self.assertEqual(0, len(
db_api.host_get_all_by_queries(['memory_mb lt 2048'])))
def test_search_for_hosts_by_cpu_info(self):
"""Create one host and search within cpu_info."""
db_api.host_create(_get_fake_host_values())
self.assertEqual(1, len(
db_api.host_get_all_by_queries(['cpu_info like %Westmere%'])))
def test_search_for_hosts_by_composed_queries(self):
"""Create one host and test composed queries."""
db_api.host_create(_get_fake_host_values(mem=8192))
self.assertEqual(1, len(
db_api.host_get_all_by_queries(['memory_mb > 2048',
'cpu_info like %Westmere%'])))
self.assertEqual(0, len(
db_api.host_get_all_by_queries(['memory_mb < 2048',
'cpu_info like %Westmere%'])))
self.assertRaises(RuntimeError,
db_api.host_get_all_by_queries, ['memory_mb <'])
self.assertRaises(RuntimeError,
db_api.host_get_all_by_queries, ['apples < 2048'])
self.assertRaises(RuntimeError,
db_api.host_get_all_by_queries,
['memory_mb wrongop 2048'])
self.assertEqual(1, len(
db_api.host_get_all_by_queries(['memory_mb in 4096,8192'])))
self.assertEqual(1, len(
db_api.host_get_all_by_queries(['memory_mb != null'])))
def test_list_hosts(self):
db_api.host_create(_get_fake_host_values(id=1))
db_api.host_create(_get_fake_host_values(id=2))
self.assertEqual(2, len(db_api.host_list()))
def test_get_hosts_per_filter(self):
db_api.host_create(_get_fake_host_values(id=1))
db_api.host_create(_get_fake_host_values(id=2))
filters = {'status': 'free'}
self.assertEqual(2, len(
db_api.host_get_all_by_filters(filters)))
def test_update_host(self):
db_api.host_create(_get_fake_host_values(id=1))
db_api.host_update(1, {'status': 'updated'})
self.assertEqual('updated', db_api.host_get(1)['status'])
def test_delete_host(self):
db_api.host_create(_get_fake_host_values(id=1))
db_api.host_destroy(1)
self.assertEqual(None, db_api.host_get(1))
self.assertRaises(RuntimeError, db_api.host_destroy, 2)
def test_create_host_extra_capability(self):
result = db_api.host_extra_capability_create(
_get_fake_host_extra_capabilities(id=1))
self.assertEqual(result['id'], _get_fake_host_values(id='1')['id'])
# Making sure we still raise a DuplicateDBEntry
self.assertRaises(RuntimeError, db_api.host_extra_capability_create,
_get_fake_host_extra_capabilities(id='1'))
def test_get_host_extra_capability_per_id(self):
db_api.host_extra_capability_create(
_get_fake_host_extra_capabilities(id='1'))
result = db_api.host_extra_capability_get('1')
self.assertEqual('1', result['id'])
def test_host_extra_capability_get_all_per_host(self):
db_api.host_extra_capability_create(
_get_fake_host_extra_capabilities(id='1', computehost_id='1'))
db_api.host_extra_capability_create(
_get_fake_host_extra_capabilities(id='2', computehost_id='1'))
res = db_api.host_extra_capability_get_all_per_host('1')
self.assertEqual(2, len(res))
def test_update_host_extra_capability(self):
db_api.host_extra_capability_create(
_get_fake_host_extra_capabilities(id='1'))
db_api.host_extra_capability_update('1', {'capability_value': '2'})
res = db_api.host_extra_capability_get('1')
self.assertEqual('2', res['capability_value'])
def test_delete_host_extra_capability(self):
db_api.host_extra_capability_create(
_get_fake_host_extra_capabilities(id='1'))
db_api.host_extra_capability_destroy('1')
self.assertEqual(None, db_api.host_extra_capability_get('1'))
self.assertRaises(RuntimeError,
db_api.host_extra_capability_destroy, '1')