Add snapshot ORM to reddwarf

- snapshot ORM model (migrate script)
- unittest for ORM

blueprint consistent-snapshots

Change-Id: I26045ff6da322802e79bbb1e6467bfe14d05688f

reddwarf/backup/__init__.py

@@ -0,0 +1,13 @@
+#Copyright 2013 Hewlett-Packard Development Company, L.P.
+
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.

reddwarf/backup/models.py

@@ -0,0 +1,115 @@
+#Copyright [2013] Hewlett-Packard Development Company, L.P.
+
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
+"""Model classes that form the core of snapshots functionality."""
+
+from reddwarf.common import cfg
+from reddwarf.common import exception
+from reddwarf.common import utils
+
+from reddwarf.db.models import DatabaseModelBase
+
+from reddwarf.openstack.common import log as logging
+
+CONF = cfg.CONF
+LOG = logging.getLogger(__name__)
+SWIFT_CONTAINER = CONF.backup_swift_container
+
+
+class BackupState(object):
+    NEW = "NEW"
+    BUILDING = "BUILDING"
+    SAVING = "SAVING"
+    COMPLETED = "COMPLETED"
+    FAILED = "FAILED"
+
+
+class Backup(object):
+
+    @classmethod
+    def create(cls, context, instance_id, name, description=None):
+        """
+        create db record for Backup
+        :param cls:
+        :param context: tenant_id included
+        :param instance_id:
+        :param name:
+        :param note:
+        :return:
+        """
+        try:
+            db_info = DBBackup.create(name=name,
+                                      description=description,
+                                      tenant_id=context.tenant,
+                                      state=BackupState.NEW,
+                                      instance_id=instance_id,
+                                      deleted=False)
+            return db_info
+        except exception.InvalidModelError as ex:
+            LOG.exception("Unable to create Backup record:")
+            raise exception.BackupCreationError(str(ex))
+
+    @classmethod
+    def list(cls, context):
+        """
+        list all live Backups belong to given tenant
+        :param cls:
+        :param context: tenant_id included
+        :return:
+        """
+        db_info = DBBackup.find_all(tenant_id=context.tenant,
+                                    deleted=False)
+        return db_info
+
+    @classmethod
+    def list_for_instance(cls, instance_id):
+        """
+        list all live Backups associated with given instance
+        :param cls:
+        :param instance_id:
+        :return:
+        """
+        db_info = DBBackup.find_all(instance_id=instance_id,
+                                    deleted=False)
+        return db_info
+
+    @classmethod
+    def delete(cls, id):
+        """
+        update Backup table on deleted flag for given Backup
+        :param cls:
+        :param id: Backup uuid
+        :return:
+        """
+        #TODO: api (service.py) might take care of actual deletion
+        # on remote swift
+        try:
+            db_info = DBBackup.find_by(id=id, deleted=False)
+            db_info.update(deleted=True, deleted_at=utils.utcnow())
+        except exception.ReddwarfError as ex:
+            LOG.exception("Backup record cannot be updated for "
+                          "backup %s :") % id
+            raise exception.BackupUpdateError(str(ex))
+
+
+def persisted_models():
+    return {'backups': DBBackup}
+
+
+class DBBackup(DatabaseModelBase):
+    """A table for Backup records"""
+    _data_fields = ['id', 'name', 'description', 'location', 'backup_type',
+                    'size', 'tenant_id', 'state', 'instance_id',
+                    'backup_timestamp', 'deleted', 'created',
+                    'updated', 'deleted_at']

reddwarf/common/cfg.py

@@ -43,6 +43,7 @@ common_opts = [
     cfg.StrOpt('nova_compute_url', default='http://localhost:8774/v2'),
     cfg.StrOpt('nova_volume_url', default='http://localhost:8776/v2'),
     cfg.StrOpt('reddwarf_auth_url', default='http://0.0.0.0:5000/v2.0'),
+    cfg.StrOpt('backup_swift_container', default='DBaaS-backup'),
     cfg.StrOpt('host', default='0.0.0.0'),
     cfg.IntOpt('report_interval', default=10),
     cfg.IntOpt('periodic_interval', default=60),

reddwarf/common/exception.py

@@ -20,6 +20,7 @@ from reddwarf.openstack.common import log as logging
 from reddwarf.openstack.common import exception as openstack_exception
 from reddwarf.openstack.common import processutils
 from reddwarf.openstack.common.gettextutils import _
+
 from webob import exc
 
 
@@ -213,3 +214,19 @@ class TenantQuotaNotFound(QuotaNotFound):
 
 class QuotaResourceUnknown(QuotaNotFound):
     message = _("Unknown quota resources %(unknown)s.")
+
+
+class BackupUploadError(ReddwarfError):
+    message = _("Unable to upload Backup onto swift")
+
+
+class BackupDownloadError(ReddwarfError):
+    message = _("Unable to download Backup from swift")
+
+
+class BackupCreationError(ReddwarfError):
+    message = _("Unable to create Backup")
+
+
+class BackupUpdateError(ReddwarfError):
+    message = _("Unable to update Backup table in db")

reddwarf/db/models.py

@@ -54,6 +54,13 @@ class DatabaseModelBase(models.ModelBase):
                   (self.__class__.__name__, self.__dict__))
         return self.db_api.delete(self)
 
+    def update(self, **values):
+        for key in values:
+            if hasattr(self, key):
+                setattr(self, key, values[key])
+        self['updated'] = utils.utcnow()
+        return self.db_api.save(self)
+
     def __init__(self, **kwargs):
         self.merge_attributes(kwargs)
         if not self.is_valid():

reddwarf/db/sqlalchemy/mappers.py

@@ -44,6 +44,8 @@ def map(engine, models):
                Table('quota_usages', meta, autoload=True))
     orm.mapper(models['reservations'],
                Table('reservations', meta, autoload=True))
+    orm.mapper(models['backups'],
+               Table('backups', meta, autoload=True))
 
 
 def mapping_exists(model):

reddwarf/db/sqlalchemy/migrate_repo/schema.py

@@ -45,6 +45,8 @@ Integer = lambda: sqlalchemy.types.Integer()
 
 BigInteger = lambda: sqlalchemy.types.BigInteger()
 
+Float = lambda: sqlalchemy.types.Float()
+
 
 def create_tables(tables):
     for table in tables:

reddwarf/db/sqlalchemy/migrate_repo/versions/012_backup.py

@@ -0,0 +1,53 @@
+#Copyright [2013] Hewlett-Packard Development Company, L.P.
+
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
+from sqlalchemy.schema import Column
+from sqlalchemy.schema import MetaData
+from sqlalchemy.schema import UniqueConstraint
+
+from reddwarf.db.sqlalchemy.migrate_repo.schema import create_tables
+from reddwarf.db.sqlalchemy.migrate_repo.schema import DateTime
+from reddwarf.db.sqlalchemy.migrate_repo.schema import drop_tables
+from reddwarf.db.sqlalchemy.migrate_repo.schema import Float
+from reddwarf.db.sqlalchemy.migrate_repo.schema import String
+from reddwarf.db.sqlalchemy.migrate_repo.schema import Table
+from reddwarf.db.sqlalchemy.migrate_repo.schema import Boolean
+
+meta = MetaData()
+
+backups = Table('backups', meta,
+                Column('id', String(36), primary_key=True, nullable=False),
+                Column('name', String(255), nullable=False),
+                Column('description', String(512)),
+                Column('location', String(1024)),
+                Column('backup_type', String(32)),
+                Column('size', Float()),
+                Column('tenant_id', String(36)),
+                Column('state', String(32), nullable=False),
+                Column('instance_id', String(36)),
+                Column('backup_timestamp', DateTime()),
+                Column('deleted', Boolean()),
+                Column('created', DateTime()),
+                Column('updated', DateTime()),
+                Column('deleted_at', DateTime()))
+
+
+def upgrade(migrate_engine):
+    meta.bind = migrate_engine
+    create_tables([backups, ])
+
+
+def downgrade(migrate_engine):
+    meta.bind = migrate_engine
+    drop_tables([backups, ])

reddwarf/db/sqlalchemy/session.py

@@ -46,6 +46,7 @@ def configure_db(options, models_mapper=None):
         from reddwarf.extensions.mysql import models as mysql_models
         from reddwarf.guestagent import models as agent_models
         from reddwarf.quota import models as quota_models
+        from reddwarf.backup import models as backup_models
 
         model_modules = [
             base_models,
@@ -53,6 +54,7 @@ def configure_db(options, models_mapper=None):
             mysql_models,
             agent_models,
             quota_models,
+            backup_models,
         ]
 
         models = {}

reddwarf/tests/unittests/backup/__init__.py

@@ -0,0 +1,13 @@
+#Copyright 2013 Hewlett-Packard Development Company, L.P.
+
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.

reddwarf/tests/unittests/backup/test_backup_models.py

@@ -0,0 +1,92 @@
+#Copyright 2013 Hewlett-Packard Development Company, L.P.
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
+
+import testtools
+from reddwarf.backup import models
+from reddwarf.tests.unittests.util import util
+from reddwarf.common import utils
+from reddwarf.common.context import ReddwarfContext
+
+
+def _prep_conf(current_time):
+    current_time = str(current_time)
+    context = ReddwarfContext(tenant='TENANT-' + current_time)
+    instance_id = 'INSTANCE-' + current_time
+    return context, instance_id
+
+BACKUP_NAME = 'WORKS'
+BACKUP_NAME_2 = 'IT-WORKS'
+BACKUP_STATE = "NEW"
+
+
+class BackupCreateTest(testtools.TestCase):
+    def setUp(self):
+        super(BackupCreateTest, self).setUp()
+        util.init_db()
+        self.context, self.instance_id = _prep_conf(utils.utcnow())
+        self.created = False
+
+    def tearDown(self):
+        super(BackupCreateTest, self).tearDown()
+        if self.created:
+            models.DBBackup.find_by(
+                tenant_id=self.context.tenant).delete()
+
+    def test_create(self):
+        models.Backup.create(
+            self.context, self.instance_id, BACKUP_NAME)
+        self.created = True
+        db_record = models.DBBackup.find_by(
+            tenant_id=self.context.tenant)
+        self.assertEqual(self.instance_id, db_record['instance_id'])
+
+
+class BackupORMTest(testtools.TestCase):
+    def setUp(self):
+        super(BackupORMTest, self).setUp()
+        util.init_db()
+        self.context, self.instance_id = _prep_conf(utils.utcnow())
+        models.DBBackup.create(tenant_id=self.context.tenant,
+                               name=BACKUP_NAME,
+                               state=BACKUP_STATE,
+                               instance_id=self.instance_id,
+                               deleted=False)
+        self.deleted = False
+
+    def tearDown(self):
+        super(BackupORMTest, self).tearDown()
+        if not self.deleted:
+            models.DBBackup.find_by(tenant_id=self.context.tenant).delete()
+
+    def test_list(self):
+        db_record = models.Backup.list(self.context)
+        self.assertEqual(1, db_record.count())
+
+    def test_list_for_instance(self):
+        models.DBBackup.create(tenant_id=self.context.tenant,
+                               name=BACKUP_NAME_2,
+                               state=BACKUP_STATE,
+                               instance_id=self.instance_id,
+                               deleted=False)
+        db_record = models.Backup.list_for_instance(self.instance_id)
+        self.assertEqual(2, db_record.count())
+
+    def test_delete(self):
+        db_record = models.DBBackup.find_by(tenant_id=self.context.tenant)
+        uuid = db_record['id']
+        print uuid
+        models.Backup.delete(uuid)
+        self.deleted = True
+        db_record = models.DBBackup.find_by(id=uuid, deleted=True)
+        self.assertEqual(self.instance_id, db_record['instance_id'])

reddwarf/tests/unittests/guestagent/test_models.py

@@ -24,9 +24,21 @@ from datetime import datetime
 class AgentHeartBeatTest(testtools.TestCase):
     def setUp(self):
         super(AgentHeartBeatTest, self).setUp()
+        self.origin_get_db_api = dbmodels.get_db_api
+        self.origin_utcnow = utils.utcnow
+        self.origin_DatabaseModelBase = dbmodels.DatabaseModelBase
+        self.origin_db_api_save = dbapi.save
+        self.origin_is_valid = dbmodels.DatabaseModelBase.is_valid
+        self.origin_generate_uuid = utils.generate_uuid
 
     def tearDown(self):
         super(AgentHeartBeatTest, self).tearDown()
+        dbmodels.get_db_api = self.origin_get_db_api
+        utils.utcnow = self.origin_utcnow
+        dbmodels.DatabaseModelBase = self.origin_DatabaseModelBase
+        dbapi.save = self.origin_db_api_save
+        dbmodels.DatabaseModelBase.is_valid = self.origin_is_valid
+        utils.generate_uuid = self.origin_generate_uuid
 
     def test_create(self):
         utils.generate_uuid = Mock()