[Share Groups] Squash SGS member and SS instances DB tables

Because the former was implemented just because the latter
was absent yet at that moment. Main reason is that they serve
same goals and should be consistent to each other. And it is
much easier to support one single table instead of two separate
tables syncing them each time we update one of them.

Change-Id: I81ba26437554d02a79e4b536d781617f46ce2f07
Partially-Implements bp manila-share-groups
This commit is contained in:
Valeriy Ponomaryov 2017-03-06 21:56:51 +03:00 committed by vponomaryov
parent ff9f489580
commit 54b030a3f2
8 changed files with 331 additions and 80 deletions

View File

@ -0,0 +1,148 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Squash 'share_group_snapshot_members' and 'share_snapshot_instances' models.
Revision ID: 31252d671ae5
Revises: 5237b6625330
Create Date: 2017-02-28 15:35:27.500063
"""
# revision identifiers, used by Alembic.
revision = '31252d671ae5'
down_revision = '5237b6625330'
from alembic import op
import sqlalchemy as sa
from manila.db.migrations import utils
SSI_TABLE_NAME = 'share_snapshot_instances'
SGSM_TABLE_NAME = 'share_group_snapshot_members'
def upgrade():
# Update 'share_snapshot_instance' table with new fields
op.add_column(SSI_TABLE_NAME, sa.Column('user_id', sa.String(255)))
op.add_column(SSI_TABLE_NAME, sa.Column('project_id', sa.String(255)))
op.add_column(SSI_TABLE_NAME, sa.Column('size', sa.Integer))
op.add_column(SSI_TABLE_NAME, sa.Column('share_proto', sa.String(255)))
op.add_column(SSI_TABLE_NAME, sa.Column('share_id', sa.String(36)))
op.add_column(
SSI_TABLE_NAME, sa.Column('share_group_snapshot_id', sa.String(36)))
# Drop FK for 'snapshot_id' because it will be null in case of SGS member
op.drop_constraint('ssi_snapshot_fk', SSI_TABLE_NAME, type_='foreignkey')
# Move existing SG snapshot members to share snapshot instance table
connection = op.get_bind()
ssi_table = utils.load_table(SSI_TABLE_NAME, connection)
ssgm_table = utils.load_table(SGSM_TABLE_NAME, connection)
ported_data = []
for ssgm_record in connection.execute(ssgm_table.select()):
ported_data.append({
"id": ssgm_record.id,
"share_group_snapshot_id": ssgm_record.share_group_snapshot_id,
"share_id": ssgm_record.share_id,
"share_instance_id": ssgm_record.share_instance_id,
"size": ssgm_record.size,
"status": ssgm_record.status,
"share_proto": ssgm_record.share_proto,
"user_id": ssgm_record.user_id,
"project_id": ssgm_record.project_id,
"provider_location": ssgm_record.provider_location,
"created_at": ssgm_record.created_at,
"updated_at": ssgm_record.updated_at,
"deleted_at": ssgm_record.deleted_at,
"deleted": ssgm_record.deleted,
})
op.bulk_insert(ssi_table, ported_data)
# Delete 'share_group_snapshot_members' table
op.drop_table(SGSM_TABLE_NAME)
def downgrade():
# Create 'share_group_snapshot_members' table
op.create_table(
SGSM_TABLE_NAME,
sa.Column('id', sa.String(36), primary_key=True, nullable=False),
sa.Column('created_at', sa.DateTime),
sa.Column('updated_at', sa.DateTime),
sa.Column('deleted_at', sa.DateTime),
sa.Column('deleted', sa.String(36), default='False'),
sa.Column('user_id', sa.String(length=255), nullable=False),
sa.Column('project_id', sa.String(length=255), nullable=False),
sa.Column(
'share_group_snapshot_id', sa.String(length=36),
sa.ForeignKey(
'share_group_snapshots.id', name='fk_gsm_group_snapshot_id'),
nullable=False),
sa.Column(
'share_instance_id', sa.String(length=36),
sa.ForeignKey(
'share_instances.id', name='fk_gsm_share_instance_id'),
nullable=False),
sa.Column(
'share_id', sa.String(length=36),
sa.ForeignKey('shares.id', name='fk_gsm_share_id'),
nullable=False),
sa.Column('size', sa.Integer),
sa.Column('status', sa.String(length=255)),
sa.Column('share_proto', sa.String(length=255)),
sa.Column('provider_location', sa.String(255), nullable=True),
mysql_engine='InnoDB',
mysql_charset='utf8')
# Select all share snapshot instances that
# have not null 'share_snapshot_group_id' to new table
connection = op.get_bind()
ssi_table = utils.load_table(SSI_TABLE_NAME, connection)
ssgm_table = utils.load_table(SGSM_TABLE_NAME, connection)
ported_data = []
for ssi_record in connection.execute(ssi_table.select().where(
ssi_table.c.share_group_snapshot_id.isnot(None))):
ported_data.append({
"id": ssi_record.id,
"share_group_snapshot_id": ssi_record.share_group_snapshot_id,
"share_id": ssi_record.share_id,
"share_instance_id": ssi_record.share_instance_id,
"size": ssi_record.size,
"status": ssi_record.status,
"share_proto": ssi_record.share_proto,
"user_id": ssi_record.user_id,
"project_id": ssi_record.project_id,
"provider_location": ssi_record.provider_location,
"created_at": ssi_record.created_at,
"updated_at": ssi_record.updated_at,
"deleted_at": ssi_record.deleted_at,
"deleted": ssi_record.deleted or "False",
})
# Copy share group snapshot members to new table
op.bulk_insert(ssgm_table, ported_data)
# Remove copied records from source table
connection.execute(
ssi_table.delete().where(
ssi_table.c.share_group_snapshot_id.isnot(None)))
# Remove redundant fields from 'share_snapshot_instance' table
for column_name in ('user_id', 'project_id', 'size', 'share_proto',
'share_id', 'share_group_snapshot_id'):
op.drop_column(SSI_TABLE_NAME, column_name)
# Add back FK for 'snapshot_id' field
op.create_foreign_key(
'ssi_snapshot_fk', SSI_TABLE_NAME, 'share_snapshots',
['snapshot_id'], ['id'])

View File

@ -2069,6 +2069,7 @@ def _share_snapshot_instance_get_with_filters(context, instance_ids=None,
if statuses is not None:
query = query.filter(models.ShareSnapshotInstance.status.in_(statuses))
query = query.options(joinedload('share_group_snapshot'))
return query
@ -2142,6 +2143,7 @@ def share_snapshot_get(context, snapshot_id, session=None):
project_only=True).\
filter_by(id=snapshot_id).\
options(joinedload('share')).\
options(joinedload('instances')).\
first()
if not result:
@ -2164,6 +2166,7 @@ def _share_snapshot_get_all_with_filters(context, project_id=None,
if share_id:
query = query.filter_by(share_id=share_id)
query = query.options(joinedload('share'))
query = query.options(joinedload('instances'))
# Apply filters
if 'usage' in filters:
@ -4015,7 +4018,7 @@ def count_share_group_snapshot_members_in_share(context, share_id,
session=None):
session = session or get_session()
return model_query(
context, models.ShareGroupSnapshotMember, session=session,
context, models.ShareSnapshotInstance, session=session,
project_only=True, read_deleted="no",
).filter_by(
share_id=share_id,
@ -4150,7 +4153,7 @@ def share_group_snapshot_destroy(context, share_group_snapshot_id):
share_group_snap_ref = _share_group_snapshot_get(
context, share_group_snapshot_id, session=session)
share_group_snap_ref.soft_delete(session)
session.query(models.ShareGroupSnapshotMember).filter_by(
session.query(models.ShareSnapshotInstance).filter_by(
share_group_snapshot_id=share_group_snapshot_id).soft_delete()
@ -4159,7 +4162,7 @@ def share_group_snapshot_members_get_all(context, share_group_snapshot_id,
session=None):
session = session or get_session()
query = model_query(
context, models.ShareGroupSnapshotMember, session=session,
context, models.ShareSnapshotInstance, session=session,
read_deleted='no',
).filter_by(share_group_snapshot_id=share_group_snapshot_id)
return query.all()
@ -4168,7 +4171,7 @@ def share_group_snapshot_members_get_all(context, share_group_snapshot_id,
@require_context
def share_group_snapshot_member_get(context, member_id, session=None):
result = model_query(
context, models.ShareGroupSnapshotMember, session=session,
context, models.ShareSnapshotInstance, session=session,
project_only=True, read_deleted='no',
).filter_by(id=member_id).first()
if not result:
@ -4178,7 +4181,7 @@ def share_group_snapshot_member_get(context, member_id, session=None):
@require_context
def share_group_snapshot_member_create(context, values):
member = models.ShareGroupSnapshotMember()
member = models.ShareSnapshotInstance()
if not values.get('id'):
values['id'] = six.text_type(uuidutils.generate_uuid())

View File

@ -700,23 +700,11 @@ class ShareSnapshot(BASE, ManilaBase):
'ShareSnapshot.share_id == Share.id,'
'ShareSnapshot.deleted == "False")')
instances = orm.relationship(
"ShareSnapshotInstance",
lazy='immediate',
primaryjoin=(
'and_('
'ShareSnapshot.id == ShareSnapshotInstance.snapshot_id, '
'ShareSnapshotInstance.deleted == "False")'
),
viewonly=True,
join_depth=2,
)
class ShareSnapshotInstance(BASE, ManilaBase):
"""Represents a snapshot of a share."""
__tablename__ = 'share_snapshot_instances'
_extra_keys = ['name', 'share_id', 'share_name']
_extra_keys = ['name', 'share_name']
@property
def name(self):
@ -726,29 +714,20 @@ class ShareSnapshotInstance(BASE, ManilaBase):
def share_name(self):
return CONF.share_name_template % self.share_instance_id
@property
def share_id(self):
# NOTE(u_glide): This property required for compatibility
# with share drivers
return self.share_instance_id
id = Column(String(36), primary_key=True)
deleted = Column(String(36), default='False')
snapshot_id = Column(
String(36), ForeignKey('share_snapshots.id'), nullable=False)
snapshot_id = Column(String(36), nullable=True)
share_instance_id = Column(
String(36), ForeignKey('share_instances.id'), nullable=False)
status = Column(String(255))
progress = Column(String(255))
provider_location = Column(String(255))
share_instance = orm.relationship(
ShareInstance, backref="snapshot_instances",
lazy='immediate',
primaryjoin=(
'and_('
'ShareSnapshotInstance.share_instance_id == ShareInstance.id,'
'ShareSnapshotInstance.deleted == "False")')
)
share_proto = Column(String(255))
size = Column(Integer)
share_id = Column(String(36), nullable=True)
share_group_snapshot_id = Column(String(36), nullable=True)
user_id = Column(String(255))
project_id = Column(String(255))
export_locations = orm.relationship(
"ShareSnapshotInstanceExportLocation",
@ -760,6 +739,37 @@ class ShareSnapshotInstance(BASE, ManilaBase):
'ShareSnapshotInstanceExportLocation.deleted == "False")'
)
)
share_instance = orm.relationship(
ShareInstance, backref="snapshot_instances",
lazy='immediate',
primaryjoin=(
'and_('
'ShareSnapshotInstance.share_instance_id == ShareInstance.id,'
'ShareSnapshotInstance.deleted == "False")')
)
snapshot = orm.relationship(
"ShareSnapshot",
lazy="immediate",
foreign_keys=snapshot_id,
backref="instances",
primaryjoin=(
'and_('
'ShareSnapshot.id == ShareSnapshotInstance.snapshot_id, '
'ShareSnapshotInstance.deleted == "False")'
),
viewonly=True,
join_depth=2,
)
share_group_snapshot = orm.relationship(
"ShareGroupSnapshot",
lazy="immediate",
foreign_keys=share_group_snapshot_id,
backref="share_group_snapshot_members",
primaryjoin=('ShareGroupSnapshot.id == '
'ShareSnapshotInstance.share_group_snapshot_id'),
viewonly=True,
join_depth=2,
)
class ShareSnapshotAccessMapping(BASE, ManilaBase):
@ -1162,39 +1172,6 @@ class ShareGroupShareTypeMapping(BASE, ManilaBase):
'ShareGroupShareTypeMapping.deleted == "False")')
)
# TODO(vponomaryov): add 'share_group_snapshot_member_export_locations' model
# to support mountable share group snapshots and add its relationship to
# 'share_group_snapshot_members' table.
class ShareGroupSnapshotMember(BASE, ManilaBase):
"""Represents the share snapshots in a share group snapshot."""
__tablename__ = 'share_group_snapshot_members'
id = Column(String(36), primary_key=True)
# TODO(vponomaryov): make 'share_group_snapshot_id' not nullable.
share_group_snapshot_id = Column(
String(36), ForeignKey('share_group_snapshots.id'))
share_id = Column(String(36), ForeignKey('shares.id'))
# TODO(vponomaryov): make 'share_instance_id' not nullable.
share_instance_id = Column(String(36), ForeignKey('share_instances.id'))
size = Column(Integer)
status = Column(String(255))
share_proto = Column(String(255))
user_id = Column(String(255))
project_id = Column(String(255))
deleted = Column(String(36), default='False')
provider_location = Column(String(255))
# TODO(vponomaryov): add relationship to source share instance as it is
# done for share snapshot instances.
# TODO(vponomaryov): add share group snapshot member export locations
# relationship.
share_group_snapshot = orm.relationship(
ShareGroupSnapshot,
backref="share_group_snapshot_members",
foreign_keys=share_group_snapshot_id,
primaryjoin='ShareGroupSnapshot.id == '
'ShareGroupSnapshotMember.share_group_snapshot_id')
def register_models():
"""Register Models and create metadata.

View File

@ -751,7 +751,8 @@ class GenericShareDriver(driver.ExecuteMixin, driver.ShareDriver):
def create_snapshot(self, context, snapshot, share_server=None):
"""Creates a snapshot."""
model_update = {}
volume = self._get_volume(self.admin_context, snapshot['share_id'])
volume = self._get_volume(
self.admin_context, snapshot['share_instance_id'])
volume_snapshot_name = (self.configuration.
volume_snapshot_name_template % snapshot['id'])
volume_snapshot = self.volume_api.create_snapshot_force(

View File

@ -164,9 +164,6 @@ class API(base.Base):
share = self.db.share_get(context, member['share_id'])
share_type = share_types.get_share_type(
context, share['share_type_id'])
member['share_instance'] = self.db.share_instance_get(
context, member['share_instance_id'],
with_share_data=True)
self.share_api.create(
context,
member['share_proto'],

View File

@ -2249,3 +2249,119 @@ class ShareGroupNewAvailabilityZoneIDColumnChecks(BaseMigrationChecks):
self.test_case.assertEqual(1, db_result.rowcount)
for sg in db_result:
self.test_case.assertFalse(hasattr(sg, self.new_attr_name))
@map_to_migration('31252d671ae5')
class SquashSGSnapshotMembersAndSSIModelsChecks(BaseMigrationChecks):
old_table_name = 'share_group_snapshot_members'
new_table_name = 'share_snapshot_instances'
share_group_type_id = uuidutils.generate_uuid()
share_group_id = uuidutils.generate_uuid()
share_id = uuidutils.generate_uuid()
share_instance_id = uuidutils.generate_uuid()
share_group_snapshot_id = uuidutils.generate_uuid()
share_group_snapshot_member_id = uuidutils.generate_uuid()
keys = (
'user_id', 'project_id', 'size', 'share_proto', 'share_id',
'share_group_snapshot_id',
)
def setup_upgrade_data(self, engine):
# Setup share group type
sgt_data = {
'id': self.share_group_type_id,
'name': uuidutils.generate_uuid(),
}
sgt_table = utils.load_table('share_group_types', engine)
engine.execute(sgt_table.insert(sgt_data))
# Setup share group
sg_data = {
'id': self.share_group_id,
'project_id': 'fake_project_id',
'user_id': 'fake_user_id',
'share_group_type_id': self.share_group_type_id,
}
sg_table = utils.load_table('share_groups', engine)
engine.execute(sg_table.insert(sg_data))
# Setup shares
share_data = {
'id': self.share_id,
'share_group_id': self.share_group_id,
}
s_table = utils.load_table('shares', engine)
engine.execute(s_table.insert(share_data))
# Setup share instances
share_instance_data = {
'id': self.share_instance_id,
'share_id': share_data['id'],
'cast_rules_to_readonly': False,
}
si_table = utils.load_table('share_instances', engine)
engine.execute(si_table.insert(share_instance_data))
# Setup share group snapshot
sgs_data = {
'id': self.share_group_snapshot_id,
'share_group_id': self.share_group_id,
'project_id': 'fake_project_id',
'user_id': 'fake_user_id',
}
sgs_table = utils.load_table('share_group_snapshots', engine)
engine.execute(sgs_table.insert(sgs_data))
# Setup share group snapshot member
sgsm_data = {
'id': self.share_group_snapshot_member_id,
'share_group_snapshot_id': self.share_group_snapshot_id,
'share_id': self.share_id,
'share_instance_id': self.share_instance_id,
'project_id': 'fake_project_id',
'user_id': 'fake_user_id',
}
sgsm_table = utils.load_table(self.old_table_name, engine)
engine.execute(sgsm_table.insert(sgsm_data))
def check_upgrade(self, engine, data):
ssi_table = utils.load_table(self.new_table_name, engine)
db_result = engine.execute(ssi_table.select().where(
ssi_table.c.id == self.share_group_snapshot_member_id))
self.test_case.assertEqual(1, db_result.rowcount)
for ssi in db_result:
for key in self.keys:
self.test_case.assertTrue(hasattr(ssi, key))
# Check that we can write string data to the new fields
engine.execute(ssi_table.update().where(
ssi_table.c.id == self.share_group_snapshot_member_id,
).values({
'user_id': ('u' * 255),
'project_id': ('p' * 255),
'share_proto': ('s' * 255),
'size': 123456789,
'share_id': self.share_id,
'share_group_snapshot_id': self.share_group_snapshot_id,
}))
# Check that table 'share_group_snapshot_members' does not
# exist anymore
self.test_case.assertRaises(
sa_exc.NoSuchTableError,
utils.load_table, 'share_group_snapshot_members', engine)
def check_downgrade(self, engine):
sgsm_table = utils.load_table(self.old_table_name, engine)
db_result = engine.execute(sgsm_table.select().where(
sgsm_table.c.id == self.share_group_snapshot_member_id))
self.test_case.assertEqual(1, db_result.rowcount)
for sgsm in db_result:
for key in self.keys:
self.test_case.assertTrue(hasattr(sgsm, key))
# Check that create SGS member is absent in SSI table
ssi_table = utils.load_table(self.new_table_name, engine)
db_result = engine.execute(ssi_table.select().where(
ssi_table.c.id == self.share_group_snapshot_member_id))
self.test_case.assertEqual(0, db_result.rowcount)

View File

@ -885,9 +885,11 @@ class ShareGroupDatabaseAPITestCase(test.TestCase):
def test_share_group_snapshot_members_get_all(self):
sg = db_utils.create_share_group()
share = db_utils.create_share(share_group_id=sg['id'])
si = db_utils.create_share_instance(share_id=share['id'])
sg_snap = db_utils.create_share_group_snapshot(sg['id'])
expected_member = db_utils.create_share_group_snapshot_member(
sg_snap['id'])
sg_snap['id'], share_instance_id=si['id'])
members = db_api.share_group_snapshot_members_get_all(
self.ctxt, sg_snap['id'])
@ -896,14 +898,16 @@ class ShareGroupDatabaseAPITestCase(test.TestCase):
self.assertDictMatch(dict(expected_member), dict(members[0]))
def test_count_share_group_snapshot_members_in_share(self):
share = db_utils.create_share()
share2 = db_utils.create_share()
sg = db_utils.create_share_group()
share = db_utils.create_share(share_group_id=sg['id'])
si = db_utils.create_share_instance(share_id=share['id'])
share2 = db_utils.create_share(share_group_id=sg['id'])
si2 = db_utils.create_share_instance(share_id=share2['id'])
sg_snap = db_utils.create_share_group_snapshot(sg['id'])
db_utils.create_share_group_snapshot_member(
sg_snap['id'], share_id=share['id'])
sg_snap['id'], share_id=share['id'], share_instance_id=si['id'])
db_utils.create_share_group_snapshot_member(
sg_snap['id'], share_id=share2['id'])
sg_snap['id'], share_id=share2['id'], share_instance_id=si2['id'])
count = db_api.count_share_group_snapshot_members_in_share(
self.ctxt, share['id'])
@ -912,9 +916,11 @@ class ShareGroupDatabaseAPITestCase(test.TestCase):
def test_share_group_snapshot_members_get(self):
sg = db_utils.create_share_group()
share = db_utils.create_share(share_group_id=sg['id'])
si = db_utils.create_share_instance(share_id=share['id'])
sg_snap = db_utils.create_share_group_snapshot(sg['id'])
expected_member = db_utils.create_share_group_snapshot_member(
sg_snap['id'])
sg_snap['id'], share_instance_id=si['id'])
member = db_api.share_group_snapshot_member_get(
self.ctxt, expected_member['id'])
@ -928,9 +934,11 @@ class ShareGroupDatabaseAPITestCase(test.TestCase):
def test_share_group_snapshot_member_update(self):
sg = db_utils.create_share_group()
share = db_utils.create_share(share_group_id=sg['id'])
si = db_utils.create_share_instance(share_id=share['id'])
sg_snap = db_utils.create_share_group_snapshot(sg['id'])
expected_member = db_utils.create_share_group_snapshot_member(
sg_snap['id'])
sg_snap['id'], share_instance_id=si['id'])
db_api.share_group_snapshot_member_update(
self.ctxt, expected_member['id'],

View File

@ -1060,7 +1060,8 @@ class GenericShareDriverTestCase(test.TestCase):
def test_create_snapshot(self):
fake_vol = fake_volume.FakeVolume()
fake_vol_snap = fake_volume.FakeVolumeSnapshot(share_id=fake_vol['id'])
fake_vol_snap = fake_volume.FakeVolumeSnapshot(
share_instance_id=fake_vol['id'])
self.mock_object(self._driver, '_get_volume',
mock.Mock(return_value=fake_vol))
self.mock_object(self._driver.volume_api, 'create_snapshot_force',
@ -1070,7 +1071,7 @@ class GenericShareDriverTestCase(test.TestCase):
share_server=self.server)
self._driver._get_volume.assert_called_once_with(
self._driver.admin_context, fake_vol_snap['share_id'])
self._driver.admin_context, fake_vol_snap['share_instance_id'])
self._driver.volume_api.create_snapshot_force.assert_called_once_with(
self._context,
fake_vol['id'],