diff --git a/glance/db/sqlalchemy/migrate_repo/versions/042_add_changes_to_reinstall_unique_metadef_constraints.py b/glance/db/sqlalchemy/migrate_repo/versions/042_add_changes_to_reinstall_unique_metadef_constraints.py new file mode 100755 index 0000000000..836e953bc5 --- /dev/null +++ b/glance/db/sqlalchemy/migrate_repo/versions/042_add_changes_to_reinstall_unique_metadef_constraints.py @@ -0,0 +1,604 @@ + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import migrate +import sqlalchemy +from sqlalchemy import (func, Index, inspect, orm, String, Table, type_coerce) + + +# The _upgrade...get_duplicate() def's are separate functions to +# accommodate sqlite which locks the database against updates as long as +# db_recs is active. +# In addition, sqlite doesn't support the function 'concat' between +# Strings and Integers, so, the updating of records is also adjusted. +def _upgrade_metadef_namespaces_get_duplicates(migrate_engine): + meta = sqlalchemy.schema.MetaData(migrate_engine) + metadef_namespaces = Table('metadef_namespaces', meta, autoload=True) + + session = orm.sessionmaker(bind=migrate_engine)() + db_recs = (session.query(func.min(metadef_namespaces.c.id), + metadef_namespaces.c.namespace) + .group_by(metadef_namespaces.c.namespace) + .having(func.count(metadef_namespaces.c.namespace) > 1)) + dbrecs = [] + for row in db_recs: + dbrecs.append({'id': row[0], 'namespace': row[1]}) + session.close() + + return dbrecs + + +def _upgrade_metadef_objects_get_duplicates(migrate_engine): + meta = sqlalchemy.schema.MetaData(migrate_engine) + metadef_objects = Table('metadef_objects', meta, autoload=True) + + session = orm.sessionmaker(bind=migrate_engine)() + db_recs = (session.query(func.min(metadef_objects.c.id), + metadef_objects.c.namespace_id, + metadef_objects.c.name) + .group_by(metadef_objects.c.namespace_id, + metadef_objects.c.name) + .having(func.count() > 1)) + dbrecs = [] + for row in db_recs: + dbrecs.append({'id': row[0], 'namespace_id': row[1], 'name': row[2]}) + session.close() + + return dbrecs + + +def _upgrade_metadef_properties_get_duplicates(migrate_engine): + meta = sqlalchemy.schema.MetaData(migrate_engine) + metadef_properties = Table('metadef_properties', meta, autoload=True) + + session = orm.sessionmaker(bind=migrate_engine)() + db_recs = (session.query(func.min(metadef_properties.c.id), + metadef_properties.c.namespace_id, + metadef_properties.c.name) + .group_by(metadef_properties.c.namespace_id, + metadef_properties.c.name) + .having(func.count() > 1)) + dbrecs = [] + for row in db_recs: + dbrecs.append({'id': row[0], 'namespace_id': row[1], 'name': row[2]}) + session.close() + + return dbrecs + + +def _upgrade_metadef_tags_get_duplicates(migrate_engine): + meta = sqlalchemy.schema.MetaData(migrate_engine) + metadef_tags = Table('metadef_tags', meta, autoload=True) + + session = orm.sessionmaker(bind=migrate_engine)() + db_recs = (session.query(func.min(metadef_tags.c.id), + metadef_tags.c.namespace_id, + metadef_tags.c.name) + .group_by(metadef_tags.c.namespace_id, + metadef_tags.c.name) + .having(func.count() > 1)) + dbrecs = [] + for row in db_recs: + dbrecs.append({'id': row[0], 'namespace_id': row[1], 'name': row[2]}) + session.close() + + return dbrecs + + +def _upgrade_metadef_resource_types_get_duplicates(migrate_engine): + meta = sqlalchemy.schema.MetaData(migrate_engine) + metadef_resource_types = Table('metadef_resource_types', meta, + autoload=True) + + session = orm.sessionmaker(bind=migrate_engine)() + db_recs = (session.query(func.min(metadef_resource_types.c.id), + metadef_resource_types.c.name) + .group_by(metadef_resource_types.c.name) + .having(func.count(metadef_resource_types.c.name) > 1)) + dbrecs = [] + for row in db_recs: + dbrecs.append({'id': row[0], 'name': row[1]}) + session.close() + + return dbrecs + + +def _upgrade_data(migrate_engine): + # Rename duplicates to be unique. + meta = sqlalchemy.schema.MetaData(migrate_engine) + + # ORM tables + metadef_namespaces = Table('metadef_namespaces', meta, autoload=True) + metadef_objects = Table('metadef_objects', meta, autoload=True) + metadef_properties = Table('metadef_properties', meta, autoload=True) + metadef_tags = Table('metadef_tags', meta, autoload=True) + metadef_resource_types = Table('metadef_resource_types', meta, + autoload=True) + + # Fix duplicate metadef_namespaces + # Update the non-first record(s) with an unique namespace value + dbrecs = _upgrade_metadef_namespaces_get_duplicates(migrate_engine) + for row in dbrecs: + s = (metadef_namespaces.update() + .where(metadef_namespaces.c.id > row['id']) + .where(metadef_namespaces.c.namespace == row['namespace']) + ) + if migrate_engine.name == 'sqlite': + s = (s.values(namespace=(row['namespace'] + '-DUPL-' + + type_coerce(metadef_namespaces.c.id, + String)), + display_name=(row['namespace'] + '-DUPL-' + + type_coerce(metadef_namespaces.c.id, + String)))) + else: + s = s.values(namespace=func.concat(row['namespace'], + '-DUPL-', + metadef_namespaces.c.id), + display_name=func.concat(row['namespace'], + '-DUPL-', + metadef_namespaces.c.id)) + s.execute() + + # Fix duplicate metadef_objects + dbrecs = _upgrade_metadef_objects_get_duplicates(migrate_engine) + for row in dbrecs: + s = (metadef_objects.update() + .where(metadef_objects.c.id > row['id']) + .where(metadef_objects.c.namespace_id == row['namespace_id']) + .where(metadef_objects.c.name == str(row['name'])) + ) + if migrate_engine.name == 'sqlite': + s = (s.values(name=(row['name'] + '-DUPL-' + + type_coerce(metadef_objects.c.id, String)))) + else: + s = s.values(name=func.concat(row['name'], '-DUPL-', + metadef_objects.c.id)) + s.execute() + + # Fix duplicate metadef_properties + dbrecs = _upgrade_metadef_properties_get_duplicates(migrate_engine) + for row in dbrecs: + s = (metadef_properties.update() + .where(metadef_properties.c.id > row['id']) + .where(metadef_properties.c.namespace_id == row['namespace_id']) + .where(metadef_properties.c.name == str(row['name'])) + ) + if migrate_engine.name == 'sqlite': + s = (s.values(name=(row['name'] + '-DUPL-' + + type_coerce(metadef_properties.c.id, String))) + ) + else: + s = s.values(name=func.concat(row['name'], '-DUPL-', + metadef_properties.c.id)) + s.execute() + + # Fix duplicate metadef_tags + dbrecs = _upgrade_metadef_tags_get_duplicates(migrate_engine) + for row in dbrecs: + s = (metadef_tags.update() + .where(metadef_tags.c.id > row['id']) + .where(metadef_tags.c.namespace_id == row['namespace_id']) + .where(metadef_tags.c.name == str(row['name'])) + ) + if migrate_engine.name == 'sqlite': + s = (s.values(name=(row['name'] + '-DUPL-' + + type_coerce(metadef_tags.c.id, String))) + ) + else: + s = s.values(name=func.concat(row['name'], '-DUPL-', + metadef_tags.c.id)) + s.execute() + + # Fix duplicate metadef_resource_types + dbrecs = _upgrade_metadef_resource_types_get_duplicates(migrate_engine) + for row in dbrecs: + s = (metadef_resource_types.update() + .where(metadef_resource_types.c.id > row['id']) + .where(metadef_resource_types.c.name == str(row['name'])) + ) + if migrate_engine.name == 'sqlite': + s = (s.values(name=(row['name'] + '-DUPL-' + + type_coerce(metadef_resource_types.c.id, + String))) + ) + else: + s = s.values(name=func.concat(row['name'], '-DUPL-', + metadef_resource_types.c.id)) + s.execute() + + +def _update_sqlite_namespace_id_name_constraint(metadef, metadef_namespaces, + new_constraint_name, + new_fk_name): + migrate.UniqueConstraint( + metadef.c.namespace_id, metadef.c.name).drop() + migrate.UniqueConstraint( + metadef.c.namespace_id, metadef.c.name, + name=new_constraint_name).create() + migrate.ForeignKeyConstraint( + [metadef.c.namespace_id], + [metadef_namespaces.c.id], + name=new_fk_name).create() + + +def _downgrade_sqlite_namespace_id_name_constraint(metadef, + metadef_namespaces, + constraint_name, + fk_name): + migrate.UniqueConstraint( + metadef.c.namespace_id, + metadef.c.name, + name=constraint_name).drop() + migrate.UniqueConstraint( + metadef.c.namespace_id, + metadef.c.name).create() + + migrate.ForeignKeyConstraint( + [metadef.c.namespace_id], + [metadef_namespaces.c.id], + name=fk_name).drop() + migrate.ForeignKeyConstraint( + [metadef.c.namespace_id], + [metadef_namespaces.c.id]).create() + + +def _drop_unique_constraint_if_exists(inspector, table_name, metadef): + name = _get_unique_constraint_name(inspector, + table_name, + ['namespace_id', 'name']) + if name: + migrate.UniqueConstraint(metadef.c.namespace_id, + metadef.c.name, + name=name).drop() + + +def _drop_index_with_fk_constraint(metadef, metadef_namespaces, + index_name, + fk_old_name, fk_new_name): + + fkc = migrate.ForeignKeyConstraint([metadef.c.namespace_id], + [metadef_namespaces.c.id], + name=fk_old_name) + fkc.drop() + + if index_name: + Index(index_name, metadef.c.namespace_id).drop() + + # Rename the fk for consistency across all db's + fkc = migrate.ForeignKeyConstraint([metadef.c.namespace_id], + [metadef_namespaces.c.id], + name=fk_new_name) + fkc.create() + + +def _downgrade_constraint_with_fk(metadef, metadef_namespaces, + constraint_name, + fk_curr_name, fk_next_name): + + fkc = migrate.ForeignKeyConstraint([metadef.c.namespace_id], + [metadef_namespaces.c.id], + name=fk_curr_name) + fkc.drop() + + migrate.UniqueConstraint(metadef.c.namespace_id, metadef.c.name, + name=constraint_name).drop() + + fkc = migrate.ForeignKeyConstraint([metadef.c.namespace_id], + [metadef_namespaces.c.id], + name=fk_next_name) + fkc.create() + + +def _get_unique_constraint_name(inspector, table_name, columns): + constraints = inspector.get_unique_constraints(table_name) + for constraint in constraints: + if set(constraint['column_names']) == set(columns): + return constraint['name'] + return None + + +def _get_fk_constraint_name(inspector, table_name, columns): + constraints = inspector.get_foreign_keys(table_name) + for constraint in constraints: + if set(constraint['constrained_columns']) == set(columns): + return constraint['name'] + return None + + +def upgrade(migrate_engine): + + _upgrade_data(migrate_engine) + + meta = sqlalchemy.MetaData() + meta.bind = migrate_engine + inspector = inspect(migrate_engine) + + # ORM tables + metadef_namespaces = Table('metadef_namespaces', meta, autoload=True) + metadef_objects = Table('metadef_objects', meta, autoload=True) + metadef_properties = Table('metadef_properties', meta, autoload=True) + metadef_tags = Table('metadef_tags', meta, autoload=True) + metadef_ns_res_types = Table('metadef_namespace_resource_types', + meta, autoload=True) + metadef_resource_types = Table('metadef_resource_types', meta, + autoload=True) + + # Drop the bad, non-unique indices. + if migrate_engine.name == 'sqlite': + # For sqlite: + # Only after the unique constraints have been added should the indices + # be dropped. If done the other way, sqlite complains during + # constraint adding/dropping that the index does/does not exist. + # Note: The _get_unique_constraint_name, _get_fk_constraint_name + # return None for constraints that do in fact exist. Also, + # get_index_names returns names, but, the names can not be used with + # the Index(name, blah).drop() command, so, putting sqlite into + # it's own section. + + # Objects + _update_sqlite_namespace_id_name_constraint( + metadef_objects, metadef_namespaces, + 'uq_metadef_objects_namespace_id_name', + 'metadef_objects_fk_1') + + # Properties + _update_sqlite_namespace_id_name_constraint( + metadef_properties, metadef_namespaces, + 'uq_metadef_properties_namespace_id_name', + 'metadef_properties_fk_1') + + # Tags + _update_sqlite_namespace_id_name_constraint( + metadef_tags, metadef_namespaces, + 'uq_metadef_tags_namespace_id_name', + 'metadef_tags_fk_1') + + # Namespaces + migrate.UniqueConstraint( + metadef_namespaces.c.namespace).drop() + migrate.UniqueConstraint( + metadef_namespaces.c.namespace, + name='uq_metadef_namespaces_namespace').create() + + # ResourceTypes + migrate.UniqueConstraint( + metadef_resource_types.c.name).drop() + migrate.UniqueConstraint( + metadef_resource_types.c.name, + name='uq_metadef_resource_types_name').create() + + # Now drop the bad indices + Index('ix_metadef_objects_namespace_id', + metadef_objects.c.namespace_id, + metadef_objects.c.name).drop() + Index('ix_metadef_properties_namespace_id', + metadef_properties.c.namespace_id, + metadef_properties.c.name).drop() + Index('ix_metadef_tags_namespace_id', + metadef_tags.c.namespace_id, + metadef_tags.c.name).drop() + else: + # First drop the bad non-unique indices. + # To do that (for mysql), must first drop foreign key constraints + # BY NAME and then drop the bad indices. + # Finally, re-create the foreign key constraints with a consistent + # name. + + # DB2 still has unique constraints, but, they are badly named. + # Drop them, they will be recreated at the final step. + name = _get_unique_constraint_name(inspector, 'metadef_namespaces', + ['namespace']) + if name: + migrate.UniqueConstraint(metadef_namespaces.c.namespace, + name=name).drop() + _drop_unique_constraint_if_exists(inspector, 'metadef_objects', + metadef_objects) + _drop_unique_constraint_if_exists(inspector, 'metadef_properties', + metadef_properties) + _drop_unique_constraint_if_exists(inspector, 'metadef_tags', + metadef_tags) + name = _get_unique_constraint_name(inspector, 'metadef_resource_types', + ['name']) + if name: + migrate.UniqueConstraint(metadef_resource_types.c.name, + name=name).drop() + + # Objects + _drop_index_with_fk_constraint( + metadef_objects, metadef_namespaces, + 'ix_metadef_objects_namespace_id', + _get_fk_constraint_name( + inspector, 'metadef_objects', ['namespace_id']), + 'metadef_objects_fk_1') + + # Properties + _drop_index_with_fk_constraint( + metadef_properties, metadef_namespaces, + 'ix_metadef_properties_namespace_id', + _get_fk_constraint_name( + inspector, 'metadef_properties', ['namespace_id']), + 'metadef_properties_fk_1') + + # Tags + _drop_index_with_fk_constraint( + metadef_tags, metadef_namespaces, + 'ix_metadef_tags_namespace_id', + _get_fk_constraint_name( + inspector, 'metadef_tags', ['namespace_id']), + 'metadef_tags_fk_1') + + # Drop Others without fk constraints. + Index('ix_metadef_namespaces_namespace', + metadef_namespaces.c.namespace).drop() + + # The next two don't exist in ibm_db_sa, but, drop them everywhere else. + if migrate_engine.name != 'ibm_db_sa': + Index('ix_metadef_resource_types_name', + metadef_resource_types.c.name).drop() + # Not needed due to primary key on same columns + Index('ix_metadef_ns_res_types_res_type_id_ns_id', + metadef_ns_res_types.c.resource_type_id, + metadef_ns_res_types.c.namespace_id).drop() + + # Now, add back the dropped indexes as unique constraints + if migrate_engine.name != 'sqlite': + # Namespaces + migrate.UniqueConstraint( + metadef_namespaces.c.namespace, + name='uq_metadef_namespaces_namespace').create() + + # Objects + migrate.UniqueConstraint( + metadef_objects.c.namespace_id, + metadef_objects.c.name, + name='uq_metadef_objects_namespace_id_name').create() + + # Properties + migrate.UniqueConstraint( + metadef_properties.c.namespace_id, + metadef_properties.c.name, + name='uq_metadef_properties_namespace_id_name').create() + + # Tags + migrate.UniqueConstraint( + metadef_tags.c.namespace_id, + metadef_tags.c.name, + name='uq_metadef_tags_namespace_id_name').create() + + # Resource Types + migrate.UniqueConstraint( + metadef_resource_types.c.name, + name='uq_metadef_resource_types_name').create() + + +def downgrade(migrate_engine): + meta = sqlalchemy.MetaData() + meta.bind = migrate_engine + + # ORM tables + metadef_namespaces = Table('metadef_namespaces', meta, autoload=True) + metadef_objects = Table('metadef_objects', meta, autoload=True) + metadef_properties = Table('metadef_properties', meta, autoload=True) + metadef_tags = Table('metadef_tags', meta, autoload=True) + metadef_resource_types = Table('metadef_resource_types', meta, + autoload=True) + metadef_ns_res_types = Table('metadef_namespace_resource_types', + meta, autoload=True) + + # Drop the unique constraints + if migrate_engine.name == 'sqlite': + # Objects + _downgrade_sqlite_namespace_id_name_constraint( + metadef_objects, metadef_namespaces, + 'uq_metadef_objects_namespace_id_name', + 'metadef_objects_fk_1') + + # Properties + _downgrade_sqlite_namespace_id_name_constraint( + metadef_properties, metadef_namespaces, + 'uq_metadef_properties_namespace_id_name', + 'metadef_properties_fk_1') + + # Tags + _downgrade_sqlite_namespace_id_name_constraint( + metadef_tags, metadef_namespaces, + 'uq_metadef_tags_namespace_id_name', + 'metadef_tags_fk_1') + + # Namespaces + migrate.UniqueConstraint( + metadef_namespaces.c.namespace, + name='uq_metadef_namespaces_namespace').drop() + migrate.UniqueConstraint( + metadef_namespaces.c.namespace).create() + + # ResourceTypes + migrate.UniqueConstraint( + metadef_resource_types.c.name, + name='uq_metadef_resource_types_name').drop() + migrate.UniqueConstraint( + metadef_resource_types.c.name).create() + else: + # For mysql, must drop foreign key constraints before dropping the + # unique constraint. So drop the fkc, then drop the constraints, + # then recreate the fkc. + + # Objects + _downgrade_constraint_with_fk( + metadef_objects, metadef_namespaces, + 'uq_metadef_objects_namespace_id_name', + 'metadef_objects_fk_1', None) + + # Properties + _downgrade_constraint_with_fk( + metadef_properties, metadef_namespaces, + 'uq_metadef_properties_namespace_id_name', + 'metadef_properties_fk_1', None) + + # Tags + _downgrade_constraint_with_fk( + metadef_tags, metadef_namespaces, + 'uq_metadef_tags_namespace_id_name', + 'metadef_tags_fk_1', 'metadef_tags_namespace_id_fkey') + + # Namespaces + migrate.UniqueConstraint( + metadef_namespaces.c.namespace, + name='uq_metadef_namespaces_namespace').drop() + + # Resource_types + migrate.UniqueConstraint( + metadef_resource_types.c.name, + name='uq_metadef_resource_types_name').drop() + + # Create dropped unique constraints as bad, non-unique indexes + Index('ix_metadef_objects_namespace_id', + metadef_objects.c.namespace_id).create() + Index('ix_metadef_properties_namespace_id', + metadef_properties.c.namespace_id).create() + + # These need to be done before the metadef_tags and metadef_namespaces + # unique constraints are created to avoid 'tuple out of range' errors + # in db2. + Index('ix_metadef_tags_namespace_id', + metadef_tags.c.namespace_id, + metadef_tags.c.name).create() + Index('ix_metadef_namespaces_namespace', + metadef_namespaces.c.namespace).create() + + # Create these everywhere, except for db2 + if migrate_engine.name != 'ibm_db_sa': + Index('ix_metadef_resource_types_name', + metadef_resource_types.c.name).create() + Index('ix_metadef_ns_res_types_res_type_id_ns_id', + metadef_ns_res_types.c.resource_type_id, + metadef_ns_res_types.c.namespace_id).create() + else: + # Recreate the badly named unique constraints in db2 + migrate.UniqueConstraint( + metadef_namespaces.c.namespace, + name='ix_namespaces_namespace').create() + migrate.UniqueConstraint( + metadef_objects.c.namespace_id, + metadef_objects.c.name, + name='ix_objects_namespace_id_name').create() + migrate.UniqueConstraint( + metadef_properties.c.namespace_id, + metadef_properties.c.name, + name='ix_metadef_properties_namespace_id_name').create() + migrate.UniqueConstraint( + metadef_tags.c.namespace_id, + metadef_tags.c.name).create() + migrate.UniqueConstraint( + metadef_resource_types.c.name, + name='ix_metadef_resource_types_name').create() diff --git a/glance/db/sqlalchemy/models_metadef.py b/glance/db/sqlalchemy/models_metadef.py index caf2bf8941..4ff7631911 100644 --- a/glance/db/sqlalchemy/models_metadef.py +++ b/glance/db/sqlalchemy/models_metadef.py @@ -28,6 +28,7 @@ from sqlalchemy import Integer from sqlalchemy.orm import relationship from sqlalchemy import String from sqlalchemy import Text +from sqlalchemy import UniqueConstraint from glance.db.sqlalchemy.models import JSONEncodedDict @@ -65,8 +66,11 @@ class GlanceMetadefBase(models.TimestampMixin): class MetadefNamespace(BASE_DICT, GlanceMetadefBase): """Represents a metadata-schema namespace in the datastore.""" __tablename__ = 'metadef_namespaces' - __table_args__ = (Index('ix_metadef_namespaces_namespace', 'namespace'), - Index('ix_metadef_namespaces_owner', 'owner')) + __table_args__ = (UniqueConstraint('namespace', + name='uq_metadef_namespaces' + '_namespace'), + Index('ix_metadef_namespaces_owner', 'owner') + ) id = Column(Integer, primary_key=True, nullable=False) namespace = Column(String(80), nullable=False) @@ -80,8 +84,11 @@ class MetadefNamespace(BASE_DICT, GlanceMetadefBase): class MetadefObject(BASE_DICT, GlanceMetadefBase): """Represents a metadata-schema object in the datastore.""" __tablename__ = 'metadef_objects' - __table_args__ = (Index('ix_metadef_objects_namespace_id', 'namespace_id'), - Index('ix_metadef_objects_name', 'name')) + __table_args__ = (UniqueConstraint('namespace_id', 'name', + name='uq_metadef_objects_namespace_id' + '_name'), + Index('ix_metadef_objects_name', 'name') + ) id = Column(Integer, primary_key=True, nullable=False) namespace_id = Column(Integer(), ForeignKey('metadef_namespaces.id'), @@ -95,9 +102,11 @@ class MetadefObject(BASE_DICT, GlanceMetadefBase): class MetadefProperty(BASE_DICT, GlanceMetadefBase): """Represents a metadata-schema namespace-property in the datastore.""" __tablename__ = 'metadef_properties' - __table_args__ = (Index('ix_metadef_properties_namespace_id', - 'namespace_id'), - Index('ix_metadef_properties_name', 'name')) + __table_args__ = (UniqueConstraint('namespace_id', 'name', + name='uq_metadef_properties_namespace' + '_id_name'), + Index('ix_metadef_properties_name', 'name') + ) id = Column(Integer, primary_key=True, nullable=False) namespace_id = Column(Integer(), ForeignKey('metadef_namespaces.id'), @@ -109,10 +118,9 @@ class MetadefProperty(BASE_DICT, GlanceMetadefBase): class MetadefNamespaceResourceType(BASE_DICT, GlanceMetadefBase): """Represents a metadata-schema namespace-property in the datastore.""" __tablename__ = 'metadef_namespace_resource_types' - __table_args__ = (Index('ix_metadef_ns_res_types_res_type_id_ns_id', - 'resource_type_id', 'namespace_id'), - Index('ix_metadef_ns_res_types_namespace_id', - 'namespace_id')) + __table_args__ = (Index('ix_metadef_ns_res_types_namespace_id', + 'namespace_id'), + ) resource_type_id = Column(Integer, ForeignKey('metadef_resource_types.id'), @@ -126,7 +134,9 @@ class MetadefNamespaceResourceType(BASE_DICT, GlanceMetadefBase): class MetadefResourceType(BASE_DICT, GlanceMetadefBase): """Represents a metadata-schema resource type in the datastore.""" __tablename__ = 'metadef_resource_types' - __table_args__ = (Index('ix_metadef_resource_types_name', 'name'), ) + __table_args__ = (UniqueConstraint('name', + name='uq_metadef_resource_types_name'), + ) id = Column(Integer, primary_key=True, nullable=False) name = Column(String(80), nullable=False) @@ -140,9 +150,11 @@ class MetadefResourceType(BASE_DICT, GlanceMetadefBase): class MetadefTag(BASE_DICT, GlanceMetadefBase): """Represents a metadata-schema tag in the data store.""" __tablename__ = 'metadef_tags' - __table_args__ = (Index('ix_metadef_tags_namespace_id', - 'namespace_id', 'name'), - Index('ix_metadef_tags_name', 'name')) + __table_args__ = (UniqueConstraint('namespace_id', 'name', + name='uq_metadef_tags_namespace_id' + '_name'), + Index('ix_metadef_tags_name', 'name') + ) id = Column(Integer, primary_key=True, nullable=False) namespace_id = Column(Integer(), ForeignKey('metadef_namespaces.id'), diff --git a/glance/tests/functional/db/base_metadef.py b/glance/tests/functional/db/base_metadef.py index 986e229cde..87dc813bee 100644 --- a/glance/tests/functional/db/base_metadef.py +++ b/glance/tests/functional/db/base_metadef.py @@ -123,6 +123,15 @@ class MetadefNamespaceTests(object): self.assertIsNotNone(created) self._assert_saved_fields(fixture, created) + def test_namespace_create_duplicate(self): + fixture = build_namespace_fixture() + created = self.db_api.metadef_namespace_create(self.context, fixture) + self.assertIsNotNone(created) + self._assert_saved_fields(fixture, created) + self.assertRaises(exception.Duplicate, + self.db_api.metadef_namespace_create, + self.context, fixture) + def test_namespace_get(self): fixture = build_namespace_fixture() created = self.db_api.metadef_namespace_create(self.context, fixture) @@ -222,6 +231,22 @@ class MetadefPropertyTests(object): self.context, created_ns['namespace'], fixture_prop) self._assert_saved_fields(fixture_prop, created_prop) + def test_property_create_duplicate(self): + fixture = build_namespace_fixture() + created_ns = self.db_api.metadef_namespace_create( + self.context, fixture) + self.assertIsNotNone(created_ns) + self._assert_saved_fields(fixture, created_ns) + + fixture_prop = build_property_fixture(namespace_id=created_ns['id']) + created_prop = self.db_api.metadef_property_create( + self.context, created_ns['namespace'], fixture_prop) + self._assert_saved_fields(fixture_prop, created_prop) + + self.assertRaises(exception.Duplicate, + self.db_api.metadef_property_create, + self.context, created_ns['namespace'], fixture_prop) + def test_property_get(self): fixture_ns = build_namespace_fixture() created_ns = self.db_api.metadef_namespace_create( @@ -332,6 +357,23 @@ class MetadefObjectTests(object): self.context, created_ns['namespace'], fixture_object) self._assert_saved_fields(fixture_object, created_object) + def test_object_create_duplicate(self): + fixture = build_namespace_fixture() + created_ns = self.db_api.metadef_namespace_create(self.context, + fixture) + self.assertIsNotNone(created_ns) + self._assert_saved_fields(fixture, created_ns) + + fixture_object = build_object_fixture(namespace_id=created_ns['id']) + created_object = self.db_api.metadef_object_create( + self.context, created_ns['namespace'], fixture_object) + self._assert_saved_fields(fixture_object, created_object) + + self.assertRaises(exception.Duplicate, + self.db_api.metadef_object_create, + self.context, created_ns['namespace'], + fixture_object) + def test_object_get(self): fixture_ns = build_namespace_fixture() created_ns = self.db_api.metadef_namespace_create(self.context, @@ -442,6 +484,24 @@ class MetadefResourceTypeAssociationTests(object): self.assertIsNotNone(assn_created) self._assert_saved_fields(assn_fixture, assn_created) + def test_association_create_duplicate(self): + ns_fixture = build_namespace_fixture() + ns_created = self.db_api.metadef_namespace_create( + self.context, ns_fixture) + self.assertIsNotNone(ns_created) + self._assert_saved_fields(ns_fixture, ns_created) + + assn_fixture = build_association_fixture() + assn_created = self.db_api.metadef_resource_type_association_create( + self.context, ns_created['namespace'], assn_fixture) + self.assertIsNotNone(assn_created) + self._assert_saved_fields(assn_fixture, assn_created) + + self.assertRaises(exception.Duplicate, + self.db_api. + metadef_resource_type_association_create, + self.context, ns_created['namespace'], assn_fixture) + def test_association_delete(self): ns_fixture = build_namespace_fixture() ns_created = self.db_api.metadef_namespace_create( @@ -499,6 +559,23 @@ class MetadefTagTests(object): self.context, created_ns['namespace'], fixture_tag) self._assert_saved_fields(fixture_tag, created_tag) + def test_tag_create_duplicate(self): + fixture = build_namespace_fixture() + created_ns = self.db_api.metadef_namespace_create(self.context, + fixture) + self.assertIsNotNone(created_ns) + self._assert_saved_fields(fixture, created_ns) + + fixture_tag = build_tag_fixture(namespace_id=created_ns['id']) + created_tag = self.db_api.metadef_tag_create( + self.context, created_ns['namespace'], fixture_tag) + self._assert_saved_fields(fixture_tag, created_tag) + + self.assertRaises(exception.Duplicate, + self.db_api.metadef_tag_create, + self.context, created_ns['namespace'], + fixture_tag) + def test_tag_create_tags(self): fixture = build_namespace_fixture() created_ns = self.db_api.metadef_namespace_create(self.context, @@ -513,6 +590,35 @@ class MetadefTagTests(object): expected = set(['Tag1', 'Tag2', 'Tag3']) self.assertEqual(expected, actual) + def test_tag_create_duplicate_tags_1(self): + fixture = build_namespace_fixture() + created_ns = self.db_api.metadef_namespace_create(self.context, + fixture) + self.assertIsNotNone(created_ns) + self._assert_saved_fields(fixture, created_ns) + + tags = build_tags_fixture(['Tag1', 'Tag2', 'Tag3', 'Tag2']) + self.assertRaises(exception.Duplicate, + self.db_api.metadef_tag_create_tags, + self.context, created_ns['namespace'], + tags) + + def test_tag_create_duplicate_tags_2(self): + fixture = build_namespace_fixture() + created_ns = self.db_api.metadef_namespace_create(self.context, + fixture) + self.assertIsNotNone(created_ns) + self._assert_saved_fields(fixture, created_ns) + + tags = build_tags_fixture(['Tag1', 'Tag2', 'Tag3']) + self.db_api.metadef_tag_create_tags(self.context, + created_ns['namespace'], tags) + dup_tag = build_tag_fixture(namespace_id=created_ns['id'], + name='Tag3') + self.assertRaises(exception.Duplicate, + self.db_api.metadef_tag_create, + self.context, created_ns['namespace'], dup_tag) + def test_tag_get(self): fixture_ns = build_namespace_fixture() created_ns = self.db_api.metadef_namespace_create(self.context, diff --git a/glance/tests/functional/v2/test_metadef_namespaces.py b/glance/tests/functional/v2/test_metadef_namespaces.py index f1d6de0cdd..fbacadc7e8 100644 --- a/glance/tests/functional/v2/test_metadef_namespaces.py +++ b/glance/tests/functional/v2/test_metadef_namespaces.py @@ -96,6 +96,10 @@ class TestNamespaces(functional.FunctionalTest): for key, value in expected_namespace.items(): self.assertEqual(namespace[key], value, key) + # Attempt to insert a duplicate + response = requests.post(path, headers=headers, data=data) + self.assertEqual(409, response.status_code) + # Get the namespace using the returned Location header response = requests.get(namespace_loc_header, headers=self._headers()) self.assertEqual(200, response.status_code) diff --git a/glance/tests/functional/v2/test_metadef_objects.py b/glance/tests/functional/v2/test_metadef_objects.py index 18e15fcafd..8a14d0ce62 100644 --- a/glance/tests/functional/v2/test_metadef_objects.py +++ b/glance/tests/functional/v2/test_metadef_objects.py @@ -107,6 +107,10 @@ class TestMetadefObjects(functional.FunctionalTest): response = requests.post(path, headers=headers, data=data) self.assertEqual(201, response.status_code) + # Attempt to insert a duplicate + response = requests.post(path, headers=headers, data=data) + self.assertEqual(409, response.status_code) + # Get the metadata object created above path = self._url('/v2/metadefs/namespaces/%s/objects/%s' % (namespace_name, metadata_object_name)) diff --git a/glance/tests/functional/v2/test_metadef_properties.py b/glance/tests/functional/v2/test_metadef_properties.py index d754b70faa..9909bd0d3f 100644 --- a/glance/tests/functional/v2/test_metadef_properties.py +++ b/glance/tests/functional/v2/test_metadef_properties.py @@ -99,6 +99,10 @@ class TestNamespaceProperties(functional.FunctionalTest): response = requests.post(path, headers=headers, data=data) self.assertEqual(201, response.status_code) + # Attempt to insert a duplicate + response = requests.post(path, headers=headers, data=data) + self.assertEqual(409, response.status_code) + # Get the property created above path = self._url('/v2/metadefs/namespaces/%s/properties/%s' % (namespace_name, property_name)) diff --git a/glance/tests/functional/v2/test_metadef_tags.py b/glance/tests/functional/v2/test_metadef_tags.py index 3fba764ba8..d6a75a04c8 100644 --- a/glance/tests/functional/v2/test_metadef_tags.py +++ b/glance/tests/functional/v2/test_metadef_tags.py @@ -105,6 +105,11 @@ class TestMetadefTags(functional.FunctionalTest): if(key in checked_values): self.assertEqual(metadata_tag[key], value, key) + # Try to create a duplicate metadata tag + headers = self._headers({'content-type': 'application/json'}) + response = requests.post(path, headers=headers) + self.assertEqual(409, response.status_code) + # The metadata_tag should be mutable path = self._url('/v2/metadefs/namespaces/%s/tags/%s' % (namespace_name, metadata_tag_name)) diff --git a/glance/tests/unit/test_migrations.py b/glance/tests/unit/test_migrations.py index be70cfe27d..503dbc2a0c 100644 --- a/glance/tests/unit/test_migrations.py +++ b/glance/tests/unit/test_migrations.py @@ -1621,6 +1621,208 @@ class MigrationsMixin(test_migrations.WalkVersionsMixin): self.assert_table(engine, 'artifact_blob_locations', locations_indices, locations_columns) + def _pre_upgrade_042(self, engine): + meta = sqlalchemy.MetaData() + meta.bind = engine + + metadef_namespaces = sqlalchemy.Table('metadef_namespaces', meta, + autoload=True) + metadef_objects = sqlalchemy.Table('metadef_objects', meta, + autoload=True) + metadef_properties = sqlalchemy.Table('metadef_properties', meta, + autoload=True) + metadef_tags = sqlalchemy.Table('metadef_tags', meta, autoload=True) + metadef_resource_types = sqlalchemy.Table('metadef_resource_types', + meta, autoload=True) + metadef_ns_res_types = sqlalchemy.Table( + 'metadef_namespace_resource_types', + meta, autoload=True) + + # These will be dropped and recreated as unique constraints. + self.assertTrue(index_exist('ix_metadef_namespaces_namespace', + metadef_namespaces.name, engine)) + self.assertTrue(index_exist('ix_metadef_objects_namespace_id', + metadef_objects.name, engine)) + self.assertTrue(index_exist('ix_metadef_properties_namespace_id', + metadef_properties.name, engine)) + self.assertTrue(index_exist('ix_metadef_tags_namespace_id', + metadef_tags.name, engine)) + self.assertTrue(index_exist('ix_metadef_resource_types_name', + metadef_resource_types.name, engine)) + + # This one will be dropped - not needed + self.assertTrue(index_exist( + 'ix_metadef_ns_res_types_res_type_id_ns_id', + metadef_ns_res_types.name, engine)) + + # The rest must remain + self.assertTrue(index_exist('ix_metadef_namespaces_owner', + metadef_namespaces.name, engine)) + self.assertTrue(index_exist('ix_metadef_objects_name', + metadef_objects.name, engine)) + self.assertTrue(index_exist('ix_metadef_properties_name', + metadef_properties.name, engine)) + self.assertTrue(index_exist('ix_metadef_tags_name', + metadef_tags.name, engine)) + self.assertTrue(index_exist('ix_metadef_ns_res_types_namespace_id', + metadef_ns_res_types.name, engine)) + + # To be created + self.assertFalse(unique_constraint_exist + ('uq_metadef_objects_namespace_id_name', + metadef_objects.name, engine) + ) + self.assertFalse(unique_constraint_exist + ('uq_metadef_properties_namespace_id_name', + metadef_properties.name, engine) + ) + self.assertFalse(unique_constraint_exist + ('uq_metadef_tags_namespace_id_name', + metadef_tags.name, engine) + ) + self.assertFalse(unique_constraint_exist + ('uq_metadef_namespaces_namespace', + metadef_namespaces.name, engine) + ) + self.assertFalse(unique_constraint_exist + ('uq_metadef_resource_types_name', + metadef_resource_types.name, engine) + ) + + def _check_042(self, engine, data): + meta = sqlalchemy.MetaData() + meta.bind = engine + + metadef_namespaces = sqlalchemy.Table('metadef_namespaces', meta, + autoload=True) + metadef_objects = sqlalchemy.Table('metadef_objects', meta, + autoload=True) + metadef_properties = sqlalchemy.Table('metadef_properties', meta, + autoload=True) + metadef_tags = sqlalchemy.Table('metadef_tags', meta, autoload=True) + metadef_resource_types = sqlalchemy.Table('metadef_resource_types', + meta, autoload=True) + metadef_ns_res_types = sqlalchemy.Table( + 'metadef_namespace_resource_types', + meta, autoload=True) + + # Dropped for unique constraints + self.assertFalse(index_exist('ix_metadef_namespaces_namespace', + metadef_namespaces.name, engine)) + self.assertFalse(index_exist('ix_metadef_objects_namespace_id', + metadef_objects.name, engine)) + self.assertFalse(index_exist('ix_metadef_properties_namespace_id', + metadef_properties.name, engine)) + self.assertFalse(index_exist('ix_metadef_tags_namespace_id', + metadef_tags.name, engine)) + self.assertFalse(index_exist('ix_metadef_resource_types_name', + metadef_resource_types.name, engine)) + + # Dropped - not needed because of the existing primary key + self.assertFalse(index_exist( + 'ix_metadef_ns_res_types_res_type_id_ns_id', + metadef_ns_res_types.name, engine)) + + # Still exist as before + self.assertTrue(index_exist('ix_metadef_namespaces_owner', + metadef_namespaces.name, engine)) + self.assertTrue(index_exist('ix_metadef_ns_res_types_namespace_id', + metadef_ns_res_types.name, engine)) + self.assertTrue(index_exist('ix_metadef_objects_name', + metadef_objects.name, engine)) + self.assertTrue(index_exist('ix_metadef_properties_name', + metadef_properties.name, engine)) + self.assertTrue(index_exist('ix_metadef_tags_name', + metadef_tags.name, engine)) + + self.assertTrue(unique_constraint_exist + ('uq_metadef_namespaces_namespace', + metadef_namespaces.name, engine) + ) + self.assertTrue(unique_constraint_exist + ('uq_metadef_objects_namespace_id_name', + metadef_objects.name, engine) + ) + self.assertTrue(unique_constraint_exist + ('uq_metadef_properties_namespace_id_name', + metadef_properties.name, engine) + ) + self.assertTrue(unique_constraint_exist + ('uq_metadef_tags_namespace_id_name', + metadef_tags.name, engine) + ) + self.assertTrue(unique_constraint_exist + ('uq_metadef_resource_types_name', + metadef_resource_types.name, engine) + ) + + def _post_downgrade_042(self, engine): + meta = sqlalchemy.MetaData() + meta.bind = engine + + metadef_namespaces = sqlalchemy.Table('metadef_namespaces', meta, + autoload=True) + metadef_objects = sqlalchemy.Table('metadef_objects', meta, + autoload=True) + metadef_properties = sqlalchemy.Table('metadef_properties', meta, + autoload=True) + metadef_tags = sqlalchemy.Table('metadef_tags', meta, autoload=True) + metadef_resource_types = sqlalchemy.Table('metadef_resource_types', + meta, autoload=True) + metadef_ns_res_types = sqlalchemy.Table( + 'metadef_namespace_resource_types', + meta, autoload=True) + + # These have been recreated + self.assertTrue(index_exist('ix_metadef_namespaces_namespace', + metadef_namespaces.name, engine)) + self.assertTrue(index_exist('ix_metadef_objects_namespace_id', + metadef_objects.name, engine)) + self.assertTrue(index_exist('ix_metadef_properties_namespace_id', + metadef_properties.name, engine)) + self.assertTrue(index_exist('ix_metadef_tags_namespace_id', + metadef_tags.name, engine)) + self.assertTrue(index_exist('ix_metadef_resource_types_name', + metadef_resource_types.name, engine)) + + self.assertTrue(index_exist( + 'ix_metadef_ns_res_types_res_type_id_ns_id', + metadef_ns_res_types.name, engine)) + + # The rest must remain + self.assertTrue(index_exist('ix_metadef_namespaces_owner', + metadef_namespaces.name, engine)) + self.assertTrue(index_exist('ix_metadef_objects_name', + metadef_objects.name, engine)) + self.assertTrue(index_exist('ix_metadef_properties_name', + metadef_properties.name, engine)) + self.assertTrue(index_exist('ix_metadef_tags_name', + metadef_tags.name, engine)) + self.assertTrue(index_exist('ix_metadef_ns_res_types_namespace_id', + metadef_ns_res_types.name, engine)) + + # Dropped + self.assertFalse(unique_constraint_exist + ('uq_metadef_objects_namespace_id_name', + metadef_objects.name, engine) + ) + self.assertFalse(unique_constraint_exist + ('uq_metadef_properties_namespace_id_name', + metadef_properties.name, engine) + ) + self.assertFalse(unique_constraint_exist + ('uq_metadef_tags_namespace_id_name', + metadef_tags.name, engine) + ) + self.assertFalse(unique_constraint_exist + ('uq_metadef_namespaces_namespace', + metadef_namespaces.name, engine) + ) + self.assertFalse(unique_constraint_exist + ('uq_metadef_resource_types_name', + metadef_resource_types.name, engine) + ) + def assert_table(self, engine, table_name, indices, columns): table = db_utils.get_table(engine, table_name) index_data = [(index.name, index.columns.keys()) for index in diff --git a/glance/tests/unit/v2/test_metadef_resources.py b/glance/tests/unit/v2/test_metadef_resources.py index 0751d1b23f..bb68f3b803 100644 --- a/glance/tests/unit/v2/test_metadef_resources.py +++ b/glance/tests/unit/v2/test_metadef_resources.py @@ -605,6 +605,17 @@ class TestMetadefsControllers(base.IsolatedUnitTest): namespace = self.namespace_controller.show(request, NAMESPACE4) self.assertEqual(NAMESPACE4, namespace.namespace) + def test_namespace_create_duplicate(self): + request = unit_test_utils.get_fake_request() + + namespace = namespaces.Namespace() + namespace.namespace = 'new-namespace' + new_ns = self.namespace_controller.create(request, namespace) + self.assertEqual('new-namespace', new_ns.namespace) + self.assertRaises(webob.exc.HTTPConflict, + self.namespace_controller.create, + request, namespace) + def test_namespace_create_different_owner(self): request = unit_test_utils.get_fake_request() @@ -1036,6 +1047,20 @@ class TestMetadefsControllers(base.IsolatedUnitTest): property) self.assertNotificationsLog([]) + def test_property_create_duplicate(self): + request = unit_test_utils.get_fake_request() + + property = properties.PropertyType() + property.name = 'new-property' + property.type = 'string' + property.title = 'title' + new_property = self.property_controller.create(request, NAMESPACE1, + property) + self.assertEqual('new-property', new_property.name) + self.assertRaises(webob.exc.HTTPConflict, + self.property_controller.create, request, + NAMESPACE1, property) + def test_property_update(self): request = unit_test_utils.get_fake_request(tenant=TENANT3) @@ -1254,6 +1279,19 @@ class TestMetadefsControllers(base.IsolatedUnitTest): self.assertEqual([], object.required) self.assertEqual({}, object.properties) + def test_object_create_duplicate(self): + request = unit_test_utils.get_fake_request() + + object = objects.MetadefObject() + object.name = 'New-Object' + object.required = [] + object.properties = {} + new_obj = self.object_controller.create(request, object, NAMESPACE3) + self.assertEqual('New-Object', new_obj.name) + self.assertRaises(webob.exc.HTTPConflict, + self.object_controller.create, request, object, + NAMESPACE3) + def test_object_create_conflict(self): request = unit_test_utils.get_fake_request()