Glance metadef tables need unique constraints.

Sometime during Kilo, the unique constraints on
metadef_namespaces (namespace)
metadef_objects(namespace_id, name)
metadef_properties(namespace_id, name)
metadef_tags(namespace_id, name)
metadef_resource_types(name)
were replaced with non-unique indices.

I believe this was done erroneously to make the migrate_repo/versions/scripts
match the db/sqlalchemy/models_metadef.py definitions. Unfortunately, the
schema scripts were correct with the unique constraints and what should have
changed was models_metadef.py.

This bug, puts one more migrate script in place which will rename any
duplicate records it finds to make them unique and then re-establishes
the unique constraints while dropping the non-unique indices. It also,
fixes models_metadef.py and adds in tests to attempt to create duplicates
which should result in an HTTPConflict.

Change-Id: Idf3569a27d64abea3ed6ec92fb77b36a4d6d5fd5
Closes-Bug: 1468946
This commit is contained in:
Wayne Okuma 2015-06-25 17:07:01 -07:00
parent b2c2ecee50
commit 5369e86e8d
9 changed files with 994 additions and 15 deletions

View File

@ -0,0 +1,604 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import migrate
import sqlalchemy
from sqlalchemy import (func, Index, inspect, orm, String, Table, type_coerce)
# The _upgrade...get_duplicate() def's are separate functions to
# accommodate sqlite which locks the database against updates as long as
# db_recs is active.
# In addition, sqlite doesn't support the function 'concat' between
# Strings and Integers, so, the updating of records is also adjusted.
def _upgrade_metadef_namespaces_get_duplicates(migrate_engine):
meta = sqlalchemy.schema.MetaData(migrate_engine)
metadef_namespaces = Table('metadef_namespaces', meta, autoload=True)
session = orm.sessionmaker(bind=migrate_engine)()
db_recs = (session.query(func.min(metadef_namespaces.c.id),
metadef_namespaces.c.namespace)
.group_by(metadef_namespaces.c.namespace)
.having(func.count(metadef_namespaces.c.namespace) > 1))
dbrecs = []
for row in db_recs:
dbrecs.append({'id': row[0], 'namespace': row[1]})
session.close()
return dbrecs
def _upgrade_metadef_objects_get_duplicates(migrate_engine):
meta = sqlalchemy.schema.MetaData(migrate_engine)
metadef_objects = Table('metadef_objects', meta, autoload=True)
session = orm.sessionmaker(bind=migrate_engine)()
db_recs = (session.query(func.min(metadef_objects.c.id),
metadef_objects.c.namespace_id,
metadef_objects.c.name)
.group_by(metadef_objects.c.namespace_id,
metadef_objects.c.name)
.having(func.count() > 1))
dbrecs = []
for row in db_recs:
dbrecs.append({'id': row[0], 'namespace_id': row[1], 'name': row[2]})
session.close()
return dbrecs
def _upgrade_metadef_properties_get_duplicates(migrate_engine):
meta = sqlalchemy.schema.MetaData(migrate_engine)
metadef_properties = Table('metadef_properties', meta, autoload=True)
session = orm.sessionmaker(bind=migrate_engine)()
db_recs = (session.query(func.min(metadef_properties.c.id),
metadef_properties.c.namespace_id,
metadef_properties.c.name)
.group_by(metadef_properties.c.namespace_id,
metadef_properties.c.name)
.having(func.count() > 1))
dbrecs = []
for row in db_recs:
dbrecs.append({'id': row[0], 'namespace_id': row[1], 'name': row[2]})
session.close()
return dbrecs
def _upgrade_metadef_tags_get_duplicates(migrate_engine):
meta = sqlalchemy.schema.MetaData(migrate_engine)
metadef_tags = Table('metadef_tags', meta, autoload=True)
session = orm.sessionmaker(bind=migrate_engine)()
db_recs = (session.query(func.min(metadef_tags.c.id),
metadef_tags.c.namespace_id,
metadef_tags.c.name)
.group_by(metadef_tags.c.namespace_id,
metadef_tags.c.name)
.having(func.count() > 1))
dbrecs = []
for row in db_recs:
dbrecs.append({'id': row[0], 'namespace_id': row[1], 'name': row[2]})
session.close()
return dbrecs
def _upgrade_metadef_resource_types_get_duplicates(migrate_engine):
meta = sqlalchemy.schema.MetaData(migrate_engine)
metadef_resource_types = Table('metadef_resource_types', meta,
autoload=True)
session = orm.sessionmaker(bind=migrate_engine)()
db_recs = (session.query(func.min(metadef_resource_types.c.id),
metadef_resource_types.c.name)
.group_by(metadef_resource_types.c.name)
.having(func.count(metadef_resource_types.c.name) > 1))
dbrecs = []
for row in db_recs:
dbrecs.append({'id': row[0], 'name': row[1]})
session.close()
return dbrecs
def _upgrade_data(migrate_engine):
# Rename duplicates to be unique.
meta = sqlalchemy.schema.MetaData(migrate_engine)
# ORM tables
metadef_namespaces = Table('metadef_namespaces', meta, autoload=True)
metadef_objects = Table('metadef_objects', meta, autoload=True)
metadef_properties = Table('metadef_properties', meta, autoload=True)
metadef_tags = Table('metadef_tags', meta, autoload=True)
metadef_resource_types = Table('metadef_resource_types', meta,
autoload=True)
# Fix duplicate metadef_namespaces
# Update the non-first record(s) with an unique namespace value
dbrecs = _upgrade_metadef_namespaces_get_duplicates(migrate_engine)
for row in dbrecs:
s = (metadef_namespaces.update()
.where(metadef_namespaces.c.id > row['id'])
.where(metadef_namespaces.c.namespace == row['namespace'])
)
if migrate_engine.name == 'sqlite':
s = (s.values(namespace=(row['namespace'] + '-DUPL-' +
type_coerce(metadef_namespaces.c.id,
String)),
display_name=(row['namespace'] + '-DUPL-' +
type_coerce(metadef_namespaces.c.id,
String))))
else:
s = s.values(namespace=func.concat(row['namespace'],
'-DUPL-',
metadef_namespaces.c.id),
display_name=func.concat(row['namespace'],
'-DUPL-',
metadef_namespaces.c.id))
s.execute()
# Fix duplicate metadef_objects
dbrecs = _upgrade_metadef_objects_get_duplicates(migrate_engine)
for row in dbrecs:
s = (metadef_objects.update()
.where(metadef_objects.c.id > row['id'])
.where(metadef_objects.c.namespace_id == row['namespace_id'])
.where(metadef_objects.c.name == str(row['name']))
)
if migrate_engine.name == 'sqlite':
s = (s.values(name=(row['name'] + '-DUPL-'
+ type_coerce(metadef_objects.c.id, String))))
else:
s = s.values(name=func.concat(row['name'], '-DUPL-',
metadef_objects.c.id))
s.execute()
# Fix duplicate metadef_properties
dbrecs = _upgrade_metadef_properties_get_duplicates(migrate_engine)
for row in dbrecs:
s = (metadef_properties.update()
.where(metadef_properties.c.id > row['id'])
.where(metadef_properties.c.namespace_id == row['namespace_id'])
.where(metadef_properties.c.name == str(row['name']))
)
if migrate_engine.name == 'sqlite':
s = (s.values(name=(row['name'] + '-DUPL-' +
type_coerce(metadef_properties.c.id, String)))
)
else:
s = s.values(name=func.concat(row['name'], '-DUPL-',
metadef_properties.c.id))
s.execute()
# Fix duplicate metadef_tags
dbrecs = _upgrade_metadef_tags_get_duplicates(migrate_engine)
for row in dbrecs:
s = (metadef_tags.update()
.where(metadef_tags.c.id > row['id'])
.where(metadef_tags.c.namespace_id == row['namespace_id'])
.where(metadef_tags.c.name == str(row['name']))
)
if migrate_engine.name == 'sqlite':
s = (s.values(name=(row['name'] + '-DUPL-' +
type_coerce(metadef_tags.c.id, String)))
)
else:
s = s.values(name=func.concat(row['name'], '-DUPL-',
metadef_tags.c.id))
s.execute()
# Fix duplicate metadef_resource_types
dbrecs = _upgrade_metadef_resource_types_get_duplicates(migrate_engine)
for row in dbrecs:
s = (metadef_resource_types.update()
.where(metadef_resource_types.c.id > row['id'])
.where(metadef_resource_types.c.name == str(row['name']))
)
if migrate_engine.name == 'sqlite':
s = (s.values(name=(row['name'] + '-DUPL-' +
type_coerce(metadef_resource_types.c.id,
String)))
)
else:
s = s.values(name=func.concat(row['name'], '-DUPL-',
metadef_resource_types.c.id))
s.execute()
def _update_sqlite_namespace_id_name_constraint(metadef, metadef_namespaces,
new_constraint_name,
new_fk_name):
migrate.UniqueConstraint(
metadef.c.namespace_id, metadef.c.name).drop()
migrate.UniqueConstraint(
metadef.c.namespace_id, metadef.c.name,
name=new_constraint_name).create()
migrate.ForeignKeyConstraint(
[metadef.c.namespace_id],
[metadef_namespaces.c.id],
name=new_fk_name).create()
def _downgrade_sqlite_namespace_id_name_constraint(metadef,
metadef_namespaces,
constraint_name,
fk_name):
migrate.UniqueConstraint(
metadef.c.namespace_id,
metadef.c.name,
name=constraint_name).drop()
migrate.UniqueConstraint(
metadef.c.namespace_id,
metadef.c.name).create()
migrate.ForeignKeyConstraint(
[metadef.c.namespace_id],
[metadef_namespaces.c.id],
name=fk_name).drop()
migrate.ForeignKeyConstraint(
[metadef.c.namespace_id],
[metadef_namespaces.c.id]).create()
def _drop_unique_constraint_if_exists(inspector, table_name, metadef):
name = _get_unique_constraint_name(inspector,
table_name,
['namespace_id', 'name'])
if name:
migrate.UniqueConstraint(metadef.c.namespace_id,
metadef.c.name,
name=name).drop()
def _drop_index_with_fk_constraint(metadef, metadef_namespaces,
index_name,
fk_old_name, fk_new_name):
fkc = migrate.ForeignKeyConstraint([metadef.c.namespace_id],
[metadef_namespaces.c.id],
name=fk_old_name)
fkc.drop()
if index_name:
Index(index_name, metadef.c.namespace_id).drop()
# Rename the fk for consistency across all db's
fkc = migrate.ForeignKeyConstraint([metadef.c.namespace_id],
[metadef_namespaces.c.id],
name=fk_new_name)
fkc.create()
def _downgrade_constraint_with_fk(metadef, metadef_namespaces,
constraint_name,
fk_curr_name, fk_next_name):
fkc = migrate.ForeignKeyConstraint([metadef.c.namespace_id],
[metadef_namespaces.c.id],
name=fk_curr_name)
fkc.drop()
migrate.UniqueConstraint(metadef.c.namespace_id, metadef.c.name,
name=constraint_name).drop()
fkc = migrate.ForeignKeyConstraint([metadef.c.namespace_id],
[metadef_namespaces.c.id],
name=fk_next_name)
fkc.create()
def _get_unique_constraint_name(inspector, table_name, columns):
constraints = inspector.get_unique_constraints(table_name)
for constraint in constraints:
if set(constraint['column_names']) == set(columns):
return constraint['name']
return None
def _get_fk_constraint_name(inspector, table_name, columns):
constraints = inspector.get_foreign_keys(table_name)
for constraint in constraints:
if set(constraint['constrained_columns']) == set(columns):
return constraint['name']
return None
def upgrade(migrate_engine):
_upgrade_data(migrate_engine)
meta = sqlalchemy.MetaData()
meta.bind = migrate_engine
inspector = inspect(migrate_engine)
# ORM tables
metadef_namespaces = Table('metadef_namespaces', meta, autoload=True)
metadef_objects = Table('metadef_objects', meta, autoload=True)
metadef_properties = Table('metadef_properties', meta, autoload=True)
metadef_tags = Table('metadef_tags', meta, autoload=True)
metadef_ns_res_types = Table('metadef_namespace_resource_types',
meta, autoload=True)
metadef_resource_types = Table('metadef_resource_types', meta,
autoload=True)
# Drop the bad, non-unique indices.
if migrate_engine.name == 'sqlite':
# For sqlite:
# Only after the unique constraints have been added should the indices
# be dropped. If done the other way, sqlite complains during
# constraint adding/dropping that the index does/does not exist.
# Note: The _get_unique_constraint_name, _get_fk_constraint_name
# return None for constraints that do in fact exist. Also,
# get_index_names returns names, but, the names can not be used with
# the Index(name, blah).drop() command, so, putting sqlite into
# it's own section.
# Objects
_update_sqlite_namespace_id_name_constraint(
metadef_objects, metadef_namespaces,
'uq_metadef_objects_namespace_id_name',
'metadef_objects_fk_1')
# Properties
_update_sqlite_namespace_id_name_constraint(
metadef_properties, metadef_namespaces,
'uq_metadef_properties_namespace_id_name',
'metadef_properties_fk_1')
# Tags
_update_sqlite_namespace_id_name_constraint(
metadef_tags, metadef_namespaces,
'uq_metadef_tags_namespace_id_name',
'metadef_tags_fk_1')
# Namespaces
migrate.UniqueConstraint(
metadef_namespaces.c.namespace).drop()
migrate.UniqueConstraint(
metadef_namespaces.c.namespace,
name='uq_metadef_namespaces_namespace').create()
# ResourceTypes
migrate.UniqueConstraint(
metadef_resource_types.c.name).drop()
migrate.UniqueConstraint(
metadef_resource_types.c.name,
name='uq_metadef_resource_types_name').create()
# Now drop the bad indices
Index('ix_metadef_objects_namespace_id',
metadef_objects.c.namespace_id,
metadef_objects.c.name).drop()
Index('ix_metadef_properties_namespace_id',
metadef_properties.c.namespace_id,
metadef_properties.c.name).drop()
Index('ix_metadef_tags_namespace_id',
metadef_tags.c.namespace_id,
metadef_tags.c.name).drop()
else:
# First drop the bad non-unique indices.
# To do that (for mysql), must first drop foreign key constraints
# BY NAME and then drop the bad indices.
# Finally, re-create the foreign key constraints with a consistent
# name.
# DB2 still has unique constraints, but, they are badly named.
# Drop them, they will be recreated at the final step.
name = _get_unique_constraint_name(inspector, 'metadef_namespaces',
['namespace'])
if name:
migrate.UniqueConstraint(metadef_namespaces.c.namespace,
name=name).drop()
_drop_unique_constraint_if_exists(inspector, 'metadef_objects',
metadef_objects)
_drop_unique_constraint_if_exists(inspector, 'metadef_properties',
metadef_properties)
_drop_unique_constraint_if_exists(inspector, 'metadef_tags',
metadef_tags)
name = _get_unique_constraint_name(inspector, 'metadef_resource_types',
['name'])
if name:
migrate.UniqueConstraint(metadef_resource_types.c.name,
name=name).drop()
# Objects
_drop_index_with_fk_constraint(
metadef_objects, metadef_namespaces,
'ix_metadef_objects_namespace_id',
_get_fk_constraint_name(
inspector, 'metadef_objects', ['namespace_id']),
'metadef_objects_fk_1')
# Properties
_drop_index_with_fk_constraint(
metadef_properties, metadef_namespaces,
'ix_metadef_properties_namespace_id',
_get_fk_constraint_name(
inspector, 'metadef_properties', ['namespace_id']),
'metadef_properties_fk_1')
# Tags
_drop_index_with_fk_constraint(
metadef_tags, metadef_namespaces,
'ix_metadef_tags_namespace_id',
_get_fk_constraint_name(
inspector, 'metadef_tags', ['namespace_id']),
'metadef_tags_fk_1')
# Drop Others without fk constraints.
Index('ix_metadef_namespaces_namespace',
metadef_namespaces.c.namespace).drop()
# The next two don't exist in ibm_db_sa, but, drop them everywhere else.
if migrate_engine.name != 'ibm_db_sa':
Index('ix_metadef_resource_types_name',
metadef_resource_types.c.name).drop()
# Not needed due to primary key on same columns
Index('ix_metadef_ns_res_types_res_type_id_ns_id',
metadef_ns_res_types.c.resource_type_id,
metadef_ns_res_types.c.namespace_id).drop()
# Now, add back the dropped indexes as unique constraints
if migrate_engine.name != 'sqlite':
# Namespaces
migrate.UniqueConstraint(
metadef_namespaces.c.namespace,
name='uq_metadef_namespaces_namespace').create()
# Objects
migrate.UniqueConstraint(
metadef_objects.c.namespace_id,
metadef_objects.c.name,
name='uq_metadef_objects_namespace_id_name').create()
# Properties
migrate.UniqueConstraint(
metadef_properties.c.namespace_id,
metadef_properties.c.name,
name='uq_metadef_properties_namespace_id_name').create()
# Tags
migrate.UniqueConstraint(
metadef_tags.c.namespace_id,
metadef_tags.c.name,
name='uq_metadef_tags_namespace_id_name').create()
# Resource Types
migrate.UniqueConstraint(
metadef_resource_types.c.name,
name='uq_metadef_resource_types_name').create()
def downgrade(migrate_engine):
meta = sqlalchemy.MetaData()
meta.bind = migrate_engine
# ORM tables
metadef_namespaces = Table('metadef_namespaces', meta, autoload=True)
metadef_objects = Table('metadef_objects', meta, autoload=True)
metadef_properties = Table('metadef_properties', meta, autoload=True)
metadef_tags = Table('metadef_tags', meta, autoload=True)
metadef_resource_types = Table('metadef_resource_types', meta,
autoload=True)
metadef_ns_res_types = Table('metadef_namespace_resource_types',
meta, autoload=True)
# Drop the unique constraints
if migrate_engine.name == 'sqlite':
# Objects
_downgrade_sqlite_namespace_id_name_constraint(
metadef_objects, metadef_namespaces,
'uq_metadef_objects_namespace_id_name',
'metadef_objects_fk_1')
# Properties
_downgrade_sqlite_namespace_id_name_constraint(
metadef_properties, metadef_namespaces,
'uq_metadef_properties_namespace_id_name',
'metadef_properties_fk_1')
# Tags
_downgrade_sqlite_namespace_id_name_constraint(
metadef_tags, metadef_namespaces,
'uq_metadef_tags_namespace_id_name',
'metadef_tags_fk_1')
# Namespaces
migrate.UniqueConstraint(
metadef_namespaces.c.namespace,
name='uq_metadef_namespaces_namespace').drop()
migrate.UniqueConstraint(
metadef_namespaces.c.namespace).create()
# ResourceTypes
migrate.UniqueConstraint(
metadef_resource_types.c.name,
name='uq_metadef_resource_types_name').drop()
migrate.UniqueConstraint(
metadef_resource_types.c.name).create()
else:
# For mysql, must drop foreign key constraints before dropping the
# unique constraint. So drop the fkc, then drop the constraints,
# then recreate the fkc.
# Objects
_downgrade_constraint_with_fk(
metadef_objects, metadef_namespaces,
'uq_metadef_objects_namespace_id_name',
'metadef_objects_fk_1', None)
# Properties
_downgrade_constraint_with_fk(
metadef_properties, metadef_namespaces,
'uq_metadef_properties_namespace_id_name',
'metadef_properties_fk_1', None)
# Tags
_downgrade_constraint_with_fk(
metadef_tags, metadef_namespaces,
'uq_metadef_tags_namespace_id_name',
'metadef_tags_fk_1', 'metadef_tags_namespace_id_fkey')
# Namespaces
migrate.UniqueConstraint(
metadef_namespaces.c.namespace,
name='uq_metadef_namespaces_namespace').drop()
# Resource_types
migrate.UniqueConstraint(
metadef_resource_types.c.name,
name='uq_metadef_resource_types_name').drop()
# Create dropped unique constraints as bad, non-unique indexes
Index('ix_metadef_objects_namespace_id',
metadef_objects.c.namespace_id).create()
Index('ix_metadef_properties_namespace_id',
metadef_properties.c.namespace_id).create()
# These need to be done before the metadef_tags and metadef_namespaces
# unique constraints are created to avoid 'tuple out of range' errors
# in db2.
Index('ix_metadef_tags_namespace_id',
metadef_tags.c.namespace_id,
metadef_tags.c.name).create()
Index('ix_metadef_namespaces_namespace',
metadef_namespaces.c.namespace).create()
# Create these everywhere, except for db2
if migrate_engine.name != 'ibm_db_sa':
Index('ix_metadef_resource_types_name',
metadef_resource_types.c.name).create()
Index('ix_metadef_ns_res_types_res_type_id_ns_id',
metadef_ns_res_types.c.resource_type_id,
metadef_ns_res_types.c.namespace_id).create()
else:
# Recreate the badly named unique constraints in db2
migrate.UniqueConstraint(
metadef_namespaces.c.namespace,
name='ix_namespaces_namespace').create()
migrate.UniqueConstraint(
metadef_objects.c.namespace_id,
metadef_objects.c.name,
name='ix_objects_namespace_id_name').create()
migrate.UniqueConstraint(
metadef_properties.c.namespace_id,
metadef_properties.c.name,
name='ix_metadef_properties_namespace_id_name').create()
migrate.UniqueConstraint(
metadef_tags.c.namespace_id,
metadef_tags.c.name).create()
migrate.UniqueConstraint(
metadef_resource_types.c.name,
name='ix_metadef_resource_types_name').create()

View File

@ -28,6 +28,7 @@ from sqlalchemy import Integer
from sqlalchemy.orm import relationship from sqlalchemy.orm import relationship
from sqlalchemy import String from sqlalchemy import String
from sqlalchemy import Text from sqlalchemy import Text
from sqlalchemy import UniqueConstraint
from glance.db.sqlalchemy.models import JSONEncodedDict from glance.db.sqlalchemy.models import JSONEncodedDict
@ -65,8 +66,11 @@ class GlanceMetadefBase(models.TimestampMixin):
class MetadefNamespace(BASE_DICT, GlanceMetadefBase): class MetadefNamespace(BASE_DICT, GlanceMetadefBase):
"""Represents a metadata-schema namespace in the datastore.""" """Represents a metadata-schema namespace in the datastore."""
__tablename__ = 'metadef_namespaces' __tablename__ = 'metadef_namespaces'
__table_args__ = (Index('ix_metadef_namespaces_namespace', 'namespace'), __table_args__ = (UniqueConstraint('namespace',
Index('ix_metadef_namespaces_owner', 'owner')) name='uq_metadef_namespaces'
'_namespace'),
Index('ix_metadef_namespaces_owner', 'owner')
)
id = Column(Integer, primary_key=True, nullable=False) id = Column(Integer, primary_key=True, nullable=False)
namespace = Column(String(80), nullable=False) namespace = Column(String(80), nullable=False)
@ -80,8 +84,11 @@ class MetadefNamespace(BASE_DICT, GlanceMetadefBase):
class MetadefObject(BASE_DICT, GlanceMetadefBase): class MetadefObject(BASE_DICT, GlanceMetadefBase):
"""Represents a metadata-schema object in the datastore.""" """Represents a metadata-schema object in the datastore."""
__tablename__ = 'metadef_objects' __tablename__ = 'metadef_objects'
__table_args__ = (Index('ix_metadef_objects_namespace_id', 'namespace_id'), __table_args__ = (UniqueConstraint('namespace_id', 'name',
Index('ix_metadef_objects_name', 'name')) name='uq_metadef_objects_namespace_id'
'_name'),
Index('ix_metadef_objects_name', 'name')
)
id = Column(Integer, primary_key=True, nullable=False) id = Column(Integer, primary_key=True, nullable=False)
namespace_id = Column(Integer(), ForeignKey('metadef_namespaces.id'), namespace_id = Column(Integer(), ForeignKey('metadef_namespaces.id'),
@ -95,9 +102,11 @@ class MetadefObject(BASE_DICT, GlanceMetadefBase):
class MetadefProperty(BASE_DICT, GlanceMetadefBase): class MetadefProperty(BASE_DICT, GlanceMetadefBase):
"""Represents a metadata-schema namespace-property in the datastore.""" """Represents a metadata-schema namespace-property in the datastore."""
__tablename__ = 'metadef_properties' __tablename__ = 'metadef_properties'
__table_args__ = (Index('ix_metadef_properties_namespace_id', __table_args__ = (UniqueConstraint('namespace_id', 'name',
'namespace_id'), name='uq_metadef_properties_namespace'
Index('ix_metadef_properties_name', 'name')) '_id_name'),
Index('ix_metadef_properties_name', 'name')
)
id = Column(Integer, primary_key=True, nullable=False) id = Column(Integer, primary_key=True, nullable=False)
namespace_id = Column(Integer(), ForeignKey('metadef_namespaces.id'), namespace_id = Column(Integer(), ForeignKey('metadef_namespaces.id'),
@ -109,10 +118,9 @@ class MetadefProperty(BASE_DICT, GlanceMetadefBase):
class MetadefNamespaceResourceType(BASE_DICT, GlanceMetadefBase): class MetadefNamespaceResourceType(BASE_DICT, GlanceMetadefBase):
"""Represents a metadata-schema namespace-property in the datastore.""" """Represents a metadata-schema namespace-property in the datastore."""
__tablename__ = 'metadef_namespace_resource_types' __tablename__ = 'metadef_namespace_resource_types'
__table_args__ = (Index('ix_metadef_ns_res_types_res_type_id_ns_id', __table_args__ = (Index('ix_metadef_ns_res_types_namespace_id',
'resource_type_id', 'namespace_id'), 'namespace_id'),
Index('ix_metadef_ns_res_types_namespace_id', )
'namespace_id'))
resource_type_id = Column(Integer, resource_type_id = Column(Integer,
ForeignKey('metadef_resource_types.id'), ForeignKey('metadef_resource_types.id'),
@ -126,7 +134,9 @@ class MetadefNamespaceResourceType(BASE_DICT, GlanceMetadefBase):
class MetadefResourceType(BASE_DICT, GlanceMetadefBase): class MetadefResourceType(BASE_DICT, GlanceMetadefBase):
"""Represents a metadata-schema resource type in the datastore.""" """Represents a metadata-schema resource type in the datastore."""
__tablename__ = 'metadef_resource_types' __tablename__ = 'metadef_resource_types'
__table_args__ = (Index('ix_metadef_resource_types_name', 'name'), ) __table_args__ = (UniqueConstraint('name',
name='uq_metadef_resource_types_name'),
)
id = Column(Integer, primary_key=True, nullable=False) id = Column(Integer, primary_key=True, nullable=False)
name = Column(String(80), nullable=False) name = Column(String(80), nullable=False)
@ -140,9 +150,11 @@ class MetadefResourceType(BASE_DICT, GlanceMetadefBase):
class MetadefTag(BASE_DICT, GlanceMetadefBase): class MetadefTag(BASE_DICT, GlanceMetadefBase):
"""Represents a metadata-schema tag in the data store.""" """Represents a metadata-schema tag in the data store."""
__tablename__ = 'metadef_tags' __tablename__ = 'metadef_tags'
__table_args__ = (Index('ix_metadef_tags_namespace_id', __table_args__ = (UniqueConstraint('namespace_id', 'name',
'namespace_id', 'name'), name='uq_metadef_tags_namespace_id'
Index('ix_metadef_tags_name', 'name')) '_name'),
Index('ix_metadef_tags_name', 'name')
)
id = Column(Integer, primary_key=True, nullable=False) id = Column(Integer, primary_key=True, nullable=False)
namespace_id = Column(Integer(), ForeignKey('metadef_namespaces.id'), namespace_id = Column(Integer(), ForeignKey('metadef_namespaces.id'),

View File

@ -123,6 +123,15 @@ class MetadefNamespaceTests(object):
self.assertIsNotNone(created) self.assertIsNotNone(created)
self._assert_saved_fields(fixture, created) self._assert_saved_fields(fixture, created)
def test_namespace_create_duplicate(self):
fixture = build_namespace_fixture()
created = self.db_api.metadef_namespace_create(self.context, fixture)
self.assertIsNotNone(created)
self._assert_saved_fields(fixture, created)
self.assertRaises(exception.Duplicate,
self.db_api.metadef_namespace_create,
self.context, fixture)
def test_namespace_get(self): def test_namespace_get(self):
fixture = build_namespace_fixture() fixture = build_namespace_fixture()
created = self.db_api.metadef_namespace_create(self.context, fixture) created = self.db_api.metadef_namespace_create(self.context, fixture)
@ -222,6 +231,22 @@ class MetadefPropertyTests(object):
self.context, created_ns['namespace'], fixture_prop) self.context, created_ns['namespace'], fixture_prop)
self._assert_saved_fields(fixture_prop, created_prop) self._assert_saved_fields(fixture_prop, created_prop)
def test_property_create_duplicate(self):
fixture = build_namespace_fixture()
created_ns = self.db_api.metadef_namespace_create(
self.context, fixture)
self.assertIsNotNone(created_ns)
self._assert_saved_fields(fixture, created_ns)
fixture_prop = build_property_fixture(namespace_id=created_ns['id'])
created_prop = self.db_api.metadef_property_create(
self.context, created_ns['namespace'], fixture_prop)
self._assert_saved_fields(fixture_prop, created_prop)
self.assertRaises(exception.Duplicate,
self.db_api.metadef_property_create,
self.context, created_ns['namespace'], fixture_prop)
def test_property_get(self): def test_property_get(self):
fixture_ns = build_namespace_fixture() fixture_ns = build_namespace_fixture()
created_ns = self.db_api.metadef_namespace_create( created_ns = self.db_api.metadef_namespace_create(
@ -332,6 +357,23 @@ class MetadefObjectTests(object):
self.context, created_ns['namespace'], fixture_object) self.context, created_ns['namespace'], fixture_object)
self._assert_saved_fields(fixture_object, created_object) self._assert_saved_fields(fixture_object, created_object)
def test_object_create_duplicate(self):
fixture = build_namespace_fixture()
created_ns = self.db_api.metadef_namespace_create(self.context,
fixture)
self.assertIsNotNone(created_ns)
self._assert_saved_fields(fixture, created_ns)
fixture_object = build_object_fixture(namespace_id=created_ns['id'])
created_object = self.db_api.metadef_object_create(
self.context, created_ns['namespace'], fixture_object)
self._assert_saved_fields(fixture_object, created_object)
self.assertRaises(exception.Duplicate,
self.db_api.metadef_object_create,
self.context, created_ns['namespace'],
fixture_object)
def test_object_get(self): def test_object_get(self):
fixture_ns = build_namespace_fixture() fixture_ns = build_namespace_fixture()
created_ns = self.db_api.metadef_namespace_create(self.context, created_ns = self.db_api.metadef_namespace_create(self.context,
@ -442,6 +484,24 @@ class MetadefResourceTypeAssociationTests(object):
self.assertIsNotNone(assn_created) self.assertIsNotNone(assn_created)
self._assert_saved_fields(assn_fixture, assn_created) self._assert_saved_fields(assn_fixture, assn_created)
def test_association_create_duplicate(self):
ns_fixture = build_namespace_fixture()
ns_created = self.db_api.metadef_namespace_create(
self.context, ns_fixture)
self.assertIsNotNone(ns_created)
self._assert_saved_fields(ns_fixture, ns_created)
assn_fixture = build_association_fixture()
assn_created = self.db_api.metadef_resource_type_association_create(
self.context, ns_created['namespace'], assn_fixture)
self.assertIsNotNone(assn_created)
self._assert_saved_fields(assn_fixture, assn_created)
self.assertRaises(exception.Duplicate,
self.db_api.
metadef_resource_type_association_create,
self.context, ns_created['namespace'], assn_fixture)
def test_association_delete(self): def test_association_delete(self):
ns_fixture = build_namespace_fixture() ns_fixture = build_namespace_fixture()
ns_created = self.db_api.metadef_namespace_create( ns_created = self.db_api.metadef_namespace_create(
@ -499,6 +559,23 @@ class MetadefTagTests(object):
self.context, created_ns['namespace'], fixture_tag) self.context, created_ns['namespace'], fixture_tag)
self._assert_saved_fields(fixture_tag, created_tag) self._assert_saved_fields(fixture_tag, created_tag)
def test_tag_create_duplicate(self):
fixture = build_namespace_fixture()
created_ns = self.db_api.metadef_namespace_create(self.context,
fixture)
self.assertIsNotNone(created_ns)
self._assert_saved_fields(fixture, created_ns)
fixture_tag = build_tag_fixture(namespace_id=created_ns['id'])
created_tag = self.db_api.metadef_tag_create(
self.context, created_ns['namespace'], fixture_tag)
self._assert_saved_fields(fixture_tag, created_tag)
self.assertRaises(exception.Duplicate,
self.db_api.metadef_tag_create,
self.context, created_ns['namespace'],
fixture_tag)
def test_tag_create_tags(self): def test_tag_create_tags(self):
fixture = build_namespace_fixture() fixture = build_namespace_fixture()
created_ns = self.db_api.metadef_namespace_create(self.context, created_ns = self.db_api.metadef_namespace_create(self.context,
@ -513,6 +590,35 @@ class MetadefTagTests(object):
expected = set(['Tag1', 'Tag2', 'Tag3']) expected = set(['Tag1', 'Tag2', 'Tag3'])
self.assertEqual(expected, actual) self.assertEqual(expected, actual)
def test_tag_create_duplicate_tags_1(self):
fixture = build_namespace_fixture()
created_ns = self.db_api.metadef_namespace_create(self.context,
fixture)
self.assertIsNotNone(created_ns)
self._assert_saved_fields(fixture, created_ns)
tags = build_tags_fixture(['Tag1', 'Tag2', 'Tag3', 'Tag2'])
self.assertRaises(exception.Duplicate,
self.db_api.metadef_tag_create_tags,
self.context, created_ns['namespace'],
tags)
def test_tag_create_duplicate_tags_2(self):
fixture = build_namespace_fixture()
created_ns = self.db_api.metadef_namespace_create(self.context,
fixture)
self.assertIsNotNone(created_ns)
self._assert_saved_fields(fixture, created_ns)
tags = build_tags_fixture(['Tag1', 'Tag2', 'Tag3'])
self.db_api.metadef_tag_create_tags(self.context,
created_ns['namespace'], tags)
dup_tag = build_tag_fixture(namespace_id=created_ns['id'],
name='Tag3')
self.assertRaises(exception.Duplicate,
self.db_api.metadef_tag_create,
self.context, created_ns['namespace'], dup_tag)
def test_tag_get(self): def test_tag_get(self):
fixture_ns = build_namespace_fixture() fixture_ns = build_namespace_fixture()
created_ns = self.db_api.metadef_namespace_create(self.context, created_ns = self.db_api.metadef_namespace_create(self.context,

View File

@ -96,6 +96,10 @@ class TestNamespaces(functional.FunctionalTest):
for key, value in expected_namespace.items(): for key, value in expected_namespace.items():
self.assertEqual(namespace[key], value, key) self.assertEqual(namespace[key], value, key)
# Attempt to insert a duplicate
response = requests.post(path, headers=headers, data=data)
self.assertEqual(409, response.status_code)
# Get the namespace using the returned Location header # Get the namespace using the returned Location header
response = requests.get(namespace_loc_header, headers=self._headers()) response = requests.get(namespace_loc_header, headers=self._headers())
self.assertEqual(200, response.status_code) self.assertEqual(200, response.status_code)

View File

@ -107,6 +107,10 @@ class TestMetadefObjects(functional.FunctionalTest):
response = requests.post(path, headers=headers, data=data) response = requests.post(path, headers=headers, data=data)
self.assertEqual(201, response.status_code) self.assertEqual(201, response.status_code)
# Attempt to insert a duplicate
response = requests.post(path, headers=headers, data=data)
self.assertEqual(409, response.status_code)
# Get the metadata object created above # Get the metadata object created above
path = self._url('/v2/metadefs/namespaces/%s/objects/%s' % path = self._url('/v2/metadefs/namespaces/%s/objects/%s' %
(namespace_name, metadata_object_name)) (namespace_name, metadata_object_name))

View File

@ -99,6 +99,10 @@ class TestNamespaceProperties(functional.FunctionalTest):
response = requests.post(path, headers=headers, data=data) response = requests.post(path, headers=headers, data=data)
self.assertEqual(201, response.status_code) self.assertEqual(201, response.status_code)
# Attempt to insert a duplicate
response = requests.post(path, headers=headers, data=data)
self.assertEqual(409, response.status_code)
# Get the property created above # Get the property created above
path = self._url('/v2/metadefs/namespaces/%s/properties/%s' % path = self._url('/v2/metadefs/namespaces/%s/properties/%s' %
(namespace_name, property_name)) (namespace_name, property_name))

View File

@ -105,6 +105,11 @@ class TestMetadefTags(functional.FunctionalTest):
if(key in checked_values): if(key in checked_values):
self.assertEqual(metadata_tag[key], value, key) self.assertEqual(metadata_tag[key], value, key)
# Try to create a duplicate metadata tag
headers = self._headers({'content-type': 'application/json'})
response = requests.post(path, headers=headers)
self.assertEqual(409, response.status_code)
# The metadata_tag should be mutable # The metadata_tag should be mutable
path = self._url('/v2/metadefs/namespaces/%s/tags/%s' % path = self._url('/v2/metadefs/namespaces/%s/tags/%s' %
(namespace_name, metadata_tag_name)) (namespace_name, metadata_tag_name))

View File

@ -1621,6 +1621,208 @@ class MigrationsMixin(test_migrations.WalkVersionsMixin):
self.assert_table(engine, 'artifact_blob_locations', locations_indices, self.assert_table(engine, 'artifact_blob_locations', locations_indices,
locations_columns) locations_columns)
def _pre_upgrade_042(self, engine):
meta = sqlalchemy.MetaData()
meta.bind = engine
metadef_namespaces = sqlalchemy.Table('metadef_namespaces', meta,
autoload=True)
metadef_objects = sqlalchemy.Table('metadef_objects', meta,
autoload=True)
metadef_properties = sqlalchemy.Table('metadef_properties', meta,
autoload=True)
metadef_tags = sqlalchemy.Table('metadef_tags', meta, autoload=True)
metadef_resource_types = sqlalchemy.Table('metadef_resource_types',
meta, autoload=True)
metadef_ns_res_types = sqlalchemy.Table(
'metadef_namespace_resource_types',
meta, autoload=True)
# These will be dropped and recreated as unique constraints.
self.assertTrue(index_exist('ix_metadef_namespaces_namespace',
metadef_namespaces.name, engine))
self.assertTrue(index_exist('ix_metadef_objects_namespace_id',
metadef_objects.name, engine))
self.assertTrue(index_exist('ix_metadef_properties_namespace_id',
metadef_properties.name, engine))
self.assertTrue(index_exist('ix_metadef_tags_namespace_id',
metadef_tags.name, engine))
self.assertTrue(index_exist('ix_metadef_resource_types_name',
metadef_resource_types.name, engine))
# This one will be dropped - not needed
self.assertTrue(index_exist(
'ix_metadef_ns_res_types_res_type_id_ns_id',
metadef_ns_res_types.name, engine))
# The rest must remain
self.assertTrue(index_exist('ix_metadef_namespaces_owner',
metadef_namespaces.name, engine))
self.assertTrue(index_exist('ix_metadef_objects_name',
metadef_objects.name, engine))
self.assertTrue(index_exist('ix_metadef_properties_name',
metadef_properties.name, engine))
self.assertTrue(index_exist('ix_metadef_tags_name',
metadef_tags.name, engine))
self.assertTrue(index_exist('ix_metadef_ns_res_types_namespace_id',
metadef_ns_res_types.name, engine))
# To be created
self.assertFalse(unique_constraint_exist
('uq_metadef_objects_namespace_id_name',
metadef_objects.name, engine)
)
self.assertFalse(unique_constraint_exist
('uq_metadef_properties_namespace_id_name',
metadef_properties.name, engine)
)
self.assertFalse(unique_constraint_exist
('uq_metadef_tags_namespace_id_name',
metadef_tags.name, engine)
)
self.assertFalse(unique_constraint_exist
('uq_metadef_namespaces_namespace',
metadef_namespaces.name, engine)
)
self.assertFalse(unique_constraint_exist
('uq_metadef_resource_types_name',
metadef_resource_types.name, engine)
)
def _check_042(self, engine, data):
meta = sqlalchemy.MetaData()
meta.bind = engine
metadef_namespaces = sqlalchemy.Table('metadef_namespaces', meta,
autoload=True)
metadef_objects = sqlalchemy.Table('metadef_objects', meta,
autoload=True)
metadef_properties = sqlalchemy.Table('metadef_properties', meta,
autoload=True)
metadef_tags = sqlalchemy.Table('metadef_tags', meta, autoload=True)
metadef_resource_types = sqlalchemy.Table('metadef_resource_types',
meta, autoload=True)
metadef_ns_res_types = sqlalchemy.Table(
'metadef_namespace_resource_types',
meta, autoload=True)
# Dropped for unique constraints
self.assertFalse(index_exist('ix_metadef_namespaces_namespace',
metadef_namespaces.name, engine))
self.assertFalse(index_exist('ix_metadef_objects_namespace_id',
metadef_objects.name, engine))
self.assertFalse(index_exist('ix_metadef_properties_namespace_id',
metadef_properties.name, engine))
self.assertFalse(index_exist('ix_metadef_tags_namespace_id',
metadef_tags.name, engine))
self.assertFalse(index_exist('ix_metadef_resource_types_name',
metadef_resource_types.name, engine))
# Dropped - not needed because of the existing primary key
self.assertFalse(index_exist(
'ix_metadef_ns_res_types_res_type_id_ns_id',
metadef_ns_res_types.name, engine))
# Still exist as before
self.assertTrue(index_exist('ix_metadef_namespaces_owner',
metadef_namespaces.name, engine))
self.assertTrue(index_exist('ix_metadef_ns_res_types_namespace_id',
metadef_ns_res_types.name, engine))
self.assertTrue(index_exist('ix_metadef_objects_name',
metadef_objects.name, engine))
self.assertTrue(index_exist('ix_metadef_properties_name',
metadef_properties.name, engine))
self.assertTrue(index_exist('ix_metadef_tags_name',
metadef_tags.name, engine))
self.assertTrue(unique_constraint_exist
('uq_metadef_namespaces_namespace',
metadef_namespaces.name, engine)
)
self.assertTrue(unique_constraint_exist
('uq_metadef_objects_namespace_id_name',
metadef_objects.name, engine)
)
self.assertTrue(unique_constraint_exist
('uq_metadef_properties_namespace_id_name',
metadef_properties.name, engine)
)
self.assertTrue(unique_constraint_exist
('uq_metadef_tags_namespace_id_name',
metadef_tags.name, engine)
)
self.assertTrue(unique_constraint_exist
('uq_metadef_resource_types_name',
metadef_resource_types.name, engine)
)
def _post_downgrade_042(self, engine):
meta = sqlalchemy.MetaData()
meta.bind = engine
metadef_namespaces = sqlalchemy.Table('metadef_namespaces', meta,
autoload=True)
metadef_objects = sqlalchemy.Table('metadef_objects', meta,
autoload=True)
metadef_properties = sqlalchemy.Table('metadef_properties', meta,
autoload=True)
metadef_tags = sqlalchemy.Table('metadef_tags', meta, autoload=True)
metadef_resource_types = sqlalchemy.Table('metadef_resource_types',
meta, autoload=True)
metadef_ns_res_types = sqlalchemy.Table(
'metadef_namespace_resource_types',
meta, autoload=True)
# These have been recreated
self.assertTrue(index_exist('ix_metadef_namespaces_namespace',
metadef_namespaces.name, engine))
self.assertTrue(index_exist('ix_metadef_objects_namespace_id',
metadef_objects.name, engine))
self.assertTrue(index_exist('ix_metadef_properties_namespace_id',
metadef_properties.name, engine))
self.assertTrue(index_exist('ix_metadef_tags_namespace_id',
metadef_tags.name, engine))
self.assertTrue(index_exist('ix_metadef_resource_types_name',
metadef_resource_types.name, engine))
self.assertTrue(index_exist(
'ix_metadef_ns_res_types_res_type_id_ns_id',
metadef_ns_res_types.name, engine))
# The rest must remain
self.assertTrue(index_exist('ix_metadef_namespaces_owner',
metadef_namespaces.name, engine))
self.assertTrue(index_exist('ix_metadef_objects_name',
metadef_objects.name, engine))
self.assertTrue(index_exist('ix_metadef_properties_name',
metadef_properties.name, engine))
self.assertTrue(index_exist('ix_metadef_tags_name',
metadef_tags.name, engine))
self.assertTrue(index_exist('ix_metadef_ns_res_types_namespace_id',
metadef_ns_res_types.name, engine))
# Dropped
self.assertFalse(unique_constraint_exist
('uq_metadef_objects_namespace_id_name',
metadef_objects.name, engine)
)
self.assertFalse(unique_constraint_exist
('uq_metadef_properties_namespace_id_name',
metadef_properties.name, engine)
)
self.assertFalse(unique_constraint_exist
('uq_metadef_tags_namespace_id_name',
metadef_tags.name, engine)
)
self.assertFalse(unique_constraint_exist
('uq_metadef_namespaces_namespace',
metadef_namespaces.name, engine)
)
self.assertFalse(unique_constraint_exist
('uq_metadef_resource_types_name',
metadef_resource_types.name, engine)
)
def assert_table(self, engine, table_name, indices, columns): def assert_table(self, engine, table_name, indices, columns):
table = db_utils.get_table(engine, table_name) table = db_utils.get_table(engine, table_name)
index_data = [(index.name, index.columns.keys()) for index in index_data = [(index.name, index.columns.keys()) for index in

View File

@ -605,6 +605,17 @@ class TestMetadefsControllers(base.IsolatedUnitTest):
namespace = self.namespace_controller.show(request, NAMESPACE4) namespace = self.namespace_controller.show(request, NAMESPACE4)
self.assertEqual(NAMESPACE4, namespace.namespace) self.assertEqual(NAMESPACE4, namespace.namespace)
def test_namespace_create_duplicate(self):
request = unit_test_utils.get_fake_request()
namespace = namespaces.Namespace()
namespace.namespace = 'new-namespace'
new_ns = self.namespace_controller.create(request, namespace)
self.assertEqual('new-namespace', new_ns.namespace)
self.assertRaises(webob.exc.HTTPConflict,
self.namespace_controller.create,
request, namespace)
def test_namespace_create_different_owner(self): def test_namespace_create_different_owner(self):
request = unit_test_utils.get_fake_request() request = unit_test_utils.get_fake_request()
@ -1036,6 +1047,20 @@ class TestMetadefsControllers(base.IsolatedUnitTest):
property) property)
self.assertNotificationsLog([]) self.assertNotificationsLog([])
def test_property_create_duplicate(self):
request = unit_test_utils.get_fake_request()
property = properties.PropertyType()
property.name = 'new-property'
property.type = 'string'
property.title = 'title'
new_property = self.property_controller.create(request, NAMESPACE1,
property)
self.assertEqual('new-property', new_property.name)
self.assertRaises(webob.exc.HTTPConflict,
self.property_controller.create, request,
NAMESPACE1, property)
def test_property_update(self): def test_property_update(self):
request = unit_test_utils.get_fake_request(tenant=TENANT3) request = unit_test_utils.get_fake_request(tenant=TENANT3)
@ -1254,6 +1279,19 @@ class TestMetadefsControllers(base.IsolatedUnitTest):
self.assertEqual([], object.required) self.assertEqual([], object.required)
self.assertEqual({}, object.properties) self.assertEqual({}, object.properties)
def test_object_create_duplicate(self):
request = unit_test_utils.get_fake_request()
object = objects.MetadefObject()
object.name = 'New-Object'
object.required = []
object.properties = {}
new_obj = self.object_controller.create(request, object, NAMESPACE3)
self.assertEqual('New-Object', new_obj.name)
self.assertRaises(webob.exc.HTTPConflict,
self.object_controller.create, request, object,
NAMESPACE3)
def test_object_create_conflict(self): def test_object_create_conflict(self):
request = unit_test_utils.get_fake_request() request = unit_test_utils.get_fake_request()