openstack/zaqar
Code Issues Proposed changes

Fix sqlalchemy migration

Browse Source 
1. Fix the case of table names
2. Fix the sqlalchemy migration tests

NOTE: The sqlalchemy migrations test depends on oslo_db's migration
test which requires mysql user 'openstack_citest' with password
'openstack_citest' otherwise the test will be skipped.

Closes-Bug: #1654105

Change-Id: Ia672440a948fa4784f6dd1aa6d5fed0bc3915663
This commit is contained in:
Fei Long Wang 2017-01-05 13:26:46 +13:00
parent 828bab9d66
commit 1ddd9ca5c0
6 changed files with 82 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
doc/source/running_tests.rst
View File

@ -153,6 +153,15 @@ For example, you want to run functional tests with keystone authentication
enabled, input a valid set of credentials to ``[auth]`` section in
configuration file and set ``auth_on`` parameter to ``True``.
Using local Mysql database
^^^^^^^^^^^^^^^^^^^^^^^^^^
To use a similar testing environment with database support like upstream CI,
you can run ``zaqar/tools/test-setup.sh`` to create a required Mysql user
``openstack_citest`` with same password. The user is required by oslo.db's
test. Zaqar needs it because Zaqar's sqlalchemy database migration is
leveraging oslo.db's migration test base.
.. rubric:: Footnotes
.. [#f1] See http://docs.openstack.org/infra/system-config/jenkins.html

1
test-requirements.txt
View File

@ -11,6 +11,7 @@ mock>=2.0 # BSD
redis>=2.10.0 # MIT
pymongo!=3.1,>=3.0.2 # Apache-2.0
websocket-client>=0.32.0 # LGPLv2+
PyMySQL>=0.7.6 # MIT License
# Unit testing
coverage>=4.0 # Apache-2.0

33
tools/test-setup.sh Executable file
View File

@ -0,0 +1,33 @@
#!/bin/bash -xe
# This script will be run by OpenStack CI before unit tests are run,
# it sets up the test system as needed.
# Developers should setup their test systems in a similar way.
# This setup needs to be run as a user that can run sudo.
# The root password for the MySQL database; pass it in via
# MYSQL_ROOT_PW.
DB_ROOT_PW=${MYSQL_ROOT_PW:-insecure_slave}
# This user and its password are used by the tests, if you change it,
# your tests might fail.
DB_USER=openstack_citest
DB_PW=openstack_citest
sudo -H mysqladmin -u root password $DB_ROOT_PW
# It's best practice to remove anonymous users from the database. If
# a anonymous user exists, then it matches first for connections and
# other connections from that host will not work.
sudo -H mysql -u root -p$DB_ROOT_PW -h localhost -e "
DELETE FROM mysql.user WHERE User='';
FLUSH PRIVILEGES;
GRANT ALL PRIVILEGES ON *.*
TO '$DB_USER'@'%' identified by '$DB_PW' WITH GRANT OPTION;"
# Now create our database.
mysql -u $DB_USER -p$DB_PW -h 127.0.0.1 -e "
SET default_storage_engine=MYISAM;
DROP DATABASE IF EXISTS openstack_citest;
CREATE DATABASE openstack_citest CHARACTER SET utf8;"

16
zaqar/storage/sqlalchemy/migration/alembic_migrations/versions/001_liberty.py
View File

@ -33,20 +33,20 @@ MYSQL_CHARSET = 'utf8'
def upgrade():
op.create_table('queues',
op.create_table('Queues',
sa.Column('id', sa.INTEGER, primary_key=True),
sa.Column('project', sa.String(64)),
sa.Column('name', sa.String(64)),
sa.Column('metadata', sa.LargeBinary),
sa.UniqueConstraint('project', 'name'))
op.create_table('poolgroup',
op.create_table('PoolGroup',
sa.Column('name', sa.String(64), primary_key=True))
op.create_table('pools',
op.create_table('Pools',
sa.Column('name', sa.String(64), primary_key=True),
sa.Column('group', sa.String(64),
sa.ForeignKey('poolgroup.name',
sa.ForeignKey('PoolGroup.name',
ondelete='CASCADE'),
nullable=True),
sa.Column('uri', sa.String(255),
@ -54,18 +54,18 @@ def upgrade():
sa.Column('weight', sa.INTEGER, nullable=False),
sa.Column('options', sa.Text()))
op.create_table('flavors',
op.create_table('Flavors',
sa.Column('name', sa.String(64), primary_key=True),
sa.Column('project', sa.String(64)),
sa.Column('pool_group', sa.String(64),
sa.ForeignKey('poolgroup.name',
sa.ForeignKey('PoolGroup.name',
ondelete='CASCADE'),
nullable=False),
sa.Column('capabilities', sa.Text()))
op.create_table('catalogue',
op.create_table('Catalogue',
sa.Column('pool', sa.String(64),
sa.ForeignKey('pools.name',
sa.ForeignKey('Pools.name',
ondelete='CASCADE')),
sa.Column('project', sa.String(64)),
sa.Column('queue', sa.String(64), nullable=False),

34
zaqar/tests/unit/storage/sqlalchemy_migration/test_migrations.py
View File

@ -30,8 +30,6 @@ postgres=# create database openstack_citest with owner openstack_citest;
"""
import os
from oslo_db.sqlalchemy import test_base
from oslo_db.sqlalchemy import utils as db_utils
@ -95,17 +93,17 @@ class ZaqarMigrationsCheckers(object):
'metadata'
]
self.assertColumnsExist(
engine, 'queues', queues_columns)
engine, 'Queues', queues_columns)
self.assertColumnCount(
engine, 'queues', queues_columns)
engine, 'Queues', queues_columns)
poolgroup_columns = [
'name',
]
self.assertColumnsExist(
engine, 'poolgroup', poolgroup_columns)
engine, 'PoolGroup', poolgroup_columns)
self.assertColumnCount(
engine, 'poolgroup', poolgroup_columns)
engine, 'PoolGroup', poolgroup_columns)
pools_columns = [
'name',
@ -115,9 +113,9 @@ class ZaqarMigrationsCheckers(object):
'options',
]
self.assertColumnsExist(
engine, 'pools', pools_columns)
engine, 'Pools', pools_columns)
self.assertColumnCount(
engine, 'pools', pools_columns)
engine, 'Pools', pools_columns)
flavors_columns = [
'name',
@ -126,9 +124,9 @@ class ZaqarMigrationsCheckers(object):
'capabilities',
]
self.assertColumnsExist(
engine, 'flavors', flavors_columns)
engine, 'Flavors', flavors_columns)
self.assertColumnCount(
engine, 'flavors', flavors_columns)
engine, 'Flavors', flavors_columns)
catalogue_columns = [
'pool',
@ -136,19 +134,19 @@ class ZaqarMigrationsCheckers(object):
'queue',
]
self.assertColumnsExist(
engine, 'catalogue', catalogue_columns)
engine, 'Catalogue', catalogue_columns)
self.assertColumnCount(
engine, 'catalogue', catalogue_columns)
engine, 'Catalogue', catalogue_columns)
self._data_001(engine, data)
def _data_001(self, engine, data):
datasize = 512 * 1024 # 512kB
data = os.urandom(datasize)
t = db_utils.get_table(engine, 'job_binary_internal')
engine.execute(t.insert(), data=data, id='123', name='name')
new_data = engine.execute(t.select()).fetchone().data
self.assertEqual(data, new_data)
project = 'myproject'
t = db_utils.get_table(engine, 'Queues')
engine.execute(t.insert(), id='123', name='name', project='myproject',
metadata={})
new_project = engine.execute(t.select()).fetchone().project
self.assertEqual(project, new_project)
engine.execute(t.delete())
def _check_002(self, engine, data):

24
zaqar/tests/unit/storage/sqlalchemy_migration/test_migrations_base.py
View File

@ -25,7 +25,6 @@
import io
import os
import sqlalchemy as sa
import alembic
from alembic import command
@ -40,10 +39,17 @@ from zaqar.i18n import _LE
import zaqar.storage.sqlalchemy.migration
from zaqar.storage.sqlalchemy import tables
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
sqlalchemy_opts = [cfg.StrOpt('uri',
help='The SQLAlchemy connection string to'
' use to connect to the database.',
secret=True)]
CONF.register_opts(sqlalchemy_opts,
group='drivers:management_store:sqlalchemy')
class BaseWalkMigrationTestCase(object):
@ -63,10 +69,10 @@ class BaseWalkMigrationTestCase(object):
should use oslo_config and openstack.commom.db.sqlalchemy.session with
database functionality (reset default settings and session cleanup).
"""
CONF.set_override('uri', str(engine.url),
group='drivers:management_store:sqlalchemy',
enforce_type=True)
sa.cleanup()
def _alembic_command(self, alembic_command, engine, *args, **kwargs):
"""Most of alembic command return data into output.
@ -77,12 +83,12 @@ class BaseWalkMigrationTestCase(object):
CONF.set_override('uri', str(engine.url),
group='drivers:management_store:sqlalchemy',
enforce_type=True)
sa.cleanup()
getattr(command, alembic_command)(*args, **kwargs)
res = buf.getvalue().strip()
LOG.debug('Alembic command {command} returns: {result}'.format(
command=alembic_command, result=res))
sa.cleanup()
return res
def _get_versions(self):
@ -167,11 +173,9 @@ class TestModelsMigrationsSync(t_m.ModelsMigrationsSync):
Allows to check if the DB schema obtained by applying of migration
scripts is equal to the one produced from models definitions.
"""
mg_path = os.path.dirname(zaqar.storage.sqlalchemy.migration.__file__)
ALEMBIC_CONFIG = alembic_config.Config(
os.path.join(
os.path.dirname(zaqar.storage.sqlalchemy.migration.__file__),
'alembic.ini')
os.path.join(mg_path, 'alembic.ini')
)
ALEMBIC_CONFIG.zaqar_config = CONF
@ -182,6 +186,8 @@ class TestModelsMigrationsSync(t_m.ModelsMigrationsSync):
CONF.set_override('uri', str(engine.url),
group='drivers:management_store:sqlalchemy',
enforce_type=True)
script_location = os.path.join(self.mg_path, 'alembic_migrations')
self.ALEMBIC_CONFIG.set_main_option('script_location', script_location)
alembic.command.upgrade(self.ALEMBIC_CONFIG, 'head')
def get_metadata(self):