diff --git a/.gitignore b/.gitignore index ac51a6a709..7b00490aef 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ keystone.egg-info/ run_tests.err.log .coverage .DS_Store +test_migrations.db diff --git a/doc/source/index.rst b/doc/source/index.rst index 44898f9218..945a32daa4 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -36,6 +36,7 @@ Getting Started setup testing + migration configuration community usingkeystone diff --git a/doc/source/migration.rst b/doc/source/migration.rst new file mode 100644 index 0000000000..ffc3b88fb5 --- /dev/null +++ b/doc/source/migration.rst @@ -0,0 +1,35 @@ +================ +Using Migrations +================ + +Keystone uses sqlalchemy-migrate to manage migrations. + + +Running Migrations +====================== + +Keep backups of your db.Add your existing database to version control. + +Example:: + +$python keystone/backends/sqlalchemy/migrate_repo/manage.py version_control --url=sqlite:///bin/keystone.db --repository=keystone/backends/sqlalchemy/migrate_repo/ + + +Set your current db version to appropriate version_number. + +Version number 1 maps to diablo release. + +Example:: + +UPDATE migrate_version SET version=1; + +Perform Upgrades/Downgrades + +Example Upgrade :: + +$python keystone/backends/sqlalchemy/migrate_repo/manage.py upgrade --url=sqlite:///bin/keystone.db --repository=keystone/backends/sqlalchemy/migrate_repo/ + +Example Downgrade:: + +$python keystone/backends/sqlalchemy/migrate_repo/manage.py downgrade 1 --url=sqlite:///bin/keystone.db --repository=keystone/backends/sqlalchemy/migrate_repo/ + diff --git a/doc/source/testing.rst b/doc/source/testing.rst index 35eb16f8c4..b80241f8e6 100644 --- a/doc/source/testing.rst +++ b/doc/source/testing.rst @@ -20,6 +20,33 @@ and aborts after the first test failure (a fail-fast behavior):: $ ./run_tests.sh +Schema Migration Tests +====================== + +Schema migrations are tested using SQLAlchemy Migrate's built-in test +runner:: + +The test does not start testing from the very top.In order for the test to run, the database +that is used to test, should be up to version above the version brought forward by the latest script.:: + +This command would create the test db with a version of 0.:: + +$python keystone/backends/sqlalchemy/migrate_repo/manage.py version_control sqlite:///test.db --repository=keystone/backends/sqlalchemy/migrate_repo/ + +Use this command to move to the version that is before our latest script. + +ie if our latest script has version 3, we should move to 2.:: + +$python keystone/backends/sqlalchemy/migrate_repo/manage.py upgrade version_number --url=sqlite:///test.db --repository=keystone/backends/sqlalchemy/migrate_repo/ + +Now try:: + +$python keystone/backends/sqlalchemy/migrate_repo/manage.py test --url=sqlite:///test.db --repository=keystone/backends/sqlalchemy/migrate_repo/ + +This tests both forward and backward migrations, and should leave behind +an test sqlite database (``test.db``) that can be safely +removed or simply ignored. + Writing Tests ============= diff --git a/keystone/backends/sqlalchemy/migrate_repo/README b/keystone/backends/sqlalchemy/migrate_repo/README new file mode 100644 index 0000000000..6218f8cac4 --- /dev/null +++ b/keystone/backends/sqlalchemy/migrate_repo/README @@ -0,0 +1,4 @@ +This is a database migration repository. + +More information at +http://code.google.com/p/sqlalchemy-migrate/ diff --git a/keystone/backends/sqlalchemy/migrate_repo/__init__.py b/keystone/backends/sqlalchemy/migrate_repo/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/keystone/backends/sqlalchemy/migrate_repo/manage.py b/keystone/backends/sqlalchemy/migrate_repo/manage.py new file mode 100755 index 0000000000..2a928c84c9 --- /dev/null +++ b/keystone/backends/sqlalchemy/migrate_repo/manage.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python +from migrate.versioning.shell import main +main(debug='False') diff --git a/keystone/backends/sqlalchemy/migrate_repo/migrate.cfg b/keystone/backends/sqlalchemy/migrate_repo/migrate.cfg new file mode 100644 index 0000000000..42986cf790 --- /dev/null +++ b/keystone/backends/sqlalchemy/migrate_repo/migrate.cfg @@ -0,0 +1,20 @@ +[db_settings] +# Used to identify which repository this database is versioned under. +# You can use the name of your project. +repository_id=Keystone + +# The name of the database table used to track the schema version. +# This name shouldn't already be used by your project. +# If this is changed once a database is under version control, you'll need to +# change the table name in each database too. +version_table=migrate_version + +# When committing a change script, Migrate will attempt to generate the +# sql for all supported databases; normally, if one of them fails - probably +# because you don't have that database installed - it is ignored and the +# commit continues, perhaps ending successfully. +# Databases in this list MUST compile successfully during a commit, or the +# entire commit will fail. List the databases your application will actually +# be using to ensure your updates to that database work properly. +# This must be a list; example: ['postgres','sqlite'] +required_dbs=[] diff --git a/keystone/backends/sqlalchemy/migrate_repo/versions/001_initial_migration.py b/keystone/backends/sqlalchemy/migrate_repo/versions/001_initial_migration.py new file mode 100644 index 0000000000..9058ed1ba4 --- /dev/null +++ b/keystone/backends/sqlalchemy/migrate_repo/versions/001_initial_migration.py @@ -0,0 +1,196 @@ +# pylint: disable=C0103 + + +import sqlalchemy + + +meta = sqlalchemy.MetaData() + + +# services + +service = {} +service['id'] = sqlalchemy.Column('id', sqlalchemy.Integer, + primary_key=True, autoincrement=True) +service['name'] = sqlalchemy.Column('name', sqlalchemy.String(255), + unique=True) +service['type'] = sqlalchemy.Column('type', sqlalchemy.String(255)) +service['desc'] = sqlalchemy.Column('desc', sqlalchemy.String(255)) +services = sqlalchemy.Table('services', meta, *service.values()) + +sqlalchemy.UniqueConstraint(service['name']) + + +# roles + +role = {} +role['id'] = sqlalchemy.Column('id', sqlalchemy.Integer, + primary_key=True, autoincrement=True) +role['name'] = sqlalchemy.Column('name', sqlalchemy.String(255)) +role['desc'] = sqlalchemy.Column('desc', sqlalchemy.String(255)) +role['service_id'] = sqlalchemy.Column('service_id', sqlalchemy.Integer) +roles = sqlalchemy.Table('roles', meta, *role.values()) + +sqlalchemy.UniqueConstraint(role['name'], role['service_id']) + +sqlalchemy.ForeignKeyConstraint( + [role['service_id']], + [service['id']]) + + +# tenants + +tenant = {} +tenant['id'] = sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True, + autoincrement=True) +tenant['name'] = sqlalchemy.Column('name', sqlalchemy.String(255), unique=True) +tenant['desc'] = sqlalchemy.Column('desc', sqlalchemy.String(255)) +tenant['enabled'] = sqlalchemy.Column('enabled', sqlalchemy.Integer) +tenants = sqlalchemy.Table('tenants', meta, *tenant.values()) + +sqlalchemy.UniqueConstraint(tenant['name']) + + +# users + +user = {} +user['id'] = sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True, + autoincrement=True) +user['name'] = sqlalchemy.Column('name', sqlalchemy.String(255), unique=True) +user['password'] = sqlalchemy.Column('password', sqlalchemy.String(255)) +user['email'] = sqlalchemy.Column('email', sqlalchemy.String(255)) +user['enabled'] = sqlalchemy.Column('enabled', sqlalchemy.Integer) +user['tenant_id'] = sqlalchemy.Column('tenant_id', sqlalchemy.Integer) +users = sqlalchemy.Table('users', meta, *user.values()) + +sqlalchemy.UniqueConstraint(user['name']) + +sqlalchemy.ForeignKeyConstraint( + [user['tenant_id']], + [tenant['id']]) + + +# credentials + +credential = {} +credential['id'] = sqlalchemy.Column('id', sqlalchemy.Integer, + primary_key=True, autoincrement=True) +credential['user_id'] = sqlalchemy.Column('user_id', sqlalchemy.Integer) +credential['tenant_id'] = sqlalchemy.Column('tenant_id', sqlalchemy.Integer, + nullable=True) +credential['type'] = sqlalchemy.Column('type', sqlalchemy.String(20)) +credential['key'] = sqlalchemy.Column('key', sqlalchemy.String(255)) +credential['secret'] = sqlalchemy.Column('secret', sqlalchemy.String(255)) +credentials = sqlalchemy.Table('credentials', meta, *credential.values()) + +sqlalchemy.ForeignKeyConstraint( + [credential['user_id']], + [user['id']]) +sqlalchemy.ForeignKeyConstraint( + [credential['tenant_id']], + [tenant['id']]) + + +# tokens + +token = {} +token['id'] = sqlalchemy.Column('id', sqlalchemy.String(255), primary_key=True, + unique=True) +token['user_id'] = sqlalchemy.Column('user_id', sqlalchemy.Integer) +token['tenant_id'] = sqlalchemy.Column('tenant_id', sqlalchemy.Integer) +token['expires'] = sqlalchemy.Column('expires', sqlalchemy.DateTime) +tokens = sqlalchemy.Table('token', meta, *token.values()) + + +# endpoint_templates + +endpoint_template = {} +endpoint_template['id'] = sqlalchemy.Column('id', sqlalchemy.Integer, + primary_key=True) +endpoint_template['region'] = sqlalchemy.Column('region', + sqlalchemy.String(255)) +endpoint_template['service_id'] = sqlalchemy.Column('service_id', + sqlalchemy.Integer) +endpoint_template['public_url'] = sqlalchemy.Column('public_url', + sqlalchemy.String(2000)) +endpoint_template['admin_url'] = sqlalchemy.Column('admin_url', + sqlalchemy.String(2000)) +endpoint_template['internal_url'] = sqlalchemy.Column('internal_url', + sqlalchemy.String(2000)) +endpoint_template['enabled'] = sqlalchemy.Column('enabled', + sqlalchemy.Boolean) +endpoint_template['is_global'] = sqlalchemy.Column('is_global', + sqlalchemy.Boolean) +endpoint_templates = sqlalchemy.Table('endpoint_templates', meta, + *endpoint_template.values()) + +sqlalchemy.ForeignKeyConstraint( + [endpoint_template['service_id']], [service['id']]) + + +# endpoints + +endpoint = {} +endpoint['id'] = sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True) +endpoint['tenant_id'] = sqlalchemy.Column('tenant_id', sqlalchemy.Integer) +endpoint['endpoint_template_id'] = sqlalchemy.Column('endpoint_template_id', + sqlalchemy.Integer) +endpoints = sqlalchemy.Table('endpoints', meta, *endpoint.values()) + +sqlalchemy.UniqueConstraint( + endpoint['endpoint_template_id'], endpoint['tenant_id']) + +sqlalchemy.ForeignKeyConstraint( + [endpoint['endpoint_template_id']], + [endpoint_template['id']]) + + +# user_roles + +user_role = {} +user_role['id'] = sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True) +user_role['user_id'] = sqlalchemy.Column('user_id', sqlalchemy.Integer) +user_role['role_id'] = sqlalchemy.Column('role_id', sqlalchemy.Integer) +user_role['tenant_id'] = sqlalchemy.Column('tenant_id', sqlalchemy.Integer) +user_roles = sqlalchemy.Table('user_roles', meta, *user_role.values()) + +sqlalchemy.UniqueConstraint( + user_role['user_id'], user_role['role_id'], user_role['tenant_id']) + +sqlalchemy.ForeignKeyConstraint( + [user_role['user_id']], + [user['id']]) +sqlalchemy.ForeignKeyConstraint( + [user_role['role_id']], + [role['id']]) +sqlalchemy.ForeignKeyConstraint( + [user_role['tenant_id']], + [tenant['id']]) + + +def upgrade(migrate_engine): + meta.bind = migrate_engine + + user_roles.create() + endpoints.create() + roles.create() + services.create() + tenants.create() + users.create() + credentials.create() + tokens.create() + endpoint_templates.create() + + +def downgrade(migrate_engine): + meta.bind = migrate_engine + + user_roles.drop() + endpoints.drop() + roles.drop() + services.drop() + tenants.drop() + users.drop() + credentials.drop() + tokens.drop() + endpoint_templates.drop() diff --git a/keystone/backends/sqlalchemy/migrate_repo/versions/002_rename_token_table.py b/keystone/backends/sqlalchemy/migrate_repo/versions/002_rename_token_table.py new file mode 100644 index 0000000000..1d15d9dac4 --- /dev/null +++ b/keystone/backends/sqlalchemy/migrate_repo/versions/002_rename_token_table.py @@ -0,0 +1,24 @@ +""" +Addresses bug 854425 + +Renames the 'token' table to 'tokens', +in order to appear more consistent with +other table names. +""" +# pylint: disable=C0103 + + +import sqlalchemy + + +meta = sqlalchemy.MetaData() + + +def upgrade(migrate_engine): + meta.bind = migrate_engine + sqlalchemy.Table('token', meta).rename('tokens') + + +def downgrade(migrate_engine): + meta.bind = migrate_engine + sqlalchemy.Table('tokens', meta).rename('token') diff --git a/keystone/backends/sqlalchemy/migrate_repo/versions/003_add_endpoint_template_versions.py b/keystone/backends/sqlalchemy/migrate_repo/versions/003_add_endpoint_template_versions.py new file mode 100644 index 0000000000..95ac175d08 --- /dev/null +++ b/keystone/backends/sqlalchemy/migrate_repo/versions/003_add_endpoint_template_versions.py @@ -0,0 +1,64 @@ +""" +Adds support for versioning endpoint templates +""" +# pylint: disable=C0103 + + +import sqlalchemy +import migrate + + +meta = sqlalchemy.MetaData() + +endpoint_template = {} +endpoint_template['id'] = sqlalchemy.Column('id', sqlalchemy.Integer, + primary_key=True) +endpoint_template['region'] = sqlalchemy.Column('region', + sqlalchemy.String(255)) +endpoint_template['service_id'] = sqlalchemy.Column('service_id', + sqlalchemy.Integer) +endpoint_template['public_url'] = sqlalchemy.Column('public_url', + sqlalchemy.String(2000)) +endpoint_template['admin_url'] = sqlalchemy.Column('admin_url', + sqlalchemy.String(2000)) +endpoint_template['internal_url'] = sqlalchemy.Column('internal_url', + sqlalchemy.String(2000)) +endpoint_template['enabled'] = sqlalchemy.Column('enabled', + sqlalchemy.Boolean) +endpoint_template['is_global'] = sqlalchemy.Column('is_global', + sqlalchemy.Boolean) +endpoint_templates = sqlalchemy.Table('endpoint_templates', meta, + *endpoint_template.values()) + +version_id = sqlalchemy.Column('version_id', sqlalchemy.String(20), + nullable=True) +version_list = sqlalchemy.Column('version_list', sqlalchemy.String(2000), + nullable=True) +version_info = sqlalchemy.Column('version_info', sqlalchemy.String(500), + nullable=True) + + +def upgrade(migrate_engine): + meta.bind = migrate_engine + + migrate.create_column(version_id, endpoint_templates) + assert endpoint_templates.c.version_id is version_id + + migrate.create_column(version_list, endpoint_templates) + assert endpoint_templates.c.version_list is version_list + + migrate.create_column(version_info, endpoint_templates) + assert endpoint_templates.c.version_info is version_info + + +def downgrade(migrate_engine): + meta.bind = migrate_engine + + migrate.drop_column(version_id, endpoint_templates) + assert not hasattr(endpoint_templates.c, 'version_id') + + migrate.drop_column(version_list, endpoint_templates) + assert not hasattr(endpoint_templates.c, 'version_list') + + migrate.drop_column(version_info, endpoint_templates) + assert not hasattr(endpoint_templates.c, 'version_info') diff --git a/keystone/backends/sqlalchemy/migrate_repo/versions/__init__.py b/keystone/backends/sqlalchemy/migrate_repo/versions/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/pip-requires b/tools/pip-requires index 1400093dbb..b87f164788 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -10,6 +10,7 @@ pastescript # command line frontend webob # wsgi framework Routes # URL matching / controller routing sqlalchemy # core backend +sqlalchemy-migrate # database migrations pysqlite # default backend database lib lxml # xml library providing ElementTree API passlib # password hashing