Track post-Diablo database evolution using migrations (BP: database-migrations)

- Added SQLAlchemy Migrations as a dependency (pip install sqlalchemy-migrate)
- Added empty migration repo
- Made migration manage.py executable
- Added initial migration matching diablo-tagged release
- Added incremental migrations to catch up to trunk
- Added usage docs for contributors
- Added doc that explains how to migrate.

Change-Id: I4fa8f930645ac6b7f82d0472361fe114b71bb489

.gitignore

@@ -20,3 +20,4 @@ keystone.egg-info/
 run_tests.err.log
 .coverage
 .DS_Store
+test_migrations.db

doc/source/index.rst

@@ -36,6 +36,7 @@ Getting Started
 
     setup
     testing
+    migration
     configuration
     community
     usingkeystone

doc/source/migration.rst

@@ -0,0 +1,35 @@
+================
+Using Migrations
+================
+
+Keystone uses sqlalchemy-migrate to manage migrations.
+
+
+Running Migrations
+======================
+
+Keep backups of your db.Add your existing database to version control.
+
+Example::
+
+$python keystone/backends/sqlalchemy/migrate_repo/manage.py version_control  --url=sqlite:///bin/keystone.db --repository=keystone/backends/sqlalchemy/migrate_repo/
+
+
+Set your current db version to appropriate version_number.
+
+Version number 1 maps to diablo release.
+
+Example::
+ 
+UPDATE migrate_version SET version=1; 
+
+Perform Upgrades/Downgrades
+
+Example Upgrade ::
+
+$python keystone/backends/sqlalchemy/migrate_repo/manage.py upgrade  --url=sqlite:///bin/keystone.db --repository=keystone/backends/sqlalchemy/migrate_repo/
+
+Example Downgrade::
+
+$python keystone/backends/sqlalchemy/migrate_repo/manage.py downgrade 1   --url=sqlite:///bin/keystone.db --repository=keystone/backends/sqlalchemy/migrate_repo/
+

doc/source/testing.rst

@@ -20,6 +20,33 @@ and aborts after the first test failure (a fail-fast behavior)::
 
     $ ./run_tests.sh
 
+Schema Migration Tests
+======================
+
+Schema migrations are tested using SQLAlchemy Migrate's built-in test
+runner::
+
+The test does not start testing from the very top.In order for the test to run, the database 
+that is used to test, should be up to version above the version brought forward by the latest script.::
+
+This command would create the test db with a version of 0.::
+
+$python keystone/backends/sqlalchemy/migrate_repo/manage.py version_control sqlite:///test.db   --repository=keystone/backends/sqlalchemy/migrate_repo/
+
+Use this command to move to the version that is before our latest script.
+
+ie if our latest script has version 3, we should move to 2.::
+
+$python keystone/backends/sqlalchemy/migrate_repo/manage.py upgrade version_number --url=sqlite:///test.db   --repository=keystone/backends/sqlalchemy/migrate_repo/
+
+Now try::
+
+$python keystone/backends/sqlalchemy/migrate_repo/manage.py test  --url=sqlite:///test.db --repository=keystone/backends/sqlalchemy/migrate_repo/
+
+This tests both forward and backward migrations, and should leave behind
+an test sqlite database (``test.db``) that can be safely
+removed or simply ignored.
+
 Writing Tests
 =============
 

keystone/backends/sqlalchemy/migrate_repo/README

@@ -0,0 +1,4 @@
+This is a database migration repository.
+
+More information at
+http://code.google.com/p/sqlalchemy-migrate/

keystone/backends/sqlalchemy/migrate_repo/manage.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python
+from migrate.versioning.shell import main
+main(debug='False')

keystone/backends/sqlalchemy/migrate_repo/migrate.cfg

@@ -0,0 +1,20 @@
+[db_settings]
+# Used to identify which repository this database is versioned under.
+# You can use the name of your project.
+repository_id=Keystone
+
+# The name of the database table used to track the schema version.
+# This name shouldn't already be used by your project.
+# If this is changed once a database is under version control, you'll need to 
+# change the table name in each database too. 
+version_table=migrate_version
+
+# When committing a change script, Migrate will attempt to generate the 
+# sql for all supported databases; normally, if one of them fails - probably
+# because you don't have that database installed - it is ignored and the 
+# commit continues, perhaps ending successfully. 
+# Databases in this list MUST compile successfully during a commit, or the 
+# entire commit will fail. List the databases your application will actually 
+# be using to ensure your updates to that database work properly.
+# This must be a list; example: ['postgres','sqlite']
+required_dbs=[]

keystone/backends/sqlalchemy/migrate_repo/versions/001_initial_migration.py

@@ -0,0 +1,196 @@
+# pylint: disable=C0103
+
+
+import sqlalchemy
+
+
+meta = sqlalchemy.MetaData()
+
+
+# services
+
+service = {}
+service['id'] = sqlalchemy.Column('id', sqlalchemy.Integer,
+        primary_key=True, autoincrement=True)
+service['name'] = sqlalchemy.Column('name', sqlalchemy.String(255),
+        unique=True)
+service['type'] = sqlalchemy.Column('type', sqlalchemy.String(255))
+service['desc'] = sqlalchemy.Column('desc', sqlalchemy.String(255))
+services = sqlalchemy.Table('services', meta, *service.values())
+
+sqlalchemy.UniqueConstraint(service['name'])
+
+
+# roles
+
+role = {}
+role['id'] = sqlalchemy.Column('id', sqlalchemy.Integer,
+        primary_key=True, autoincrement=True)
+role['name'] = sqlalchemy.Column('name', sqlalchemy.String(255))
+role['desc'] = sqlalchemy.Column('desc', sqlalchemy.String(255))
+role['service_id'] = sqlalchemy.Column('service_id', sqlalchemy.Integer)
+roles = sqlalchemy.Table('roles', meta, *role.values())
+
+sqlalchemy.UniqueConstraint(role['name'], role['service_id'])
+
+sqlalchemy.ForeignKeyConstraint(
+        [role['service_id']],
+        [service['id']])
+
+
+# tenants
+
+tenant = {}
+tenant['id'] = sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True,
+        autoincrement=True)
+tenant['name'] = sqlalchemy.Column('name', sqlalchemy.String(255), unique=True)
+tenant['desc'] = sqlalchemy.Column('desc', sqlalchemy.String(255))
+tenant['enabled'] = sqlalchemy.Column('enabled', sqlalchemy.Integer)
+tenants = sqlalchemy.Table('tenants', meta, *tenant.values())
+
+sqlalchemy.UniqueConstraint(tenant['name'])
+
+
+# users
+
+user = {}
+user['id'] = sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True,
+        autoincrement=True)
+user['name'] = sqlalchemy.Column('name', sqlalchemy.String(255), unique=True)
+user['password'] = sqlalchemy.Column('password', sqlalchemy.String(255))
+user['email'] = sqlalchemy.Column('email', sqlalchemy.String(255))
+user['enabled'] = sqlalchemy.Column('enabled', sqlalchemy.Integer)
+user['tenant_id'] = sqlalchemy.Column('tenant_id', sqlalchemy.Integer)
+users = sqlalchemy.Table('users', meta, *user.values())
+
+sqlalchemy.UniqueConstraint(user['name'])
+
+sqlalchemy.ForeignKeyConstraint(
+        [user['tenant_id']],
+        [tenant['id']])
+
+
+# credentials
+
+credential = {}
+credential['id'] = sqlalchemy.Column('id', sqlalchemy.Integer,
+        primary_key=True, autoincrement=True)
+credential['user_id'] = sqlalchemy.Column('user_id', sqlalchemy.Integer)
+credential['tenant_id'] = sqlalchemy.Column('tenant_id', sqlalchemy.Integer,
+        nullable=True)
+credential['type'] = sqlalchemy.Column('type', sqlalchemy.String(20))
+credential['key'] = sqlalchemy.Column('key', sqlalchemy.String(255))
+credential['secret'] = sqlalchemy.Column('secret', sqlalchemy.String(255))
+credentials = sqlalchemy.Table('credentials', meta, *credential.values())
+
+sqlalchemy.ForeignKeyConstraint(
+        [credential['user_id']],
+        [user['id']])
+sqlalchemy.ForeignKeyConstraint(
+        [credential['tenant_id']],
+        [tenant['id']])
+
+
+# tokens
+
+token = {}
+token['id'] = sqlalchemy.Column('id', sqlalchemy.String(255), primary_key=True,
+        unique=True)
+token['user_id'] = sqlalchemy.Column('user_id', sqlalchemy.Integer)
+token['tenant_id'] = sqlalchemy.Column('tenant_id', sqlalchemy.Integer)
+token['expires'] = sqlalchemy.Column('expires', sqlalchemy.DateTime)
+tokens = sqlalchemy.Table('token', meta, *token.values())
+
+
+# endpoint_templates
+
+endpoint_template = {}
+endpoint_template['id'] = sqlalchemy.Column('id', sqlalchemy.Integer,
+        primary_key=True)
+endpoint_template['region'] = sqlalchemy.Column('region',
+        sqlalchemy.String(255))
+endpoint_template['service_id'] = sqlalchemy.Column('service_id',
+        sqlalchemy.Integer)
+endpoint_template['public_url'] = sqlalchemy.Column('public_url',
+        sqlalchemy.String(2000))
+endpoint_template['admin_url'] = sqlalchemy.Column('admin_url',
+        sqlalchemy.String(2000))
+endpoint_template['internal_url'] = sqlalchemy.Column('internal_url',
+        sqlalchemy.String(2000))
+endpoint_template['enabled'] = sqlalchemy.Column('enabled',
+        sqlalchemy.Boolean)
+endpoint_template['is_global'] = sqlalchemy.Column('is_global',
+        sqlalchemy.Boolean)
+endpoint_templates = sqlalchemy.Table('endpoint_templates', meta,
+        *endpoint_template.values())
+
+sqlalchemy.ForeignKeyConstraint(
+        [endpoint_template['service_id']], [service['id']])
+
+
+# endpoints
+
+endpoint = {}
+endpoint['id'] = sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True)
+endpoint['tenant_id'] = sqlalchemy.Column('tenant_id', sqlalchemy.Integer)
+endpoint['endpoint_template_id'] = sqlalchemy.Column('endpoint_template_id',
+        sqlalchemy.Integer)
+endpoints = sqlalchemy.Table('endpoints', meta, *endpoint.values())
+
+sqlalchemy.UniqueConstraint(
+        endpoint['endpoint_template_id'], endpoint['tenant_id'])
+
+sqlalchemy.ForeignKeyConstraint(
+        [endpoint['endpoint_template_id']],
+        [endpoint_template['id']])
+
+
+# user_roles
+
+user_role = {}
+user_role['id'] = sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True)
+user_role['user_id'] = sqlalchemy.Column('user_id', sqlalchemy.Integer)
+user_role['role_id'] = sqlalchemy.Column('role_id', sqlalchemy.Integer)
+user_role['tenant_id'] = sqlalchemy.Column('tenant_id', sqlalchemy.Integer)
+user_roles = sqlalchemy.Table('user_roles', meta, *user_role.values())
+
+sqlalchemy.UniqueConstraint(
+        user_role['user_id'], user_role['role_id'], user_role['tenant_id'])
+
+sqlalchemy.ForeignKeyConstraint(
+        [user_role['user_id']],
+        [user['id']])
+sqlalchemy.ForeignKeyConstraint(
+        [user_role['role_id']],
+        [role['id']])
+sqlalchemy.ForeignKeyConstraint(
+        [user_role['tenant_id']],
+        [tenant['id']])
+
+
+def upgrade(migrate_engine):
+    meta.bind = migrate_engine
+
+    user_roles.create()
+    endpoints.create()
+    roles.create()
+    services.create()
+    tenants.create()
+    users.create()
+    credentials.create()
+    tokens.create()
+    endpoint_templates.create()
+
+
+def downgrade(migrate_engine):
+    meta.bind = migrate_engine
+
+    user_roles.drop()
+    endpoints.drop()
+    roles.drop()
+    services.drop()
+    tenants.drop()
+    users.drop()
+    credentials.drop()
+    tokens.drop()
+    endpoint_templates.drop()

keystone/backends/sqlalchemy/migrate_repo/versions/002_rename_token_table.py

@@ -0,0 +1,24 @@
+"""
+Addresses bug 854425
+
+Renames the 'token' table to 'tokens',
+in order to appear more consistent with
+other table names.
+"""
+# pylint: disable=C0103
+
+
+import sqlalchemy
+
+
+meta = sqlalchemy.MetaData()
+
+
+def upgrade(migrate_engine):
+    meta.bind = migrate_engine
+    sqlalchemy.Table('token', meta).rename('tokens')
+
+
+def downgrade(migrate_engine):
+    meta.bind = migrate_engine
+    sqlalchemy.Table('tokens', meta).rename('token')

keystone/backends/sqlalchemy/migrate_repo/versions/003_add_endpoint_template_versions.py

@@ -0,0 +1,64 @@
+"""
+Adds support for versioning endpoint templates
+"""
+# pylint: disable=C0103
+
+
+import sqlalchemy
+import migrate
+
+
+meta = sqlalchemy.MetaData()
+
+endpoint_template = {}
+endpoint_template['id'] = sqlalchemy.Column('id', sqlalchemy.Integer,
+        primary_key=True)
+endpoint_template['region'] = sqlalchemy.Column('region',
+        sqlalchemy.String(255))
+endpoint_template['service_id'] = sqlalchemy.Column('service_id',
+        sqlalchemy.Integer)
+endpoint_template['public_url'] = sqlalchemy.Column('public_url',
+        sqlalchemy.String(2000))
+endpoint_template['admin_url'] = sqlalchemy.Column('admin_url',
+        sqlalchemy.String(2000))
+endpoint_template['internal_url'] = sqlalchemy.Column('internal_url',
+        sqlalchemy.String(2000))
+endpoint_template['enabled'] = sqlalchemy.Column('enabled',
+        sqlalchemy.Boolean)
+endpoint_template['is_global'] = sqlalchemy.Column('is_global',
+        sqlalchemy.Boolean)
+endpoint_templates = sqlalchemy.Table('endpoint_templates', meta,
+        *endpoint_template.values())
+
+version_id = sqlalchemy.Column('version_id', sqlalchemy.String(20),
+        nullable=True)
+version_list = sqlalchemy.Column('version_list', sqlalchemy.String(2000),
+        nullable=True)
+version_info = sqlalchemy.Column('version_info', sqlalchemy.String(500),
+        nullable=True)
+
+
+def upgrade(migrate_engine):
+    meta.bind = migrate_engine
+
+    migrate.create_column(version_id, endpoint_templates)
+    assert endpoint_templates.c.version_id is version_id
+
+    migrate.create_column(version_list, endpoint_templates)
+    assert endpoint_templates.c.version_list is version_list
+
+    migrate.create_column(version_info, endpoint_templates)
+    assert endpoint_templates.c.version_info is version_info
+
+
+def downgrade(migrate_engine):
+    meta.bind = migrate_engine
+
+    migrate.drop_column(version_id, endpoint_templates)
+    assert not hasattr(endpoint_templates.c, 'version_id')
+
+    migrate.drop_column(version_list, endpoint_templates)
+    assert not hasattr(endpoint_templates.c, 'version_list')
+
+    migrate.drop_column(version_info, endpoint_templates)
+    assert not hasattr(endpoint_templates.c, 'version_info')

tools/pip-requires

@@ -10,6 +10,7 @@ pastescript # command line frontend
 webob # wsgi framework
 Routes # URL matching / controller routing
 sqlalchemy # core backend
+sqlalchemy-migrate # database migrations
 pysqlite # default backend database lib
 lxml # xml library providing ElementTree API
 passlib # password hashing