From 893f88c07889535b265340561cd2945aab5037f4 Mon Sep 17 00:00:00 2001 From: tonytan4ever Date: Tue, 11 Nov 2014 13:37:00 -0500 Subject: [PATCH] Add cassandra schema migration framework Implements: blueprint cassandra-schema-versioning Change-Id: I37ac447116d69b465052063269f55d8487142124 --- README.rst | 1 + docker/api/poppy.conf | 2 + docker/api_ci/poppy.conf | 2 + docker/api_dev/poppy.conf | 2 + etc/poppy.conf | 2 + poppy/storage/cassandra/driver.py | 19 ++++-- .../001_initial_schema.cql} | 5 +- poppy/storage/cassandra/schema.py | 58 ------------------- requirements/storage/cassandra.txt | 3 +- .../storage/cassandra/schema_migration.cql | 8 +++ tests/etc/default_functional.conf | 1 + tests/unit/storage/cassandra/test_driver.py | 11 ++++ tests/unit/storage/cassandra/test_flavors.py | 7 +++ tests/unit/storage/cassandra/test_services.py | 6 ++ 14 files changed, 60 insertions(+), 67 deletions(-) rename poppy/storage/cassandra/{schema.cql => migrations/001_initial_schema.cql} (88%) delete mode 100644 poppy/storage/cassandra/schema.py create mode 100644 scripts/storage/cassandra/schema_migration.cql diff --git a/README.rst b/README.rst index 85908baf..57059de2 100644 --- a/README.rst +++ b/README.rst @@ -71,6 +71,7 @@ installed and running. [drivers:storage:cassandra] cluster = "localhost" keyspace = poppy + migrations_path = /home/poppy/poppy/storage/cassandra/migrations 4. By using cassandra storage plugin, you will need to create the default keyspace "poppy" on your cassandra host/cluster. So log into cqlsh, do:: diff --git a/docker/api/poppy.conf b/docker/api/poppy.conf index 0dab3665..fd99f20d 100644 --- a/docker/api/poppy.conf +++ b/docker/api/poppy.conf @@ -48,6 +48,8 @@ port = 8081 [drivers:storage:cassandra] cluster = "cassandra" keyspace = poppy +# Path to directory containing CQL migration scripts +migrations_path = /home/poppy/poppy/storage/cassandra/migrations [drivers:dns:rackspace] username = DNS_USERNAME diff --git a/docker/api_ci/poppy.conf b/docker/api_ci/poppy.conf index a72b78b1..bc946647 100644 --- a/docker/api_ci/poppy.conf +++ b/docker/api_ci/poppy.conf @@ -57,6 +57,8 @@ email = "email@example.com" [drivers:storage:cassandra] cluster = "cassandra" keyspace = poppy +# Path to directory containing CQL migration scripts +migrations_path = /home/poppy/poppy/storage/cassandra/migrations [drivers:provider:akamai] policy_api_client_token = POLICY-API-CLIENT-TOKEN diff --git a/docker/api_dev/poppy.conf b/docker/api_dev/poppy.conf index d807361b..b7261917 100644 --- a/docker/api_dev/poppy.conf +++ b/docker/api_dev/poppy.conf @@ -48,6 +48,8 @@ port = 8081 [drivers:storage:cassandra] cluster = "cassandra" keyspace = poppy +# Path to directory containing CQL migration scripts +migrations_path = /home/poppy/poppy/storage/cassandra/migrations [drivers:dns:rackspace] username = DNS_USERNAME diff --git a/etc/poppy.conf b/etc/poppy.conf index 19b089c8..59cab1dc 100644 --- a/etc/poppy.conf +++ b/etc/poppy.conf @@ -76,6 +76,8 @@ keyspace = poppy # `map` as show in the syntax here: http://www.datastax.com/documentation/cql/3 # .1/cql/cql_reference/create_keyspace_r.html replication_strategy = class:SimpleStrategy, replication_factor:1 +# Path to directory containing CQL migration scripts +migrations_path = /poppy/storage/cassandra/migrations [drivers:storage:mockdb] database = poppy diff --git a/poppy/storage/cassandra/driver.py b/poppy/storage/cassandra/driver.py index 7fbe6a42..34eaa68b 100644 --- a/poppy/storage/cassandra/driver.py +++ b/poppy/storage/cassandra/driver.py @@ -23,12 +23,12 @@ from cassandra import auth from cassandra import cluster from cassandra import policies from cassandra import query +from cdeploy import migrator from oslo.config import cfg from poppy.openstack.common import log as logging from poppy.storage import base from poppy.storage.cassandra import controllers -from poppy.storage.cassandra import schema LOG = logging.getLogger(__name__) @@ -62,6 +62,11 @@ CASSANDRA_OPTIONS = [ }, help='Replication strategy for Cassandra cluster' ), + cfg.StrOpt( + 'migrations_path', + default='./poppy/storage/cassandra/migrations', + help='Path to directory containing CQL migration scripts', + ), cfg.BoolOpt('archive_on_delete', default=True, help='Archive services on delete?'), ] @@ -112,6 +117,8 @@ def _connection(conf, datacenter, keyspace=None): except cassandra.InvalidRequest: _create_keyspace(session, keyspace, conf.replication_strategy) + _run_migrations(conf.migrations_path, session) + session.row_factory = query.dict_factory return session @@ -123,6 +130,8 @@ def _create_keyspace(session, keyspace, replication_strategy): :param keyspace :param replication_strategy """ + LOG.debug('Creating keyspace: ' + keyspace) + # replication factor will come in as a string with quotes already session.execute( "CREATE KEYSPACE " + keyspace + " " + @@ -130,10 +139,12 @@ def _create_keyspace(session, keyspace, replication_strategy): ) session.set_keyspace(keyspace) - for statement in schema.schema_statements: - session.execute(statement) - LOG.debug('Creating keyspace: ' + keyspace) +def _run_migrations(migrations_path, session): + LOG.debug('Running schema migration(s)') + + schema_migrator = migrator.Migrator(migrations_path, session) + schema_migrator.run_migrations() class CassandraStorageDriver(base.Driver): diff --git a/poppy/storage/cassandra/schema.cql b/poppy/storage/cassandra/migrations/001_initial_schema.cql similarity index 88% rename from poppy/storage/cassandra/schema.cql rename to poppy/storage/cassandra/migrations/001_initial_schema.cql index fbe90101..82fbc248 100644 --- a/poppy/storage/cassandra/schema.cql +++ b/poppy/storage/cassandra/migrations/001_initial_schema.cql @@ -1,6 +1,3 @@ -CREATE KEYSPACE poppy WITH REPLICATION = { 'class' : 'SimpleStrategy' , 'replication_factor' : 1} ; -USE poppy; - CREATE TABLE services ( project_id VARCHAR, service_id UUID, @@ -39,4 +36,4 @@ CREATE TABLE flavors ( flavor_id VARCHAR, providers MAP, PRIMARY KEY (flavor_id) -); +); \ No newline at end of file diff --git a/poppy/storage/cassandra/schema.py b/poppy/storage/cassandra/schema.py deleted file mode 100644 index f3001f3d..00000000 --- a/poppy/storage/cassandra/schema.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright (c) 2014 Rackspace, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -schema_statements = [ - '''CREATE TABLE services ( - project_id VARCHAR, - service_id UUID, - service_name VARCHAR, - flavor_id VARCHAR, - domains LIST, - origins LIST, - caching_rules LIST, - restrictions LIST, - provider_details MAP, - PRIMARY KEY (project_id, service_id) - ); - ''', - '''CREATE TABLE flavors ( - flavor_id VARCHAR, - providers MAP, - PRIMARY KEY (flavor_id) - ); - ''', - '''CREATE TABLE domain_names ( - project_id VARCHAR, - service_id UUID, - domain_name VARCHAR, - PRIMARY KEY (domain_name) - ); - ''', - '''CREATE TABLE archives ( - project_id VARCHAR, - service_id UUID, - service_name VARCHAR, - flavor_id VARCHAR, - domains LIST, - origins LIST, - caching_rules LIST, - restrictions LIST, - provider_details MAP, - archived_time timestamp, - PRIMARY KEY (project_id, service_id, archived_time) - ); - ''' -] diff --git a/requirements/storage/cassandra.txt b/requirements/storage/cassandra.txt index 8a01411f..57f5443c 100644 --- a/requirements/storage/cassandra.txt +++ b/requirements/storage/cassandra.txt @@ -1 +1,2 @@ -cassandra-driver>=2.1.3 \ No newline at end of file +cassandra-driver>=2.1.3 +cdeploy \ No newline at end of file diff --git a/scripts/storage/cassandra/schema_migration.cql b/scripts/storage/cassandra/schema_migration.cql new file mode 100644 index 00000000..699c33a8 --- /dev/null +++ b/scripts/storage/cassandra/schema_migration.cql @@ -0,0 +1,8 @@ +CREATE TABLE IF NOT EXISTS schema_migrations( + type text, + version int, + PRIMARY KEY(type, version)) + WITH COMMENT = 'Schema migration history' + AND CLUSTERING ORDER BY (version DESC); + +INSERT INTO schema_migrations (type, version) VALUES ('migration', 1); \ No newline at end of file diff --git a/tests/etc/default_functional.conf b/tests/etc/default_functional.conf index 4cb0b901..ba550084 100644 --- a/tests/etc/default_functional.conf +++ b/tests/etc/default_functional.conf @@ -8,6 +8,7 @@ dns = default [drivers:storage:cassandra] cluster = "192.168.59.103" keyspace = poppy +migrations_path = ../poppy/storage/cassandra/migrations [drivers:provider:fastly] apikey = "MYAPIKEY" diff --git a/tests/unit/storage/cassandra/test_driver.py b/tests/unit/storage/cassandra/test_driver.py index 8ee2ef31..6f3cadf2 100644 --- a/tests/unit/storage/cassandra/test_driver.py +++ b/tests/unit/storage/cassandra/test_driver.py @@ -54,6 +54,11 @@ CASSANDRA_OPTIONS = [ }, help='Replication strategy for Cassandra cluster' ), + cfg.StrOpt( + 'migrations_path', + default='./poppy/storage/cassandra/migrations', + help='Path to directory containing CQL migration scripts', + ), cfg.BoolOpt('archive_on_delete', default=True, help='Archive services on delete?'), ] @@ -75,6 +80,12 @@ class CassandraStorageDriverTests(base.TestCase): group=driver.CASSANDRA_GROUP) self.cassandra_driver = driver.CassandraStorageDriver(conf) + migrations_patcher = mock.patch( + 'cdeploy.migrator.Migrator' + ) + migrations_patcher.start() + self.addCleanup(migrations_patcher.stop) + def test_storage_driver(self): # assert that the configs are set up based on what was passed in self.assertEqual(self.cassandra_driver.cassandra_conf['cluster'], diff --git a/tests/unit/storage/cassandra/test_flavors.py b/tests/unit/storage/cassandra/test_flavors.py index d3940432..d2e6df4b 100644 --- a/tests/unit/storage/cassandra/test_flavors.py +++ b/tests/unit/storage/cassandra/test_flavors.py @@ -44,8 +44,15 @@ class CassandraStorageFlavorsTests(base.TestCase): help='datacenter where the C* cluster hosted')) conf.register_opts(driver.CASSANDRA_OPTIONS, group=driver.CASSANDRA_GROUP) + cassandra_driver = driver.CassandraStorageDriver(conf) + migrations_patcher = mock.patch( + 'cdeploy.migrator.Migrator' + ) + migrations_patcher.start() + self.addCleanup(migrations_patcher.stop) + # stubbed cassandra driver self.fc = flavors.FlavorsController(cassandra_driver) diff --git a/tests/unit/storage/cassandra/test_services.py b/tests/unit/storage/cassandra/test_services.py index 864f9bb0..f435288c 100644 --- a/tests/unit/storage/cassandra/test_services.py +++ b/tests/unit/storage/cassandra/test_services.py @@ -54,6 +54,12 @@ class CassandraStorageServiceTests(base.TestCase): group=driver.CASSANDRA_GROUP) cassandra_driver = driver.CassandraStorageDriver(conf) + migrations_patcher = mock.patch( + 'cdeploy.migrator.Migrator' + ) + migrations_patcher.start() + self.addCleanup(migrations_patcher.stop) + # stubbed cassandra driver self.sc = services.ServicesController(cassandra_driver)