Add cassandra schema migration framework

Implements: blueprint cassandra-schema-versioning

Change-Id: I37ac447116d69b465052063269f55d8487142124

README.rst

@@ -71,6 +71,7 @@ installed and running.
     [drivers:storage:cassandra]
     cluster = "localhost"
     keyspace = poppy
+	migrations_path = /home/poppy/poppy/storage/cassandra/migrations
 
 4. By using cassandra storage plugin, you will need to create the default 
    keyspace "poppy" on your cassandra host/cluster. So log into cqlsh, do::

docker/api/poppy.conf

@@ -48,6 +48,8 @@ port = 8081
 [drivers:storage:cassandra]
 cluster = "cassandra"
 keyspace = poppy
+# Path to directory containing CQL migration scripts
+migrations_path = /home/poppy/poppy/storage/cassandra/migrations
 
 [drivers:dns:rackspace]
 username = DNS_USERNAME

docker/api_ci/poppy.conf

@@ -57,6 +57,8 @@ email = "email@example.com"
 [drivers:storage:cassandra]
 cluster = "cassandra"
 keyspace = poppy
+# Path to directory containing CQL migration scripts
+migrations_path = /home/poppy/poppy/storage/cassandra/migrations
 
 [drivers:provider:akamai]
 policy_api_client_token = POLICY-API-CLIENT-TOKEN

docker/api_dev/poppy.conf

@@ -48,6 +48,8 @@ port = 8081
 [drivers:storage:cassandra]
 cluster = "cassandra"
 keyspace = poppy
+# Path to directory containing CQL migration scripts
+migrations_path = /home/poppy/poppy/storage/cassandra/migrations
 
 [drivers:dns:rackspace]
 username = DNS_USERNAME

etc/poppy.conf

@@ -76,6 +76,8 @@ keyspace = poppy
 # `map` as show in the syntax here: http://www.datastax.com/documentation/cql/3
 # .1/cql/cql_reference/create_keyspace_r.html
 replication_strategy = class:SimpleStrategy, replication_factor:1
+# Path to directory containing CQL migration scripts
+migrations_path = <poppy_code_path>/poppy/storage/cassandra/migrations
 
 [drivers:storage:mockdb]
 database = poppy

poppy/storage/cassandra/driver.py

@@ -23,12 +23,12 @@ from cassandra import auth
 from cassandra import cluster
 from cassandra import policies
 from cassandra import query
+from cdeploy import migrator
 from oslo.config import cfg
 
 from poppy.openstack.common import log as logging
 from poppy.storage import base
 from poppy.storage.cassandra import controllers
-from poppy.storage.cassandra import schema
 
 
 LOG = logging.getLogger(__name__)
@@ -62,6 +62,11 @@ CASSANDRA_OPTIONS = [
         },
         help='Replication strategy for Cassandra cluster'
     ),
+    cfg.StrOpt(
+        'migrations_path',
+        default='./poppy/storage/cassandra/migrations',
+        help='Path to directory containing CQL migration scripts',
+    ),
     cfg.BoolOpt('archive_on_delete', default=True,
                 help='Archive services on delete?'),
 ]
@@ -112,6 +117,8 @@ def _connection(conf, datacenter, keyspace=None):
     except cassandra.InvalidRequest:
         _create_keyspace(session, keyspace, conf.replication_strategy)
 
+    _run_migrations(conf.migrations_path, session)
+
     session.row_factory = query.dict_factory
 
     return session
@@ -123,6 +130,8 @@ def _create_keyspace(session, keyspace, replication_strategy):
     :param keyspace
     :param replication_strategy
     """
+    LOG.debug('Creating keyspace: ' + keyspace)
+
     # replication factor will come in as a string with quotes already
     session.execute(
         "CREATE KEYSPACE " + keyspace + " " +
@@ -130,10 +139,12 @@ def _create_keyspace(session, keyspace, replication_strategy):
     )
     session.set_keyspace(keyspace)
 
-    for statement in schema.schema_statements:
-        session.execute(statement)
 
-    LOG.debug('Creating keyspace: ' + keyspace)
+def _run_migrations(migrations_path, session):
+    LOG.debug('Running schema migration(s)')
+
+    schema_migrator = migrator.Migrator(migrations_path, session)
+    schema_migrator.run_migrations()
 
 
 class CassandraStorageDriver(base.Driver):

poppy/storage/cassandra/migrations/001_initial_schema.cql

@@ -1,6 +1,3 @@
-CREATE KEYSPACE poppy WITH REPLICATION = { 'class' : 'SimpleStrategy' , 'replication_factor' :  1}  ;
-USE poppy;
-
 CREATE TABLE services (
     project_id VARCHAR,
     service_id UUID,
@@ -39,4 +36,4 @@ CREATE TABLE flavors (
     flavor_id VARCHAR,
     providers MAP<TEXT, TEXT>,
     PRIMARY KEY (flavor_id)
-);
+);

poppy/storage/cassandra/schema.py

@@ -1,58 +0,0 @@
-# Copyright (c) 2014 Rackspace, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-schema_statements = [
-    '''CREATE TABLE services (
-    project_id VARCHAR,
-    service_id UUID,
-    service_name VARCHAR,
-    flavor_id VARCHAR,
-    domains LIST<TEXT>,
-    origins LIST<TEXT>,
-    caching_rules LIST<TEXT>,
-    restrictions LIST<TEXT>,
-    provider_details MAP<TEXT, TEXT>,
-    PRIMARY KEY (project_id, service_id)
-    );
-    ''',
-    '''CREATE TABLE flavors (
-    flavor_id VARCHAR,
-    providers MAP<TEXT, TEXT>,
-    PRIMARY KEY (flavor_id)
-    );
-    ''',
-    '''CREATE TABLE domain_names (
-    project_id VARCHAR,
-    service_id UUID,
-    domain_name VARCHAR,
-    PRIMARY KEY (domain_name)
-    );
-    ''',
-    '''CREATE TABLE archives (
-    project_id VARCHAR,
-    service_id UUID,
-    service_name VARCHAR,
-    flavor_id VARCHAR,
-    domains LIST<TEXT>,
-    origins LIST<TEXT>,
-    caching_rules LIST<TEXT>,
-    restrictions LIST<TEXT>,
-    provider_details MAP<TEXT, TEXT>,
-    archived_time timestamp,
-    PRIMARY KEY (project_id, service_id, archived_time)
-    );
-    '''
-]

requirements/storage/cassandra.txt

@@ -1 +1,2 @@
-cassandra-driver>=2.1.3
+cassandra-driver>=2.1.3
+cdeploy

scripts/storage/cassandra/schema_migration.cql

@@ -0,0 +1,8 @@
+CREATE TABLE IF NOT EXISTS schema_migrations(
+    type text,
+    version int,
+    PRIMARY KEY(type, version))
+    WITH COMMENT = 'Schema migration history'
+    AND CLUSTERING ORDER BY (version DESC);
+
+INSERT INTO schema_migrations (type, version) VALUES ('migration', 1);

tests/etc/default_functional.conf

@@ -8,6 +8,7 @@ dns = default
 [drivers:storage:cassandra]
 cluster = "192.168.59.103"
 keyspace = poppy
+migrations_path = ../poppy/storage/cassandra/migrations
 
 [drivers:provider:fastly]
 apikey = "MYAPIKEY"

tests/unit/storage/cassandra/test_driver.py

@@ -54,6 +54,11 @@ CASSANDRA_OPTIONS = [
         },
         help='Replication strategy for Cassandra cluster'
     ),
+    cfg.StrOpt(
+        'migrations_path',
+        default='./poppy/storage/cassandra/migrations',
+        help='Path to directory containing CQL migration scripts',
+    ),
     cfg.BoolOpt('archive_on_delete', default=True,
                 help='Archive services on delete?'),
 ]
@@ -75,6 +80,12 @@ class CassandraStorageDriverTests(base.TestCase):
                            group=driver.CASSANDRA_GROUP)
         self.cassandra_driver = driver.CassandraStorageDriver(conf)
 
+        migrations_patcher = mock.patch(
+            'cdeploy.migrator.Migrator'
+        )
+        migrations_patcher.start()
+        self.addCleanup(migrations_patcher.stop)
+
     def test_storage_driver(self):
         # assert that the configs are set up based on what was passed in
         self.assertEqual(self.cassandra_driver.cassandra_conf['cluster'],

tests/unit/storage/cassandra/test_flavors.py

@@ -44,8 +44,15 @@ class CassandraStorageFlavorsTests(base.TestCase):
                 help='datacenter where the C* cluster hosted'))
         conf.register_opts(driver.CASSANDRA_OPTIONS,
                            group=driver.CASSANDRA_GROUP)
+
         cassandra_driver = driver.CassandraStorageDriver(conf)
 
+        migrations_patcher = mock.patch(
+            'cdeploy.migrator.Migrator'
+        )
+        migrations_patcher.start()
+        self.addCleanup(migrations_patcher.stop)
+
         # stubbed cassandra driver
         self.fc = flavors.FlavorsController(cassandra_driver)
 

tests/unit/storage/cassandra/test_services.py

@@ -54,6 +54,12 @@ class CassandraStorageServiceTests(base.TestCase):
                            group=driver.CASSANDRA_GROUP)
         cassandra_driver = driver.CassandraStorageDriver(conf)
 
+        migrations_patcher = mock.patch(
+            'cdeploy.migrator.Migrator'
+        )
+        migrations_patcher.start()
+        self.addCleanup(migrations_patcher.stop)
+
         # stubbed cassandra driver
         self.sc = services.ServicesController(cassandra_driver)