Migrations for extensions should not depend on Core's db schema

There should be a way to use and test extensions when
there is no core database schema.

* perform data migration from the core only when there
  is buffer table
* add basic tests which are run separately from core's
  tests

Change-Id: Ide0d2e619913675ff4fff3febc9707bec3bfb112
Closes-bug: #1477020

nailgun/nailgun/extensions/utils.py

@@ -0,0 +1,27 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from sqlalchemy.engine.reflection import Inspector
+
+from nailgun.extensions.consts import extensions_migration_buffer_table_name
+
+
+def is_buffer_table_exist(connection):
+    """Performs check if buffer table exists in the database.
+
+    :returns: True if table exists, False otherwise
+    """
+    inspector = Inspector.from_engine(connection)
+    return (extensions_migration_buffer_table_name in
+            inspector.get_table_names())

nailgun/nailgun/extensions/volume_manager/alembic_migrations/migrations/versions/001_add_volumes_table.py

@@ -24,22 +24,37 @@ Create Date: 2015-06-19 16:16:44.513714
 revision = '086cde3de7cf'
 down_revision = None
 
+import logging
+
 from alembic import context
 from alembic import op
 from oslo.serialization import jsonutils
-
 import sqlalchemy as sa
 
 from nailgun.db.sqlalchemy.models.fields import JSON
 from nailgun.extensions.consts import extensions_migration_buffer_table_name
+from nailgun.extensions.utils import is_buffer_table_exist
 
-
+logger = logging.getLogger('alembic.migration')
 config = context.config
 table_prefix = config.get_main_option('table_prefix')
 table_volumes_name = '{0}node_volumes'.format(table_prefix)
 
 
 def migrate_data_from_core(connection):
+    if not is_buffer_table_exist(connection):
+        # NOTE(eli): if there is no buffer table it means that there
+        # is no core database we should not run data migrations includes
+        # this case because extension might be installed and used
+        # separately from Nailgun core and its database
+        logger.warn(
+            "Cannot find buffer table '{0}'. "
+            "Don't run data migrations from buffer table, "
+            "because extension might be installed and used "
+            "separately from Nailgun core and its database".format(
+                extensions_migration_buffer_table_name))
+        return
+
     ext_name = 'volume_manager'
 
     select_query = sa.sql.text(

nailgun/nailgun/extensions/volume_manager/tests/test_db_migrations.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import alembic
+
+from ..extension import VolumeManagerExtension
+from nailgun import db
+from nailgun.db.migration import make_alembic_config_from_extension
+from nailgun.test import base
+
+
+_test_revision = '086cde3de7cf'
+
+
+def setup_module(module):
+    alembic_config = make_alembic_config_from_extension(VolumeManagerExtension)
+    db.dropdb()
+    alembic.command.upgrade(alembic_config, _test_revision)
+
+
+class TestAddVolumes(base.BaseAlembicMigrationTest):
+
+    def test_works_without_core_migrations(self):
+        columns = [
+            t.name for t in
+            self.meta.tables['volume_manager_node_volumes'].columns]
+
+        self.assertItemsEqual(columns, ['id', 'node_id', 'volumes'])

run_tests.sh

@@ -33,6 +33,8 @@ function usage {
   echo "  -U, --no-upgrade            Don't run tests for UPGRADE system"
   echo "  -w, --webui                 Run all UI tests"
   echo "  -W, --no-webui              Don't run all UI tests"
+  echo "  -e, --extensions            Run EXTENSIONS unit/integration tests"
+  echo "  -E, --no-extensions         Don't run EXTENSIONS unit/integration tests"
   echo "      --ui-lint               Run UI linting tasks"
   echo "      --no-ui-lint            Don't run UI linting tasks"
   echo "      --ui-unit               Run UI unit tests"
@@ -62,6 +64,8 @@ function process_options {
       -P|--no-flake8) no_flake8_checks=1;;
       -w|--webui) ui_lint_checks=1; ui_unit_tests=1; ui_func_tests=1;;
       -W|--no-webui) no_ui_lint_checks=1; no_ui_unit_tests=1; no_ui_func_tests=1;;
+      -e|--extensions) extensions_tests=1;;
+      -E|--no-extensions) no_extensions_tests=1;;
       --ui-lint) ui_lint_checks=1;;
       --no-ui-lint) no_ui_lint_checks=1;;
       --ui-unit) ui_unit_tests=1;;
@@ -94,6 +98,7 @@ testropts="--with-timer --timer-warning=10 --timer-ok=2 --timer-top-n=10"
 NAILGUN_XUNIT=${NAILGUN_XUNIT:-"$ROOT/nailgun.xml"}
 FUELUPGRADE_XUNIT=${FUELUPGRADE_XUNIT:-"$ROOT/fuelupgrade.xml"}
 SHOTGUN_XUNIT=${SHOTGUN_XUNIT:-"$ROOT/shotgun.xml"}
+EXTENSIONS_XUNIT=${EXTENSIONS_XUNIT:-"$ROOT/extensions.xml"}
 UI_SERVER_PORT=${UI_SERVER_PORT:-5544}
 NAILGUN_PORT=${NAILGUN_PORT:-8003}
 TEST_NAILGUN_DB=${TEST_NAILGUN_DB:-nailgun}
@@ -121,6 +126,8 @@ ui_unit_tests=0
 no_ui_unit_tests=0
 ui_func_tests=0
 no_ui_func_tests=0
+extensions_tests=0
+no_extensions_tests=0
 certain_tests=0
 ui_func_selenium_tests=0
 
@@ -154,15 +161,18 @@ function run_tests {
       $ui_func_selenium_tests -eq 0 && \
       $upgrade_system -eq 0 && \
       $shotgun_tests -eq 0 && \
+      $extensions_tests -eq 0 && \
       $flake8_checks -eq 0 ]]; then
 
-    if [ $no_nailgun_tests -ne 1 ];  then nailgun_tests=1;  fi
+    if [ $no_nailgun_tests -ne 1 ];    then nailgun_tests=1;  fi
-    if [ $no_ui_lint_checks -ne 1 ]; then ui_lint_checks=1; fi
+    if [ $no_ui_lint_checks -ne 1 ];   then ui_lint_checks=1; fi
-    if [ $no_ui_unit_tests -ne 1 ];  then ui_unit_tests=1;  fi
+    if [ $no_ui_unit_tests -ne 1 ];    then ui_unit_tests=1;  fi
-    if [ $no_ui_func_tests -ne 1 ];  then ui_func_tests=1;  fi
+    if [ $no_ui_func_tests -ne 1 ];    then ui_func_tests=1;  fi
-    if [ $no_upgrade_system -ne 1 ]; then upgrade_system=1; fi
+    if [ $no_upgrade_system -ne 1 ];   then upgrade_system=1; fi
-    if [ $no_shotgun_tests -ne 1 ];  then shotgun_tests=1;  fi
+    if [ $no_shotgun_tests -ne 1 ];    then shotgun_tests=1;  fi
-    if [ $no_flake8_checks -ne 1 ];  then flake8_checks=1;  fi
+    if [ $no_flake8_checks -ne 1 ];    then flake8_checks=1;  fi
+    if [ $no_extensions_tests -ne 1 ]; then extensions_tests=1; fi
+
   fi
 
   # Run all enabled tests
@@ -206,6 +216,11 @@ function run_tests {
     run_shotgun_tests || errors+=" shotgun_tests"
   fi
 
+  if [ $extensions_tests -eq 1 ]; then
+    echo "Starting Extensions tests..."
+    run_extensions_tests || errors+=" extensions_tests"
+  fi
+
   # print failed tests
   if [ -n "$errors" ]; then
     echo Failed tests: $errors
@@ -440,6 +455,21 @@ function run_flake8_subproject {
   return $result
 }
 
+
+# Run tests for Nailgun extensions
+function run_extensions_tests {
+  local EXTENSIONS_PATH="$ROOT/nailgun/nailgun/extensions/"
+  local NAILGUN_PATH="$ROOT/nailgun/"
+  local result=0
+
+  pushd "${NAILGUN_PATH}" >> /dev/null
+  tox -epy26 -- -vv "${EXTENSIONS_PATH}" --junit-xml $EXTENSIONS_XUNIT || result=1
+  popd >> /dev/null
+
+  return $result
+}
+
+
 # Check python code with flake8 and pep8.
 #
 # Some settings description: