Introduce extension to upgrade clusters

The patch adds an extension which implements the procedure of upgrading
clusters from one major release to another. As a first step of the
procedure the extension provides an ability to create a seed cluster
with the same settings as the original.

Implements blueprint: nailgun-api-env-upgrade-extensions
Change-Id: I22d51a3ffd51a7c88bdcbde0eef6f47b65def1c8

cluster_upgrade/alembic_migrations/alembic.ini

@@ -0,0 +1,54 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = migrations
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+sqlalchemy.url = driver://user:pass@localhost/dbname
+
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

cluster_upgrade/alembic_migrations/migrations/env.py

@@ -0,0 +1,90 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from __future__ import with_statement
+
+from alembic import context
+from logging.config import fileConfig
+from sqlalchemy import engine_from_config
+from sqlalchemy import pool
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = None
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    context.configure(
+        url=config.get_main_option('sqlalchemy.url'),
+        version_table=config.get_main_option('version_table'))
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    engine = engine_from_config(
+        config.get_section(config.config_ini_section),
+        prefix='sqlalchemy.',
+        poolclass=pool.NullPool)
+    connection = engine.connect()
+
+    context.configure(
+        connection=connection,
+        target_metadata=target_metadata,
+        version_table=config.get_main_option('version_table'))
+
+    try:
+        with context.begin_transaction():
+            context.run_migrations()
+    finally:
+        connection.close()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

cluster_upgrade/alembic_migrations/migrations/script.py.mako

@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

cluster_upgrade/alembic_migrations/migrations/versions/001_add_upgrade_relations_table.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""cluster_upgrade
+
+Revision ID: 3b5d115d7e49
+Revises: None
+Create Date: 2015-07-17 19:46:59.579553
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '3b5d115d7e49'
+down_revision = None
+
+
+from alembic import context
+from alembic import op
+
+import sqlalchemy as sa
+
+table_prefix = context.config.get_main_option('table_prefix')
+table_upgrade_relation_name = '{0}relations'.format(table_prefix)
+
+
+def upgrade():
+    op.create_table(
+        table_upgrade_relation_name,
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('orig_cluster_id', sa.Integer(), nullable=False),
+        sa.Column('seed_cluster_id', sa.Integer(), nullable=False),
+        sa.PrimaryKeyConstraint('id'),
+        sa.UniqueConstraint('orig_cluster_id'),
+        sa.UniqueConstraint('seed_cluster_id'))
+
+
+def downgrade():
+    op.drop_table(table_upgrade_relation_name)

cluster_upgrade/extension.py

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+
+from nailgun import extensions
+
+from . import handlers
+
+
+class ClusterUpgradeExtension(extensions.BaseExtension):
+    name = 'cluster_upgrade'
+    version = '0.0.1'
+
+    urls = [
+        {'uri': r'/clusters/(?P<cluster_id>\d+)/upgrade/clone/?$',
+         'handler': handlers.ClusterUpgradeHandler},
+    ]
+
+    @classmethod
+    def alembic_migrations_path(cls):
+        return os.path.join(os.path.dirname(__file__),
+                            'alembic_migrations', 'migrations')
+
+    @classmethod
+    def on_cluster_delete(cls, cluster):
+        from .objects import relations
+
+        relations.UpgradeRelationObject.delete_relation(cluster.id)

cluster_upgrade/handlers.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+from nailgun.api.v1.handlers import base
+from nailgun import objects
+
+from . import upgrade
+from . import validators
+from .objects import adapters
+
+
+class ClusterUpgradeHandler(base.BaseHandler):
+    single = objects.Cluster
+    validator = validators.ClusterUpgradeValidator
+
+    @base.content
+    def POST(self, cluster_id):
+        """Initialize the upgrade of the cluster.
+
+        Creates a new cluster with specified name and release_id. The
+        new cluster is created with parameters that are copied from the
+        cluster with the given cluster_id. The values of the generated
+        and editable attributes are just copied from one to the other.
+
+        :param cluster_id: ID of the cluster from which parameters would
+                           be copied
+        :returns: JSON representation of the created cluster
+        :http: * 200 (OK)
+               * 400 (upgrade parameters are invalid)
+               * 404 (node or release not found in db)
+        """
+        orig_cluster = adapters.NailgunClusterAdapter(
+            self.get_object_or_404(self.single, cluster_id))
+        request_data = self.checked_data(cluster=orig_cluster)
+        new_cluster = upgrade.UpgradeHelper.clone_cluster(orig_cluster,
+                                                          request_data)
+        return new_cluster.to_json()

cluster_upgrade/models.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from sqlalchemy import Column
+from sqlalchemy import Integer
+
+from nailgun.db.sqlalchemy.models.base import Base
+
+from . import extension
+
+
+class UpgradeRelation(Base):
+    __tablename__ = '{0}relations'.format(
+        extension.ClusterUpgradeExtension.table_prefix())
+
+    id = Column(Integer, primary_key=True)
+    orig_cluster_id = Column(Integer, unique=True, nullable=False)
+    seed_cluster_id = Column(Integer, unique=True, nullable=False)

cluster_upgrade/objects/adapters.py

@@ -0,0 +1,108 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from nailgun import objects
+
+
+class NailgunClusterAdapter(object):
+    def __init__(self, cluster):
+        self.cluster = cluster
+
+    @classmethod
+    def create(cls, data):
+        cluster = objects.Cluster.create(data)
+        return cls(cluster)
+
+    @property
+    def id(self):
+        return self.cluster.id
+
+    @property
+    def name(self):
+        return self.cluster.name
+
+    @property
+    def net_provider(self):
+        return self.cluster.net_provider
+
+    @property
+    def release(self):
+        return NailgunReleaseAdapter(self.cluster.release)
+
+    @property
+    def generated_attrs(self):
+        return self.cluster.attributes.generated
+
+    @generated_attrs.setter
+    def generated_attrs(self, attrs):
+        self.cluster.attributes.generated = attrs
+
+    @property
+    def editable_attrs(self):
+        return self.cluster.attributes.editable
+
+    @editable_attrs.setter
+    def editable_attrs(self, attrs):
+        self.cluster.attributes.editable = attrs
+
+    def get_create_data(self):
+        return objects.Cluster.get_create_data(self.cluster)
+
+    def get_network_manager(self):
+        net_manager = objects.Cluster.get_network_manager(
+            instance=self.cluster)
+        return NailgunNetworkManager(self.cluster, net_manager)
+
+    def to_json(self):
+        return objects.Cluster.to_json(self.cluster)
+
+
+class NailgunReleaseAdapter(object):
+    def __init__(self, release):
+        self.release = release
+
+    @classmethod
+    def get_by_uid(cls, uid, fail_if_not_found=False):
+        release = objects.Release.get_by_uid(
+            uid, fail_if_not_found=fail_if_not_found)
+        return release
+
+    @property
+    def is_deployable(self):
+        return self.release.is_deployable
+
+    def __cmp__(self, other):
+        if isinstance(other, NailgunReleaseAdapter):
+            other = other.release
+        return self.release.__cmp__(other)
+
+
+class NailgunNetworkManager(object):
+    def __init__(self, cluster, net_manager):
+        self.cluster = cluster
+        self.net_manager = net_manager
+
+    def update(self, network_configuration):
+        self.net_manager.update(self.cluster, network_configuration)
+
+    def get_assigned_vips(self):
+        return self.net_manager.get_assigned_vips(self.cluster)
+
+    def assign_vips_for_net_groups(self):
+        return self.net_manager.assign_vips_for_net_groups(self.cluster)
+
+    def assign_given_vips_for_net_groups(self, vips):
+        self.net_manager.assign_given_vips_for_net_groups(self.cluster, vips)

cluster_upgrade/objects/relations.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from nailgun.db import db
+
+from .. import models
+
+
+class UpgradeRelationObject(object):
+    @staticmethod
+    def _query_cluster_relations(cluster_id):
+        return db.query(models.UpgradeRelation).filter(
+            (models.UpgradeRelation.orig_cluster_id == cluster_id) |
+            (models.UpgradeRelation.seed_cluster_id == cluster_id))
+
+    @classmethod
+    def get_cluster_relation(cls, cluster_id):
+        return cls._query_cluster_relations(cluster_id).first()
+
+    @classmethod
+    def delete_relation(cls, cluster_id):
+        cls._query_cluster_relations(cluster_id).delete()
+
+    @classmethod
+    def is_cluster_in_upgrade(cls, cluster_id):
+        query = cls._query_cluster_relations(cluster_id).exists()
+        return db.query(query).scalar()
+
+    @classmethod
+    def create_relation(cls, orig_cluster_id, seed_cluster_id):
+        relation = models.UpgradeRelation(
+            orig_cluster_id=orig_cluster_id,
+            seed_cluster_id=seed_cluster_id)
+        db.add(relation)
+        db.flush()

cluster_upgrade/tests/__init__.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2013 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+EXTENSION = "nailgun.extensions.cluster_upgrade."

cluster_upgrade/tests/base.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from nailgun import consts
+from nailgun.test import base as nailgun_test_base
+
+from .. import upgrade
+from ..objects import adapters
+
+
+class BaseCloneClusterTest(nailgun_test_base.BaseIntegrationTest):
+    helper = upgrade.UpgradeHelper
+
+    def setUp(self):
+        super(BaseCloneClusterTest, self).setUp()
+        self.release_61 = self.env.create_release(
+            operating_system=consts.RELEASE_OS.ubuntu,
+            version="2014.2.2-6.1",
+            is_deployable=False,
+        )
+        self.release_70 = self.env.create_release(
+            operating_system=consts.RELEASE_OS.ubuntu,
+            version="2015.1.0-7.0",
+        )
+        self.cluster_61_db = self.env.create_cluster(
+            api=False,
+            release_id=self.release_61.id,
+            net_provider=consts.CLUSTER_NET_PROVIDERS.neutron,
+            net_l23_provider=consts.NEUTRON_L23_PROVIDERS.ovs,
+        )
+        self.cluster_61 = adapters.NailgunClusterAdapter(
+            self.cluster_61_db)
+        self.data = {
+            "name": "cluster-clone-{0}".format(self.cluster_61.id),
+            "release_id": self.release_70.id,
+        }

cluster_upgrade/tests/test_db_migrations.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import alembic
+
+from nailgun import db
+from nailgun.db.migration import make_alembic_config_from_extension
+from nailgun.test import base
+
+from .. import extension
+
+
+_test_revision = '3b5d115d7e49'
+
+
+def setup_module(module):
+    alembic_config = make_alembic_config_from_extension(
+        extension.ClusterUpgradeExtension)
+    db.dropdb()
+    alembic.command.upgrade(alembic_config, _test_revision)
+
+
+class TestAddRelations(base.BaseAlembicMigrationTest):
+
+    def test_works_without_core_migrations(self):
+        columns = [
+            t.name for t in
+            self.meta.tables['cluster_upgrade_relations'].columns]
+
+        self.assertItemsEqual(columns, [
+            'id',
+            'orig_cluster_id',
+            'seed_cluster_id',
+        ])

cluster_upgrade/tests/test_extension.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+from nailgun.test.base import BaseTestCase
+
+from .. import extension
+from ..objects import relations
+
+
+class TestExtension(BaseTestCase):
+    @mock.patch.object(relations.UpgradeRelationObject, "delete_relation")
+    def test_on_cluster_delete(self, mock_on_cluster_delete):
+        cluster = mock.Mock(id=42)
+        extension.ClusterUpgradeExtension.on_cluster_delete(cluster)
+        mock_on_cluster_delete.assert_called_once_with(42)

cluster_upgrade/tests/test_handlers.py

@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_serialization import jsonutils
+
+from nailgun.utils import reverse
+
+from . import base as tests_base
+
+
+class TestClusterUpgradeHandler(tests_base.BaseCloneClusterTest):
+    def test_clone(self):
+        resp = self.app.post(
+            reverse("ClusterUpgradeHandler",
+                    kwargs={"cluster_id": self.cluster_61.id}),
+            jsonutils.dumps(self.data),
+            headers=self.default_headers)
+        body = resp.json_body
+        self.assertEqual(resp.status_code, 200)
+        self.assertEqual(body["name"],
+                         "cluster-clone-{0}".format(self.cluster_61.id))
+        self.assertEqual(body["release_id"], self.release_70.id)
+
+    def test_clone_cluster_not_found_error(self):
+        resp = self.app.post(
+            reverse("ClusterUpgradeHandler",
+                    kwargs={"cluster_id": 42}),
+            jsonutils.dumps(self.data),
+            headers=self.default_headers,
+            expect_errors=True)
+        self.assertEqual(resp.status_code, 404)
+        self.assertEqual(resp.json_body["message"], "Cluster not found")
+
+    def test_clone_cluster_already_in_upgrade_error(self):
+        self.app.post(
+            reverse("ClusterUpgradeHandler",
+                    kwargs={"cluster_id": self.cluster_61.id}),
+            jsonutils.dumps(self.data),
+            headers=self.default_headers)
+        resp = self.app.post(
+            reverse("ClusterUpgradeHandler",
+                    kwargs={"cluster_id": self.cluster_61.id}),
+            jsonutils.dumps(self.data),
+            headers=self.default_headers,
+            expect_errors=True)
+        self.assertEqual(resp.status_code, 400)
+
+    def test_clone_cluster_name_already_exists_error(self):
+        data = dict(self.data, name=self.cluster_61.name)
+        resp = self.app.post(
+            reverse("ClusterUpgradeHandler",
+                    kwargs={"cluster_id": self.cluster_61.id}),
+            jsonutils.dumps(data),
+            headers=self.default_headers,
+            expect_errors=True)
+        self.assertEqual(resp.status_code, 409)

cluster_upgrade/tests/test_objects.py

@@ -0,0 +1,48 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from nailgun.test.base import BaseIntegrationTest
+
+from .. import models
+from ..objects import relations as objects
+
+
+class TestUpgradeRelationObject(BaseIntegrationTest):
+    def test_get_and_create_relation(self):
+        objects.UpgradeRelationObject.create_relation(1, 2)
+        rel0 = objects.UpgradeRelationObject.get_cluster_relation(1)
+        self.assertEqual(rel0.orig_cluster_id, 1)
+        self.assertEqual(rel0.seed_cluster_id, 2)
+        rel1 = objects.UpgradeRelationObject.get_cluster_relation(2)
+        self.assertEqual(rel1.orig_cluster_id, 1)
+        self.assertEqual(rel1.seed_cluster_id, 2)
+
+    def test_is_cluster_in_upgrade(self):
+        objects.UpgradeRelationObject.create_relation(1, 2)
+        in_upgrade = objects.UpgradeRelationObject.is_cluster_in_upgrade
+        self.assertTrue(in_upgrade(1))
+        self.assertTrue(in_upgrade(2))
+
+    def test_is_cluster_not_in_upgrade(self):
+        self.assertEqual(self.db.query(models.UpgradeRelation).count(), 0)
+        in_upgrade = objects.UpgradeRelationObject.is_cluster_in_upgrade
+        self.assertFalse(in_upgrade(1))
+        self.assertFalse(in_upgrade(2))
+
+    def test_delete_relation(self):
+        objects.UpgradeRelationObject.create_relation(1, 2)
+        objects.UpgradeRelationObject.delete_relation(1)
+        self.assertEqual(self.db.query(models.UpgradeRelation).count(), 0)

cluster_upgrade/tests/test_upgrade.py

@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import copy
+import six
+
+from nailgun import consts
+from nailgun.objects.serializers import network_configuration
+
+from . import base as base_tests
+from ..objects import relations
+
+
+class TestUpgradeHelperCloneCluster(base_tests.BaseCloneClusterTest):
+    def test_create_cluster_clone(self):
+        new_cluster = self.helper.create_cluster_clone(self.cluster_61,
+                                                       self.data)
+        cluster_61_data = self.cluster_61.get_create_data()
+        new_cluster_data = new_cluster.get_create_data()
+        for key, value in cluster_61_data.items():
+            if key in ("name", "release_id"):
+                continue
+            self.assertEqual(value, new_cluster_data[key])
+
+    def test_copy_attributes(self):
+        new_cluster = self.helper.create_cluster_clone(self.cluster_61,
+                                                       self.data)
+        self.assertNotEqual(self.cluster_61.generated_attrs,
+                            new_cluster.generated_attrs)
+
+        # Do some unordinary changes
+        attrs = copy.deepcopy(self.cluster_61.editable_attrs)
+        attrs["access"]["user"]["value"] = "operator"
+        attrs["access"]["password"]["value"] = "secrete"
+        self.cluster_61.editable_attrs = attrs
+
+        self.helper.copy_attributes(self.cluster_61, new_cluster)
+
+        self.assertEqual(self.cluster_61.generated_attrs,
+                         new_cluster.generated_attrs)
+        editable_attrs = self.cluster_61.editable_attrs
+        for section, params in six.iteritems(new_cluster.editable_attrs):
+            if section == "repo_setup":
+                continue
+            for key, value in six.iteritems(params):
+                if key == "metadata":
+                    continue
+                self.assertEqual(editable_attrs[section][key]["value"],
+                                 value["value"])
+
+    def test_copy_network_config(self):
+        new_cluster = self.helper.create_cluster_clone(self.cluster_61,
+                                                       self.data)
+        orig_net_manager = self.cluster_61.get_network_manager()
+        new_net_manager = new_cluster.get_network_manager()
+
+        # Do some unordinary changes
+        nets = network_configuration.NeutronNetworkConfigurationSerializer.\
+            serialize_for_cluster(self.cluster_61.cluster)
+        nets["networks"][0].update({
+            "cidr": "172.16.42.0/24",
+            "gateway": "172.16.42.1",
+            "ip_ranges": [["172.16.42.2", "172.16.42.126"]],
+        })
+        orig_net_manager.update(nets)
+        orig_net_manager.assign_vips_for_net_groups()
+
+        self.helper.copy_network_config(self.cluster_61, new_cluster)
+
+        orig_vips = orig_net_manager.get_assigned_vips()
+        new_vips = new_net_manager.get_assigned_vips()
+        for net_name in (consts.NETWORKS.public,
+                         consts.NETWORKS.management):
+            for vip_type in consts.NETWORK_VIP_TYPES:
+                self.assertEqual(orig_vips[net_name][vip_type],
+                                 new_vips[net_name][vip_type])
+
+    def test_clone_cluster(self):
+        orig_net_manager = self.cluster_61.get_network_manager()
+        orig_net_manager.assign_vips_for_net_groups()
+        new_cluster = self.helper.clone_cluster(self.cluster_61, self.data)
+        relation = relations.UpgradeRelationObject.get_cluster_relation(
+            self.cluster_61.id)
+        self.assertEqual(relation.orig_cluster_id, self.cluster_61.id)
+        self.assertEqual(relation.seed_cluster_id, new_cluster.id)

cluster_upgrade/tests/test_validators.py

@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_serialization import jsonutils
+
+from nailgun import consts
+from nailgun.errors import errors
+
+from .. import validators
+from . import base as tests_base
+from ..objects import relations
+
+
+class TestClusterUpgradeValidator(tests_base.BaseCloneClusterTest):
+    validator = validators.ClusterUpgradeValidator
+
+    def test_validate_release_upgrade(self):
+        self.validator.validate_release_upgrade(self.release_61,
+                                                self.release_70)
+
+    def test_validate_release_upgrade_deprecated_release(self):
+        release_511 = self.env.create_release(
+            operating_system=consts.RELEASE_OS.ubuntu,
+            version="2014.1.3-5.1.1",
+            is_deployable=False,
+        )
+        msg = "^Upgrade to the given release \({0}\).*is deprecated and " \
+              "cannot be installed\.$".format(self.release_61.id)
+        with self.assertRaisesRegexp(errors.InvalidData, msg):
+            self.validator.validate_release_upgrade(release_511,
+                                                    self.release_61)
+
+    def test_validate_release_upgrade_to_older_release(self):
+        self.release_61.is_deployable = True
+        msg = "^Upgrade to the given release \({0}\).*release is equal or " \
+              "lower than the release of the original cluster\.$" \
+              .format(self.release_61.id)
+        with self.assertRaisesRegexp(errors.InvalidData, msg):
+            self.validator.validate_release_upgrade(self.release_70,
+                                                    self.release_61)
+
+    def test_validate_cluster_name(self):
+        self.validator.validate_cluster_name("cluster-42")
+
+    def test_validate_cluster_name_already_exists(self):
+        msg = "^Environment with this name '{0}' already exists\.$"\
+              .format(self.cluster_61.name)
+        with self.assertRaisesRegexp(errors.AlreadyExists, msg):
+            self.validator.validate_cluster_name(self.cluster_61.name)
+
+    def test_validate_cluster_status(self):
+        self.validator.validate_cluster_status(self.cluster_61)
+
+    def test_validate_cluster_status_invalid(self):
+        cluster_70 = self.env.create_cluster(
+            api=False,
+            release_id=self.release_70.id,
+        )
+        relations.UpgradeRelationObject.create_relation(self.cluster_61.id,
+                                                        cluster_70.id)
+        msg = "^Upgrade is not possible because of the original cluster " \
+              "\({0}\) is already involved in the upgrade routine\.$" \
+              .format(self.cluster_61.id)
+        with self.assertRaisesRegexp(errors.InvalidData, msg):
+            self.validator.validate_cluster_status(self.cluster_61)
+
+    def test_validate(self):
+        data = jsonutils.dumps(self.data)
+        self.validator.validate(data, self.cluster_61)
+
+    def test_validate_invalid_data(self):
+        data = "{}"
+        with self.assertRaises(errors.InvalidData):
+            self.validator.validate(data, self.cluster_61)

cluster_upgrade/upgrade.py

@@ -0,0 +1,125 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import copy
+import six
+
+from nailgun import consts
+from nailgun.objects.serializers import network_configuration
+from nailgun import utils
+
+from .objects import adapters
+
+
+def merge_attributes(a, b):
+    """Merge values of editable attributes.
+
+    The values of the b attributes have precedence over the values
+    of the a attributes.
+    """
+    attrs = copy.deepcopy(b)
+    for section, pairs in six.iteritems(attrs):
+        if section == "repo_setup" or section not in a:
+            continue
+        a_values = a[section]
+        for key, values in six.iteritems(pairs):
+            if key != "metadata" and key in a_values:
+                values["value"] = a_values[key]["value"]
+    return attrs
+
+
+def merge_nets(a, b):
+    new_settings = copy.deepcopy(b)
+    source_networks = dict((n["name"], n) for n in a["networks"])
+    for net in new_settings["networks"]:
+        if net["name"] not in source_networks:
+            continue
+        source_net = source_networks[net["name"]]
+        for key, value in six.iteritems(net):
+            if (key not in ("cluster_id", "id", "meta", "group_id") and
+                    key in source_net):
+                net[key] = source_net[key]
+    networking_params = new_settings["networking_parameters"]
+    source_params = a["networking_parameters"]
+    for key, value in six.iteritems(networking_params):
+        if key not in source_params:
+            continue
+        networking_params[key] = source_params[key]
+    return new_settings
+
+
+class UpgradeHelper(object):
+    network_serializers = {
+        consts.CLUSTER_NET_PROVIDERS.neutron:
+        network_configuration.NeutronNetworkConfigurationSerializer,
+        consts.CLUSTER_NET_PROVIDERS.nova_network:
+        network_configuration.NovaNetworkConfigurationSerializer,
+    }
+
+    @classmethod
+    def clone_cluster(cls, orig_cluster, data):
+        from .objects import relations
+
+        new_cluster = cls.create_cluster_clone(orig_cluster, data)
+        cls.copy_attributes(orig_cluster, new_cluster)
+        cls.copy_network_config(orig_cluster, new_cluster)
+        relations.UpgradeRelationObject.create_relation(orig_cluster.id,
+                                                        new_cluster.id)
+        return new_cluster
+
+    @classmethod
+    def create_cluster_clone(cls, orig_cluster, data):
+        create_data = orig_cluster.get_create_data()
+        create_data["name"] = data["name"]
+        create_data["release_id"] = data["release_id"]
+        new_cluster = adapters.NailgunClusterAdapter.create(create_data)
+        return new_cluster
+
+    @classmethod
+    def copy_attributes(cls, orig_cluster, new_cluster):
+        # TODO(akscram): Attributes should be copied including
+        #                borderline cases when some parameters are
+        #                renamed or moved into plugins. Also, we should
+        #                to keep special steps in copying of parameters
+        #                that know how to translate parameters from one
+        #                version to another. A set of this kind of steps
+        #                should define an upgrade path of a particular
+        #                cluster.
+        new_cluster.generated_attrs = utils.dict_merge(
+            new_cluster.generated_attrs,
+            orig_cluster.generated_attrs)
+        new_cluster.editable_attrs = merge_attributes(
+            orig_cluster.editable_attrs,
+            new_cluster.editable_attrs)
+
+    @classmethod
+    def copy_network_config(cls, orig_cluster, new_cluster):
+        nets_serializer = cls.network_serializers[orig_cluster.net_provider]
+        nets = merge_nets(
+            nets_serializer.serialize_for_cluster(orig_cluster.cluster),
+            nets_serializer.serialize_for_cluster(new_cluster.cluster))
+
+        orig_net_manager = orig_cluster.get_network_manager()
+        new_net_manager = new_cluster.get_network_manager()
+
+        new_net_manager.update(nets)
+        vips = orig_net_manager.get_assigned_vips()
+        for ng_name in vips:
+            if ng_name not in (consts.NETWORKS.public,
+                               consts.NETWORKS.management):
+                vips.pop(ng_name)
+        new_net_manager.assign_given_vips_for_net_groups(vips)
+        new_net_manager.assign_vips_for_net_groups()

cluster_upgrade/validators.py

@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from nailgun.api.v1.validators import base
+from nailgun.errors import errors
+from nailgun import objects
+
+from .objects import adapters
+
+
+class ClusterUpgradeValidator(base.BasicValidator):
+    schema = {
+        "$schema": "http://json-schema.org/draft-04/schema#",
+        "title": "Start upgrade procedure for a cluster",
+        "description": "Serialized parameters to upgrade a cluster.",
+        "type": "object",
+        "properties": {
+            "name": {"type": "string"},
+            "release_id": {"type": "number"},
+        },
+        "required": ["name", "release_id"],
+    }
+
+    @classmethod
+    def validate(cls, data, cluster):
+        cluster = adapters.NailgunClusterAdapter(cluster)
+        data = super(ClusterUpgradeValidator, cls).validate(data)
+        cls.validate_schema(data, cls.schema)
+        cls.validate_cluster_status(cluster)
+        cls.validate_cluster_name(data["name"])
+        release = adapters.NailgunReleaseAdapter.get_by_uid(
+            data["release_id"], fail_if_not_found=True)
+        cls.validate_release_upgrade(cluster.release, release)
+        return data
+
+    @classmethod
+    def validate_release_upgrade(cls, orig_release, new_release):
+        if not new_release.is_deployable:
+            raise errors.InvalidData(
+                "Upgrade to the given release ({0}) is not possible because "
+                "this release is deprecated and cannot be installed."
+                .format(new_release.id),
+                log_message=True)
+        if orig_release >= new_release:
+            raise errors.InvalidData(
+                "Upgrade to the given release ({0}) is not possible because "
+                "this release is equal or lower than the release of the "
+                "original cluster.".format(new_release.id),
+                log_message=True)
+
+    @classmethod
+    def validate_cluster_name(cls, cluster_name):
+        clusters = objects.ClusterCollection.filter_by(None,
+                                                       name=cluster_name)
+        if clusters.first():
+            raise errors.AlreadyExists(
+                "Environment with this name '{0}' already exists."
+                .format(cluster_name),
+                log_message=True)
+
+    @classmethod
+    def validate_cluster_status(cls, cluster):
+        from .objects.relations import UpgradeRelationObject
+
+        if UpgradeRelationObject.is_cluster_in_upgrade(cluster.id):
+            raise errors.InvalidData(
+                "Upgrade is not possible because of the original cluster ({0})"
+                " is already involved in the upgrade routine."
+                .format(cluster.id),
+                log_message=True)