From eded14892c7979337c70a1b46ce1bcedf8b27731 Mon Sep 17 00:00:00 2001 From: Ryan Moe Date: Mon, 29 Jun 2015 14:55:02 -0700 Subject: [PATCH] Add API handler for handling network template YAML network template can be uploaded/got/deleted via a call to api/clusters//network_configuration/template Change-Id: I5c9ae51558dab2301e8ad8dffcf68a63ed0abdb4 Implements: blueprint templates-for-networking --- nailgun/nailgun/api/v1/handlers/base.py | 2 +- nailgun/nailgun/api/v1/handlers/cluster.py | 2 +- .../api/v1/handlers/network_configuration.py | 40 +++++++++ nailgun/nailgun/api/v1/urls.py | 4 + .../alembic_migrations/versions/fuel_7_0.py | 22 +++++ .../db/sqlalchemy/models/network_config.py | 2 + nailgun/nailgun/db/sqlalchemy/models/node.py | 2 + nailgun/nailgun/objects/cluster.py | 17 ++++ nailgun/nailgun/objects/node.py | 55 ++++++++++++ .../serializers/network_configuration.py | 3 +- .../test/integration/test_network_models.py | 3 +- .../test/unit/test_migration_fuel_7_0.py | 34 +++++++ .../unit/test_network_template_handler.py | 81 +++++++++++++++++ .../test/unit/test_node_assignment_handler.py | 89 +++++++++++++++++++ 14 files changed, 352 insertions(+), 4 deletions(-) create mode 100644 nailgun/nailgun/test/unit/test_network_template_handler.py diff --git a/nailgun/nailgun/api/v1/handlers/base.py b/nailgun/nailgun/api/v1/handlers/base.py index 8b9669c315..6c749a9840 100644 --- a/nailgun/nailgun/api/v1/handlers/base.py +++ b/nailgun/nailgun/api/v1/handlers/base.py @@ -248,7 +248,7 @@ class BaseHandler(object): def content_json(func, cls, *args, **kwargs): json_resp = lambda data: ( jsonutils.dumps(data) - if isinstance(data, (dict, list)) else data + if isinstance(data, (dict, list)) or not data else data ) request_validate_needed = True diff --git a/nailgun/nailgun/api/v1/handlers/cluster.py b/nailgun/nailgun/api/v1/handlers/cluster.py index 401641af5a..2ce3a390f8 100644 --- a/nailgun/nailgun/api/v1/handlers/cluster.py +++ b/nailgun/nailgun/api/v1/handlers/cluster.py @@ -301,7 +301,7 @@ class VmwareAttributesHandler(BaseHandler): :http: * 200 (OK) * 400 (wrong attributes data specified | cluster doesn't accept vmware configuration) - * 403 (attriutes can't be changed) + * 403 (attributes can't be changed) * 404 (cluster not found in db | cluster has no vmware attributes) """ diff --git a/nailgun/nailgun/api/v1/handlers/network_configuration.py b/nailgun/nailgun/api/v1/handlers/network_configuration.py index 06946db3f0..1533eb28e1 100644 --- a/nailgun/nailgun/api/v1/handlers/network_configuration.py +++ b/nailgun/nailgun/api/v1/handlers/network_configuration.py @@ -195,6 +195,46 @@ class NeutronNetworkConfigurationHandler(ProviderHandler): self.raise_task(task) +class TemplateNetworkConfigurationHandler(ProviderHandler): + """Neutron Network configuration handler + """ + + @content + def GET(self, cluster_id): + """:returns: network template for cluster (json format) + :http: * 200 (OK) + * 404 (cluster not found in db) + """ + cluster = self.get_object_or_404(objects.Cluster, cluster_id) + return cluster.network_config.configuration_template + + @content + def PUT(self, cluster_id): + """:returns: {} + :http: * 200 (OK) + * 400 (invalid object data specified) + * 403 (change of configuration is forbidden) + * 404 (cluster not found in db) + """ + template = jsonutils.loads(web.data()) + + cluster = self.get_object_or_404(objects.Cluster, cluster_id) + self.check_if_network_configuration_locked(cluster) + objects.Cluster.set_network_template(cluster, template) + raise self.http(200) + + def DELETE(self, cluster_id): + """:returns: {} + :http: * 204 (object successfully deleted) + * 403 (change of configuration is forbidden) + * 404 (cluster not found in db) + """ + cluster = self.get_object_or_404(objects.Cluster, cluster_id) + self.check_if_network_configuration_locked(cluster) + objects.Cluster.set_network_template(cluster, None) + raise self.http(204) + + class NetworkConfigurationVerifyHandler(ProviderHandler): """Network configuration verify handler base """ diff --git a/nailgun/nailgun/api/v1/urls.py b/nailgun/nailgun/api/v1/urls.py index ae3aebbec5..4e0d4f5920 100644 --- a/nailgun/nailgun/api/v1/urls.py +++ b/nailgun/nailgun/api/v1/urls.py @@ -54,6 +54,8 @@ from nailgun.api.v1.handlers.network_configuration \ import NovaNetworkConfigurationHandler from nailgun.api.v1.handlers.network_configuration \ import NovaNetworkConfigurationVerifyHandler +from nailgun.api.v1.handlers.network_configuration \ + import TemplateNetworkConfigurationHandler from nailgun.api.v1.handlers.node import NodeAgentHandler from nailgun.api.v1.handlers.node import NodeCollectionHandler @@ -153,6 +155,8 @@ urls = ( r'/clusters/(?P\d+)/network_configuration/' 'neutron/verify/?$', NeutronNetworkConfigurationVerifyHandler, + r'/clusters/(?P\d+)/network_configuration/template/?$', + TemplateNetworkConfigurationHandler, r'/clusters/(?P\d+)/orchestrator/deployment/?$', DeploymentInfo, diff --git a/nailgun/nailgun/db/migration/alembic_migrations/versions/fuel_7_0.py b/nailgun/nailgun/db/migration/alembic_migrations/versions/fuel_7_0.py index 9af26b2852..5b179f80cb 100644 --- a/nailgun/nailgun/db/migration/alembic_migrations/versions/fuel_7_0.py +++ b/nailgun/nailgun/db/migration/alembic_migrations/versions/fuel_7_0.py @@ -86,6 +86,7 @@ def upgrade(): upgrade_node_roles_metadata() node_roles_as_plugin_upgrade() migrate_volumes_into_extension_upgrade() + networking_templates_upgrade() extend_releases_model_upgrade() upgrade_task_names() vms_conf_upgrade() @@ -98,6 +99,7 @@ def downgrade(): downgrade_cluster_ui_settings() extend_nic_model_downgrade() extend_releases_model_downgrade() + networking_templates_downgrade() migrate_volumes_into_extension_downgrade() node_roles_as_plugin_downgrade() extend_plugin_model_downgrade() @@ -240,6 +242,26 @@ def extend_plugin_model_upgrade(): ) +def networking_templates_upgrade(): + op.add_column( + 'networking_configs', + sa.Column( + 'configuration_template', fields.JSON(), + nullable=True, server_default=None) + ) + op.add_column( + 'nodes', + sa.Column( + 'network_template', fields.JSON(), nullable=True, + server_default=None) + ) + + +def networking_templates_downgrade(): + op.drop_column('nodes', 'network_template') + op.drop_column('networking_configs', 'configuration_template') + + def extend_ip_addrs_model_downgrade(): vrouter_enum = sa.Enum('haproxy', 'vrouter', name='network_vip_types') diff --git a/nailgun/nailgun/db/sqlalchemy/models/network_config.py b/nailgun/nailgun/db/sqlalchemy/models/network_config.py index eecc479251..f797f1d78c 100644 --- a/nailgun/nailgun/db/sqlalchemy/models/network_config.py +++ b/nailgun/nailgun/db/sqlalchemy/models/network_config.py @@ -40,6 +40,8 @@ class NetworkingConfig(Base): "8.8.8.8" ]) floating_ranges = Column(JSON, default=[]) + configuration_template = Column(JSON, default=None, server_default=None, + nullable=True) __mapper_args__ = { 'polymorphic_on': discriminator diff --git a/nailgun/nailgun/db/sqlalchemy/models/node.py b/nailgun/nailgun/db/sqlalchemy/models/node.py index 51b9579e2e..96ed792033 100755 --- a/nailgun/nailgun/db/sqlalchemy/models/node.py +++ b/nailgun/nailgun/db/sqlalchemy/models/node.py @@ -103,6 +103,8 @@ class Node(Base): ip_addrs = relationship("IPAddr", viewonly=True) replaced_deployment_info = Column(JSON, default=[]) replaced_provisioning_info = Column(JSON, default={}) + network_template = Column(JSON, default=None, server_default=None, + nullable=True) @property def interfaces(self): diff --git a/nailgun/nailgun/objects/cluster.py b/nailgun/nailgun/objects/cluster.py index 620ab6d644..c1c62978f9 100644 --- a/nailgun/nailgun/objects/cluster.py +++ b/nailgun/nailgun/objects/cluster.py @@ -498,10 +498,14 @@ class Cluster(NailgunObject): ) cls.replace_provisioning_info_on_nodes(instance, [], nodes_to_remove) cls.replace_deployment_info_on_nodes(instance, [], nodes_to_remove) + from nailgun.objects import NodeCollection + NodeCollection.reset_network_template(nodes_to_remove) + map( net_manager.assign_networks_by_default, nodes_to_add ) + cls.update_nodes_network_template(instance, nodes_to_add) db().flush() @classmethod @@ -916,6 +920,19 @@ class Cluster(NailgunObject): """ return instance.release.network_roles_metadata + @classmethod + def set_network_template(cls, instance, template): + instance.network_config.configuration_template = template + cls.update_nodes_network_template(instance, instance.nodes) + db().flush() + + @classmethod + def update_nodes_network_template(cls, instance, nodes): + from nailgun.objects import Node + template = instance.network_config.configuration_template + for node in nodes: + Node.apply_network_template(node, template) + class ClusterCollection(NailgunCollection): """Cluster collection diff --git a/nailgun/nailgun/objects/node.py b/nailgun/nailgun/objects/node.py index f6c5c78306..e6e1244c15 100644 --- a/nailgun/nailgun/objects/node.py +++ b/nailgun/nailgun/objects/node.py @@ -18,7 +18,9 @@ Node-related objects and collections """ import itertools +import jinja2 import operator +from oslo.serialization import jsonutils import traceback from datetime import datetime @@ -693,6 +695,13 @@ class Node(NailgunObject): network_manager = Cluster.get_network_manager(instance.cluster) network_manager.assign_networks_by_default(instance) cls.add_pending_change(instance, consts.CLUSTER_CHANGES.interfaces) + cls.set_network_template(instance) + + @classmethod + def set_network_template(cls, instance): + template = instance.cluster.network_config.configuration_template + cls.apply_network_template(instance, template) + db().flush() @classmethod def add_pending_change(cls, instance, change): @@ -786,6 +795,7 @@ class Node(NailgunObject): def remove_replaced_params(cls, instance): instance.replaced_deployment_info = [] instance.replaced_provisioning_info = {} + instance.network_template = None @classmethod def all_roles(cls, instance): @@ -797,6 +807,46 @@ class Node(NailgunObject): return sorted(roles | primary_roles) + @classmethod + def apply_network_template(cls, instance, template): + if template is None: + instance.network_template = None + return + + template_body = template['adv_net_template'] + # Get the correct nic_mapping for this node so we can + # dynamically replace any interface references in any + # template for this node. + node_group = instance.group_id + if node_group not in template_body: + node_group = 'default' + + node_name = cls.make_slave_name(instance) + nic_mapping = template_body[node_group]['nic_mapping'] + if node_name not in nic_mapping: + node_name = 'default' + + nic_mapping = nic_mapping[node_name] + env = jinja2.Environment(variable_start_string='<%', + variable_end_string='%>') + + # Replace interface references and re-parse JSON + template_object = env.from_string(jsonutils.dumps(template_body)) + node_template = template_object.render(nic_mapping) + parsed_template = jsonutils.loads(node_template) + + output = parsed_template[node_group] + output['templates'] = output.pop('network_scheme') + output['roles'] = {} + output['endpoints'] = {} + for v in output['templates'].values(): + for endpoint in v['endpoints']: + output['endpoints'][endpoint] = {} + for role, ep in v['roles'].items(): + output['roles'][role] = ep + + instance.network_template = output + class NodeCollection(NailgunCollection): """Node collection @@ -885,3 +935,8 @@ class NodeCollection(NailgunCollection): @classmethod def get_by_ids(cls, ids): return db.query(models.Node).filter(models.Node.id.in_(ids)).all() + + @classmethod + def reset_network_template(cls, instances): + for instance in instances: + instance.network_template = None diff --git a/nailgun/nailgun/objects/serializers/network_configuration.py b/nailgun/nailgun/objects/serializers/network_configuration.py index 975ffe82c8..6f0c6744b3 100644 --- a/nailgun/nailgun/objects/serializers/network_configuration.py +++ b/nailgun/nailgun/objects/serializers/network_configuration.py @@ -77,7 +77,8 @@ class NeutronNetworkConfigurationSerializer(NetworkConfigurationSerializer): network_cfg_fields = ( 'dns_nameservers', 'segmentation_type', 'net_l23_provider', 'floating_ranges', 'vlan_range', 'gre_id_range', - 'base_mac', 'internal_cidr', 'internal_gateway') + 'base_mac', 'internal_cidr', 'internal_gateway', + 'configuration_template') @classmethod def serialize_for_cluster(cls, cluster): diff --git a/nailgun/nailgun/test/integration/test_network_models.py b/nailgun/nailgun/test/integration/test_network_models.py index 2eb541d174..2f13eaae1c 100644 --- a/nailgun/nailgun/test/integration/test_network_models.py +++ b/nailgun/nailgun/test/integration/test_network_models.py @@ -110,7 +110,8 @@ class TestNetworkModels(BaseIntegrationTest): "internal_gateway": "192.168.111.1", "floating_ranges": [["172.16.0.130", "172.16.0.254"]], "dns_nameservers": ["8.8.4.4", "8.8.8.8"], - "cluster_id": cluster.id + "cluster_id": cluster.id, + "configuration_template": {} } nc = NeutronConfig(**kw) self.db.add(nc) diff --git a/nailgun/nailgun/test/unit/test_migration_fuel_7_0.py b/nailgun/nailgun/test/unit/test_migration_fuel_7_0.py index 635ac6bfd9..86487d56ff 100644 --- a/nailgun/nailgun/test/unit/test_migration_fuel_7_0.py +++ b/nailgun/nailgun/test/unit/test_migration_fuel_7_0.py @@ -161,6 +161,27 @@ def prepare(): 'fuel_version': '6.1', }]) + result = db.execute( + meta.tables['networking_configs'].insert(), + [{ + 'cluster_id': None, + 'dns_nameservers': ['8.8.8.8'], + 'floating_ranges': [], + 'configuration_template': None, + }]) + db.execute( + meta.tables['neutron_config'].insert(), + [{ + 'id': result.inserted_primary_key[0], + 'vlan_range': [], + 'gre_id_range': [], + 'base_mac': '00:00:00:00:00:00', + 'internal_cidr': '10.10.10.00/24', + 'internal_gateway': '10.10.10.01', + 'segmentation_type': 'vlan', + 'net_l23_provider': 'ovs' + }]) + result = db.execute( meta.tables['nodes'].insert(), [ @@ -483,6 +504,19 @@ class TestInterfacesOffloadingModesMigration(base.BaseAlembicMigrationTest): jsonutils.loads(result.fetchone()[0]), []) +class TestNetworkingTemplatesMigration(base.BaseAlembicMigrationTest): + def test_new_fields_exists_and_empty(self): + result = db.execute( + sa.select([self.meta.tables['networking_configs'] + .c.configuration_template])) + self.assertIsNone(result.fetchone()[0]) + + result = db.execute( + sa.select([self.meta.tables['nodes'] + .c.network_template])) + self.assertIsNone(result.fetchone()[0]) + + class TestInterfacesPxePropertyMigration(base.BaseAlembicMigrationTest): def test_old_fields_exists(self): diff --git a/nailgun/nailgun/test/unit/test_network_template_handler.py b/nailgun/nailgun/test/unit/test_network_template_handler.py new file mode 100644 index 0000000000..f001ed86cc --- /dev/null +++ b/nailgun/nailgun/test/unit/test_network_template_handler.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- + +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo.serialization import jsonutils + +from nailgun.test.base import BaseIntegrationTest +from nailgun.utils import reverse + + +class TestHandlers(BaseIntegrationTest): + + def get_template(self, cluster_id, expect_errors=False): + resp = self.app.get( + reverse( + 'TemplateNetworkConfigurationHandler', + kwargs={'cluster_id': cluster_id} + ), + headers=self.default_headers, + expect_errors=expect_errors + ) + + return resp + + def test_network_template_upload(self): + cluster = self.env.create_cluster(api=False) + template = {'template': 'test'} + resp = self.app.put( + reverse( + 'TemplateNetworkConfigurationHandler', + kwargs={'cluster_id': cluster.id}, + ), + jsonutils.dumps(template), + headers=self.default_headers + ) + self.assertEqual(200, resp.status_code) + + resp = self.get_template(cluster.id) + self.assertEqual(200, resp.status_code) + self.assertEqual('test', resp.json_body.get('template')) + + def test_template_not_set(self): + resp = self.get_template(1, expect_errors=True) + self.assertEqual(404, resp.status_code) + + def test_delete_template(self): + cluster = self.env.create_cluster(api=False) + template = {'template': 'test'} + resp = self.app.put( + reverse( + 'TemplateNetworkConfigurationHandler', + kwargs={'cluster_id': cluster.id}, + ), + jsonutils.dumps(template), + headers=self.default_headers + ) + self.assertEquals(200, resp.status_code) + + resp = self.app.delete( + reverse( + 'TemplateNetworkConfigurationHandler', + kwargs={'cluster_id': cluster.id}, + ), + headers=self.default_headers + ) + self.assertEquals(204, resp.status_code) + + resp = self.get_template(cluster.id) + self.assertEquals(None, resp.json_body) diff --git a/nailgun/nailgun/test/unit/test_node_assignment_handler.py b/nailgun/nailgun/test/unit/test_node_assignment_handler.py index ae682c1cb8..a89c227880 100644 --- a/nailgun/nailgun/test/unit/test_node_assignment_handler.py +++ b/nailgun/nailgun/test/unit/test_node_assignment_handler.py @@ -163,6 +163,95 @@ class TestAssignmentHandlers(BaseIntegrationTest): self.assertEquals(404, resp.status_code) + def test_add_node_with_cluster_network_template(self): + net_template = """ + { + "adv_net_template": { + "default": { + "network_assignments": { + "management": { + "ep": "br-mgmt" + }, + "storage": { + "ep": "br-storage" + }, + "public": { + "ep": "br-ex" + }, + "private": { + "ep": "br-prv" + }, + "fuelweb_admin": { + "ep": "br-fw-admin" + } + }, + "templates_for_node_role": { + "controller": [ + "common" + ] + }, + "nic_mapping": { + "default": { + "if4": "eth3", + "if1": "eth0", + "if2": "eth1", + "if3": "eth2" + } + }, + "network_scheme": { + "common": { + "endpoints": [ + "br-mgmt" + ], + "transformations": [ + { + "action": "add-br", + "name": "br-mgmt" + }, + { + "action": "add-port", + "bridge": "br-mgmt", + "name": "<% if2 %>" + } + ], + "roles": { + "management": "br-mgmt" + } + } + } + } + } + } + """ + net_template = jsonutils.loads(net_template) + cluster = self.env.create_cluster(api=False) + cluster.network_config.configuration_template = net_template + + node = self.env.create_node() + assignment_data = [ + { + "id": node.id, + "roles": ['controller'] + } + ] + self.app.post( + reverse( + 'NodeAssignmentHandler', + kwargs={'cluster_id': cluster.id} + ), + jsonutils.dumps(assignment_data), + headers=self.default_headers + ) + net_scheme = node.network_template['templates']['common'] + self.assertNotEqual({}, node.network_template) + self.assertEquals(['br-mgmt'], net_scheme['endpoints']) + self.assertEquals({'management': 'br-mgmt'}, net_scheme['roles']) + + # The order of transformations matters + self.assertIn('add-br', net_scheme['transformations'][0].values()) + self.assertIn('add-port', net_scheme['transformations'][1].values()) + self.assertEquals('eth1', net_scheme['transformations'][1]['name']) + class TestClusterStateUnassigment(BaseIntegrationTest):