openstack/deb-sahara
Code Issues Proposed changes

Merge "Add volume type support to sahara"

Browse Source
This commit is contained in:
Jenkins 2014-10-14 07:39:44 +00:00 committed by Gerrit Code Review
commit 56bd690b2c
17 changed files with 109 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
sahara/conductor/manager.py
View File

@ -39,6 +39,7 @@ NODE_GROUP_DEFAULTS = {
"volumes_size": 0,
"volumes_availability_zone": None,
"volume_mount_prefix": "/volumes/disk",
"volume_type": None,
"floating_ip_pool": None,
"security_groups": None,
"auto_security_group": False,

2
sahara/conductor/objects.py
View File

@ -77,6 +77,7 @@ class NodeGroup(object):
volumes_availability_zone - name of Cinder availability zone
where to spawn volumes
volume_mount_prefix
volume_type
floating_ip_pool - Floating IP Pool name used to assign Floating IPs to
instances in this Node Group
security_groups - List of security groups for instances in this Node Group
@ -178,6 +179,7 @@ class NodeGroupTemplate(object):
volumes_size
volumes_availability_zone
volume_mount_prefix
volume_type
floating_ip_pool
security_groups
auto_security_group

47
sahara/db/migration/alembic_migrations/versions/014_add_volume_type.py Normal file
View File

@ -0,0 +1,47 @@
# Copyright 2014 OpenStack Foundation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""add_volume_type
Revision ID: 014
Revises: 013
Create Date: 2014-10-09 12:47:17.871520
"""
# revision identifiers, used by Alembic.
revision = '014'
down_revision = '013'
from alembic import op
import sqlalchemy as sa
def upgrade():
op.add_column('node_group_templates',
sa.Column('volume_type', sa.String(length=255),
nullable=True))
op.add_column('node_groups',
sa.Column('volume_type', sa.String(length=255),
nullable=True))
op.add_column('templates_relations',
sa.Column('volume_type', sa.String(length=255),
nullable=True))
def downgrade():
op.drop_column('templates_relations', 'volume_type')
op.drop_column('node_groups', 'volume_type')
op.drop_column('node_group_templates', 'volume_type')

3
sahara/db/sqlalchemy/models.py
View File

@ -101,6 +101,7 @@ class NodeGroup(mb.SaharaBase):
volumes_size = sa.Column(sa.Integer)
volumes_availability_zone = sa.Column(sa.String(255))
volume_mount_prefix = sa.Column(sa.String(80))
volume_type = sa.Column(sa.String(255))
count = sa.Column(sa.Integer, nullable=False)
instances = relationship('Instance', cascade="all,delete",
backref='node_group',
@ -197,6 +198,7 @@ class NodeGroupTemplate(mb.SaharaBase):
volumes_size = sa.Column(sa.Integer)
volumes_availability_zone = sa.Column(sa.String(255))
volume_mount_prefix = sa.Column(sa.String(80))
volume_type = sa.Column(sa.String(255))
floating_ip_pool = sa.Column(sa.String(36))
security_groups = sa.Column(st.JsonListType())
auto_security_group = sa.Column(sa.Boolean())
@ -222,6 +224,7 @@ class TemplatesRelation(mb.SaharaBase):
volumes_size = sa.Column(sa.Integer)
volumes_availability_zone = sa.Column(sa.String(255))
volume_mount_prefix = sa.Column(sa.String(80))
volume_type = sa.Column(sa.String(255))
count = sa.Column(sa.Integer, nullable=False)
cluster_template_id = sa.Column(sa.String(36),
sa.ForeignKey('cluster_templates.id'))

3
sahara/resources/volume.heat
View File

@ -3,7 +3,8 @@
"Properties" : {
"name" : "%(volume_name)s",
%(availability_zone)s
"size" : "%(volumes_size)s"
"size" : "%(volumes_size)s",
%(volume_type)s
}
},
"%(volume_attach_name)s" : {

11
sahara/service/validations/base.py
View File

@ -135,6 +135,9 @@ def check_node_group_basic_fields(plugin_name, hadoop_version, ng,
check_volume_availability_zone_exist(
ng['volumes_availability_zone'])
if ng.get('volume_type'):
check_volume_type_exists(ng['volume_type'])
if ng.get('floating_ip_pool'):
check_floatingip_pool_exists(ng['name'], ng['floating_ip_pool'])
@ -225,6 +228,14 @@ def check_volume_availability_zone_exist(az):
% az)
def check_volume_type_exists(volume_type):
volume_types = cinder.client().volume_types.list(search_opts={'name':
volume_type})
if len(volume_types) == 1 and volume_types[0] == volume_type:
return
raise ex.NotFoundException(_("Volume type '%s' not found") % volume_type)
# Cluster creation related checks
def check_cluster_unique_name(name):

3
sahara/service/validations/node_group_templates.py
View File

@ -59,6 +59,9 @@ NODE_GROUP_TEMPLATE_SCHEMA = {
"type": "integer",
"minimum": 1,
},
"volume_type": {
"type": "string"
},
"volumes_availability_zone": {
"type": "string",
},

6
sahara/service/volumes.py
View File

@ -67,11 +67,12 @@ def _await_attach_volumes(instance, devices):
def _attach_volumes_to_node(node_group, instance):
ctx = context.ctx()
size = node_group.volumes_size
volume_type = node_group.volume_type
devices = []
for idx in range(1, node_group.volumes_per_node + 1):
display_name = "volume_" + instance.instance_name + "_" + str(idx)
device = _create_attach_volume(
ctx, instance, size, display_name,
ctx, instance, size, volume_type, display_name,
node_group.volumes_availability_zone)
devices.append(device)
LOG.debug("Attached volume %s to instance %s" %
@ -82,13 +83,14 @@ def _attach_volumes_to_node(node_group, instance):
_mount_volumes_to_node(instance, devices)
def _create_attach_volume(ctx, instance, size, name=None,
def _create_attach_volume(ctx, instance, size, volume_type, name=None,
availability_zone=None):
if CONF.cinder_api_version == 1:
kwargs = {'size': size, 'display_name': name}
else:
kwargs = {'size': size, 'name': name}
kwargs['volume_type'] = volume_type
if availability_zone is not None:
kwargs['availability_zone'] = availability_zone

1
sahara/tests/unit/conductor/manager/test_clusters.py
View File

@ -115,6 +115,7 @@ class ClusterTest(test_base.ConductorManagerTestCase):
ng.pop("volumes_size")
ng.pop("volumes_per_node")
ng.pop("volumes_availability_zone")
ng.pop("volume_type")
ng.pop("floating_ip_pool")
ng.pop("image_username")
ng.pop("open_ports")

1
sahara/tests/unit/conductor/manager/test_templates.py
View File

@ -190,6 +190,7 @@ class ClusterTemplates(test_base.ConductorManagerTestCase):
ng.pop("volumes_size")
ng.pop("volumes_per_node")
ng.pop("volumes_availability_zone")
ng.pop("volume_type")
ng.pop("auto_security_group")
self.assertEqual(SAMPLE_CLT["node_groups"],

5
sahara/tests/unit/db/migration/test_migrations.py
View File

@ -407,3 +407,8 @@ class TestMigrations(base.BaseWalkMigrationTestCase, base.CommonTestsMixIn):
self.assertColumnExists(engine, 'node_groups', 'availability_zone')
self.assertColumnExists(engine, 'templates_relations',
'availability_zone')
def _check_014(self, engine, data):
self.assertColumnExists(engine, 'node_group_templates', 'volume_type')
self.assertColumnExists(engine, 'node_groups', 'volume_type')
self.assertColumnExists(engine, 'templates_relations', 'volume_type')

6
sahara/tests/unit/resources/test_serialize_resources_use_neutron.heat
View File

@ -35,7 +35,8 @@
"Type" : "OS::Cinder::Volume",
"Properties" : {
"name" : "cluster-worker-001-volume-0",
"size" : "10"
"size" : "10",
"volume_type" : "vol_type"
}
},
"cluster-worker-001-volume-attachment-0" : {
@ -50,7 +51,8 @@
"Type" : "OS::Cinder::Volume",
"Properties" : {
"name" : "cluster-worker-001-volume-1",
"size" : "10"
"size" : "10",
"volume_type": "vol_type"
}
},
"cluster-worker-001-volume-attachment-1" : {

6
sahara/tests/unit/resources/test_serialize_resources_use_nn_with_autoassignment.heat
View File

@ -20,7 +20,8 @@
"Type" : "OS::Cinder::Volume",
"Properties" : {
"name" : "cluster-worker-001-volume-0",
"size" : "10"
"size" : "10",
"volume_type": null
}
},
"cluster-worker-001-volume-attachment-0" : {
@ -35,7 +36,8 @@
"Type" : "OS::Cinder::Volume",
"Properties" : {
"name" : "cluster-worker-001-volume-1",
"size" : "10"
"size" : "10",
"volume_type": null
}
},
"cluster-worker-001-volume-attachment-1" : {

6
sahara/tests/unit/resources/test_serialize_resources_use_nn_without_autoassignment.heat
View File

@ -33,7 +33,8 @@
"Type" : "OS::Cinder::Volume",
"Properties" : {
"name" : "cluster-worker-001-volume-0",
"size" : "10"
"size" : "10",
"volume_type": null
}
},
"cluster-worker-001-volume-attachment-0" : {
@ -48,7 +49,8 @@
"Type" : "OS::Cinder::Volume",
"Properties" : {
"name" : "cluster-worker-001-volume-1",
"size" : "10"
"size" : "10",
"volume_type": null
}
},
"cluster-worker-001-volume-attachment-1" : {

1
sahara/tests/unit/service/test_volumes.py
View File

@ -98,6 +98,7 @@ class TestAttachVolume(base.SaharaWithDbTestCase):
'volumes_size': 2,
'volumes_availability_zone': None,
'volume_mount_prefix': '/mnt/vols',
'volume_type': None,
'name': 'master',
'instances': [instance1, instance2]}

8
sahara/tests/unit/utils/test_heat.py
View File

@ -58,15 +58,15 @@ class TestClusterTemplate(base.SaharaWithDbTestCase):
into Heat templates.
"""
def _make_node_groups(self, floating_ip_pool=None):
def _make_node_groups(self, floating_ip_pool=None, volume_type=None):
ng1 = tu.make_ng_dict('master', 42, ['namenode'], 1,
floating_ip_pool=floating_ip_pool, image_id=None,
volumes_per_node=0, volumes_size=0, id=1,
image_username='root')
image_username='root', volume_type=None)
ng2 = tu.make_ng_dict('worker', 42, ['datanode'], 1,
floating_ip_pool=floating_ip_pool, image_id=None,
volumes_per_node=2, volumes_size=10, id=2,
image_username='root')
image_username='root', volume_type=volume_type)
return ng1, ng2
def _make_cluster(self, mng_network, ng1, ng2, anti_affinity=[]):
@ -110,7 +110,7 @@ class TestClusterTemplate(base.SaharaWithDbTestCase):
'worker' with 2 attached volumes 10GB size each
"""
ng1, ng2 = self._make_node_groups('floating')
ng1, ng2 = self._make_node_groups('floating', 'vol_type')
cluster = self._make_cluster('private_net', ng1, ng2)
heat_template = self._make_heat_template(cluster, ng1, ng2)
self.override_config("use_neutron", True)

15
sahara/utils/openstack/heat.py
View File

@ -244,7 +244,8 @@ class ClusterTemplate(object):
for idx in range(0, ng.volumes_per_node):
yield self._serialize_volume(inst_name, idx, ng.volumes_size,
ng.volumes_availability_zone)
ng.volumes_availability_zone,
ng.volume_type)
def _serialize_port(self, port_name, fixed_net_id, security_groups):
fields = {'port_name': port_name,
@ -272,14 +273,22 @@ class ClusterTemplate(object):
return _load_template('nova-floating.heat', fields)
def _serialize_volume_type(self, volume_type):
property = '"volume_type" : %s'
if volume_type is None:
return property % 'null'
else:
return property % ('"%s"' % volume_type)
def _serialize_volume(self, inst_name, volume_idx, volumes_size,
volumes_availability_zone):
volumes_availability_zone, volume_type):
fields = {'volume_name': _get_volume_name(inst_name, volume_idx),
'volumes_size': volumes_size,
'volume_attach_name': _get_volume_attach_name(inst_name,
volume_idx),
'availability_zone': '',
'instance_name': inst_name}
'instance_name': inst_name,
'volume_type': self._serialize_volume_type(volume_type)}
if volumes_availability_zone:
# Use json.dumps to escape volumes_availability_zone