Add rexray volume driver to Swarm

* Volumes are created using the bay's trusstee user
  scoped by the bay's trustID.
* In volume-service.sh it is checked if rexray exists
  in the image, so if you have built an image with
  rexray it will configure and start rexray service.

This change replaces:
Ie5e78e6b9d06fd2c92915b8635b8062c008e9696

Change-Id: I11043d5edcf73a87220c86320f9938b9347c118f
Partially-Implements: blueprint magnum-integrate-with-cinder
This commit is contained in:
Spyros Trigazis 2016-07-12 16:44:46 +02:00
parent 7fc4b27607
commit 1466f93b21
10 changed files with 175 additions and 9 deletions

View File

@ -59,6 +59,8 @@ class AtomicSwarmTemplateDefinition(template_def.BaseTemplateDefinition):
cluster_template_attr='flavor_id')
self.add_parameter('docker_volume_size',
cluster_template_attr='docker_volume_size')
self.add_parameter('volume_driver',
cluster_template_attr='volume_driver')
self.add_parameter('external_network',
cluster_template_attr='external_network_id',
required=True)
@ -98,7 +100,9 @@ class AtomicSwarmTemplateDefinition(template_def.BaseTemplateDefinition):
extra_params['magnum_url'] = osc.magnum_url()
label_list = ['flannel_network_cidr', 'flannel_backend',
'flannel_network_subnetlen']
'flannel_network_subnetlen', 'rexray_preempt']
extra_params['auth_url'] = context.auth_url
for label in label_list:
extra_params[label] = cluster_template.labels.get(label)

View File

@ -212,6 +212,21 @@ parameters:
size fo the data segments for the swift dynamic large objects
default: 5242880
volume_driver:
type: string
description: volume driver to use for container storage
default: ""
constraints:
- allowed_values: ["","rexray"]
rexray_preempt:
type: string
description: >
enables any host to take control of a volume irrespective of whether
other hosts are using the volume
default: "false"
resources:
######################################################################
@ -384,6 +399,8 @@ resources:
trustee_password: {get_param: trustee_password}
trust_id: {get_param: trust_id}
auth_url: {get_param: auth_url}
volume_driver: {get_param: volume_driver}
rexray_preempt: {get_param: rexray_preempt}
swarm_nodes:
type: "OS::Heat::ResourceGroup"
@ -427,6 +444,8 @@ resources:
registry_container: {get_param: registry_container}
registry_insecure: {get_param: registry_insecure}
registry_chunksize: {get_param: registry_chunksize}
volume_driver: {get_param: volume_driver}
rexray_preempt: {get_param: rexray_preempt}
outputs:

View File

@ -0,0 +1,72 @@
#!/bin/sh
# Add rexray volume driver support for Swarm
. /etc/sysconfig/heat-params
set -e
set -x
# if no voulume driver is selected don't do any configuration
if [ -z "$VOLUME_DRIVER" ]; then
exit 0
fi
mkdir -p /etc/rexray
mkdir -p /var/log/rexray
mkdir -p /var/run/rexray
mkdir -p /var/lib/rexray
REXRAY_CONFIG=/etc/rexray/config.yml
# Add rexray configuration
cat > $REXRAY_CONFIG <<EOF
libstorage:
logging:
level: info
service: openstack
integration:
volume:
operations:
mount:
preempt: $REXRAY_PREEMPT
openstack:
authUrl: $AUTH_URL
userID: $TRUSTEE_USER_ID
password: $TRUSTEE_PASSWORD
trustID: $TRUST_ID
EOF
if [ ! -f /usr/bin/rexray ]; then
# If rexray is not installed, run it in a docker container
cat > /etc/systemd/system/rexray.service <<EOF
[Unit]
Description=Rexray container
Requires=docker.service
After=docker.service
[Service]
RemainAfterExit=yes
ExecStartPre=-/usr/bin/docker pull openstackmagnum/rexray:alpine
ExecStart=/usr/bin/rm -f /var/run/rexray/rexray.pid && \
/usr/bin/docker run -d --name=rexray --privileged -p 7979:7979 \
-v /run/docker/plugins:/run/docker/plugins \
-v /var/lib/rexray:/var/lib/rexray:z \
-v /var/log/rexray:/var/log/rexray \
-v /var/run/rexray:/var/run/rexray \
-v /dev:/dev \
-v /etc/rexray/config.yml:/etc/rexray/config.yml \
openstackmagnum/rexray:alpine
ExecStop=/usr/bin/docker stop rexray
[Install]
WantedBy=multi-user.target
EOF
chown root:root /etc/systemd/system/rexray.service
chmod 644 /etc/systemd/system/rexray.service
systemctl daemon-reload
fi
echo "starting rexray..."
systemctl enable rexray
systemctl --no-block start rexray

View File

@ -30,3 +30,5 @@ write_files:
TRUSTEE_PASSWORD="$TRUSTEE_PASSWORD"
TRUST_ID="$TRUST_ID"
AUTH_URL="$AUTH_URL"
VOLUME_DRIVER="$VOLUME_DRIVER"
REXRAY_PREEMPT="$REXRAY_PREEMPT"

View File

@ -34,3 +34,5 @@ write_files:
REGISTRY_CONTAINER="$REGISTRY_CONTAINER"
REGISTRY_INSECURE="$REGISTRY_INSECURE"
REGISTRY_CHUNKSIZE="$REGISTRY_CHUNKSIZE"
VOLUME_DRIVER="$VOLUME_DRIVER"
REXRAY_PREEMPT="$REXRAY_PREEMPT"

View File

@ -143,6 +143,18 @@ parameters:
type: string
description: url for keystone
volume_driver:
type: string
description: volume driver to use for container storage
default: ""
rexray_preempt:
type: string
description: >
enables any host to take control of a volume irrespective of whether
other hosts are using the volume
default: "false"
resources:
master_wait_handle:
@ -213,6 +225,8 @@ resources:
"$TRUSTEE_PASSWORD": {get_param: trustee_password}
"$TRUST_ID": {get_param: trust_id}
"$AUTH_URL": {get_param: auth_url}
"$VOLUME_DRIVER": {get_param: volume_driver}
"$REXRAY_PREEMPT": {get_param: rexray_preempt}
write_network_config:
type: "OS::Heat::SoftwareConfig"
@ -330,6 +344,12 @@ resources:
group: ungrouped
config: {get_file: fragments/add-proxy.sh}
volume_service:
type: "OS::Heat::SoftwareConfig"
properties:
group: ungrouped
config: {get_file: fragments/volume-service.sh}
swarm_master_init:
type: "OS::Heat::MultipartMime"
properties:
@ -350,6 +370,7 @@ resources:
- config: {get_resource: add_proxy}
- config: {get_resource: enable_services}
- config: {get_resource: cfn_signal}
- config: {get_resource: volume_service}
######################################################################
#

View File

@ -149,6 +149,18 @@ parameters:
description: >
size fo the data segments for the swift dynamic large objects
volume_driver:
type: string
description: volume driver to use for container storage
default: ""
rexray_preempt:
type: string
description: >
enables any host to take control of a volume irrespective of whether
other hosts are using the volume
default: "false"
resources:
node_wait_handle:
@ -202,6 +214,8 @@ resources:
"$REGISTRY_CONTAINER": {get_param: registry_container}
"$REGISTRY_INSECURE": {get_param: registry_insecure}
"$REGISTRY_CHUNKSIZE": {get_param: registry_chunksize}
"$VOLUME_DRIVER": {get_param: volume_driver}
"$REXRAY_PREEMPT": {get_param: rexray_preempt}
remove_docker_key:
type: "OS::Heat::SoftwareConfig"
@ -301,6 +315,12 @@ resources:
group: ungrouped
config: {get_file: fragments/add-proxy.sh}
volume_service:
type: "OS::Heat::SoftwareConfig"
properties:
group: ungrouped
config: {get_file: fragments/volume-service.sh}
swarm_node_init:
type: "OS::Heat::MultipartMime"
properties:
@ -320,6 +340,7 @@ resources:
- config: {get_resource: enable_docker_registry}
- config: {get_resource: enable_services}
- config: {get_resource: cfn_signal}
- config: {get_resource: volume_service}
swarm_node:
type: "OS::Nova::Server"

View File

@ -46,8 +46,10 @@ class TestBayConductorWithSwarm(base.TestCase):
'network_driver': 'network_driver',
'labels': {'flannel_network_cidr': '10.101.0.0/16',
'flannel_network_subnetlen': '26',
'flannel_backend': 'vxlan'},
'flannel_backend': 'vxlan',
'rexray_preempt': 'False'},
'master_lb_enabled': False,
'volume_driver': 'rexray'
}
self.bay_dict = {
'id': 1,
@ -128,7 +130,9 @@ class TestBayConductorWithSwarm(base.TestCase):
'trustee_user_id': '7b489f04-b458-4541-8179-6a48a553e656',
'trust_id': 'bd11efc5-d4e2-4dac-bbce-25e348ddf7de',
'auth_url': 'http://192.168.10.10:5000/v3',
'swarm_version': 'fake-version'
'swarm_version': 'fake-version',
'volume_driver': 'rexray',
'rexray_preempt': 'False'
}
self.assertEqual(expected, definition)
self.assertEqual(
@ -193,7 +197,9 @@ class TestBayConductorWithSwarm(base.TestCase):
'trust_id': 'bd11efc5-d4e2-4dac-bbce-25e348ddf7de',
'auth_url': 'http://192.168.10.10:5000/v3',
'docker_storage_driver': 'devicemapper',
'swarm_version': 'fake-version'
'swarm_version': 'fake-version',
'volume_driver': 'rexray',
'rexray_preempt': 'False'
}
self.assertEqual(expected, definition)
self.assertEqual(
@ -210,7 +216,8 @@ class TestBayConductorWithSwarm(base.TestCase):
not_required = ['image_id', 'flavor_id', 'dns_nameserver',
'docker_volume_size', 'fixed_network', 'http_proxy',
'https_proxy', 'no_proxy', 'network_driver',
'master_flavor_id', 'docker_storage_driver']
'master_flavor_id', 'docker_storage_driver',
'volume_driver', 'rexray_preempt']
for key in not_required:
self.cluster_template_dict[key] = None
self.bay_dict['discovery_url'] = 'https://discovery.etcd.io/test'
@ -250,7 +257,8 @@ class TestBayConductorWithSwarm(base.TestCase):
'trustee_user_id': '7b489f04-b458-4541-8179-6a48a553e656',
'trust_id': 'bd11efc5-d4e2-4dac-bbce-25e348ddf7de',
'auth_url': 'http://192.168.10.10:5000/v3',
'swarm_version': 'fake-version'
'swarm_version': 'fake-version',
'rexray_preempt': 'False'
}
self.assertEqual(expected, definition)
self.assertEqual(
@ -309,7 +317,9 @@ class TestBayConductorWithSwarm(base.TestCase):
'trustee_user_id': '7b489f04-b458-4541-8179-6a48a553e656',
'trust_id': 'bd11efc5-d4e2-4dac-bbce-25e348ddf7de',
'auth_url': 'http://192.168.10.10:5000/v3',
'swarm_version': 'fake-version'
'swarm_version': 'fake-version',
'volume_driver': 'rexray',
'rexray_preempt': 'False'
}
self.assertEqual(expected, definition)
self.assertEqual(
@ -369,7 +379,9 @@ class TestBayConductorWithSwarm(base.TestCase):
'trustee_user_id': '7b489f04-b458-4541-8179-6a48a553e656',
'trust_id': 'bd11efc5-d4e2-4dac-bbce-25e348ddf7de',
'auth_url': 'http://192.168.10.10:5000/v3',
'swarm_version': 'fake-version'
'swarm_version': 'fake-version',
'volume_driver': 'rexray',
'rexray_preempt': 'False'
}
self.assertEqual(expected, definition)
self.assertEqual(

View File

@ -716,6 +716,7 @@ class AtomicSwarmTemplateDefinitionTestCase(base.TestCase):
flannel_cidr = mock_baymodel.labels.get('flannel_network_cidr')
flannel_subnet = mock_baymodel.labels.get('flannel_network_subnetlen')
flannel_backend = mock_baymodel.labels.get('flannel_backend')
rexray_preempt = mock_baymodel.labels.get('rexray_preempt')
swarm_def = swarm_tdef.AtomicSwarmTemplateDefinition()
@ -726,7 +727,9 @@ class AtomicSwarmTemplateDefinitionTestCase(base.TestCase):
'magnum_url': mock_osc.magnum_url.return_value,
'flannel_network_cidr': flannel_cidr,
'flannel_backend': flannel_backend,
'flannel_network_subnetlen': flannel_subnet}}
'flannel_network_subnetlen': flannel_subnet,
'auth_url': 'http://192.168.10.10:5000/v3',
'rexray_preempt': rexray_preempt}}
mock_get_params.assert_called_once_with(mock_context, mock_baymodel,
mock_bay, **expected_kwargs)

View File

@ -0,0 +1,10 @@
---
features:
- Integrate Docker Swarm Fedora Atomic driver with the
Block Storage Service (cinder). The rexray volume
driver was added based on rexray v0.4. Users can
create and attach volumes using docker's navive
client and they will authenticate using the per
cluster trustee user. Rexray can be either added
in the Fedora Atomic image or can be used running
in a container.