Add docker-storage-driver attribute to baymodel

* This parameter will be optional and users will be able to select
  a supported driver, otherwise the default configuration will be
  used.
* Add docker storage driver enum field to baymodel
* Add db upgrade file
* Update heat templates for kubernetes and swarm allowing only
  devicemapper and overlay as docker_storage_driver values.
* Add configuration for OverlayFS on Fedora Atomic, if overlay is
  incompatible bay creation will result a CREATE_FAILED status.
* Factor out configuration of docker storage drivers
* Update tests
* Add Release Notes

Partially-Implements: blueprint support-for-different-docker-storage-driver
Change-Id: Ib58cb734c4e9c90d5d83574852213d2e97359e92
This commit is contained in:
Spyros Trigazis 2016-02-25 14:06:08 +01:00
parent 599614eb47
commit dec85b538f
24 changed files with 223 additions and 20 deletions

View File

@ -134,6 +134,9 @@ class BayModel(base.APIBase):
insecure_registry = wtypes.StringType(min_length=1, max_length=255)
"""insecure registry url when create baymodel """
docker_storage_driver = wtypes.Enum(str, *fields.DockerStorageDriver.ALL)
"""Docker storage driver"""
def __init__(self, **kwargs):
self.fields = []
for field in objects.BayModel.fields:
@ -173,6 +176,7 @@ class BayModel(base.APIBase):
volume_driver='cinder',
apiserver_port=8080,
docker_volume_size=25,
docker_storage_driver='devicemapper',
cluster_distro='fedora-atomic',
coe=fields.BayType.KUBERNETES,
http_proxy='http://proxy.com:123',

View File

@ -522,6 +522,8 @@ class AtomicK8sTemplateDefinition(K8sTemplateDefinition):
super(AtomicK8sTemplateDefinition, self).__init__()
self.add_parameter('docker_volume_size',
baymodel_attr='docker_volume_size')
self.add_parameter('docker_storage_driver',
baymodel_attr='docker_storage_driver')
def get_params(self, context, baymodel, bay, **kwargs):
extra_params = kwargs.pop('extra_params', {})
@ -583,6 +585,8 @@ class AtomicSwarmTemplateDefinition(BaseTemplateDefinition):
required=True)
self.add_parameter('registry_enabled',
baymodel_attr='registry_enabled')
self.add_parameter('docker_storage_driver',
baymodel_attr='docker_storage_driver')
self.add_output('api_address',
bay_attr='api_address',
mapping_type=SwarmApiAddressOutputMapping)

View File

@ -0,0 +1,35 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Add docker storage driver column
Revision ID: a1136d335540
Revises: d072f58ab240
Create Date: 2016-03-07 19:00:28.738486
"""
# revision identifiers, used by Alembic.
revision = 'a1136d335540'
down_revision = 'd072f58ab240'
from alembic import op
import sqlalchemy as sa
docker_storage_driver_enum = sa.Enum('devicemapper', 'overlay',
name='docker_storage_driver')
def upgrade():
op.add_column('baymodel', sa.Column('docker_storage_driver',
docker_storage_driver_enum,
nullable=True))

View File

@ -160,6 +160,7 @@ class BayModel(Base):
dns_nameserver = Column(String(255))
apiserver_port = Column(Integer())
docker_volume_size = Column(Integer())
docker_storage_driver = Column(String(255))
cluster_distro = Column(String(255))
coe = Column(String(255))
http_proxy = Column(String(255))

View File

@ -35,7 +35,8 @@ class BayModel(base.MagnumPersistentObject, base.MagnumObject,
# Version 1.9: Added 'volume_driver' field
# Version 1.10: Removed 'ssh_authorized_key' field
# Version 1.11: Added 'insecure_registry' field
VERSION = '1.11'
# Version 1.12: Added 'docker_storage_driver' field
VERSION = '1.12'
dbapi = dbapi.get_instance()
@ -56,6 +57,8 @@ class BayModel(base.MagnumPersistentObject, base.MagnumObject,
'volume_driver': fields.StringField(nullable=True),
'apiserver_port': fields.IntegerField(nullable=True),
'docker_volume_size': fields.IntegerField(nullable=True),
'docker_storage_driver': m_fields.DockerStorageDriverField(
nullable=True),
'cluster_distro': fields.StringField(nullable=True),
'coe': m_fields.BayTypeField(nullable=True),
'http_proxy': fields.StringField(nullable=True),

View File

@ -65,6 +65,18 @@ class BayType(fields.Enum):
super(BayType, self).__init__(valid_values=BayType.ALL)
class DockerStorageDriver(fields.Enum):
ALL = (
DEVICEMAPPER, OVERLAY,
) = (
'devicemapper', 'overlay',
)
def __init__(self):
super(DockerStorageDriver, self).__init__(
valid_values=DockerStorageDriver.ALL)
class ListOfDictsField(fields.AutoTypedField):
AUTO_TYPE = fields.List(fields.Dict(fields.FieldType()))
@ -79,3 +91,7 @@ class ContainerStatusField(fields.BaseEnumField):
class BayTypeField(fields.BaseEnumField):
AUTO_TYPE = BayType()
class DockerStorageDriverField(fields.BaseEnumField):
AUTO_TYPE = DockerStorageDriver()

View File

@ -0,0 +1,28 @@
configure_overlay () {
rm -rf /var/lib/docker/*
mkfs.xfs ${device_path}
echo "${device_path} /var/lib/docker xfs defaults 0 0" >> /etc/fstab
mount -a
echo "STORAGE_DRIVER=overlay" > /etc/sysconfig/docker-storage-setup
# SELinux must be enabled and in enforcing mode on the physical
# machine, but must be disabled in the container when performing
# container separation
sed -i "/^OPTIONS=/ s/--selinux-enabled/--selinux-enabled=false/" /etc/sysconfig/docker
}
configure_devicemapper () {
pvcreate ${device_path}
vgcreate docker ${device_path}
echo "VG=docker" > /etc/sysconfig/docker-storage-setup
STORAGE_CONF="--storage-driver devicemapper \
--storage-opt dm.fs=xfs \
--storage-opt dm.thinpooldev=/dev/mapper/docker-docker--pool \
--storage-opt dm.use_deferred_removal=true"
sed -i "/^DOCKER_STORAGE_OPTIONS=/ s#=.*#=${STORAGE_CONF}#" /etc/sysconfig/docker-storage
}

View File

@ -20,11 +20,18 @@ if [ -z "${device_name}" ]; then
fi
device_path=/dev/disk/by-id/${device_name}
pvcreate ${device_path}
vgcreate docker ${device_path}
cat > /etc/sysconfig/docker-storage-setup << EOF
VG=docker
EOF
$configure_docker_storage_driver
sed -i '/^DOCKER_STORAGE_OPTIONS=/ s/=.*/=--storage-driver devicemapper --storage-opt dm.fs=xfs --storage-opt dm.thinpooldev=\/dev\/mapper\/docker-docker--pool --storage-opt dm.use_deferred_removal=true/' /etc/sysconfig/docker-storage
if [ "$DOCKER_STORAGE_DRIVER" = "overlay" ]; then
if [ $(echo -e "$(uname -r)\n3.18" | sort -V | head -1) \
= $(uname -r) ]; then
ERROR_MESSAGE="OverlayFS requires at least Linux kernel 3.18. Bay node kernel version: $(uname -r)"
echo "ERROR: ${ERROR_MESSAGE}" >&2
sh -c "${WAIT_CURL} --data-binary '{\"status\": \"FAILURE\", \"reason\": \"${ERROR_MESSAGE}\"}'"
else
configure_overlay
fi
else
configure_devicemapper
fi

View File

@ -11,6 +11,7 @@ write_files:
KUBE_NODE_IP="$KUBE_NODE_IP"
KUBE_ALLOW_PRIV="$KUBE_ALLOW_PRIV"
DOCKER_VOLUME="$DOCKER_VOLUME"
DOCKER_STORAGE_DRIVER="$DOCKER_STORAGE_DRIVER"
NETWORK_DRIVER="$NETWORK_DRIVER"
FLANNEL_NETWORK_CIDR="$FLANNEL_NETWORK_CIDR"
FLANNEL_NETWORK_SUBNETLEN="$FLANNEL_NETWORK_SUBNETLEN"

View File

@ -11,6 +11,7 @@ write_files:
KUBE_NODE_IP="$KUBE_NODE_IP"
ETCD_SERVER_IP="$ETCD_SERVER_IP"
DOCKER_VOLUME="$DOCKER_VOLUME"
DOCKER_STORAGE_DRIVER="$DOCKER_STORAGE_DRIVER"
NETWORK_DRIVER="$NETWORK_DRIVER"
REGISTRY_ENABLED="$REGISTRY_ENABLED"
REGISTRY_PORT="$REGISTRY_PORT"

View File

@ -94,6 +94,13 @@ parameters:
storage
default: 25
docker_storage_driver:
type: string
description: docker storage driver name
default: "devicemapper"
constraints:
- allowed_values: ["devicemapper", "overlay"]
wait_condition_timeout:
type: number
description: >
@ -409,6 +416,7 @@ resources:
external_network: {get_param: external_network}
kube_allow_priv: {get_param: kube_allow_priv}
docker_volume_size: {get_param: docker_volume_size}
docker_storage_driver: {get_param: docker_storage_driver}
wait_condition_timeout: {get_param: wait_condition_timeout}
network_driver: {get_param: network_driver}
flannel_network_cidr: {get_param: flannel_network_cidr}
@ -467,6 +475,7 @@ resources:
external_network: {get_param: external_network}
kube_allow_priv: {get_param: kube_allow_priv}
docker_volume_size: {get_param: docker_volume_size}
docker_storage_driver: {get_param: docker_storage_driver}
wait_condition_timeout: {get_param: wait_condition_timeout}
registry_enabled: {get_param: registry_enabled}
registry_port: {get_param: registry_port}

View File

@ -41,6 +41,13 @@ parameters:
size of a cinder volume to allocate to docker for container/image
storage
docker_storage_driver:
type: string
description: docker storage driver name
default: "devicemapper"
constraints:
- allowed_values: ["devicemapper", "overlay"]
flannel_network_cidr:
type: string
description: network range for flannel overlay network
@ -206,6 +213,7 @@ resources:
"$KUBE_NODE_IP": {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
"$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv}
"$DOCKER_VOLUME": {get_resource: docker_volume}
"$DOCKER_STORAGE_DRIVER": {get_param: docker_storage_driver}
"$NETWORK_DRIVER": {get_param: network_driver}
"$FLANNEL_NETWORK_CIDR": {get_param: flannel_network_cidr}
"$FLANNEL_NETWORK_SUBNETLEN": {get_param: flannel_network_subnetlen}
@ -240,7 +248,11 @@ resources:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: fragments/configure-docker-storage.sh}
config:
str_replace:
params:
$configure_docker_storage_driver: {get_file: ../common/fragments/configure_docker_storage_driver_atomic.sh}
template: {get_file: fragments/configure-docker-storage.sh}
configure_etcd:
type: OS::Heat::SoftwareConfig

View File

@ -36,6 +36,13 @@ parameters:
size of a cinder volume to allocate to docker for container/image
storage
docker_storage_driver:
type: string
description: docker storage driver name
default: "devicemapper"
constraints:
- allowed_values: ["devicemapper", "overlay"]
tls_disabled:
type: boolean
description: whether or not to enable TLS
@ -223,6 +230,7 @@ resources:
$KUBE_NODE_IP: {get_attr: [kube_minion_eth0, fixed_ips, 0, ip_address]}
$ETCD_SERVER_IP: {get_param: etcd_server_ip}
$DOCKER_VOLUME: {get_resource: docker_volume}
$DOCKER_STORAGE_DRIVER: {get_param: docker_storage_driver}
$NETWORK_DRIVER: {get_param: network_driver}
$REGISTRY_ENABLED: {get_param: registry_enabled}
$REGISTRY_PORT: {get_param: registry_port}
@ -268,7 +276,11 @@ resources:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: fragments/configure-docker-storage.sh}
config:
str_replace:
params:
$configure_docker_storage_driver: {get_file: ../common/fragments/configure_docker_storage_driver_atomic.sh}
template: {get_file: fragments/configure-docker-storage.sh}
configure_docker_registry:
type: OS::Heat::SoftwareConfig

View File

@ -21,11 +21,18 @@ if [ -z "${device_name}" ]; then
fi
device_path=/dev/disk/by-id/${device_name}
pvcreate ${device_path}
vgcreate docker ${device_path}
cat > /etc/sysconfig/docker-storage-setup << EOF
VG=docker
EOF
$configure_docker_storage_driver
sed -i '/^DOCKER_STORAGE_OPTIONS=/ s/=.*/=--storage-driver devicemapper --storage-opt dm.fs=xfs --storage-opt dm.thinpooldev=\/dev\/mapper\/docker-docker--pool --storage-opt dm.use_deferred_removal=true/' /etc/sysconfig/docker-storage
if [ "$DOCKER_STORAGE_DRIVER" = "overlay" ]; then
if [ $(echo -e "$(uname -r)\n3.18" | sort -V | head -1) \
= $(uname -r) ]; then
ERROR_MESSAGE="OverlayFS requires at least Linux kernel 3.18. Bay node kernel version: $(uname -r)"
echo "ERROR: ${ERROR_MESSAGE}" >&2
sh -c "${WAIT_CURL} --data-binary '{\"status\": \"FAILURE\", \"reason\": \"${ERROR_MESSAGE}\"}'"
else
configure_overlay
fi
else
configure_devicemapper
fi

View File

@ -8,6 +8,7 @@ write_files:
WAIT_HANDLE="$WAIT_HANDLE"
ETCD_DISCOVERY_URL="$ETCD_DISCOVERY_URL"
DOCKER_VOLUME="$DOCKER_VOLUME"
DOCKER_STORAGE_DRIVER="$DOCKER_STORAGE_DRIVER"
HTTP_PROXY="$HTTP_PROXY"
HTTPS_PROXY="$HTTPS_PROXY"
NO_PROXY="$NO_PROXY"

View File

@ -7,6 +7,7 @@ write_files:
content: |
WAIT_HANDLE="$WAIT_HANDLE"
DOCKER_VOLUME="$DOCKER_VOLUME"
DOCKER_STORAGE_DRIVER="$DOCKER_STORAGE_DRIVER"
HTTP_PROXY="$HTTP_PROXY"
HTTPS_PROXY="$HTTPS_PROXY"
NO_PROXY="$NO_PROXY"

View File

@ -120,6 +120,13 @@ parameters:
storage
default: 25
docker_storage_driver:
type: string
description: docker storage driver name
default: "devicemapper"
constraints:
- allowed_values: ["devicemapper", "overlay"]
loadbalancing_protocol:
type: string
description: >
@ -330,6 +337,7 @@ resources:
server_image: {get_param: server_image}
server_flavor: {get_param: master_flavor}
docker_volume_size: {get_param: docker_volume_size}
docker_storage_driver: {get_param: docker_storage_driver}
fixed_network_id: {get_resource: fixed_network}
fixed_subnet_id: {get_resource: fixed_subnet}
external_network: {get_param: external_network}
@ -370,6 +378,7 @@ resources:
server_image: {get_param: server_image}
server_flavor: {get_param: node_flavor}
docker_volume_size: {get_param: docker_volume_size}
docker_storage_driver: {get_param: docker_storage_driver}
fixed_network_id: {get_resource: fixed_network}
fixed_subnet_id: {get_resource: fixed_subnet}
external_network: {get_param: external_network}

View File

@ -16,6 +16,12 @@ parameters:
size of a cinder volume to allocate to docker for container/image
storage
docker_storage_driver:
type: string
description: docker storage driver name
constraints:
- allowed_values: ["devicemapper", "overlay"]
external_network:
type: string
description: uuid/name of a network to use for floating ip addresses
@ -175,6 +181,7 @@ resources:
params:
"$WAIT_HANDLE": {get_resource: cloud_init_wait_handle}
"$DOCKER_VOLUME": {get_resource: docker_volume}
"$DOCKER_STORAGE_DRIVER": {get_param: docker_storage_driver}
"$ETCD_DISCOVERY_URL": {get_param: discovery_url}
"$HTTP_PROXY": {get_param: http_proxy}
"$HTTPS_PROXY": {get_param: https_proxy}
@ -230,7 +237,11 @@ resources:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: fragments/configure-docker-storage.sh}
config:
str_replace:
params:
$configure_docker_storage_driver: {get_file: ../common/fragments/configure_docker_storage_driver_atomic.sh}
template: {get_file: fragments/configure-docker-storage.sh}
make_cert:
type: "OS::Heat::SoftwareConfig"

View File

@ -25,6 +25,12 @@ parameters:
size of a cinder volume to allocate to docker for container/image
storage
docker_storage_driver:
type: string
description: docker storage driver name
constraints:
- allowed_values: ["devicemapper", "overlay"]
external_network:
type: string
description: uuid/name of a network to use for floating ip addresses
@ -187,6 +193,7 @@ resources:
params:
"$WAIT_HANDLE": {get_resource: node_cloud_init_wait_handle}
"$DOCKER_VOLUME": {get_resource: docker_volume}
"$DOCKER_STORAGE_DRIVER": {get_param: docker_storage_driver}
"$HTTP_PROXY": {get_param: http_proxy}
"$HTTPS_PROXY": {get_param: https_proxy}
"$NO_PROXY": {get_param: no_proxy}
@ -229,7 +236,11 @@ resources:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: fragments/configure-docker-storage.sh}
config:
str_replace:
params:
$configure_docker_storage_driver: {get_file: ../common/fragments/configure_docker_storage_driver_atomic.sh}
template: {get_file: fragments/configure-docker-storage.sh}
configure_docker_registry:
type: OS::Heat::SoftwareConfig

View File

@ -34,6 +34,7 @@ class TestBayConductorWithK8s(base.TestCase):
'network_driver': 'network_driver',
'volume_driver': 'volume_driver',
'docker_volume_size': 20,
'docker_storage_driver': 'devicemapper',
'cluster_distro': 'fedora-atomic',
'coe': 'kubernetes',
'token': None,
@ -108,6 +109,7 @@ class TestBayConductorWithK8s(base.TestCase):
'image_id': 'server_image',
'flavor_id': 'minion_flavor',
'docker_volume_size': 'docker_volume_size',
'docker_storage_driver': 'docker_storage_driver',
'network_driver': 'network_driver',
'volume_driver': 'volume_driver',
'master_flavor_id': 'master_flavor',
@ -138,6 +140,7 @@ class TestBayConductorWithK8s(base.TestCase):
'number_of_minions': 1,
'number_of_masters': 1,
'docker_volume_size': 20,
'docker_storage_driver': 'devicemapper',
'discovery_url': 'https://discovery.etcd.io/test',
'flannel_network_cidr': '10.101.0.0/16',
'flannel_network_subnetlen': '26',
@ -187,6 +190,7 @@ class TestBayConductorWithK8s(base.TestCase):
'bay_uuid': '5d12f6fd-a196-4bf0-ae4c-1f639a523a52',
'discovery_url': 'https://discovery.etcd.io/test',
'dns_nameserver': 'dns_nameserver',
'docker_storage_driver': 'devicemapper',
'docker_volume_size': 20,
'external_network': 'external_network_id',
'flannel_backend': 'vxlan',
@ -348,6 +352,14 @@ class TestBayConductorWithK8s(base.TestCase):
mock_objects_baymodel_get_by_uuid,
missing_attr='docker_volume_size')
@patch('magnum.objects.BayModel.get_by_uuid')
def test_extract_template_definition_without_docker_storage_driver(
self,
mock_objects_baymodel_get_by_uuid):
self._test_extract_template_definition(
mock_objects_baymodel_get_by_uuid,
missing_attr='docker_storage_driver')
@patch('magnum.objects.BayModel.get_by_uuid')
def test_extract_template_definition_without_master_flavor(
self,
@ -414,6 +426,7 @@ class TestBayConductorWithK8s(base.TestCase):
'network_driver': 'network_driver',
'volume_driver': 'volume_driver',
'docker_volume_size': 20,
'docker_storage_driver': 'devicemapper',
'discovery_url': 'https://address/token',
'http_proxy': 'http_proxy',
'https_proxy': 'https_proxy',

View File

@ -33,6 +33,7 @@ class TestBayConductorWithSwarm(base.TestCase):
'keypair_id': 'keypair_id',
'dns_nameserver': 'dns_nameserver',
'docker_volume_size': 20,
'docker_storage_driver': 'devicemapper',
'external_network_id': 'external_network_id',
'cluster_distro': 'fedora-atomic',
'coe': 'swarm',
@ -96,6 +97,7 @@ class TestBayConductorWithSwarm(base.TestCase):
'number_of_masters': 1,
'number_of_nodes': 1,
'docker_volume_size': 20,
'docker_storage_driver': 'devicemapper',
'discovery_url': 'https://discovery.test.io/123456789',
'http_proxy': 'http_proxy',
'https_proxy': 'https_proxy',
@ -163,7 +165,8 @@ class TestBayConductorWithSwarm(base.TestCase):
'trustee_password': 'fake_trustee_password',
'trustee_user_id': '7b489f04-b458-4541-8179-6a48a553e656',
'trust_id': 'bd11efc5-d4e2-4dac-bbce-25e348ddf7de',
'auth_url': 'http://192.168.10.10:5000/v3'
'auth_url': 'http://192.168.10.10:5000/v3',
'docker_storage_driver': 'devicemapper'
}
self.assertEqual(expected, definition)
@ -175,7 +178,7 @@ class TestBayConductorWithSwarm(base.TestCase):
not_required = ['image_id', 'flavor_id', 'dns_nameserver',
'docker_volume_size', 'fixed_network', 'http_proxy',
'https_proxy', 'no_proxy', 'network_driver',
'master_flavor_id']
'master_flavor_id', 'docker_storage_driver']
for key in not_required:
self.baymodel_dict[key] = None
self.bay_dict['discovery_url'] = 'https://discovery.etcd.io/test'

View File

@ -37,6 +37,8 @@ def get_test_baymodel(**kw):
'dns_nameserver': kw.get('dns_nameserver', '8.8.1.1'),
'apiserver_port': kw.get('apiserver_port', 8080),
'docker_volume_size': kw.get('docker_volume_size', 20),
'docker_storage_driver': kw.get('docker_storage_driver',
'devicemapper'),
'cluster_distro': kw.get('cluster_distro', 'fedora-atomic'),
'coe': kw.get('coe', 'swarm'),
'created_at': kw.get('created_at'),

View File

@ -358,7 +358,7 @@ class _TestObject(object):
# http://docs.openstack.org/developer/magnum/objects.html
object_data = {
'Bay': '1.5-a3b9292ef5d35175b93ca46ba3baec2d',
'BayModel': '1.11-ca90ce57d0c7e6ac4716dc2b7ed28e42',
'BayModel': '1.12-ab6910eba29f1af6e6660020228d31d0',
'Certificate': '1.0-2aff667971b85c1edf8d15684fd7d5e2',
'Container': '1.3-e2d9d2e8a8844d421148cd9fde6c6bd6',
'MyObj': '1.0-b43567e512438205e32f4e95ca616697',

View File

@ -0,0 +1,12 @@
---
features:
- Add docker-storage-driver parameter to baymodel to
allow user select from the supported drivers. Until
now, only devicemapper was supported. This release
adds support for OverlayFS on Fedora Atomic hosts with
kernel version >= 3.18 (Fedora 22 or higher) resulting
significant performance improvement. To use OverlayFS,
SELinux must be enabled and in enforcing mode on the
physical machine, but must be disabled in the container.
Thus, if you select overlay for docker-storage-driver
SELinux will be disable inside the containers.