add a glusterfs driver + heketi layout backend variant
Change-Id: I4bd7abc83605687fc5d990b54e728a113b4f37fe
This commit is contained in:
parent
2cfbc1db61
commit
701ce2bb77
|
@ -342,9 +342,48 @@ function _configure_manila_glusterfs_native {
|
||||||
iniset $MANILA_CONF DEFAULT enabled_share_backends $group_name
|
iniset $MANILA_CONF DEFAULT enabled_share_backends $group_name
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function _setup_rootssh {
|
||||||
|
mkdir -p "$HOME"/.ssh
|
||||||
|
chmod 700 "$HOME"/.ssh
|
||||||
|
sudo mkdir -p /root/.ssh
|
||||||
|
sudo chmod 700 /root/.ssh
|
||||||
|
yes n | ssh-keygen -f "$HOME"/.ssh/id_rsa -N ''
|
||||||
|
sudo sh -c "cat >> /root/.ssh/authorized_keys" < "$HOME"/.ssh/id_rsa.pub
|
||||||
|
sudo chmod 600 /root/.ssh/authorized_keys
|
||||||
|
}
|
||||||
|
|
||||||
|
function _configure_setup_heketi {
|
||||||
|
# get Heketi and start service
|
||||||
|
wget "$HEKETI_V1_PACKAGE"
|
||||||
|
tar xvf "$(basename "$HEKETI_V1_PACKAGE")"
|
||||||
|
( ./heketi/heketi -config "$GLUSTERFS_PLUGIN_DIR"/extras/heketi.json &>/dev/null & ) &
|
||||||
|
|
||||||
|
# basic Heketi setup
|
||||||
|
$GLUSTERFS_PLUGIN_DIR/extras/heketisetup.py -s 1T -n 3 -v -D $(hostname)
|
||||||
|
}
|
||||||
|
|
||||||
|
function _configure_manila_glusterfs_heketi {
|
||||||
|
_setup_rootssh
|
||||||
|
_configure_setup_heketi
|
||||||
|
|
||||||
|
# Manila config
|
||||||
|
local share_driver=manila.share.drivers.glusterfs.GlusterfsShareDriver
|
||||||
|
local group_name=glusterheketi1
|
||||||
|
|
||||||
|
iniset $MANILA_CONF $group_name share_driver $share_driver
|
||||||
|
iniset $MANILA_CONF $group_name share_backend_name GLUSTERFSHEKETI
|
||||||
|
iniset $MANILA_CONF $group_name driver_handles_share_servers False
|
||||||
|
iniset $MANILA_CONF $group_name glusterfs_share_layout layout_heketi.GlusterfsHeketiLayout
|
||||||
|
iniset $MANILA_CONF $group_name glusterfs_heketi_url http://localhost:8080
|
||||||
|
iniset $MANILA_CONF $group_name glusterfs_heketi_nodeadmin_username root
|
||||||
|
iniset $MANILA_CONF $group_name glusterfs_heketi_volume_replica 1
|
||||||
|
}
|
||||||
|
|
||||||
# Configure GlusterFS as a backend for Manila
|
# Configure GlusterFS as a backend for Manila
|
||||||
function configure_manila_backend_glusterfs {
|
function configure_manila_backend_glusterfs {
|
||||||
if [[ "${GLUSTERFS_MANILA_DRIVER_TYPE}" == "glusterfs" ]]; then
|
if [[ "${GLUSTERFS_MANILA_DRIVER_TYPE}" == "glusterfs-heketi" ]]; then
|
||||||
|
_configure_manila_glusterfs_heketi
|
||||||
|
elif [[ "${GLUSTERFS_MANILA_DRIVER_TYPE}" == "glusterfs" ]]; then
|
||||||
_configure_manila_glusterfs_nfs
|
_configure_manila_glusterfs_nfs
|
||||||
else
|
else
|
||||||
_configure_manila_glusterfs_native
|
_configure_manila_glusterfs_native
|
||||||
|
|
|
@ -25,6 +25,9 @@ if [[ ${DISTRO} =~ rhel7 ]] && [[ ! -f /etc/yum.repos.d/glusterfs-epel.repo ]];
|
||||||
GLUSTERFS_CENTOS_REPO=${GLUSTERFS_CENTOS_REPO:-"http://download.gluster.org/pub/gluster/glusterfs/LATEST/CentOS/glusterfs-epel.repo"}
|
GLUSTERFS_CENTOS_REPO=${GLUSTERFS_CENTOS_REPO:-"http://download.gluster.org/pub/gluster/glusterfs/LATEST/CentOS/glusterfs-epel.repo"}
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Official Heketi 1.0.* binary
|
||||||
|
HEKETI_V1_PACKAGE="https://github.com/heketi/heketi/releases/download/1.0.2/heketi-1.0.2-release-1.0.0.linux.amd64.tar.gz"
|
||||||
|
|
||||||
TEMPEST_STORAGE_PROTOCOL=glusterfs
|
TEMPEST_STORAGE_PROTOCOL=glusterfs
|
||||||
|
|
||||||
######### Glance Specific Configuration #########
|
######### Glance Specific Configuration #########
|
||||||
|
|
|
@ -0,0 +1,35 @@
|
||||||
|
{
|
||||||
|
"_port_comment": "Heketi Server Port Number",
|
||||||
|
"port" : "8080",
|
||||||
|
|
||||||
|
"_use_auth": "Enable JWT authorization. Please enable for deployment",
|
||||||
|
"use_auth" : false,
|
||||||
|
|
||||||
|
"_jwt" : "Private keys for access",
|
||||||
|
"jwt" : {
|
||||||
|
"_admin" : "Admin has access to all APIs",
|
||||||
|
"admin" : {
|
||||||
|
"key" : "My Secret"
|
||||||
|
},
|
||||||
|
"_user" : "User only has access to /volumes endpoint",
|
||||||
|
"user" : {
|
||||||
|
"key" : "My Secret"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
"_glusterfs_comment": "GlusterFS Configuration",
|
||||||
|
"glusterfs" : {
|
||||||
|
|
||||||
|
"_executor_comment": "Execute plugin. Possible choices: mock, ssh",
|
||||||
|
"executor" : "ssh",
|
||||||
|
|
||||||
|
"_db_comment": "Database file name",
|
||||||
|
"db" : "heketi.db",
|
||||||
|
|
||||||
|
"sshexec": {
|
||||||
|
"user": "root"
|
||||||
|
},
|
||||||
|
"brick_min_size_gb": 1
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,288 @@
|
||||||
|
#!/usr/bin/env python
|
||||||
|
|
||||||
|
# Copyright (c) 2016 Red Hat, Inc.
|
||||||
|
# All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from __future__ import print_function
|
||||||
|
import datetime
|
||||||
|
import hashlib
|
||||||
|
import pipes
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
|
||||||
|
try:
|
||||||
|
import jwt
|
||||||
|
except ImportError:
|
||||||
|
print("jwt module not found, Heketi JWT auth won't work.\n"
|
||||||
|
"You can install it with 'pip install PyJWT'.\n", file=sys.stderr)
|
||||||
|
import requests
|
||||||
|
|
||||||
|
|
||||||
|
class Objlog(object):
|
||||||
|
|
||||||
|
def __init__(self, obj):
|
||||||
|
self.object = obj
|
||||||
|
self.__log__ = []
|
||||||
|
|
||||||
|
def __getattr__(self, att):
|
||||||
|
av = getattr(self.object, att)
|
||||||
|
|
||||||
|
if not hasattr(av, '__call__'):
|
||||||
|
return av
|
||||||
|
|
||||||
|
def alog(*a, **kw):
|
||||||
|
try:
|
||||||
|
ret = av(*a, **kw)
|
||||||
|
except Exception as x:
|
||||||
|
self.__log__.append(((att, a, kw), None, x))
|
||||||
|
raise x
|
||||||
|
self.__log__.append(((att, a, kw), ret))
|
||||||
|
return ret
|
||||||
|
|
||||||
|
return alog
|
||||||
|
|
||||||
|
def __reset__(self):
|
||||||
|
self.__log__ = []
|
||||||
|
|
||||||
|
|
||||||
|
def objlog_get(ol):
|
||||||
|
return ol.__log__
|
||||||
|
|
||||||
|
|
||||||
|
def reqlog(req, lower=0, upper=None):
|
||||||
|
ol = objlog_get(req)
|
||||||
|
if upper is None:
|
||||||
|
upper = len(ol)
|
||||||
|
for i in range(lower, upper):
|
||||||
|
e = ol[i]
|
||||||
|
print(i, e[0])
|
||||||
|
print(e[1], e[1].headers, e[1].text)
|
||||||
|
|
||||||
|
|
||||||
|
class HeketiException(Exception):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def jwt_token(method, path, key, issuer="admin", delta={'minutes': 5}):
|
||||||
|
method = method.upper()
|
||||||
|
path = re.sub("\A/+", "/", "/" + path)
|
||||||
|
|
||||||
|
claims = {}
|
||||||
|
|
||||||
|
# Issuer
|
||||||
|
claims['iss'] = issuer
|
||||||
|
|
||||||
|
# Issued at time
|
||||||
|
claims['iat'] = datetime.datetime.utcnow()
|
||||||
|
|
||||||
|
# Expiration time
|
||||||
|
claims['exp'] = (datetime.datetime.utcnow() +
|
||||||
|
datetime.timedelta(**delta))
|
||||||
|
|
||||||
|
# URI tampering protection
|
||||||
|
claims['qsh'] = hashlib.sha256(method + '&' + path).hexdigest()
|
||||||
|
|
||||||
|
return jwt.encode(claims, key, algorithm='HS256')
|
||||||
|
|
||||||
|
|
||||||
|
class HeketiClient(object):
|
||||||
|
"""A client class for the Heteki GlusterFS management service."""
|
||||||
|
|
||||||
|
def __init__(self, host, requests_like=requests, jwt_key=None):
|
||||||
|
host_stripped = re.sub("/+\Z", "", host)
|
||||||
|
self.__host__ = host_stripped
|
||||||
|
self.__requests__ = requests_like
|
||||||
|
self.__jwt_key__ = jwt_key
|
||||||
|
|
||||||
|
def __getattr__(self, attr):
|
||||||
|
attr_value = getattr(self.__requests__, attr)
|
||||||
|
|
||||||
|
if not hasattr(attr_value, '__call__'):
|
||||||
|
return attr_value
|
||||||
|
|
||||||
|
def _attr_value_host_injected(*a, **kw):
|
||||||
|
if len(a) == 0:
|
||||||
|
return attr_value(*a, **kw)
|
||||||
|
else:
|
||||||
|
path = a[0]
|
||||||
|
path_stripped = re.sub("\A/+", "", path)
|
||||||
|
req_url = "/".join((self.__host__, path_stripped))
|
||||||
|
if self.__jwt_key__:
|
||||||
|
token = jwt_token(attr, path, self.__jwt_key__)
|
||||||
|
kw['headers'] = {"Authorization": "bearer %s" % token}
|
||||||
|
report("Heketi request %(method)s to %(url)s" % {
|
||||||
|
'method': attr.upper(), 'url': req_url})
|
||||||
|
resp = attr_value(req_url, *a[1:], **kw)
|
||||||
|
report("Heketi response: %s" % repr(resp))
|
||||||
|
return resp
|
||||||
|
|
||||||
|
return _attr_value_host_injected
|
||||||
|
|
||||||
|
def asyncop(self, *a, **kw):
|
||||||
|
method = kw.pop('method', 'post')
|
||||||
|
retry_interval = kw.pop('retry_interval', 1)
|
||||||
|
resp = getattr(self, method)(*a, **kw)
|
||||||
|
if resp.status_code != 202:
|
||||||
|
resp.raise_for_status()
|
||||||
|
raise HeketiException((
|
||||||
|
'Unexpected Heketi async %(method)s status %(status)d'
|
||||||
|
) % {'method': method.upper(), 'status': resp.status_code})
|
||||||
|
queue = resp.headers['location']
|
||||||
|
while True:
|
||||||
|
resp = self.get(queue)
|
||||||
|
if resp.status_code == 204:
|
||||||
|
return resp
|
||||||
|
elif resp.status_code == 303:
|
||||||
|
return self.get(resp.headers['location'])
|
||||||
|
elif resp.status_code == 200:
|
||||||
|
if 'x-pending' not in resp.headers:
|
||||||
|
return resp
|
||||||
|
else:
|
||||||
|
resp.raise_for_status()
|
||||||
|
raise HeketiException((
|
||||||
|
'Unexpected Heketi async queue status %d'
|
||||||
|
) % resp.status_code)
|
||||||
|
time.sleep(retry_interval)
|
||||||
|
|
||||||
|
|
||||||
|
class ShExec(object):
|
||||||
|
|
||||||
|
def __init__(self, host, user=None, key=None, root=False):
|
||||||
|
self.host = host
|
||||||
|
self.root = root
|
||||||
|
self.args = []
|
||||||
|
if host == "localhost":
|
||||||
|
self.args = ["sh", "-c"]
|
||||||
|
else:
|
||||||
|
self.args = ["ssh", host]
|
||||||
|
if user:
|
||||||
|
self.args.extend(["-l", user])
|
||||||
|
if key:
|
||||||
|
self.args.extend(["-i", key])
|
||||||
|
|
||||||
|
def __call__(self, cmd):
|
||||||
|
if not self.root:
|
||||||
|
cmd = "sudo sh -c " + pipes.quote(cmd)
|
||||||
|
report("Running on %(host)s: %(cmd)s" % {
|
||||||
|
'cmd': cmd, 'host': self.host})
|
||||||
|
po = subprocess.Popen(self.args + [cmd],
|
||||||
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||||
|
out, err = po.communicate()
|
||||||
|
report("Command output: %s" % out, cond=out)
|
||||||
|
if po.returncode:
|
||||||
|
raise RuntimeError("%(cmd)s has failed with %(exit)d" % {
|
||||||
|
'cmd': ' '.join(self.args) + ' ' + cmd,
|
||||||
|
'exit': po.returncode})
|
||||||
|
return out, err
|
||||||
|
|
||||||
|
|
||||||
|
def setup(clusters, args, h):
|
||||||
|
cluster = args.cluster
|
||||||
|
if not cluster and clusters:
|
||||||
|
cluster = clusters[0]
|
||||||
|
if cluster not in clusters:
|
||||||
|
if re.match('[\da-f]{8}(-?[\da-f]{4}){3}-?[\da-f]{12}\Z',
|
||||||
|
cluster or '', re.I):
|
||||||
|
raise HeketiException(
|
||||||
|
"Cluster %s not found on Heketi server." % cluster)
|
||||||
|
cluster = None
|
||||||
|
if not cluster:
|
||||||
|
cluster = h.post("clusters").json()['id']
|
||||||
|
report("Using cluster %s" % cluster)
|
||||||
|
|
||||||
|
hostdevices = {}
|
||||||
|
for host in args.host:
|
||||||
|
hostdevices[host] = []
|
||||||
|
shx = ShExec(host, user=args.user, key=args.key, root=args.root)
|
||||||
|
for i in range(args.devices):
|
||||||
|
dev, _ = shx("i=0; while [ -f /LOOP%(cluster)s-$i ]; do i=$(($i+1)); done && "
|
||||||
|
"truncate -s %(size)s /LOOP%(cluster)s-$i && "
|
||||||
|
"losetup -f --show /LOOP%(cluster)s-$i" % {'size': args.size,
|
||||||
|
'cluster': cluster})
|
||||||
|
hostdevices[host].append(dev.strip())
|
||||||
|
|
||||||
|
for host, devices in hostdevices.items():
|
||||||
|
# add a node
|
||||||
|
node = h.asyncop("nodes", json={
|
||||||
|
"zone": 1,
|
||||||
|
"hostnames": {"manage": [host], "storage": [host]},
|
||||||
|
"cluster": cluster}).json()
|
||||||
|
for dev in devices:
|
||||||
|
# add a device
|
||||||
|
h.asyncop("devices", json={"node": node['id'], "name": dev})
|
||||||
|
|
||||||
|
|
||||||
|
def teardown(cliusters, args, h):
|
||||||
|
if args.cluster not in clusters:
|
||||||
|
raise HeketiException(
|
||||||
|
"Cluster %s not found on Heketi server." % args.cluster)
|
||||||
|
|
||||||
|
cluster = h.get("clusters/%s" % args.cluster).json()
|
||||||
|
for vol in cluster["volumes"]:
|
||||||
|
h.asyncop('volumes/%s' % vol, method='delete')
|
||||||
|
for nodeid in cluster["nodes"]:
|
||||||
|
node = h.get("nodes/%s" % nodeid).json()
|
||||||
|
for dev in node["devices"]:
|
||||||
|
h.asyncop('devices/%s' % dev['id'], method='delete')
|
||||||
|
h.asyncop('nodes/%s' % nodeid, method='delete')
|
||||||
|
h.delete("clusters/%s" % args.cluster)
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
import argparse
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("host", nargs='+')
|
||||||
|
parser.add_argument("-v", "--verbose", action='store_true')
|
||||||
|
parser.add_argument("-H", "--heketi", default="http://localhost:8080",
|
||||||
|
help="Heketi service URL")
|
||||||
|
parser.add_argument("-s", "--size", help="size of devices created")
|
||||||
|
parser.add_argument("-n", "--devices",
|
||||||
|
help="number of devices created per node", type=int)
|
||||||
|
parser.add_argument("-c", "--cluster", help="Heketi cluster to use")
|
||||||
|
parser.add_argument("-j", "--jwt", help="JWT key to use with Heketi auth")
|
||||||
|
parser.add_argument("-u", "--user", default="heketi",
|
||||||
|
help="user with which nodes are managed")
|
||||||
|
parser.add_argument("-k", "--key", help="SSH key used to log in remotely")
|
||||||
|
parser.add_argument("--root", action='store_true',
|
||||||
|
help="remote user has root privileges")
|
||||||
|
parser.add_argument("-A", "--action", choices=('setup', 'teardown'),
|
||||||
|
default='setup')
|
||||||
|
parser.add_argument("-D", "--debug", action='store_true')
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
if args.jwt:
|
||||||
|
jwt
|
||||||
|
if args.verbose:
|
||||||
|
def report(*a, **kw):
|
||||||
|
if not kw.pop('cond', True):
|
||||||
|
return
|
||||||
|
print(*a, **kw)
|
||||||
|
else:
|
||||||
|
report = lambda *a, **kw: None
|
||||||
|
|
||||||
|
req = requests.session()
|
||||||
|
if args.debug:
|
||||||
|
req = Objlog(req)
|
||||||
|
heketi = HeketiClient(args.heketi, requests_like=req, jwt_key=args.jwt)
|
||||||
|
|
||||||
|
try:
|
||||||
|
# get a cluster
|
||||||
|
clusters = heketi.get("clusters").json()['clusters']
|
||||||
|
getattr(sys.modules[__name__], args.action)(clusters, args, heketi)
|
||||||
|
finally:
|
||||||
|
if args.debug:
|
||||||
|
reqlog(req)
|
|
@ -35,7 +35,11 @@ if [[ "$JOB_NAME" =~ "glusterfs-native" ]]; then
|
||||||
iniset $TEMPEST_CONFIG share enable_cert_rules_for_protocols glusterfs
|
iniset $TEMPEST_CONFIG share enable_cert_rules_for_protocols glusterfs
|
||||||
iniset $TEMPEST_CONFIG share capability_snapshot_support True
|
iniset $TEMPEST_CONFIG share capability_snapshot_support True
|
||||||
else
|
else
|
||||||
local BACKEND_NAME="GLUSTERFS"
|
if [[ "$JOB_NAME" =~ "glusterfs-heketi" ]]; then
|
||||||
|
local BACKEND_NAME="GLUSTERFSHEKETI"
|
||||||
|
else
|
||||||
|
local BACKEND_NAME="GLUSTERFS"
|
||||||
|
fi
|
||||||
iniset $TEMPEST_CONFIG share enable_protocols nfs
|
iniset $TEMPEST_CONFIG share enable_protocols nfs
|
||||||
iniset $TEMPEST_CONFIG share enable_ip_rules_for_protocols nfs
|
iniset $TEMPEST_CONFIG share enable_ip_rules_for_protocols nfs
|
||||||
iniset $TEMPEST_CONFIG share storage_protocol NFS
|
iniset $TEMPEST_CONFIG share storage_protocol NFS
|
||||||
|
|
Loading…
Reference in New Issue