Browse Source

add a glusterfs driver + heketi layout backend variant

Change-Id: I4bd7abc83605687fc5d990b54e728a113b4f37fe
Csaba Henk 3 years ago
parent
commit
701ce2bb77
5 changed files with 371 additions and 2 deletions
  1. 40
    1
      devstack/gluster-functions.sh
  2. 3
    0
      devstack/settings
  3. 35
    0
      extras/heketi.json
  4. 288
    0
      extras/heketisetup.py
  5. 5
    1
      manila/post_test_hook.sh

+ 40
- 1
devstack/gluster-functions.sh View File

@@ -342,9 +342,48 @@ function _configure_manila_glusterfs_native {
342 342
     iniset $MANILA_CONF DEFAULT enabled_share_backends $group_name
343 343
 }
344 344
 
345
+function _setup_rootssh {
346
+    mkdir -p "$HOME"/.ssh
347
+    chmod 700 "$HOME"/.ssh
348
+    sudo mkdir -p /root/.ssh
349
+    sudo chmod 700 /root/.ssh
350
+    yes n | ssh-keygen -f  "$HOME"/.ssh/id_rsa -N ''
351
+    sudo sh -c "cat >> /root/.ssh/authorized_keys" < "$HOME"/.ssh/id_rsa.pub
352
+    sudo chmod 600 /root/.ssh/authorized_keys
353
+}
354
+
355
+function _configure_setup_heketi {
356
+    # get Heketi and start service
357
+    wget "$HEKETI_V1_PACKAGE"
358
+    tar xvf "$(basename "$HEKETI_V1_PACKAGE")"
359
+    ( ./heketi/heketi -config "$GLUSTERFS_PLUGIN_DIR"/extras/heketi.json &>/dev/null & ) &
360
+
361
+    # basic Heketi setup
362
+    $GLUSTERFS_PLUGIN_DIR/extras/heketisetup.py -s 1T -n 3 -v -D $(hostname)
363
+}
364
+
365
+function _configure_manila_glusterfs_heketi {
366
+    _setup_rootssh
367
+    _configure_setup_heketi
368
+
369
+    # Manila config
370
+    local share_driver=manila.share.drivers.glusterfs.GlusterfsShareDriver
371
+    local group_name=glusterheketi1
372
+
373
+    iniset $MANILA_CONF $group_name share_driver $share_driver
374
+    iniset $MANILA_CONF $group_name share_backend_name GLUSTERFSHEKETI
375
+    iniset $MANILA_CONF $group_name driver_handles_share_servers False
376
+    iniset $MANILA_CONF $group_name glusterfs_share_layout layout_heketi.GlusterfsHeketiLayout
377
+    iniset $MANILA_CONF $group_name glusterfs_heketi_url http://localhost:8080
378
+    iniset $MANILA_CONF $group_name glusterfs_heketi_nodeadmin_username root
379
+    iniset $MANILA_CONF $group_name glusterfs_heketi_volume_replica 1
380
+}
381
+
345 382
 # Configure GlusterFS as a backend for Manila
346 383
 function configure_manila_backend_glusterfs {
347
-    if [[ "${GLUSTERFS_MANILA_DRIVER_TYPE}" == "glusterfs" ]]; then
384
+    if [[ "${GLUSTERFS_MANILA_DRIVER_TYPE}" == "glusterfs-heketi" ]]; then
385
+        _configure_manila_glusterfs_heketi
386
+    elif [[ "${GLUSTERFS_MANILA_DRIVER_TYPE}" == "glusterfs" ]]; then
348 387
         _configure_manila_glusterfs_nfs
349 388
     else
350 389
         _configure_manila_glusterfs_native

+ 3
- 0
devstack/settings View File

@@ -25,6 +25,9 @@ if [[ ${DISTRO} =~ rhel7 ]] && [[ ! -f /etc/yum.repos.d/glusterfs-epel.repo ]];
25 25
     GLUSTERFS_CENTOS_REPO=${GLUSTERFS_CENTOS_REPO:-"http://download.gluster.org/pub/gluster/glusterfs/LATEST/CentOS/glusterfs-epel.repo"}
26 26
 fi
27 27
 
28
+# Official Heketi 1.0.* binary
29
+HEKETI_V1_PACKAGE="https://github.com/heketi/heketi/releases/download/1.0.2/heketi-1.0.2-release-1.0.0.linux.amd64.tar.gz"
30
+
28 31
 TEMPEST_STORAGE_PROTOCOL=glusterfs
29 32
 
30 33
 ######### Glance Specific Configuration #########

+ 35
- 0
extras/heketi.json View File

@@ -0,0 +1,35 @@
1
+{
2
+        "_port_comment": "Heketi Server Port Number",
3
+        "port" : "8080",
4
+
5
+        "_use_auth": "Enable JWT authorization. Please enable for deployment",
6
+        "use_auth" : false,
7
+
8
+        "_jwt" : "Private keys for access",
9
+        "jwt" : {
10
+                "_admin" : "Admin has access to all APIs",
11
+                "admin" : {
12
+                        "key" : "My Secret"
13
+                },
14
+                "_user" : "User only has access to /volumes endpoint",
15
+                "user" : {
16
+                        "key" : "My Secret"
17
+                }
18
+        },
19
+
20
+        "_glusterfs_comment": "GlusterFS Configuration",
21
+        "glusterfs" : {
22
+
23
+                "_executor_comment": "Execute plugin. Possible choices: mock, ssh",
24
+                "executor" : "ssh",
25
+
26
+                "_db_comment": "Database file name",
27
+                "db" : "heketi.db",
28
+
29
+                "sshexec": {
30
+                        "user": "root"
31
+                },
32
+                "brick_min_size_gb": 1
33
+
34
+        }
35
+}

+ 288
- 0
extras/heketisetup.py View File

@@ -0,0 +1,288 @@
1
+#!/usr/bin/env python
2
+
3
+# Copyright (c) 2016 Red Hat, Inc.
4
+# All Rights Reserved.
5
+#
6
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
7
+#    not use this file except in compliance with the License. You may obtain
8
+#    a copy of the License at
9
+#
10
+#         http://www.apache.org/licenses/LICENSE-2.0
11
+#
12
+#    Unless required by applicable law or agreed to in writing, software
13
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15
+#    License for the specific language governing permissions and limitations
16
+#    under the License.
17
+
18
+from __future__ import print_function
19
+import datetime
20
+import hashlib
21
+import pipes
22
+import re
23
+import subprocess
24
+import sys
25
+import time
26
+
27
+try:
28
+    import jwt
29
+except ImportError:
30
+    print("jwt module not found, Heketi JWT auth won't work.\n"
31
+          "You can install it with 'pip install PyJWT'.\n", file=sys.stderr)
32
+import requests
33
+
34
+
35
+class Objlog(object):
36
+
37
+    def __init__(self, obj):
38
+        self.object = obj
39
+        self.__log__ = []
40
+
41
+    def __getattr__(self, att):
42
+        av = getattr(self.object, att)
43
+
44
+        if not hasattr(av, '__call__'):
45
+            return av
46
+
47
+        def alog(*a, **kw):
48
+            try:
49
+                ret = av(*a, **kw)
50
+            except Exception as x:
51
+                self.__log__.append(((att, a, kw), None, x))
52
+                raise x
53
+            self.__log__.append(((att, a, kw), ret))
54
+            return ret
55
+
56
+        return alog
57
+
58
+    def __reset__(self):
59
+        self.__log__ = []
60
+
61
+
62
+def objlog_get(ol):
63
+    return ol.__log__
64
+
65
+
66
+def reqlog(req, lower=0, upper=None):
67
+    ol = objlog_get(req)
68
+    if upper is None:
69
+        upper = len(ol)
70
+    for i in range(lower, upper):
71
+        e = ol[i]
72
+        print(i, e[0])
73
+        print(e[1], e[1].headers, e[1].text)
74
+
75
+
76
+class HeketiException(Exception):
77
+    pass
78
+
79
+
80
+def jwt_token(method, path, key, issuer="admin", delta={'minutes': 5}):
81
+    method = method.upper()
82
+    path = re.sub("\A/+", "/", "/" + path)
83
+
84
+    claims = {}
85
+
86
+    # Issuer
87
+    claims['iss'] = issuer
88
+
89
+    # Issued at time
90
+    claims['iat'] = datetime.datetime.utcnow()
91
+
92
+    # Expiration time
93
+    claims['exp'] = (datetime.datetime.utcnow() +
94
+                     datetime.timedelta(**delta))
95
+
96
+    # URI tampering protection
97
+    claims['qsh'] = hashlib.sha256(method + '&' + path).hexdigest()
98
+
99
+    return jwt.encode(claims, key, algorithm='HS256')
100
+
101
+
102
+class HeketiClient(object):
103
+    """A client class for the Heteki GlusterFS management service."""
104
+
105
+    def __init__(self, host, requests_like=requests, jwt_key=None):
106
+        host_stripped = re.sub("/+\Z", "", host)
107
+        self.__host__ = host_stripped
108
+        self.__requests__ = requests_like
109
+        self.__jwt_key__ = jwt_key
110
+
111
+    def __getattr__(self, attr):
112
+        attr_value = getattr(self.__requests__, attr)
113
+
114
+        if not hasattr(attr_value, '__call__'):
115
+            return attr_value
116
+
117
+        def _attr_value_host_injected(*a, **kw):
118
+            if len(a) == 0:
119
+                return attr_value(*a, **kw)
120
+            else:
121
+                path = a[0]
122
+                path_stripped = re.sub("\A/+", "", path)
123
+                req_url = "/".join((self.__host__, path_stripped))
124
+                if self.__jwt_key__:
125
+                    token = jwt_token(attr, path, self.__jwt_key__)
126
+                    kw['headers'] = {"Authorization": "bearer %s" % token}
127
+                report("Heketi request %(method)s to %(url)s" % {
128
+                    'method': attr.upper(), 'url': req_url})
129
+                resp = attr_value(req_url, *a[1:], **kw)
130
+                report("Heketi response: %s" % repr(resp))
131
+                return resp
132
+
133
+        return _attr_value_host_injected
134
+
135
+    def asyncop(self, *a, **kw):
136
+        method = kw.pop('method', 'post')
137
+        retry_interval = kw.pop('retry_interval', 1)
138
+        resp = getattr(self, method)(*a, **kw)
139
+        if resp.status_code != 202:
140
+            resp.raise_for_status()
141
+            raise HeketiException((
142
+                'Unexpected Heketi async %(method)s status %(status)d'
143
+            ) % {'method': method.upper(), 'status': resp.status_code})
144
+        queue = resp.headers['location']
145
+        while True:
146
+            resp = self.get(queue)
147
+            if resp.status_code == 204:
148
+                return resp
149
+            elif resp.status_code == 303:
150
+                return self.get(resp.headers['location'])
151
+            elif resp.status_code == 200:
152
+                if 'x-pending' not in resp.headers:
153
+                    return resp
154
+            else:
155
+                resp.raise_for_status()
156
+                raise HeketiException((
157
+                   'Unexpected Heketi async queue status %d'
158
+                ) % resp.status_code)
159
+            time.sleep(retry_interval)
160
+
161
+
162
+class ShExec(object):
163
+
164
+    def __init__(self, host, user=None, key=None, root=False):
165
+        self.host = host
166
+        self.root = root
167
+        self.args = []
168
+        if host == "localhost":
169
+            self.args = ["sh", "-c"]
170
+        else:
171
+            self.args = ["ssh", host]
172
+            if user:
173
+                self.args.extend(["-l", user])
174
+            if key:
175
+                self.args.extend(["-i", key])
176
+
177
+    def __call__(self, cmd):
178
+        if not self.root:
179
+            cmd = "sudo sh -c " + pipes.quote(cmd)
180
+        report("Running on %(host)s: %(cmd)s" % {
181
+               'cmd': cmd, 'host': self.host})
182
+        po = subprocess.Popen(self.args + [cmd],
183
+                              stdout=subprocess.PIPE, stderr=subprocess.PIPE)
184
+        out, err = po.communicate()
185
+        report("Command output: %s" % out, cond=out)
186
+        if po.returncode:
187
+            raise RuntimeError("%(cmd)s has failed with %(exit)d" % {
188
+                'cmd': ' '.join(self.args) + ' ' + cmd,
189
+                'exit': po.returncode})
190
+        return out, err
191
+
192
+
193
+def setup(clusters, args, h):
194
+    cluster = args.cluster
195
+    if not cluster and clusters:
196
+        cluster = clusters[0]
197
+    if cluster not in clusters:
198
+        if re.match('[\da-f]{8}(-?[\da-f]{4}){3}-?[\da-f]{12}\Z',
199
+                    cluster or '', re.I):
200
+            raise HeketiException(
201
+                "Cluster %s not found on Heketi server." % cluster)
202
+        cluster = None
203
+    if not cluster:
204
+        cluster = h.post("clusters").json()['id']
205
+    report("Using cluster %s" % cluster)
206
+
207
+    hostdevices = {}
208
+    for host in args.host:
209
+        hostdevices[host] = []
210
+        shx = ShExec(host, user=args.user, key=args.key, root=args.root)
211
+        for i in range(args.devices):
212
+            dev, _ = shx("i=0; while [ -f /LOOP%(cluster)s-$i ]; do i=$(($i+1)); done && "
213
+                         "truncate -s %(size)s /LOOP%(cluster)s-$i && "
214
+                         "losetup -f --show /LOOP%(cluster)s-$i" % {'size': args.size,
215
+                         'cluster': cluster})
216
+            hostdevices[host].append(dev.strip())
217
+
218
+    for host, devices in hostdevices.items():
219
+        # add a node
220
+        node = h.asyncop("nodes", json={
221
+                  "zone": 1,
222
+                  "hostnames": {"manage": [host], "storage": [host]},
223
+                  "cluster": cluster}).json()
224
+        for dev in devices:
225
+            # add a device
226
+            h.asyncop("devices", json={"node": node['id'], "name": dev})
227
+
228
+
229
+def teardown(cliusters, args, h):
230
+    if args.cluster not in clusters:
231
+        raise HeketiException(
232
+            "Cluster %s not found on Heketi server." % args.cluster)
233
+
234
+    cluster = h.get("clusters/%s" % args.cluster).json()
235
+    for vol in cluster["volumes"]:
236
+        h.asyncop('volumes/%s' % vol, method='delete')
237
+    for nodeid in cluster["nodes"]:
238
+        node = h.get("nodes/%s" % nodeid).json()
239
+        for dev in node["devices"]:
240
+            h.asyncop('devices/%s' % dev['id'], method='delete')
241
+        h.asyncop('nodes/%s' % nodeid, method='delete')
242
+    h.delete("clusters/%s" % args.cluster)
243
+
244
+if __name__ == '__main__':
245
+    import argparse
246
+
247
+    parser = argparse.ArgumentParser()
248
+    parser.add_argument("host", nargs='+')
249
+    parser.add_argument("-v", "--verbose", action='store_true')
250
+    parser.add_argument("-H", "--heketi", default="http://localhost:8080",
251
+                        help="Heketi service URL")
252
+    parser.add_argument("-s", "--size", help="size of devices created")
253
+    parser.add_argument("-n", "--devices",
254
+                        help="number of devices created per node", type=int)
255
+    parser.add_argument("-c", "--cluster", help="Heketi cluster to use")
256
+    parser.add_argument("-j", "--jwt", help="JWT key to use with Heketi auth")
257
+    parser.add_argument("-u", "--user", default="heketi",
258
+                        help="user with which nodes are managed")
259
+    parser.add_argument("-k", "--key", help="SSH key used to log in remotely")
260
+    parser.add_argument("--root", action='store_true',
261
+                        help="remote user has root privileges")
262
+    parser.add_argument("-A", "--action", choices=('setup', 'teardown'),
263
+                        default='setup')
264
+    parser.add_argument("-D", "--debug", action='store_true')
265
+    args = parser.parse_args()
266
+
267
+    if args.jwt:
268
+        jwt
269
+    if args.verbose:
270
+        def report(*a, **kw):
271
+            if not kw.pop('cond', True):
272
+                return
273
+            print(*a, **kw)
274
+    else:
275
+        report = lambda *a, **kw: None
276
+
277
+    req = requests.session()
278
+    if args.debug:
279
+        req = Objlog(req)
280
+    heketi = HeketiClient(args.heketi, requests_like=req, jwt_key=args.jwt)
281
+
282
+    try:
283
+        # get a cluster
284
+        clusters = heketi.get("clusters").json()['clusters']
285
+        getattr(sys.modules[__name__], args.action)(clusters, args, heketi)
286
+    finally:
287
+        if args.debug:
288
+            reqlog(req)

+ 5
- 1
manila/post_test_hook.sh View File

@@ -35,7 +35,11 @@ if [[ "$JOB_NAME" =~ "glusterfs-native" ]]; then
35 35
     iniset $TEMPEST_CONFIG share enable_cert_rules_for_protocols glusterfs
36 36
     iniset $TEMPEST_CONFIG share capability_snapshot_support True
37 37
 else
38
-    local BACKEND_NAME="GLUSTERFS"
38
+    if [[ "$JOB_NAME" =~ "glusterfs-heketi" ]]; then
39
+        local BACKEND_NAME="GLUSTERFSHEKETI"
40
+    else
41
+        local BACKEND_NAME="GLUSTERFS"
42
+    fi
39 43
     iniset $TEMPEST_CONFIG share enable_protocols nfs
40 44
     iniset $TEMPEST_CONFIG share enable_ip_rules_for_protocols nfs
41 45
     iniset $TEMPEST_CONFIG share storage_protocol NFS

Loading…
Cancel
Save