Browse Source

Add mount automation example based on Zaqar

This example consists of three parts:
- Server side hook module
'contrib.share_driver_hooks.zaqar_notification' that can be enabled in Manila
- Client side consumer module
'contrib.share_driver_hooks.zaqar_notification_example_consumer' that can be
used in any user machine.
- Common module 'contrib.share_driver_hooks.zaqarclientwrapper' that is used
by server and client side modules for initialization of Zaqar client.

Details of its usage are described in file
'contrib/share_driver_hooks/README.rst'

Change-Id: I5e802ee2e2a4dd36db92865b0ba82e73c1fa86d4
tags/2.0.0.0b1
Valeriy Ponomaryov 3 years ago
parent
commit
2f4795f7fe

+ 113
- 0
contrib/share_driver_hooks/README.rst View File

@@ -0,0 +1,113 @@
1
+Manila mount automation example using share driver hooks feature
2
+================================================================
3
+
4
+Manila has feature called 'share driver hooks'. Which allows to perform
5
+actions before and after driver actions such as 'create share' or
6
+'access allow', also allows to do custom things on periodic basis.
7
+
8
+Here, we provide example of mount automation using this feature.
9
+This example uses OpenStack Zaqar project for sending notifications
10
+when operations 'access allow' and 'access deny' are performed.
11
+Server side hook will send notifications about changed access for shares
12
+after granting and prior to denying access.
13
+
14
+
15
+Possibilities of the mount automation example (consumer)
16
+--------------------------------------------------------
17
+
18
+- Supports only 'NFS' protocol.
19
+- Supports only 'IP' rules.
20
+- Supports both levels of access - 'RW' and 'RO'.
21
+- Consume interval can be configured.
22
+- Allows to choose parent mount directory.
23
+
24
+
25
+Server side setup and run
26
+-------------------------
27
+
28
+1. Place files 'zaqarclientwrapper.py' and 'zaqar_notification.py' to dir
29
+%manila_dir%/manila/share/hooks.
30
+
31
+Then update manila configuration file with following options:
32
+
33
+::
34
+
35
+    [share_backend_config_group]
36
+    hook_drivers = manila.share.hooks.zaqar_notification.ZaqarNotification
37
+    enable_pre_hooks = True
38
+    enable_post_hooks = True
39
+    enable_periodic_hooks = False
40
+
41
+    [zaqar]
42
+    zaqar_auth_url = http://%ip_of_endpoint_with_keystone%:35357/v2.0/
43
+    zaqar_region_name = %name_of_region_optional%
44
+    zaqar_username = foo_user
45
+    zaqar_password = foo_tenant
46
+    zaqar_project_name = foo_password
47
+    zaqar_queues = manila_notification
48
+
49
+2. Restart manila-share service.
50
+
51
+
52
+Consumer side setup and run
53
+---------------------------
54
+
55
+1. Place files 'zaqarclientwrapper.py' and
56
+'zaqar_notification_example_consumer.py' to any dir on user machine, but they
57
+both should be in the same dir.
58
+
59
+2. Make sure that following dependencies are installed:
60
+
61
+- PIP dependencies:
62
+
63
+  - netaddr
64
+
65
+  - oslo_concurrency
66
+
67
+  - oslo_config
68
+
69
+  - oslo_utils
70
+
71
+  - python-zaqarclient
72
+
73
+  - six
74
+
75
+- System libs that install 'mount' and 'mount.nfs' apps.
76
+
77
+3. Create file with following options:
78
+
79
+::
80
+
81
+    [zaqar]
82
+    # Consumer-related options
83
+    sleep_between_consume_attempts = 7
84
+    mount_dir = "/tmp"
85
+    expected_ip_addresses = 10.254.0.4
86
+
87
+    # Common options for consumer and server sides
88
+    zaqar_auth_url = http://%ip_of_endpoint_with_keystone%:35357/v2.0/
89
+    zaqar_region_name = %name_of_region_optional%
90
+    zaqar_username = foo_user
91
+    zaqar_password = foo_tenant
92
+    zaqar_project_name = foo_password
93
+    zaqar_queues = manila_notification
94
+
95
+Consumer options descriptions:
96
+
97
+- 'sleep_between_consume_attempts' - wait interval between consuming
98
+  notifications from message queue.
99
+
100
+- 'mount_dir' - parent mount directory that will contain all mounted shares
101
+  as subdirectories.
102
+
103
+- 'expected_ip_addresses' - list of IP addresses that are expected
104
+  to be granted access for. Could be either equal to or be part of a CIDR.
105
+  Match triggers [un]mount operations.
106
+
107
+4. Run consumer with following command:
108
+
109
+::
110
+
111
+    $ zaqar_notification_example_consumer.py --config-file path/to/config.conf
112
+
113
+5. Now create NFS share and grant IP access to consumer by its IP address.

+ 121
- 0
contrib/share_driver_hooks/zaqar_notification.py View File

@@ -0,0 +1,121 @@
1
+# Copyright (c) 2015 Mirantis, Inc.
2
+# All Rights Reserved.
3
+#
4
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
5
+#    not use this file except in compliance with the License. You may obtain
6
+#    a copy of the License at
7
+#
8
+#         http://www.apache.org/licenses/LICENSE-2.0
9
+#
10
+#    Unless required by applicable law or agreed to in writing, software
11
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+#    License for the specific language governing permissions and limitations
14
+#    under the License.
15
+
16
+from oslo_log import log
17
+from oslo_utils import timeutils
18
+
19
+from manila import exception
20
+from manila.share import api
21
+from manila.share import hook
22
+from manila.share.hooks import zaqarclientwrapper  # noqa
23
+
24
+CONF = zaqarclientwrapper.CONF
25
+LOG = log.getLogger(__name__)
26
+ZAQARCLIENT = zaqarclientwrapper.ZAQARCLIENT
27
+
28
+
29
+class ZaqarNotification(hook.HookBase):
30
+    share_api = api.API()
31
+
32
+    def _access_changed_trigger(self, context, func_name,
33
+                                access_id, share_instance_id):
34
+        access = self.share_api.access_get(context, access_id=access_id)
35
+        share = self.share_api.get(context, share_id=access.share_id)
36
+
37
+        for ins in share.instances:
38
+            if ins.id == share_instance_id:
39
+                share_instance = ins
40
+                break
41
+        else:
42
+            raise exception.InstanceNotFound(instance_id=share_instance_id)
43
+
44
+        for ins in access.instance_mappings:
45
+            if ins.share_instance_id == share_instance_id:
46
+                access_instance = ins
47
+                break
48
+        else:
49
+            raise exception.InstanceNotFound(instance_id=share_instance_id)
50
+
51
+        is_allow_operation = 'allow' in func_name
52
+        results = {
53
+            'share_id': access.share_id,
54
+            'share_instance_id': share_instance_id,
55
+            'export_locations': [
56
+                el.path for el in share_instance.export_locations],
57
+            'share_proto': share.share_proto,
58
+            'access_id': access.id,
59
+            'access_instance_id': access_instance.id,
60
+            'access_type': access.access_type,
61
+            'access_to': access.access_to,
62
+            'access_level': access.access_level,
63
+            'access_state': access_instance.state,
64
+            'is_allow_operation': is_allow_operation,
65
+            'availability_zone': share_instance.availability_zone,
66
+        }
67
+        LOG.debug(results)
68
+        return results
69
+
70
+    def _execute_pre_hook(self, context, func_name, *args, **kwargs):
71
+        LOG.debug("\n PRE zaqar notification has been called for "
72
+                  "method '%s'.\n" % func_name)
73
+        if func_name == "deny_access":
74
+            LOG.debug("\nSending notification about denied access.\n")
75
+            data = self._access_changed_trigger(
76
+                context,
77
+                func_name,
78
+                kwargs.get('access_id'),
79
+                kwargs.get('share_instance_id'),
80
+            )
81
+            self._send_notification(data)
82
+
83
+    def _execute_post_hook(self, context, func_name, pre_hook_data,
84
+                           driver_action_results, *args, **kwargs):
85
+        LOG.debug("\n POST zaqar notification has been called for "
86
+                  "method '%s'.\n" % func_name)
87
+        if func_name == "allow_access":
88
+            LOG.debug("\nSending notification about allowed access.\n")
89
+            data = self._access_changed_trigger(
90
+                context,
91
+                func_name,
92
+                kwargs.get('access_id'),
93
+                kwargs.get('share_instance_id'),
94
+            )
95
+            self._send_notification(data)
96
+
97
+    def _send_notification(self, data):
98
+        for queue_name in CONF.zaqar.zaqar_queues:
99
+            ZAQARCLIENT.queue_name = queue_name
100
+            message = {
101
+                "body": {
102
+                    "example_message": (
103
+                        "message generated at '%s'" % timeutils.utcnow()),
104
+                    "data": data,
105
+                }
106
+            }
107
+            LOG.debug(
108
+                "\n Sending message %(m)s to '%(q)s' queue using '%(u)s' user "
109
+                "and '%(p)s' project." % {
110
+                    'm': message,
111
+                    'q': queue_name,
112
+                    'u': CONF.zaqar.zaqar_username,
113
+                    'p': CONF.zaqar.zaqar_project_name,
114
+                }
115
+            )
116
+            queue = ZAQARCLIENT.queue(queue_name)
117
+            queue.post(message)
118
+
119
+    def _execute_periodic_hook(self, context, periodic_hook_data,
120
+                               *args, **kwargs):
121
+        LOG.debug("Periodic zaqar notification has been called. (Placeholder)")

+ 233
- 0
contrib/share_driver_hooks/zaqar_notification_example_consumer.py View File

@@ -0,0 +1,233 @@
1
+#!/usr/bin/env python
2
+#
3
+# Copyright (c) 2015 Mirantis, Inc.
4
+# All Rights Reserved.
5
+#
6
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
7
+#    not use this file except in compliance with the License. You may obtain
8
+#    a copy of the License at
9
+#
10
+#         http://www.apache.org/licenses/LICENSE-2.0
11
+#
12
+#    Unless required by applicable law or agreed to in writing, software
13
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15
+#    License for the specific language governing permissions and limitations
16
+#    under the License.
17
+
18
+from __future__ import print_function
19
+
20
+import os
21
+import pprint
22
+import signal
23
+import sys
24
+import time
25
+
26
+import netaddr
27
+from oslo_concurrency import processutils
28
+from oslo_config import cfg
29
+from oslo_utils import timeutils
30
+import six
31
+
32
+opts = [
33
+    cfg.IntOpt(
34
+        "consume_interval",
35
+        default=5,
36
+        deprecated_name="sleep_between_consume_attempts",
37
+        help=("Time that script will sleep between requests for consuming "
38
+              "Zaqar messages in seconds."),
39
+    ),
40
+    cfg.StrOpt(
41
+        "mount_dir",
42
+        default="/tmp",
43
+        help="Directory that will contain all mounted shares."
44
+    ),
45
+    cfg.ListOpt(
46
+        "expected_ip_addresses",
47
+        default=[],
48
+        help=("List of IP addresses that are expected to be found in access "
49
+              "rules to trigger [un]mount operation for a share.")
50
+    ),
51
+]
52
+
53
+CONF = cfg.CONF
54
+
55
+
56
+def print_with_time(data):
57
+    time = six.text_type(timeutils.utcnow())
58
+    print(time + " " + six.text_type(data))
59
+
60
+
61
+def print_pretty_dict(d):
62
+    pprint.pprint(d)
63
+
64
+
65
+def pop_zaqar_messages(client, queues_names):
66
+    if not isinstance(queues_names, (list, set, tuple)):
67
+        queues_names = (queues_names, )
68
+    try:
69
+        user = client.conf['auth_opts']['options']['os_username']
70
+        project = client.conf['auth_opts']['options']['os_project_name']
71
+        messages = []
72
+        for queue_name in queues_names:
73
+            queue = client.queue(queue_name)
74
+            messages.extend([six.text_type(m.body) for m in queue.pop()])
75
+            print_with_time(
76
+                "Received %(len)s message[s] from '%(q)s' "
77
+                "queue using '%(u)s' user and '%(p)s' project." % {
78
+                    'len': len(messages),
79
+                    'q': queue_name,
80
+                    'u': user,
81
+                    'p': project,
82
+                }
83
+            )
84
+        return messages
85
+    except Exception as e:
86
+        print_with_time("Caught exception - %s" % e)
87
+        return []
88
+
89
+
90
+def signal_handler(signal, frame):
91
+    print("")
92
+    print_with_time("Ctrl+C was pressed. Shutting down consumer.")
93
+    sys.exit(0)
94
+
95
+
96
+def parse_str_to_dict(string):
97
+    if not isinstance(string, six.string_types):
98
+        return string
99
+    result = eval(string)
100
+    return result
101
+
102
+
103
+def handle_message(data):
104
+    """Handles consumed message.
105
+
106
+    Expected structure of a message is following:
107
+        {'data': {
108
+             'access_id': u'b28268b9-36c6-40d3-a485-22534077328f',
109
+             'access_instance_id': u'd137b2cb-f549-4141-9dd7-36b2789fb973',
110
+             'access_level': u'rw',
111
+             'access_state': u'active',
112
+             'access_to': u'7.7.7.7',
113
+             'access_type': u'ip',
114
+             'availability_zone': u'nova',
115
+             'export_locations': [u'127.0.0.1:/path/to/nfs/share'],
116
+             'is_allow_operation': True,
117
+             'share_id': u'053eae9a-726f-4f7e-8502-49d7b1adf290',
118
+             'share_instance_id': u'dc33e554-e0b9-40f5-9046-c198716d73a0',
119
+             'share_proto': u'NFS'
120
+        }}
121
+    """
122
+    if 'data' in data.keys():
123
+        data = data['data']
124
+    if (data.get('access_type', '?').lower() == 'ip' and
125
+            'access_state' in data.keys() and
126
+            'error' not in data.get('access_state', '?').lower() and
127
+            data.get('share_proto', '?').lower() == 'nfs'):
128
+        is_allow_operation = data['is_allow_operation']
129
+        export_location = data['export_locations'][0]
130
+        if is_allow_operation:
131
+            mount_share(export_location, data['access_to'])
132
+        else:
133
+            unmount_share(export_location, data['access_to'])
134
+    else:
135
+        print_with_time('Do nothing with above message.')
136
+
137
+
138
+def execute(cmd):
139
+    try:
140
+        print_with_time('Executing following command: \n%s' % cmd)
141
+        cmd = cmd.split()
142
+        stdout, stderr = processutils.execute(*cmd)
143
+        if stderr:
144
+            print_with_time('Got error: %s' % stderr)
145
+        return stdout, stderr
146
+    except Exception as e:
147
+        print_with_time('Got following error: %s' % e)
148
+        return False, True
149
+
150
+
151
+def is_share_mounted(mount_point):
152
+    mounts, stderr = execute('mount')
153
+    return mount_point in mounts
154
+
155
+
156
+def rule_affects_me(ip_or_cidr):
157
+    if '/' in ip_or_cidr:
158
+        net = netaddr.IPNetwork(ip_or_cidr)
159
+        for my_ip in CONF.zaqar.expected_ip_addresses:
160
+            if netaddr.IPAddress(my_ip) in net:
161
+                return True
162
+    else:
163
+        for my_ip in CONF.zaqar.expected_ip_addresses:
164
+            if my_ip == ip_or_cidr:
165
+                return True
166
+    return False
167
+
168
+
169
+def mount_share(export_location, access_to):
170
+    data = {
171
+        'mount_point': os.path.join(CONF.zaqar.mount_dir,
172
+                                    export_location.split('/')[-1]),
173
+        'export_location': export_location,
174
+    }
175
+    if (rule_affects_me(access_to) and
176
+            not is_share_mounted(data['mount_point'])):
177
+        print_with_time(
178
+            "Mounting '%(export_location)s' share to %(mount_point)s.")
179
+        execute('sudo mkdir -p %(mount_point)s' % data)
180
+        stdout, stderr = execute(
181
+            'sudo mount.nfs %(export_location)s %(mount_point)s' % data)
182
+        if stderr:
183
+            print_with_time("Mount operation failed.")
184
+        else:
185
+            print_with_time("Mount operation went OK.")
186
+
187
+
188
+def unmount_share(export_location, access_to):
189
+    if rule_affects_me(access_to) and is_share_mounted(export_location):
190
+        print_with_time("Unmounting '%(export_location)s' share.")
191
+        stdout, stderr = execute('sudo umount %s' % export_location)
192
+        if stderr:
193
+            print_with_time("Unmount operation failed.")
194
+        else:
195
+            print_with_time("Unmount operation went OK.")
196
+
197
+
198
+def main():
199
+    # Register other local modules
200
+    cur = os.path.dirname(__file__)
201
+    pathtest = os.path.join(cur)
202
+    sys.path.append(pathtest)
203
+
204
+    # Init configuration
205
+    CONF(sys.argv[1:], project="manila_notifier", version=1.0)
206
+    CONF.register_opts(opts, group="zaqar")
207
+
208
+    # Import common config and Zaqar client
209
+    import zaqarclientwrapper
210
+
211
+    # Handle SIGINT
212
+    signal.signal(signal.SIGINT, signal_handler)
213
+
214
+    # Run consumer
215
+    print_with_time("Consumer was successfully run.")
216
+    while(True):
217
+        messages = pop_zaqar_messages(
218
+            zaqarclientwrapper.ZAQARCLIENT, CONF.zaqar.zaqar_queues)
219
+        if not messages:
220
+            message = ("No new messages in '%s' queue[s] "
221
+                       "found." % ','.join(CONF.zaqar.zaqar_queues))
222
+        else:
223
+            message = "Got following messages:"
224
+        print_with_time(message)
225
+        for message in messages:
226
+            message = parse_str_to_dict(message)
227
+            print_pretty_dict(message)
228
+            handle_message(message)
229
+        time.sleep(CONF.zaqar.consume_interval)
230
+
231
+
232
+if __name__ == '__main__':
233
+    main()

+ 86
- 0
contrib/share_driver_hooks/zaqarclientwrapper.py View File

@@ -0,0 +1,86 @@
1
+# Copyright (c) 2015 Mirantis, Inc.
2
+# All Rights Reserved.
3
+#
4
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
5
+#    not use this file except in compliance with the License. You may obtain
6
+#    a copy of the License at
7
+#
8
+#         http://www.apache.org/licenses/LICENSE-2.0
9
+#
10
+#    Unless required by applicable law or agreed to in writing, software
11
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13
+#    License for the specific language governing permissions and limitations
14
+#    under the License.
15
+
16
+from oslo_config import cfg
17
+from zaqarclient.queues import client as zaqar
18
+
19
+zaqar_notification_opts = [
20
+    cfg.StrOpt(
21
+        "zaqar_username",
22
+        help="Username that should be used for init of zaqar client.",
23
+    ),
24
+    cfg.StrOpt(
25
+        "zaqar_password",
26
+        secret=True,
27
+        help="Password for user specified in opt 'zaqar_username'.",
28
+    ),
29
+    cfg.StrOpt(
30
+        "zaqar_project_name",
31
+        help=("Project/Tenant name that is owns user specified "
32
+              "in opt 'zaqar_username'."),
33
+    ),
34
+    cfg.StrOpt(
35
+        "zaqar_auth_url",
36
+        default="http://127.0.0.1:35357/v2.0/",
37
+        help="Auth url to be used by Zaqar client.",
38
+    ),
39
+    cfg.StrOpt(
40
+        "zaqar_region_name",
41
+        help="Name of the region that should be used. Optional.",
42
+    ),
43
+    cfg.StrOpt(
44
+        "zaqar_service_type",
45
+        default="messaging",
46
+        help="Service type for Zaqar. Optional.",
47
+    ),
48
+    cfg.StrOpt(
49
+        "zaqar_endpoint_type",
50
+        default="publicURL",
51
+        help="Type of endpoint to be used for init of Zaqar client. Optional.",
52
+    ),
53
+    cfg.FloatOpt(
54
+        "zaqar_api_version",
55
+        default=1.1,
56
+        help="Version of Zaqar API to use. Optional.",
57
+    ),
58
+    cfg.ListOpt(
59
+        "zaqar_queues",
60
+        default=["manila_notification_qeueue"],
61
+        help=("List of queues names to be used for sending Manila "
62
+              "notifications. Optional."),
63
+    ),
64
+]
65
+
66
+CONF = cfg.CONF
67
+CONF.register_opts(zaqar_notification_opts, group='zaqar')
68
+
69
+ZAQARCLIENT = zaqar.Client(
70
+    version=CONF.zaqar.zaqar_api_version,
71
+    conf={
72
+        "auth_opts": {
73
+            "backend": "keystone",
74
+            "options": {
75
+                "os_username": CONF.zaqar.zaqar_username,
76
+                "os_password": CONF.zaqar.zaqar_password,
77
+                "os_project_name": CONF.zaqar.zaqar_project_name,
78
+                "os_auth_url": CONF.zaqar.zaqar_auth_url,
79
+                "os_region_name": CONF.zaqar.zaqar_region_name,
80
+                "os_service_type": CONF.zaqar.zaqar_service_type,
81
+                "os_endpoint_type": CONF.zaqar.zaqar_endpoint_type,
82
+                "insecure": True,
83
+            },
84
+        },
85
+    },
86
+)

Loading…
Cancel
Save