From 9c5cdb3de9934c074d40d48a9e4446f1d4288d3a Mon Sep 17 00:00:00 2001 From: Volodymyr Boiko Date: Wed, 10 Apr 2024 00:04:57 +0300 Subject: [PATCH] Add share driver for VastData storage This driver enables use of the VastData filesystem for Manila shares. NFS vers 3 is supported Following interfaces are supported: 1. Create/Delete share. 2. Expand/Shrink share capacity. 3. Create/Delete snapshot. 4. Allow/Deny IP(NFS) access. 5. Get share stats. Co-Authored-By: Ofer Koren Implements: bp vastdata-driver Change-Id: I7e8f81694183210b146d60a132a933eabd8b1eb1 --- doc/source/admin/index.rst | 2 + ...hare_back_ends_feature_support_mapping.rst | 8 + .../shared-file-systems/drivers.rst | 1 + .../drivers/vastdata_driver.rst | 90 +++ .../configuration/tables/manila-vastdata.inc | 32 + manila/exception.py | 17 + manila/opts.py | 2 + manila/share/drivers/vastdata/__init__.py | 0 manila/share/drivers/vastdata/driver.py | 400 ++++++++++ manila/share/drivers/vastdata/driver_util.py | 211 ++++++ manila/share/drivers/vastdata/rest.py | 332 +++++++++ .../tests/share/drivers/vastdata/__init__.py | 0 .../share/drivers/vastdata/test_driver.py | 690 ++++++++++++++++++ .../drivers/vastdata/test_driver_util.py | 267 +++++++ .../tests/share/drivers/vastdata/test_rest.py | 512 +++++++++++++ .../add-vastdriver-5a2ca79a81bc9280.yaml | 3 + requirements.txt | 2 + 17 files changed, 2569 insertions(+) create mode 100644 doc/source/configuration/shared-file-systems/drivers/vastdata_driver.rst create mode 100644 doc/source/configuration/tables/manila-vastdata.inc create mode 100644 manila/share/drivers/vastdata/__init__.py create mode 100644 manila/share/drivers/vastdata/driver.py create mode 100644 manila/share/drivers/vastdata/driver_util.py create mode 100644 manila/share/drivers/vastdata/rest.py create mode 100644 manila/tests/share/drivers/vastdata/__init__.py create mode 100644 manila/tests/share/drivers/vastdata/test_driver.py create mode 100644 manila/tests/share/drivers/vastdata/test_driver_util.py create mode 100644 manila/tests/share/drivers/vastdata/test_rest.py create mode 100644 releasenotes/notes/add-vastdriver-5a2ca79a81bc9280.yaml diff --git a/doc/source/admin/index.rst b/doc/source/admin/index.rst index c4637b0bf3..cfe6aa7ca5 100644 --- a/doc/source/admin/index.rst +++ b/doc/source/admin/index.rst @@ -104,3 +104,5 @@ each back end. nexentastor5_driver ../configuration/shared-file-systems/drivers/windows-smb-driver zadara_driver + ../configuration/shared-file-systems/drivers/vastdata_driver + diff --git a/doc/source/admin/share_back_ends_feature_support_mapping.rst b/doc/source/admin/share_back_ends_feature_support_mapping.rst index d0fa00cc5e..b4627e72fe 100644 --- a/doc/source/admin/share_back_ends_feature_support_mapping.rst +++ b/doc/source/admin/share_back_ends_feature_support_mapping.rst @@ -103,6 +103,8 @@ Mapping of share drivers and share features support +----------------------------------------+-----------------------+-----------------------+--------------------------+--------------------------+------------------------+-----------------------------------+--------------------------+--------------------+--------------------+ | Pure Storage FlashBlade | X | \- | X | X | X | \- | \- | X | \- | +----------------------------------------+-----------------------+-----------------------+--------------------------+--------------------------+------------------------+-----------------------------------+--------------------------+--------------------+--------------------+ +| Vastdata | D | \- | D | D | D | \- | \- | \- | \- | ++----------------------------------------+-----------------------+-----------------------+--------------------------+--------------------------+------------------------+-----------------------------------+--------------------------+--------------------+--------------------+ Mapping of share drivers and share access rules support ------------------------------------------------------- @@ -180,6 +182,8 @@ Mapping of share drivers and share access rules support +----------------------------------------+--------------+--------------+----------------+------------+--------------+--------------+--------------+----------------+------------+------------+ | Pure Storage FlashBlade | NFS (X) | \- | \- | \- | \- | NFS (X) | \- | \- | \- | \- | +----------------------------------------+--------------+--------------+----------------+------------+--------------+--------------+--------------+----------------+------------+------------+ +| Vastdata | NFS (D) | \- | \- | \- | \- | NFS (D) | \- | \- | \- | \- | ++----------------------------------------+--------------+--------------+----------------+------------+--------------+--------------+--------------+----------------+------------+------------+ Mapping of share drivers and security services support ------------------------------------------------------ @@ -255,6 +259,8 @@ Mapping of share drivers and security services support +----------------------------------------+------------------+-----------------+------------------+ | Pure Storage FlashBlade | \- | \- | \- | +----------------------------------------+------------------+-----------------+------------------+ +| Vastdata | \- | \- | \- | ++----------------------------------------+------------------+-----------------+------------------+ Mapping of share drivers and common capabilities ------------------------------------------------ @@ -332,6 +338,8 @@ More information: :ref:`capabilities_and_extra_specs` +----------------------------------------+-----------+------------+--------+-------------+-------------------+--------------------+-----+----------------------------+--------------------+--------------------+--------------+--------------+-------------------------+--------------------------+ | Pure Storage FlashBlade | \- | X | \- | \- | X | \- | \- | \- | X | \- | X | \- | \- | \- | +----------------------------------------+-----------+------------+--------+-------------+-------------------+--------------------+-----+----------------------------+--------------------+--------------------+--------------+--------------+-------------------------+--------------------------+ +| Vastdata | \- | D | \- | \- | \- | \- | \- | \- | \- | \- | D | \- | \- | \- | ++----------------------------------------+-----------+------------+--------+-------------+-------------------+--------------------+-----+----------------------------+--------------------+--------------------+--------------+--------------+-------------------------+--------------------------+ .. note:: diff --git a/doc/source/configuration/shared-file-systems/drivers.rst b/doc/source/configuration/shared-file-systems/drivers.rst index 31c9589c40..05c4a98d33 100644 --- a/doc/source/configuration/shared-file-systems/drivers.rst +++ b/doc/source/configuration/shared-file-systems/drivers.rst @@ -37,6 +37,7 @@ Share drivers drivers/windows-smb-driver.rst drivers/nexentastor5-driver.rst drivers/purestorage-flashblade-driver.rst + drivers/vastdata_driver.rst To use different share drivers for the Shared File Systems service, use the diff --git a/doc/source/configuration/shared-file-systems/drivers/vastdata_driver.rst b/doc/source/configuration/shared-file-systems/drivers/vastdata_driver.rst new file mode 100644 index 0000000000..4a43e87647 --- /dev/null +++ b/doc/source/configuration/shared-file-systems/drivers/vastdata_driver.rst @@ -0,0 +1,90 @@ +==================================== +Vastdata Share Driver +==================================== + +Vastdata can be used as a storage back end for the OpenStack Shared +File System service. Shares in the Shared File System service are +mapped 1:1 to Vastdata volumes. Access is provided via NFS protocol +and IP-based authentication. The `Vastdata `__ +Manila driver uses the Vastdata API service. + +Supported shared filesystems +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The driver supports NFS shares. + +Operations supported +~~~~~~~~~~~~~~~~~~~~ +The driver supports NFS shares. + +The following operations are supported: + +- Create a share. + +- Delete a share. + +- Allow share access. + +- Deny share access. + +- Extend a share. + +- Shrink a share. + + +Requirements +~~~~~~~~~~~~ + +- Trash API must be enabled on Vastdata cluster. + +Driver options +~~~~~~~~~~~~~~ + +The following table contains the configuration options specific to the +share driver. + +.. include:: ../../tables/manila-vastdata.inc + + +Vastdata driver configuration example +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following parameters shows a sample subset of the ``manila.conf`` file, +which configures two backends and the relevant ``[DEFAULT]`` options. A real +configuration would include additional ``[DEFAULT]`` options and additional +sections that are not discussed in this document: + +.. code-block:: ini + + [DEFAULT] + enabled_share_backends = vast + enabled_share_protocols = NFS + + [vast] + share_driver = manila.share.drivers.vastdata.driver.VASTShareDriver + share_backend_name = vast + driver_handles_share_servers = False + snapshot_support = True + vast_mgmt_host = {vms_ip} + vast_mgmt_port = {vms_port} + vast_mgmt_user = {mgmt_user} + vast_mgmt_password = {mgmt_password} + vast_vippool_name = {vip_pool} + vast_root_export = {root_export} + + +Restrictions +------------ + +The Vastdata driver has the following restrictions: + +- Only IP access type is supported for NFS. + + +The :mod:`manila.share.drivers.vastdata.driver` Module +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: manila.share.drivers.vastdata.driver + :noindex: + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/configuration/tables/manila-vastdata.inc b/doc/source/configuration/tables/manila-vastdata.inc new file mode 100644 index 0000000000..8f0c084aeb --- /dev/null +++ b/doc/source/configuration/tables/manila-vastdata.inc @@ -0,0 +1,32 @@ +.. + Warning: Do not edit this file. It is automatically generated from the + software project's code and your changes will be overwritten. + + The tool to generate this file lives in openstack-doc-tools repository. + + Please make any changes needed in the code, then run the + autogenerate-config-doc tool from the openstack-doc-tools repository, or + ask for help on the documentation mailing list, IRC channel or meeting. + +.. _manila-vastdata: + +.. list-table:: Description of Vastdata share driver configuration options + :header-rows: 1 + :class: config-ref-table + + * - Configuration option = Default value + - Description + * - **[DEFAULT]** + - + * - ``vast_mgmt_host`` = + - (String) Hostname or IP address VAST storage system management VIP. + * - ``vast_mgmt_port`` = ``443`` + - (String) Port for VAST management API. + * - ``vast_vippool_name`` = + - (String) Name of Virtual IP pool. + * - ``vast_root_export`` = ``manila`` + - (String) Base path for shares. + * - ``vast_mgmt_user`` = + - (String) Username for VAST management API. + * - ``vast_mgmt_password`` = + - (String) Password for VAST management API. diff --git a/manila/exception.py b/manila/exception.py index 5335ee7dcb..cb2b6e88b7 100644 --- a/manila/exception.py +++ b/manila/exception.py @@ -1186,3 +1186,20 @@ class ShareBackupSizeExceedsAvailableQuota(QuotaError): class NetappActiveIQWeigherRequiredParameter(ManilaException): message = _("%(config)s configuration of the NetAppActiveIQ weigher " "must be set.") + + +# Vastdata Storage driver +class VastApiException(ManilaException): + message = _("Rest api error: %(reason)s.") + + +class VastApiRetry(ManilaException): + message = _("Rest api retry: %(reason)s.") + + +class VastShareNotFound(ShareBackendException): + message = _("Share %(name)s could not be found.") + + +class VastDriverException(ShareBackendException): + message = _("Vast driver error: %(reason)s.") diff --git a/manila/opts.py b/manila/opts.py index ec0d81970a..54e377b971 100644 --- a/manila/opts.py +++ b/manila/opts.py @@ -83,6 +83,7 @@ import manila.share.drivers.qnap.qnap import manila.share.drivers.quobyte.quobyte import manila.share.drivers.service_instance import manila.share.drivers.tegile.tegile +import manila.share.drivers.vastdata.driver import manila.share.drivers.windows.service_instance import manila.share.drivers.windows.winrm_helper import manila.share.drivers.zfsonlinux.driver @@ -196,6 +197,7 @@ _global_opt_lists = [ manila.share.manager.share_manager_opts, manila.volume._volume_opts, manila.wsgi.eventlet_server.socket_opts, + manila.share.drivers.vastdata.driver.OPTS, ] _opts = [ diff --git a/manila/share/drivers/vastdata/__init__.py b/manila/share/drivers/vastdata/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/manila/share/drivers/vastdata/driver.py b/manila/share/drivers/vastdata/driver.py new file mode 100644 index 0000000000..5c3dda33c0 --- /dev/null +++ b/manila/share/drivers/vastdata/driver.py @@ -0,0 +1,400 @@ +# Copyright 2024 VAST Data Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +VAST's Share Driver + + +Configuration: + + +[DEFAULT] +enabled_share_backends = vast + +[vast] +share_driver = manila.share.drivers.vastdata.driver.VASTShareDriver +share_backend_name = vast +snapshot_support = true +driver_handles_share_servers = false +vast_mgmt_host = v11 +vast_vippool_name = vippool-1 +vast_root_export = manila +vast_mgmt_user = admin +vast_mgmt_password = 123456 +""" + +import collections + +import netaddr +from oslo_config import cfg +from oslo_log import log as logging +from oslo_utils import units + +from manila.common import constants +from manila import exception +from manila.i18n import _ +from manila.share import driver +from manila.share.drivers.vastdata import driver_util +import manila.share.drivers.vastdata.rest as vast_rest + + +LOG = logging.getLogger(__name__) + +OPTS = [ + cfg.HostAddressOpt( + "vast_mgmt_host", + help="Hostname or IP address VAST storage system management VIP.", + ), + cfg.PortOpt( + "vast_mgmt_port", + help="Port for VAST management", + default=443 + ), + cfg.StrOpt( + "vast_vippool_name", + help="Name of Virtual IP pool" + ), + cfg.StrOpt( + "vast_root_export", + default="manila", + help="Base path for shares" + ), + cfg.StrOpt( + "vast_mgmt_user", + help="Username for VAST management" + ), + cfg.StrOpt( + "vast_mgmt_password", + help="Password for VAST management", + secret=True + ), +] + +CONF = cfg.CONF +CONF.register_opts(OPTS) + +MANILA_TO_VAST_ACCESS_LEVEL = { + constants.ACCESS_LEVEL_RW: "nfs_read_write", + constants.ACCESS_LEVEL_RO: "nfs_read_only", +} + + +@driver_util.decorate_methods_with( + driver_util.verbose_driver_trace +) +class VASTShareDriver(driver.ShareDriver): + """Driver for the VastData Filesystem.""" + + VERSION = "1.0" # driver version + + def __init__(self, *args, **kwargs): + super().__init__(False, *args, config_opts=[OPTS], **kwargs) + + def do_setup(self, context): + """Driver initialization""" + backend_name = self.configuration.safe_get("share_backend_name") + root_export = self.configuration.vast_root_export + vip_pool_name = self.configuration.safe_get("vast_vippool_name") + if not vip_pool_name: + raise exception.VastDriverException( + reason="vast_vippool_name must be set" + ) + self._backend_name = backend_name or self.__class__.__name__ + self._vippool_name = vip_pool_name + self._root_export = "/" + root_export.strip("/") + + username = self.configuration.safe_get("vast_mgmt_user") + password = self.configuration.safe_get("vast_mgmt_password") + host = self.configuration.safe_get("vast_mgmt_host") + port = self.configuration.safe_get("vast_mgmt_port") + if not all((username, password, port)): + raise exception.VastDriverException( + reason="Not all required parameters are present." + " Make sure you specified `vast_mgmt_host`," + " `vast_mgmt_port`, and `vast_mgmt_user` " + "in manila.conf." + ) + if port: + host = f"{host}:{port}" + self.rest = vast_rest.RestApi( + host, username, password, False, self.VERSION + ) + LOG.debug("VAST Data driver setup is complete.") + + def _update_share_stats(self, data=None): + """Retrieve stats info from share group.""" + metrics_list = [ + "Capacity,drr", + "Capacity,logical_space", + "Capacity,logical_space_in_use", + "Capacity,physical_space", + "Capacity,physical_space_in_use", + ] + metrics = self.rest.capacity_metrics.get(metrics_list) + data = dict( + share_backend_name=self._backend_name, + vendor_name="VAST STORAGE", + driver_version=self.VERSION, + storage_protocol="NFS", + data_reduction=metrics.drr, + total_capacity_gb=float(metrics.logical_space) / units.Gi, + free_capacity_gb=float( + metrics.logical_space - metrics.logical_space_in_use + ) + / units.Gi, + provisioned_capacity_gb=float( + metrics.logical_space_in_use) / units.Gi, + snapshot_support=True, + create_share_from_snapshot_support=False, + mount_snapshot_support=False, + revert_to_snapshot_support=False, + ) + + super()._update_share_stats(data) + + def _to_volume_path(self, share_id, root=None): + if not root: + root = self._root_export + return f"{root}/manila-{share_id}" + + def create_share(self, context, share, share_server=None): + return self._ensure_share(share)[0] + + def delete_share(self, context, share, share_server=None): + """Called to delete a share""" + share_id = share["id"] + src = self._to_volume_path(share_id) + LOG.debug(f"Deleting '{src}'.") + self.rest.folders.delete(path=src) + self.rest.views.delete(name=share_id) + self.rest.quotas.delete(name=share_id) + self.rest.view_policies.delete(name=share_id) + + def update_access( + self, context, share, access_rules, + add_rules, delete_rules, share_server=None + ): + """Update access rules for share.""" + rule_state_map = {} + + if not (add_rules or delete_rules): + add_rules = access_rules + + if share["share_proto"] != "NFS": + LOG.error("The share protocol flavor is invalid. Please use NFS.") + return + + valid_add_rules = [] + for rule in (add_rules or []): + try: + validate_access_rule(rule) + except ( + exception.InvalidShareAccess, + exception.InvalidShareAccessLevel, + ) as exc: + rule_id = rule["access_id"] + access_level = rule["access_level"] + access_to = rule["access_to"] + LOG.exception( + f"Failed to provide {access_level} access to " + f"{access_to} (Rule ID: {rule_id}, Reason: {exc}). " + "Setting rule to 'error' state." + ) + rule_state_map[rule['id']] = {'state': 'error'} + else: + valid_add_rules.append(rule) + + share_id = share["id"] + export = self._to_volume_path(share_id) + + LOG.debug(f"Changing access on {share_id}.") + data = { + "name": share_id, + "nfs_no_squash": ["*"], + "nfs_root_squash": ["*"] + } + policy = self.rest.view_policies.one(name=share_id) + if not policy: + raise exception.VastDriverException( + reason=f"Policy not found for share {share_id}." + ) + if valid_add_rules: + policy_rules = policy_payload_from_rules( + rules=valid_add_rules, policy=policy, action="update" + ) + data.update(policy_rules) + LOG.debug(f"Changing access on {export}. Rules: {policy_rules}.") + self.rest.view_policies.update(policy.id, **data) + + if delete_rules: + policy_rules = policy_payload_from_rules( + rules=delete_rules, policy=policy, action="deny" + ) + LOG.debug(f"Changing access on {export}. Rules: {policy_rules}.") + data.update(policy_rules) + self.rest.view_policies.update(policy.id, **data) + + return rule_state_map + + def extend_share(self, share, new_size, share_server=None): + """uses resize_share to extend a share""" + self._resize_share(share, new_size) + + def shrink_share(self, share, new_size, share_server=None): + """uses resize_share to shrink a share""" + self._resize_share(share, new_size) + + def create_snapshot(self, context, snapshot, share_server=None): + """Is called to create snapshot.""" + path = self._to_volume_path(snapshot["share_instance_id"]) + self.rest.snapshots.create(path=path, name=snapshot["name"]) + + def delete_snapshot(self, context, snapshot, share_server=None): + """Is called to remove share.""" + self.rest.snapshots.delete(name=snapshot["name"]) + + def get_network_allocations_number(self): + return 0 + + def ensure_shares(self, context, shares): + updates = {} + for share in shares: + export_locations = self._ensure_share(share) + updates[share["id"]] = { + 'export_locations': export_locations + } + return updates + + def get_backend_info(self, context): + backend_info = { + "vast_vippool_name": self.configuration.vast_vippool_name, + "vast_mgmt_host": self.configuration.vast_mgmt_host, + } + return backend_info + + def _resize_share(self, share, new_size): + share_id = share["id"] + quota = self.rest.quotas.one(name=share_id) + if not quota: + raise exception.ShareNotFound( + reason="Share not found", share_id=share_id + ) + requested_capacity = new_size * units.Gi + if requested_capacity < quota.used_effective_capacity: + raise exception.ShareShrinkingPossibleDataLoss( + share_id=share['id']) + self.rest.quotas.update(quota.id, hard_limit=requested_capacity) + + def _ensure_share(self, share): + share_proto = share["share_proto"] + if share_proto != "NFS": + raise exception.InvalidShare( + reason=_( + "Invalid NAS protocol supplied: {}.".format(share_proto) + ) + ) + + vips = self.rest.vip_pools.vips(pool_name=self._vippool_name) + + share_id = share["id"] + requested_capacity = share["size"] * units.Gi + path = self._to_volume_path(share_id) + policy = self.rest.view_policies.ensure(name=share_id) + quota = self.rest.quotas.ensure( + name=share_id, path=path, + create_dir=True, hard_limit=requested_capacity + ) + if quota.hard_limit != requested_capacity: + raise exception.VastDriverException( + reason=f"Share already exists with different capacity" + f" (requested={requested_capacity}, exists={quota.hard_limit})" + ) + view = self.rest.views.ensure( + name=share_id, path=path, policy_id=policy.id + ) + if view.policy != share_id: + self.rest.views.update(view.id, policy_id=policy.id) + return [ + dict(path=f"{vip}:{path}", is_admin_only=False) for vip in vips + ] + + +def policy_payload_from_rules(rules, policy, action): + """Convert list of manila rules + + into vast compatible payload for updating/creating policy. + """ + hosts = collections.defaultdict(set) + for rule in rules: + addr_list = map( + str, netaddr.IPNetwork(rule["access_to"]).iter_hosts() + ) + hosts[ + MANILA_TO_VAST_ACCESS_LEVEL[rule["access_level"]] + ].update(addr_list) + + _default_rules = set() + + # Delete default_vast_policy on each update. + # There is no sense to keep * in list of allowed/denied hosts + # as user want to set particular ip/ips only. + _default_vast_policy = {"*"} + if action == "update": + rw = set(policy.nfs_read_write).union( + hosts.get("nfs_read_write", _default_rules) + ) + ro = set(policy.nfs_read_only).union( + hosts.get("nfs_read_only", _default_rules) + ) + elif action == "deny": + rw = set(policy.nfs_read_write).difference( + hosts.get("nfs_read_write", _default_rules) + ) + ro = set(policy.nfs_read_only).difference( + hosts.get("nfs_read_only", _default_rules) + ) + else: + raise ValueError("Invalid action") + + # When policy created default access is + # "*" for read-write and read-only operations. + # After updating any of rules (rw or ro) + # we need to delete "*" to prevent ambiguous state when + # resource available for certain ip and for all range of ip addresses. + if len(rw) > 1: + rw -= _default_vast_policy + + if len(ro) > 1: + ro -= _default_vast_policy + + return {"nfs_read_write": list(rw), "nfs_read_only": list(ro)} + + +def validate_access_rule(access_rule): + allowed_types = {"ip"} + allowed_levels = MANILA_TO_VAST_ACCESS_LEVEL.keys() + + access_type = access_rule["access_type"] + access_level = access_rule["access_level"] + if access_type not in allowed_types: + reason = _("Only {} access type allowed.").format( + ", ".join(tuple([f"'{x}'" for x in allowed_types])) + ) + raise exception.InvalidShareAccess(reason=reason) + if access_level not in allowed_levels: + raise exception.InvalidShareAccessLevel(level=access_level) + try: + netaddr.IPNetwork(access_rule["access_to"]) + except (netaddr.core.AddrFormatError, OSError) as exc: + raise exception.InvalidShareAccess(reason=str(exc)) diff --git a/manila/share/drivers/vastdata/driver_util.py b/manila/share/drivers/vastdata/driver_util.py new file mode 100644 index 0000000000..9b34e78fed --- /dev/null +++ b/manila/share/drivers/vastdata/driver_util.py @@ -0,0 +1,211 @@ +# Copyright 2024 VAST Data Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import ipaddress +import types + +from oslo_config import cfg +from oslo_log import log +from oslo_utils import timeutils + + +CONF = cfg.CONF +LOG = log.getLogger(__name__) + + +class Bunch(dict): + # from https://github.com/real-easypy/easypy + + __slots__ = ("__stop_recursing__",) + + def __getattr__(self, name): + try: + return self[name] + except KeyError: + if name[0] == "_" and name[1:].isdigit(): + return self[name[1:]] + raise AttributeError( + "%s has no attribute %r" % (self.__class__, name) + ) + + def __getitem__(self, key): + try: + return super(Bunch, self).__getitem__(key) + except KeyError: + from numbers import Integral + + if isinstance(key, Integral): + return self[str(key)] + raise + + def __setattr__(self, name, value): + self[name] = value + + def __delattr__(self, name): + try: + del self[name] + except KeyError: + raise AttributeError( + "%s has no attribute %r" % (self.__class__, name) + ) + + def __getstate__(self): + return self + + def __setstate__(self, dict): + self.update(dict) + + def __repr__(self): + if getattr(self, "__stop_recursing__", False): + items = sorted( + "%s" % k for k in self + if isinstance(k, str) and not k.startswith("__") + ) + attrs = ", ".join(items) + else: + dict.__setattr__(self, "__stop_recursing__", True) + try: + attrs = self.render() + finally: + dict.__delattr__(self, "__stop_recursing__") + return "%s(%s)" % (self.__class__.__name__, attrs) + + def render(self): + items = sorted( + "%s=%r" % (k, v) + for k, v in self.items() + if isinstance(k, str) and not k.startswith("__") + ) + return ", ".join(items) + + def to_dict(self): + return unbunchify(self) + + def to_json(self): + import json + + return json.dumps(self.to_dict()) + + def copy(self, deep=False): + if deep: + return _convert(self, self.__class__) + else: + return self.__class__(self) + + @classmethod + def from_dict(cls, d): + return _convert(d, cls) + + @classmethod + def from_json(cls, d): + import json + + return cls.from_dict(json.loads(d)) + + def __dir__(self): + members = set( + k + for k in self + if isinstance(k, str) + and (k[0] == "_" or k.replace("_", "").isalnum()) + ) + members.update(dict.__dir__(self)) + return sorted(members) + + def without(self, *keys): + "Return a shallow copy of the bunch without the specified keys" + return Bunch((k, v) for k, v in self.items() if k not in keys) + + def but_with(self, **kw): + "Return a shallow copy of the bunch with the specified keys" + return Bunch(self, **kw) + + +def _convert(d, typ): + if isinstance(d, dict): + return typ({str(k): _convert(v, typ) for k, v in d.items()}) + elif isinstance(d, (tuple, list, set)): + return type(d)(_convert(e, typ) for e in d) + else: + return d + + +def unbunchify(d): + """Recursively convert Bunches in `d` to a regular dicts.""" + return _convert(d, dict) + + +def bunchify(d=None, **kw): + """Recursively convert dicts in `d` to Bunches. + + If `kw` given, recursively convert dicts in + it to Bunches and update `d` with it. + If `d` is None, an empty Bunch is made. + """ + + d = _convert(d, Bunch) if d is not None else Bunch() + if kw: + d.update(bunchify(kw)) + return d + + +def generate_ip_range(ip_ranges): + """Generate list of ips from provided ip ranges. + + `ip_ranges` should be list of ranges where fist + ip in range represents start ip and second is end ip + eg: [["15.0.0.1", "15.0.0.4"], ["10.0.0.27", "10.0.0.30"]] + """ + return [ + ip.compressed + for start_ip, end_ip in ip_ranges + for net in ipaddress.summarize_address_range( + ipaddress.ip_address(start_ip), + ipaddress.ip_address(end_ip) + ) + for ip in net + ] + + +def decorate_methods_with(dec): + if not CONF.debug: + return lambda cls: cls + + def inner(cls): + for attr_name, attr_val in cls.__dict__.items(): + if (isinstance(attr_val, types.FunctionType) and + not attr_name.startswith("_")): + setattr(cls, attr_name, dec(attr_val)) + return cls + + return inner + + +def verbose_driver_trace(fn): + if not CONF.debug: + return fn + + def inner(self, *args, **kwargs): + start = timeutils.utcnow() + LOG.debug(f"[{fn.__name__}] >>>") + res = fn(self, *args, **kwargs) + end = timeutils.utcnow() + LOG.debug( + f"Spent {timeutils.delta_seconds(start, end)} sec. " + f"Return {res}.\n" + f"<<< [{fn.__name__}]" + ) + return res + + return inner diff --git a/manila/share/drivers/vastdata/rest.py b/manila/share/drivers/vastdata/rest.py new file mode 100644 index 0000000000..0b46f170a4 --- /dev/null +++ b/manila/share/drivers/vastdata/rest.py @@ -0,0 +1,332 @@ +# Copyright 2024 VAST Data Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from abc import ABC +import json +import pprint +import textwrap + +import cachetools +from oslo_log import log as logging +from oslo_utils import versionutils +from packaging import version as packaging_version +import requests + +from manila import exception +from manila.share.drivers.vastdata import driver_util +import manila.utils as manila_utils + +LOG = logging.getLogger(__name__) + + +class Session(requests.Session): + + def __init__(self, host, username, password, ssl_verify, plugin_version): + super().__init__() + self.base_url = f"https://{host.strip('/')}/api" + self.ssl_verify = ssl_verify + self.username = username + self.password = password + self.headers["Accept"] = "application/json" + self.headers["Content-Type"] = "application/json" + self.headers["User-Agent"] = ( + f"manila/v{plugin_version}" + f" ({requests.utils.default_user_agent()})" + ) + # will be updated on first request + self.headers["authorization"] = "Bearer" + + if not ssl_verify: + import urllib3 + + urllib3.disable_warnings() + + def refresh_auth_token(self): + try: + resp = super().request( + "POST", + f"{self.base_url}/token/", + verify=self.ssl_verify, + timeout=5, + json={"username": self.username, "password": self.password}, + ) + resp.raise_for_status() + token = resp.json()["access"] + self.headers["authorization"] = f"Bearer {token}" + except ConnectionError as e: + raise exception.VastApiException( + reason=f"The vms on the designated host {self.base_url} " + f"cannot be accessed. Please verify the specified endpoint. " + f"origin error: {e}" + ) + + @manila_utils.retry(retry_param=exception.VastApiRetry, retries=3) + def request( + self, verb, api_method, params=None, log_result=True, **kwargs + ): + verb = verb.upper() + api_method = api_method.strip("/") + url = f"{self.base_url}/{api_method}/" + log_pref = f"\n[{verb}] {url}" + + if "data" in kwargs: + kwargs["data"] = json.dumps(kwargs["data"]) + + if log_result and (params or kwargs): + payload = dict(kwargs, params=params) + formatted_request = textwrap.indent( + pprint.pformat(payload), prefix="| " + ) + LOG.debug(f"{log_pref} >>>:\n{formatted_request}") + else: + LOG.debug(f"{log_pref} >>> (request)") + + ret = super().request( + verb, url, verify=self.ssl_verify, params=params, **kwargs + ) + if ret.status_code == 403 and "Token is invalid" in ret.text: + self.refresh_auth_token() + raise exception.VastApiRetry(reason="Token is invalid or expired.") + + if ret.status_code in (400, 503) and ret.text: + raise exception.VastApiException(reason=ret.text) + + try: + ret.raise_for_status() + except Exception as exc: + raise exception.VastApiException(reason=str(exc)) + + ret = ret.json() if ret.content else {} + if ret and log_result: + formatted_response = textwrap.indent( + pprint.pformat(ret), prefix="| " + ) + LOG.debug(f"{log_pref} <<<:\n{formatted_response}") + else: + LOG.debug(f"{log_pref} <<< (response)") + return driver_util.Bunch.from_dict(ret) + + def __getattr__(self, attr): + if attr.startswith("_"): + raise AttributeError(attr) + + def func(**params): + return self.request("get", attr, params=params) + + func.__name__ = attr + setattr(self, attr, func) + return func + + +def requisite(semver: str, operation: str = None): + """Use this decorator to indicate the minimum required version cluster + + for invoking the API that is being decorated. + Decorator works in two modes: + 1. When ignore == False and version mismatch detected then + `OperationNotSupported` exception will be thrown + 2. When ignore == True and version mismatch detected then + method decorated method execution never happened + """ + + def dec(fn): + + def _args_wrapper(self, *args, **kwargs): + + version = packaging_version.parse( + self.rest.get_sw_version().replace("-", ".") + ) + sw_version = f"{version.major}.{version.minor}.{version.micro}" + + if not versionutils.is_compatible( + semver, sw_version, same_major=False + ): + op = operation or fn.__name__ + raise exception.VastDriverException( + f"Operation {op} is not supported" + f" on VAST version {sw_version}." + f" Required version is {semver}" + ) + return fn(self, *args, **kwargs) + + return _args_wrapper + + return dec + + +class VastResource(ABC): + resource_name = None + + def __init__(self, rest): + self.rest = rest # For intercommunication between resources. + self.session = rest.session + + def list(self, **params): + """Get list of entries with optional filtering params""" + return self.session.get(self.resource_name, params=params) + + def create(self, **params): + """Create new entry with provided params""" + return self.session.post(self.resource_name, data=params) + + def update(self, entry_id, **params): + """Update entry by id with provided params""" + return self.session.patch( + f"{self.resource_name}/{entry_id}", data=params + ) + + def delete(self, name): + """Delete entry by name. Skip if entry not found.""" + entry = self.one(name) + if not entry: + resource = self.__class__.__name__.lower() + LOG.warning( + f"{resource} {name} not found on VAST, skipping delete" + ) + return + return self.session.delete(f"{self.resource_name}/{entry.id}") + + def one(self, name): + """Get single entry by name. + + Raise exception if multiple entries found. + """ + entries = self.list(name=name) + if not entries: + return + if len(entries) > 1: + resource = self.__class__.__name__.lower() + "s" + raise exception.VastDriverException( + reason=f"Too many {resource} found with name {name}" + ) + return entries[0] + + def ensure(self, name, **params): + entry = self.one(name) + if not entry: + entry = self.create(name=name, **params) + return entry + + +class View(VastResource): + resource_name = "views" + + def create(self, name, path, policy_id): + data = dict( + name=name, + path=path, + policy_id=policy_id, + create_dir=True, + protocols=["NFS"], + ) + return super().create(**data) + + +class ViewPolicy(VastResource): + resource_name = "viewpolicies" + + +class CapacityMetrics(VastResource): + + def get(self, metrics, object_type="cluster", time_frame="1m"): + """Get capacity metrics for the cluster""" + params = dict( + prop_list=metrics, + object_type=object_type, time_frame=time_frame + ) + ret = self.session.get("monitors/ad_hoc_query", params=params) + last_sample = ret.data[-1] + return driver_util.Bunch( + { + name.partition(",")[-1]: value + for name, value in zip(ret.prop_list, last_sample) + } + ) + + +class Quota(VastResource): + resource_name = "quotas" + + +class VipPool(VastResource): + resource_name = "vippools" + + def vips(self, pool_name): + """Get list of ip addresses from vip pool""" + vippool = self.one(name=pool_name) + if not vippool: + raise exception.VastDriverException( + reason=f"No vip pool found with name {pool_name}" + ) + vips = driver_util.generate_ip_range(vippool.ip_ranges) + if not vips: + raise exception.VastDriverException( + reason=f"Pool {pool_name} has no available vips" + ) + return vips + + +class Snapshots(VastResource): + resource_name = "snapshots" + + +class Folders(VastResource): + resource_name = "folders" + + @requisite(semver="4.7.0") + def delete(self, path): + try: + self.session.delete( + f"{self.resource_name}/delete_folder/", data=dict(path=path) + ) + except exception.VastApiException as e: + exc_msg = str(e) + if "no such directory" in exc_msg: + LOG.debug(f"remote directory " + f"might have been removed earlier. ({e})") + elif "trash folder disabled" in exc_msg: + raise exception.VastDriverException( + reason="Trash Folder Access is disabled" + " (see Settings/Cluster/Features in VMS)" + ) + else: + # unpredictable error + raise + + +class RestApi: + + def __init__(self, host, username, password, ssl_verify, plugin_version): + self.session = Session( + host=host, + username=username, + password=password, + ssl_verify=ssl_verify, + plugin_version=plugin_version, + ) + self.views = View(self) + self.view_policies = ViewPolicy(self) + self.capacity_metrics = CapacityMetrics(self) + self.quotas = Quota(self) + self.vip_pools = VipPool(self) + self.snapshots = Snapshots(self) + self.folders = Folders(self) + + # Refresh auth token to avoid initial "forbidden" status error. + self.session.refresh_auth_token() + + @cachetools.cached(cache=cachetools.TTLCache(ttl=60 * 60, maxsize=1)) + def get_sw_version(self): + """Software version of cluster Rest API interacts with""" + return self.session.versions(status="success")[0].sys_version diff --git a/manila/tests/share/drivers/vastdata/__init__.py b/manila/tests/share/drivers/vastdata/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/manila/tests/share/drivers/vastdata/test_driver.py b/manila/tests/share/drivers/vastdata/test_driver.py new file mode 100644 index 0000000000..4c43decc89 --- /dev/null +++ b/manila/tests/share/drivers/vastdata/test_driver.py @@ -0,0 +1,690 @@ +# Copyright 2024 VAST Data Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import itertools +import unittest +from unittest import mock + +import ddt +import netaddr + +import manila.context as manila_context +import manila.exception as exception +from manila.share import configuration +from manila.share.drivers.vastdata import driver +from manila.share.drivers.vastdata import driver_util +from manila.tests import fake_share +from manila.tests.share.drivers.vastdata.test_rest import fake_metrics + + +@mock.patch( + "manila.share.drivers.vastdata.rest.Session.refresh_auth_token", + mock.MagicMock() +) +@ddt.ddt +class VASTShareDriverTestCase(unittest.TestCase): + def _create_mocked_rest_api(self): + # Create a mock RestApi instance + mock_rest_api = mock.MagicMock() + + # Create mock sub resources with their methods + subresources = [ + "views", + "view_policies", + "capacity_metrics", + "quotas", + "vip_pools", + "snapshots", + "folders", + ] + methods = [ + "list", "create", "update", + "delete", "one", "ensure", "vips" + ] + + for subresource in subresources: + mock_subresource = mock.MagicMock() + setattr(mock_rest_api, subresource, mock_subresource) + + for method in methods: + mock_method = mock.MagicMock() + setattr(mock_subresource, method, mock_method) + + return mock_rest_api + + @mock.patch( + "manila.share.drivers.vastdata.rest.Session.refresh_auth_token" + ) + def setUp(self, m_auth_token): + super().setUp() + self.fake_conf = configuration.Configuration(None) + self._context = manila_context.get_admin_context() + self._snapshot = fake_share.fake_snapshot_instance() + + self.fake_conf.set_default("driver_handles_share_servers", False) + self.fake_conf.set_default("share_backend_name", "vast") + self.fake_conf.set_default("vast_mgmt_host", "test") + self.fake_conf.set_default("vast_root_export", "/fake") + self.fake_conf.set_default("vast_vippool_name", "vippool") + self.fake_conf.set_default("vast_mgmt_user", "user") + self.fake_conf.set_default("vast_mgmt_password", "password") + self._driver = driver.VASTShareDriver( + execute=mock.MagicMock(), configuration=self.fake_conf + ) + self._driver.do_setup(self._context) + m_auth_token.assert_called_once() + + def test_do_setup(self): + session = self._driver.rest.session + self.assertEqual(self._driver._backend_name, "vast") + self.assertEqual(self._driver._vippool_name, "vippool") + self.assertEqual(self._driver._root_export, "/fake") + self.assertFalse(session.ssl_verify) + self.assertEqual(session.base_url, "https://test:443/api") + + @ddt.data("vast_mgmt_user", "vast_vippool_name") + def test_do_setup_missing_required_fields(self, missing_field): + self.fake_conf.set_default(missing_field, None) + _driver = driver.VASTShareDriver( + execute=mock.MagicMock(), configuration=self.fake_conf + ) + with self.assertRaises(exception.VastDriverException): + _driver.do_setup(self._context) + + @mock.patch( + "manila.share.drivers.vastdata.rest.Session.get", + mock.MagicMock(return_value=fake_metrics), + ) + def test_update_share_stats(self): + self._driver._update_share_stats() + result = self._driver._stats + self.assertEqual(result["share_backend_name"], "vast") + self.assertEqual(result["driver_handles_share_servers"], False) + self.assertEqual(result["vendor_name"], "VAST STORAGE") + self.assertEqual(result["driver_version"], "1.0") + self.assertEqual(result["storage_protocol"], "NFS") + self.assertEqual(result["total_capacity_gb"], 471.1061706542969) + self.assertEqual(result["free_capacity_gb"], 450.2256333641708) + self.assertEqual(result["reserved_percentage"], 0) + self.assertEqual(result["reserved_snapshot_percentage"], 0) + self.assertEqual(result["reserved_share_extend_percentage"], 0) + self.assertIs(result["qos"], False) + self.assertIsNone(result["pools"]) + self.assertIs(result["snapshot_support"], True) + self.assertIs(result["create_share_from_snapshot_support"], False) + self.assertIs(result["revert_to_snapshot_support"], False) + self.assertIs(result["mount_snapshot_support"], False) + self.assertIsNone(result["replication_domain"]) + self.assertIsNone(result["filter_function"]) + self.assertIsNone(result["goodness_function"]) + self.assertIs(result["security_service_update_support"], False) + self.assertIs(result["network_allocation_update_support"], False) + self.assertIs(result["share_server_multiple_subnet_support"], False) + self.assertIs(result["mount_point_name_support"], False) + self.assertEqual(result["data_reduction"], 1.2) + self.assertEqual(result["provisioned_capacity_gb"], 20.880537290126085) + self.assertEqual( + result["share_group_stats"], + {"consistent_snapshot_support": None} + ) + self.assertIs(result["ipv4_support"], True) + self.assertIs(result["ipv6_support"], False) + + @ddt.idata( + itertools.product( + [1073741824, 1], ["NFS", "SMB"], ["fakeid", None] + ) + ) + @ddt.unpack + def test_create_shares(self, capacity, proto, policy): + share = fake_share.fake_share(share_proto=proto) + mock_rest = self._create_mocked_rest_api() + mock_rest.view_policies.ensure.return_value = driver_util.Bunch(id=1) + mock_rest.quotas.ensure.return_value = driver_util.Bunch( + id=2, hard_limit=capacity + ) + mock_rest.views.ensure.return_value = driver_util.Bunch( + id=3, policy=policy + ) + mock_rest.vip_pools.vips.return_value = ["1.1.1.0", "1.1.1.1"] + with mock.patch.object(self._driver, "rest", mock_rest): + if proto != "NFS": + with self.assertRaises(exception.InvalidShare) as exc: + self._driver.create_share(self._context, share) + self.assertIn( + "Invalid NAS protocol supplied", + str(exc.exception) + ) + elif capacity == 1: + with self.assertRaises(exception.ManilaException) as exc: + self._driver.create_share(self._context, share) + self.assertIn( + "Share already exists with different capacity", + str(exc.exception) + ) + else: + + location = self._driver.create_share(self._context, share) + mock_rest.vip_pools.vips.assert_called_once_with( + pool_name="vippool" + ) + mock_rest.view_policies.ensure.assert_called_once_with( + name="fakeid" + ) + mock_rest.quotas.ensure.assert_called_once_with( + name="fakeid", + path="/fake/manila-fakeid", + create_dir=True, + hard_limit=capacity, + ) + mock_rest.views.ensure.assert_called_once_with( + name="fakeid", path="/fake/manila-fakeid", policy_id=1 + ) + self.assertDictEqual( + location, + { + 'path': '1.1.1.0:/fake/manila-fakeid', + 'is_admin_only': False + } + ) + if not policy: + mock_rest.views.update.assert_called_once_with( + 3, policy_id=1 + ) + else: + mock_rest.views.update.assert_not_called() + + def test_delete_share(self): + share = fake_share.fake_share(share_proto="NFS") + mock_rest = self._create_mocked_rest_api() + with mock.patch.object(self._driver, "rest", mock_rest): + self._driver.delete_share(self._context, share) + mock_rest.folders.delete.assert_called_once_with( + path="/fake/manila-fakeid" + ) + mock_rest.views.delete.assert_called_once_with(name="fakeid") + mock_rest.quotas.delete.assert_called_once_with(name="fakeid") + mock_rest.view_policies.delete.assert_called_once_with(name="fakeid") + + def test_update_access_rules_wrong_proto(self): + share = fake_share.fake_share(share_proto="SMB") + access_rules = [ + { + "access_level": "rw", + "access_to": "127.0.0.1", + "access_type": "ip" + } + ] + res = self._driver.update_access( + self._context, + share, + access_rules, + None, + None + ) + self.assertIsNone(res) + + def test_update_access_add_rules_no_policy(self): + share = fake_share.fake_share(share_proto="NFS") + mock_rest = self._create_mocked_rest_api() + mock_rest.view_policies.one.return_value = None + access_rules = [ + { + "access_level": "rw", + "access_to": "127.0.0.1", + "access_type": "ip" + } + ] + with mock.patch.object(self._driver, "rest", mock_rest): + with self.assertRaises(exception.ManilaException) as exc: + self._driver.update_access( + self._context, share, access_rules, None, None + ) + self.assertIn("Policy not found", str(exc.exception)) + + @ddt.data( + (["*"], ["10.10.10.1", "10.10.10.2"]), + (["10.10.10.1", "10.10.10.2"], []), + (["*"], []), + ) + @ddt.unpack + def test_update_access_add_rules(self, rw, ro): + share = fake_share.fake_share(share_proto="NFS") + mock_rest = self._create_mocked_rest_api() + mock_rest.view_policies.one.return_value = driver_util.Bunch( + id=1, nfs_read_write=rw, nfs_read_only=ro + ) + access_rules = [ + { + "access_level": "rw", + "access_to": "127.0.0.1", + "access_type": "ip" + } + ] + with mock.patch.object(self._driver, "rest", mock_rest): + failed_rules = self._driver.update_access( + self._context, + share, + access_rules, + None, + None + ) + + expected_ro = set(ro) + if rw == ["*"]: + expected_rw = {"127.0.0.1"} + else: + expected_rw = set(["127.0.0.1"] + rw) + kw = mock_rest.view_policies.update.call_args.kwargs + self.assertEqual(kw["name"], "fakeid") + self.assertSetEqual(set(kw["nfs_read_write"]), expected_rw) + self.assertSetEqual(set(kw["nfs_read_only"]), expected_ro) + self.assertEqual(kw["nfs_no_squash"], ["*"]) + self.assertEqual(kw["nfs_root_squash"], ["*"]) + self.assertFalse(failed_rules) + + # and the same for ro + mock_rest = self._create_mocked_rest_api() + mock_rest.view_policies.one.return_value = driver_util.Bunch( + id=1, nfs_read_write=rw, nfs_read_only=ro + ) + access_rules = [ + { + "access_level": "ro", + "access_to": "127.0.0.1", + "access_type": "ip" + } + ] + with mock.patch.object(self._driver, "rest", mock_rest): + failed_rules = self._driver.update_access( + self._context, + share, + access_rules, + None, + None + ) + + expected_rw = set(rw) + if ro == ["*"]: + expected_ro = {"127.0.0.1"} + else: + expected_ro = set(["127.0.0.1"] + ro) + kw = mock_rest.view_policies.update.call_args.kwargs + self.assertEqual(kw["name"], "fakeid") + self.assertSetEqual(set(kw["nfs_read_write"]), expected_rw) + self.assertSetEqual(set(kw["nfs_read_only"]), expected_ro) + self.assertEqual(kw["nfs_no_squash"], ["*"]) + self.assertEqual(kw["nfs_root_squash"], ["*"]) + self.assertFalse(failed_rules) + + @ddt.data( + (["*"], ["10.10.10.1", "10.10.10.2"]), + (["10.10.10.1", "10.10.10.2"], []), + (["*"], []), + ) + @ddt.unpack + def test_update_access_delete_rules(self, rw, ro): + share = fake_share.fake_share(share_proto="NFS") + mock_rest = self._create_mocked_rest_api() + mock_rest.view_policies.one.return_value = driver_util.Bunch( + id=1, nfs_read_write=rw, nfs_read_only=ro + ) + delete_rules = [ + { + "access_level": "rw", + "access_to": "10.10.10.1", + "access_type": "ip" + } + ] + with mock.patch.object(self._driver, "rest", mock_rest): + failed_rules = self._driver.update_access( + self._context, share, + None, + None, + delete_rules, + ) + + expected_ro = set(ro) + if rw == ["*"]: + expected_rw = set(rw) + else: + expected_rw = set([r for r in rw if r != "10.10.10.1"]) + kw = mock_rest.view_policies.update.call_args.kwargs + self.assertEqual(kw["name"], "fakeid") + self.assertSetEqual(set(kw["nfs_read_write"]), expected_rw) + self.assertSetEqual(set(kw["nfs_read_only"]), expected_ro) + self.assertEqual(kw["nfs_no_squash"], ["*"]) + self.assertEqual(kw["nfs_root_squash"], ["*"]) + self.assertFalse(failed_rules) + + # and the same for ro + mock_rest = self._create_mocked_rest_api() + mock_rest.view_policies.one.return_value = driver_util.Bunch( + id=1, nfs_read_write=rw, nfs_read_only=ro + ) + delete_rules = [ + { + "access_level": "ro", + "access_to": "10.10.10.1", + "access_type": "ip" + } + ] + with mock.patch.object(self._driver, "rest", mock_rest): + failed_rules = self._driver.update_access( + self._context, share, None, None, delete_rules + ) + + expected_rw = set(rw) + if ro == ["*"]: + expected_ro = set(ro) + else: + expected_ro = set([r for r in ro if r != "10.10.10.1"]) + kw = mock_rest.view_policies.update.call_args.kwargs + self.assertEqual(kw["name"], "fakeid") + self.assertSetEqual(set(kw["nfs_read_write"]), expected_rw) + self.assertSetEqual(set(kw["nfs_read_only"]), expected_ro) + self.assertEqual(kw["nfs_no_squash"], ["*"]) + self.assertEqual(kw["nfs_root_squash"], ["*"]) + self.assertFalse(failed_rules) + + def test_update_access_for_cidr(self): + share = fake_share.fake_share(share_proto="NFS") + mock_rest = self._create_mocked_rest_api() + mock_rest.view_policies.one.return_value = driver_util.Bunch( + id=1, nfs_read_write=["10.0.0.1"], nfs_read_only=["*"] + ) + access_rules = [ + { + "access_level": "ro", + "access_to": "10.0.0.1/29", + "access_type": "ip", + "access_id": 12345, + } + ] + with mock.patch.object(self._driver, "rest", mock_rest): + failed_rules = self._driver.update_access( + self._context, + share, + access_rules, + None, + None + ) + kw = mock_rest.view_policies.update.call_args.kwargs + self.assertEqual(kw["name"], "fakeid") + self.assertSetEqual(set(kw["nfs_read_write"]), {"10.0.0.1"}) + self.assertSetEqual( + set(kw["nfs_read_only"]), + { + '10.0.0.1', + '10.0.0.3', + '10.0.0.2', + '10.0.0.6', + '10.0.0.5', + '10.0.0.4' + } + ) + self.assertFalse(failed_rules) + + delete_rules = [ + { + "access_level": "ro", + "access_to": "10.0.0.1/30", + "access_type": "ip", + "access_id": 12345, + } + ] + mock_rest.view_policies.one.return_value = driver_util.Bunch( + id=1, nfs_read_write=["10.0.0.1"], + nfs_read_only=[ + '10.0.0.1', + '10.0.0.3', + '10.0.0.2', + '10.0.0.6', + '10.0.0.5', + '10.0.0.4', + ] + ) + with mock.patch.object(self._driver, "rest", mock_rest): + failed_rules = self._driver.update_access( + self._context, + share, + None, + None, + delete_rules, + ) + kw = mock_rest.view_policies.update.call_args.kwargs + self.assertEqual(kw["name"], "fakeid") + self.assertSetEqual(set(kw["nfs_read_write"]), {"10.0.0.1"}) + self.assertSetEqual( + set(kw["nfs_read_only"]), + {'10.0.0.6', '10.0.0.3', '10.0.0.4', '10.0.0.5'} + ) + self.assertFalse(failed_rules) + + def test_update_access_for_invalid_rules(self): + share = fake_share.fake_share(share_proto="NFS") + mock_rest = self._create_mocked_rest_api() + mock_rest.view_policies.one.return_value = driver_util.Bunch( + id=1, nfs_read_write=["10.0.0.1"], nfs_read_only=["*"] + ) + access_rules = [ + { + "access_level": "ry", + "access_to": "10.0.0.1", + "access_type": "ip", + "access_id": 12345, + "id": 12345, + }, + { + "access_level": "ro", + "access_to": "10.0.0.2", + "access_type": "ip", + "access_id": 12346, + "id": 12346, + }, + { + "access_level": "ro", + "access_to": "10.0.0.2/33", + "access_type": "ip", + "access_id": 12347, + "id": 12347, + }, + { + "access_level": "rw", + "access_to": "10.0.0.2.4", + "access_type": "ip", + "access_id": 12348, + "id": 12348, + } + ] + with mock.patch.object(self._driver, "rest", mock_rest): + failed_rules = self._driver.update_access( + self._context, + share, + access_rules, + None, + None + ) + kw = mock_rest.view_policies.update.call_args.kwargs + self.assertEqual(kw["name"], "fakeid") + self.assertSetEqual(set(kw["nfs_read_write"]), {"10.0.0.1"}) + self.assertSetEqual(set(kw["nfs_read_only"]), {'10.0.0.2'}) + self.assertDictEqual( + failed_rules, + { + 12345: {'state': 'error'}, + 12347: {'state': 'error'}, + 12348: {'state': 'error'} + } + ) + + def test_resize_share_quota_not_found(self): + share = fake_share.fake_share(share_proto="NFS") + mock_rest = self._create_mocked_rest_api() + mock_rest.quotas.one.return_value = None + with mock.patch.object(self._driver, "rest", mock_rest): + with self.assertRaises(exception.ShareNotFound) as exc: + self._driver.extend_share(share, 10000) + self.assertIn("could not be found", str(exc.exception)) + + def test_resize_share_ok(self): + share = fake_share.fake_share(share_proto="NFS") + mock_rest = self._create_mocked_rest_api() + mock_rest.quotas.one.return_value = driver_util.Bunch( + id=1, used_effective_capacity=1073741824 + ) + with mock.patch.object(self._driver, "rest", mock_rest): + self._driver.extend_share(share, 50) + mock_rest.quotas.update.assert_called_with( + 1, hard_limit=53687091200 + ) + mock_rest.quotas.update.reset() + self._driver.shrink_share(share, 20) + mock_rest.quotas.update.assert_called_with( + 1, hard_limit=21474836480 + ) + + def test_resize_share_exceeded_hard_limit(self): + share = fake_share.fake_share( + share_proto="NFS" + ) + mock_rest = self._create_mocked_rest_api() + mock_rest.quotas.one.return_value = driver_util.Bunch( + id=1, used_effective_capacity=10737418240 + ) # 10GB + with mock.patch.object(self._driver, "rest", mock_rest): + with self.assertRaises(exception.ShareShrinkingPossibleDataLoss): + self._driver.shrink_share(share, 9.7) + self._driver.shrink_share(share, 10) + + def test_create_snapshot(self): + snapshot = driver_util.Bunch( + name="fakesnap", share_instance_id="fakeid" + ) + mock_rest = self._create_mocked_rest_api() + with mock.patch.object(self._driver, "rest", mock_rest): + self._driver.create_snapshot(self._context, snapshot, None) + mock_rest.snapshots.create.assert_called_once_with( + path="/fake/manila-fakeid", name="fakesnap" + ) + + def test_delete_snapshot(self): + snapshot = driver_util.Bunch( + name="fakesnap", share_instance_id="fakeid" + ) + mock_rest = self._create_mocked_rest_api() + with mock.patch.object(self._driver, "rest", mock_rest): + self._driver.delete_snapshot(self._context, snapshot, None) + mock_rest.snapshots.delete.assert_called_once_with(name="fakesnap") + + def test_network_allocation_number(self): + self.assertEqual(self._driver.get_network_allocations_number(), 0) + + @ddt.data([], ['fake/path/1', 'fake/path']) + def test_ensure_shares(self, fake_export_locations): + mock_rest = self._create_mocked_rest_api() + mock_rest.view_policies.ensure.return_value = driver_util.Bunch(id=1) + mock_rest.quotas.ensure.return_value = driver_util.Bunch( + id=2, hard_limit=1073741824 + ) + mock_rest.views.ensure.return_value = driver_util.Bunch( + id=3, policy="test_policy" + ) + shares = [ + fake_share.fake_share( + id=_id, + share_id=share_id, + share_proto="NFS", + export_locations=fake_export_locations, + ) + for _id, share_id in enumerate(["123", "456", "789"], 1) + ] + mock_rest.vip_pools.vips.return_value = ["1.1.1.0", "1.1.1.1"] + with mock.patch.object(self._driver, "rest", mock_rest): + locations = self._driver.ensure_shares(self._context, shares) + + common = {"is_admin_only": False} + self.assertDictEqual( + locations, + { + 1: { + "export_locations": [ + {"path": "1.1.1.0:/fake/manila-1", **common}, + {"path": "1.1.1.1:/fake/manila-1", **common}, + ] + }, + 2: { + "export_locations": [ + {"path": "1.1.1.0:/fake/manila-2", **common}, + {"path": "1.1.1.1:/fake/manila-2", **common}, + ] + }, + 3: { + "export_locations": [ + {"path": "1.1.1.0:/fake/manila-3", **common}, + {"path": "1.1.1.1:/fake/manila-3", **common}, + ] + }, + }, + ) + + def test_backend_info(self): + backend_info = self._driver.get_backend_info(self._context) + self.assertDictEqual( + backend_info, + {'vast_vippool_name': 'vippool', 'vast_mgmt_host': 'test'} + ) + + +class TestPolicyPayloadFromRules(unittest.TestCase): + def test_policy_payload_from_rules_update(self): + rules = [{"access_level": "rw", "access_to": "127.0.0.1"}] + policy = mock.MagicMock() + policy.nfs_read_write = ["127.0.0.1"] + policy.nfs_read_only = [] + result = driver.policy_payload_from_rules(rules, policy, "update") + self.assertEqual( + result, {"nfs_read_write": ["127.0.0.1"], "nfs_read_only": []} + ) + + def test_policy_payload_from_rules_deny(self): + rules = [{"access_level": "rw", "access_to": "127.0.0.1"}] + policy = mock.MagicMock() + policy.nfs_read_write = ["127.0.0.1"] + policy.nfs_read_only = [] + result = driver.policy_payload_from_rules(rules, policy, "deny") + self.assertEqual(result, {"nfs_read_write": [], "nfs_read_only": []}) + + def test_policy_payload_from_rules_invalid_action(self): + rules = [{"access_level": "rw", "access_to": "127.0.0.1"}] + with self.assertRaises(ValueError): + driver.policy_payload_from_rules(rules, None, "invalid") + + def test_policy_payload_from_rules_invalid_ip(self): + rules = [{"access_level": "rw", "access_to": "1.0.0.257"}] + with self.assertRaises(netaddr.core.AddrFormatError): + driver.policy_payload_from_rules(rules, None, "deny") + + +class TestValidateAccessRules(unittest.TestCase): + def test_validate_access_rules_invalid_type(self): + rule = {"access_type": "INVALID", "access_level": "rw"} + with self.assertRaises(exception.InvalidShareAccess): + driver.validate_access_rule(rule) + + def test_validate_access_rules_invalid_level(self): + rule = {"access_type": "ip", "access_level": "INVALID"} + with self.assertRaises(exception.InvalidShareAccessLevel): + driver.validate_access_rule(rule) diff --git a/manila/tests/share/drivers/vastdata/test_driver_util.py b/manila/tests/share/drivers/vastdata/test_driver_util.py new file mode 100644 index 0000000000..d82f56ed0f --- /dev/null +++ b/manila/tests/share/drivers/vastdata/test_driver_util.py @@ -0,0 +1,267 @@ +# Copyright 2024 VAST Data Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import json +import pickle +from unittest import mock + +import ddt + +from manila.share.drivers.vastdata import driver_util +from manila import test + + +driver_util.CONF.debug = True + + +@ddt.ddt +class TestBunch(test.TestCase): + def setUp(self): + super(TestBunch, self).setUp() + self.bunch = driver_util.Bunch(a=1, b=2) + + def test_bunch_getattr(self): + self.assertEqual(self.bunch.a, 1) + + def test_bunch_setattr(self): + self.bunch.c = 3 + self.assertEqual(self.bunch.c, 3) + + def test_bunch_delattr(self): + del self.bunch.a + self.assertRaises(AttributeError, lambda: self.bunch.a) + + def test_bunch_to_dict(self): + self.assertEqual(self.bunch.to_dict(), {"a": 1, "b": 2}) + + def test_bunch_from_dict(self): + self.assertEqual( + driver_util.Bunch.from_dict({"a": 1, "b": 2}), self.bunch + ) + + def test_bunch_to_json(self): + self.assertEqual(self.bunch.to_json(), json.dumps({"a": 1, "b": 2})) + + def test_bunch_without(self): + self.assertEqual(self.bunch.without("a"), driver_util.Bunch(b=2)) + + def test_bunch_but_with(self): + self.assertEqual( + self.bunch.but_with(c=3), driver_util.Bunch(a=1, b=2, c=3) + ) + + def test_bunch_delattr_missing(self): + self.assertRaises( + AttributeError, + lambda: self.bunch.__delattr__("non_existing_attribute") + ) + + def test_bunch_from_json(self): + json_bunch = json.dumps({"a": 1, "b": 2}) + self.assertEqual(driver_util.Bunch.from_json(json_bunch), self.bunch) + + def test_bunch_render(self): + self.assertEqual(self.bunch.render(), "a=1, b=2") + + def test_bunch_pickle(self): + pickled_bunch = pickle.dumps(self.bunch) + unpickled_bunch = pickle.loads(pickled_bunch) + self.assertEqual(self.bunch, unpickled_bunch) + + @ddt.data(True, False) + def test_bunch_copy(self, deep): + copy_bunch = self.bunch.copy(deep=deep) + self.assertEqual(copy_bunch, self.bunch) + self.assertIsNot(copy_bunch, self.bunch) + + def test_name_starts_with_underscore_and_digit(self): + bunch = driver_util.Bunch() + bunch["1"] = "value" + self.assertEqual(bunch._1, "value") + + def test_bunch_recursion(self): + x = driver_util.Bunch( + a="a", b="b", d=driver_util.Bunch(x="axe", y="why") + ) + x.d.x = x + x.d.y = x.b + print(x) + + def test_bunch_repr(self): + self.assertEqual(repr(self.bunch), "Bunch(a=1, b=2)") + + def test_getitem_with_integral_key(self): + self.bunch["1"] = "value" + self.assertEqual(self.bunch[1], "value") + + def test_bunch_dir(self): + self.assertEqual( + set(i for i in dir(self.bunch) if not i.startswith("_")), + { + "a", + "b", + "but_with", + "clear", + "copy", + "from_dict", + "from_json", + "fromkeys", + "get", + "items", + "keys", + "pop", + "popitem", + "render", + "setdefault", + "to_dict", + "to_json", + "update", + "values", + "without", + }, + ) + + def test_bunch_edge_cases(self): + # Test edge cases for attribute access, setting, and deletion + self.bunch["key-with-special-chars_123"] = "value" + self.assertEqual(self.bunch["key-with-special-chars_123"], "value") + self.bunch["key-with-special-chars_123"] = None + self.assertIsNone(self.bunch["key-with-special-chars_123"]) + del self.bunch["key-with-special-chars_123"] + self.assertRaises( + KeyError, + lambda: self.bunch["key-with-special-chars_123"] + ) + + def test_bunch_deep_copy(self): + nested_bunch = driver_util.Bunch(x=driver_util.Bunch(y=1)) + deep_copy = nested_bunch.copy(deep=True) + self.assertIsNot(nested_bunch["x"], deep_copy["x"]) + self.assertEqual(nested_bunch["x"]["y"], deep_copy["x"]["y"]) + + def test_bunch_serialization(self): + # Test serialization with nested structures + nested_bunch = driver_util.Bunch(a=1, b=driver_util.Bunch(c=2)) + self.assertEqual(nested_bunch.to_dict(), {"a": 1, "b": {"c": 2}}) + self.assertEqual( + nested_bunch.to_json(), + json.dumps({"a": 1, "b": {"c": 2}}) + ) + + +class TestBunchify(test.TestCase): + def test_bunchify(self): + self.assertEqual( + driver_util.bunchify({"a": 1, "b": 2}, c=3), + driver_util.Bunch(a=1, b=2, c=3) + ) + x = driver_util.bunchify(dict(a=[dict(b=5), 9, (1, 2)], c=8)) + self.assertEqual(x.a[0].b, 5) + self.assertEqual(x.a[1], 9) + self.assertIsInstance(x.a[2], tuple) + self.assertEqual(x.c, 8) + self.assertEqual(x.pop("c"), 8) + + def test_bunchify_edge_cases(self): + # Test edge cases for bunchify function + self.assertEqual(driver_util.bunchify({}), driver_util.Bunch()) + + def test_bunchify_nested_structures(self): + # Test bunchify with nested structures + nested_dict = {"a": [{"b": 1}, 2]} + self.assertEqual(driver_util.bunchify(nested_dict).a[0].b, 1) + + +class TestUnbunchify(test.TestCase): + def test_unbunchify(self): + self.assertEqual( + driver_util.unbunchify(driver_util.Bunch(a=1, b=2)), + {"a": 1, "b": 2} + ) + + +@ddt.ddt +class TestGenerateIpRange(test.TestCase): + + @ddt.data( + ( + [["15.0.0.1", "15.0.0.4"], ["10.0.0.27", "10.0.0.30"]], + [ + "15.0.0.1", + "15.0.0.2", + "15.0.0.3", + "15.0.0.4", + "10.0.0.27", + "10.0.0.28", + "10.0.0.29", + "10.0.0.30", + ], + ), + ( + [["15.0.0.1", "15.0.0.1"], ["10.0.0.20", "10.0.0.20"]], + ["15.0.0.1", "10.0.0.20"], + ), + ([], []), + ) + @ddt.unpack + def test_generate_ip_range(self, ip_ranges, expected): + ips = driver_util.generate_ip_range(ip_ranges) + assert ips == expected + + def test_generate_ip_range_edge_cases(self): + # Test edge cases for generate_ip_range function + self.assertEqual(driver_util.generate_ip_range([]), []) + self.assertEqual(driver_util.generate_ip_range( + [["15.0.0.1", "15.0.0.1"]]), ["15.0.0.1"] + ) + + def test_generate_ip_range_large_range(self): + # Test with a large range of IPs + start_ip = "192.168.0.1" + end_ip = "192.168.255.255" + ips = driver_util.generate_ip_range([[start_ip, end_ip]]) + self.assertEqual(len(ips), 65535) + + +class MockClass1: + def method1(self): + return 1 + + def _private_method(self): + return 2 + + +class TestDecorateMethodsWith(test.TestCase): + + def test_decorate_methods_with(self): + decorated_cls = driver_util.decorate_methods_with( + mock.Mock())(MockClass1) + self.assertTrue(hasattr(decorated_cls, 'method1')) + self.assertTrue(hasattr(decorated_cls, '_private_method')) + + +class MockClass2: + @driver_util.verbose_driver_trace + def method1(self): + return 1 + + +class TestVerboseDriverTrace(test.TestCase): + + def test_verbose_driver_trace_debug_true(self): + mock_instance = MockClass2() + with mock.patch.object( + driver_util.LOG, 'debug') as mock_debug: + mock_instance.method1() + self.assertEqual(mock_debug.call_count, 2) diff --git a/manila/tests/share/drivers/vastdata/test_rest.py b/manila/tests/share/drivers/vastdata/test_rest.py new file mode 100644 index 0000000000..f76d3db9c0 --- /dev/null +++ b/manila/tests/share/drivers/vastdata/test_rest.py @@ -0,0 +1,512 @@ +# Copyright 2024 VAST Data Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import io +import unittest +from unittest import mock + +import ddt +import requests + +from manila import exception as manila_exception +from manila.share.drivers.vastdata import driver_util +from manila.share.drivers.vastdata import rest as vast_rest + + +fake_metrics = driver_util.Bunch.from_dict( + { + "object_ids": [1], + "prop_list": [ + "timestamp", + "object_id", + "Capacity,drr", + "Capacity,physical_space_in_use", + "Capacity,physical_space", + "Capacity,logical_space", + "Capacity,logical_space_in_use", + ], + "data": [ + [ + "2024-04-13T14:47:48Z", + 1, + 1.2, + 30635076602.0, + 711246584217.0, + 505850953728.0, + 22370924820.0, + ], + [ + "2024-04-13T14:47:38Z", + 1, + 1.2, + 30635109399.0, + 711246584217.0, + 505850134528.0, + 22370810131.0, + ], + [ + "2024-04-13T14:47:28Z", + 1, + 1.2, + 30635142195.0, + 711246584217.0, + 505849217024.0, + 22370720020.0, + ], + [ + "2024-04-13T14:47:18Z", + 1, + 1.2, + 30635174991.0, + 711246584217.0, + 505848365056.0, + 22370654484.0, + ], + [ + "2024-04-13T14:47:08Z", + 1, + 1.2, + 30635207787.0, + 711246584217.0, + 505847447552.0, + 22420396308.0, + ], + [ + "2024-04-13T14:46:58Z", + 1, + 1.2, + 30635248783.0, + 711246584217.0, + 505846398976.0, + 22420306196.0, + ], + ], + "granularity": None, + } +) + + +class TestSession(unittest.TestCase): + + def setUp(self): + self.session = vast_rest.Session( + "host", "username", + "password", False, "1.0" + ) + + @mock.patch("requests.Session.request") + def test_refresh_auth_token_success(self, mock_request): + mock_request.return_value.json.return_value = {"access": "test_token"} + self.session.refresh_auth_token() + self.assertEqual( + self.session.headers["authorization"], + "Bearer test_token" + ) + + @mock.patch("requests.Session.request") + def test_refresh_auth_token_failure(self, mock_request): + mock_request.side_effect = ConnectionError() + with self.assertRaises(manila_exception.VastApiException): + self.session.refresh_auth_token() + + @mock.patch("requests.Session.request") + def test_request_success(self, mock_request): + mock_request.return_value.status_code = 200 + self.session.request( + "GET", "test_method", + log_result=False, params={"foo": "bar"} + ) + mock_request.assert_called_once_with( + "GET", "https://host/api/test_method/", + verify=False, params={"foo": "bar"} + ) + + @mock.patch("requests.Session.request") + def test_request_failure_400(self, mock_request): + mock_request.return_value.status_code = 400 + mock_request.return_value.text = "foo/bar" + with self.assertRaises(manila_exception.VastApiException): + self.session.request( + "POST", "test_method", + data={"data": {"foo": "bar"}} + ) + + def test_request_failure_500(self): + + resp = requests.Response() + resp.status_code = 500 + resp.raw = io.BytesIO(b"Server error") + + with mock.patch( + "requests.Session.request", new=lambda *a, **k: resp + ): + with self.assertRaises(manila_exception.VastApiException) as exc: + self.session.request("GET", "test_method", log_result=False) + self.assertIn("Server Error", str(exc.exception)) + + def test_request_no_return_content(self): + resp = requests.Response() + resp.status_code = 200 + resp.raw = io.BytesIO(b"") + + with mock.patch( + "requests.Session.request", new=lambda *a, **k: resp + ): + res = self.session.request("GET", "test_method") + self.assertFalse(res) + + @mock.patch( + "manila.share.drivers.vastdata.rest.Session.refresh_auth_token", + mock.MagicMock() + ) + def test_refresh_token_retries(self): + resp = requests.Response() + resp.status_code = 403 + resp.raw = io.BytesIO(b"Token is invalid") + + with mock.patch("requests.Session.request", new=lambda *a, **k: resp): + with self.assertRaises(manila_exception.VastApiRetry): + self.session.request("POST", "test_method", foo="bar") + + def test_getattr_with_underscore(self): + with self.assertRaises(AttributeError): + self.session.__getattr__("_private") + + @mock.patch.object(vast_rest.Session, "request") + def test_getattr_without_underscore(self, mock_request): + attr = "public" + params = {"key": "value"} + self.session.__getattr__(attr)(**params) + mock_request.assert_called_once_with("get", attr, params=params) + + +class TestVastResource(unittest.TestCase): + def setUp(self): + self.mock_rest = mock.MagicMock() + self.vast_resource = vast_rest.VastResource(self.mock_rest) + + def test_list_with_filtering_params(self): + self.vast_resource.list(name="test") + self.mock_rest.session.get.assert_called_with( + self.vast_resource.resource_name, params={"name": "test"} + ) + + def test_create_with_provided_params(self): + self.vast_resource.create(name="test", size=10) + self.mock_rest.session.post.assert_called_with( + self.vast_resource.resource_name, data={"name": "test", "size": 10} + ) + + def test_update_with_provided_params(self): + self.vast_resource.update("1", name="test", size=10) + self.mock_rest.session.patch.assert_called_with( + f"{self.vast_resource.resource_name}/1", + data={"name": "test", "size": 10} + ) + + def test_delete_when_entry_not_found(self): + self.vast_resource.one = mock.MagicMock(return_value=None) + self.vast_resource.delete("test") + self.mock_rest.session.delete.assert_not_called() + + def test_delete_when_entry_found(self): + mock_entry = mock.MagicMock() + mock_entry.id = "1" + self.vast_resource.one = mock.MagicMock(return_value=mock_entry) + self.vast_resource.delete("test") + self.mock_rest.session.delete.assert_called_with( + f"{self.vast_resource.resource_name}/{mock_entry.id}" + ) + + def test_one_when_no_entries_found(self): + self.vast_resource.list = mock.MagicMock(return_value=[]) + result = self.vast_resource.one("test") + self.assertIsNone(result) + + def test_one_when_multiple_entries_found(self): + self.vast_resource.list = mock.MagicMock( + return_value=[mock.MagicMock(), mock.MagicMock()] + ) + with self.assertRaises(manila_exception.VastDriverException): + self.vast_resource.one("test") + + def test_one_when_single_entry_found(self): + mock_entry = mock.MagicMock() + self.vast_resource.list = mock.MagicMock(return_value=[mock_entry]) + result = self.vast_resource.one("test") + self.assertEqual(result, mock_entry) + + def test_ensure_when_entry_not_found(self): + self.vast_resource.one = mock.MagicMock(return_value=None) + mock_entry = mock.MagicMock() + self.vast_resource.create = mock.MagicMock(return_value=mock_entry) + result = self.vast_resource.ensure("test", size=10) + self.assertEqual(result, mock_entry) + + def test_ensure_when_entry_found(self): + mock_entry = mock.MagicMock() + self.vast_resource.one = mock.MagicMock(return_value=mock_entry) + result = self.vast_resource.ensure("test", size=10) + self.assertEqual(result, mock_entry) + + +class ViewTest(unittest.TestCase): + @mock.patch( + "manila.share.drivers.vastdata.rest.Session.refresh_auth_token", + mock.MagicMock() + ) + def test_view_create(self): + with mock.patch( + "manila.share.drivers.vastdata.rest.Session.post" + ) as mock_session: + rest_api = vast_rest.RestApi( + "host", + "username", + "password", + True, + "1.0" + ) + rest_api.views.create("test-view", "/test", 1) + + self.assertEqual(("views",), mock_session.call_args.args) + self.assertDictEqual( + { + "data": { + "name": "test-view", + "path": "/test", + "policy_id": 1, + "create_dir": True, + "protocols": ["NFS"], + } + }, + mock_session.call_args.kwargs, + ) + + +@mock.patch( + "manila.share.drivers.vastdata.rest.Session.refresh_auth_token", + mock.MagicMock() +) +@mock.patch( + "manila.share.drivers.vastdata.rest.Session.get", + mock.MagicMock(return_value=fake_metrics), +) +class TestCapacityMetrics(unittest.TestCase): + + def test_capacity_metrics(self): + metrics_list = [ + "Capacity,drr", + "Capacity,logical_space", + "Capacity,logical_space_in_use", + "Capacity,physical_space", + "Capacity,physical_space_in_use", + ] + expected = { + "": 1, + "drr": 1.2, + "physical_space_in_use": 30635248783.0, + "physical_space": 711246584217.0, + "logical_space": 505846398976.0, + "logical_space_in_use": 22420306196.0, + } + rest_api = vast_rest.RestApi( + "host", + "username", + "password", + True, + "1.0" + ) + metrics = rest_api.capacity_metrics.get(metrics_list) + self.assertDictEqual(expected, metrics) + + +@mock.patch( + "manila.share.drivers.vastdata.rest.Session.refresh_auth_token", + mock.MagicMock() +) +@ddt.ddt +class TestFolders(unittest.TestCase): + + @mock.patch( + "manila.share.drivers.vastdata.rest.Session.refresh_auth_token", + mock.MagicMock() + ) + def setUp(self): + self.rest_api = vast_rest.RestApi( + "host", "username", "password", True, "1.0" + ) + + @ddt.data( + "4.3.9", + "4.0.11.12", + "3.4.6.123.1", + "4.5.6-1", + "4.6.0", + "4.6.0-1", + "4.6.0-1.1", + "4.6.9", + ) + def test_requisite_decorator(self, cluster_version): + """Test `requisite` decorator produces exception + + when cluster version doesn't met requirements + """ + with mock.patch( + "manila.share.drivers.vastdata.rest.RestApi.get_sw_version", + new=lambda s: cluster_version, + ): + self.assertRaises( + manila_exception.VastDriverException, + lambda: self.rest_api.folders.delete("/abc") + ) + + def test_trash_api_disabled(self): + def raise_http_err(*args, **kwargs): + resp = requests.Response() + resp.status_code = 400 + resp.raw = io.BytesIO(b"trash folder disabled") + raise manila_exception.VastApiException(message=resp.text) + + with ( + mock.patch( + "manila.share.drivers.vastdata.rest.Session.delete", + side_effect=raise_http_err, + ), + mock.patch( + "manila.share.drivers.vastdata.rest.RestApi.get_sw_version", + new=lambda s: "5.0.0", + ), + ): + with self.assertRaises( + manila_exception.VastDriverException + ) as exc: + self.rest_api.folders.delete("/abc") + self.assertIn("Trash Folder Access is disabled", str(exc.exception)) + + def test_trash_api_unpredictable_error(self): + def raise_http_err(*args, **kwargs): + raise RuntimeError() + + with ( + mock.patch( + "manila.share.drivers.vastdata.rest.Session.delete", + side_effect=raise_http_err, + ), + mock.patch( + "manila.share.drivers.vastdata.rest.RestApi.get_sw_version", + new=lambda s: "5.0.0", + ), + ): + with self.assertRaises(RuntimeError): + self.rest_api.folders.delete("/abc") + + def test_double_deletion(self): + def raise_http_err(*args, **kwargs): + resp = requests.Response() + resp.status_code = 400 + resp.raw = io.BytesIO(b"no such directory") + raise manila_exception.VastApiException(message=resp.text) + + with ( + mock.patch( + "manila.share.drivers.vastdata.rest.Session.delete", + side_effect=raise_http_err, + ), + mock.patch( + "manila.share.drivers.vastdata.rest.RestApi.get_sw_version", + new=lambda s: "5.0.0", + ), + ): + with self.assertLogs(level="DEBUG") as cm: + self.rest_api.folders.delete("/abc") + self.assertIn( + "remote directory might have been removed earlier", + str(cm.output) + ) + + +class VipPoolTest(unittest.TestCase): + @mock.patch( + "manila.share.drivers.vastdata.rest.Session.refresh_auth_token", + mock.MagicMock() + ) + def setUp(self): + self.rest_api = vast_rest.RestApi( + "host", + "username", + "password", + True, + "1.0" + ) + + def test_no_vipool(self): + with mock.patch( + "manila.share.drivers.vastdata.rest.Session.get", + return_value=[] + ): + with self.assertRaises( + manila_exception.VastDriverException + ) as exc: + self.rest_api.vip_pools.vips("test-vip") + self.assertIn("No vip pool found", str(exc.exception)) + + def test_no_vips(self): + vippool = driver_util.Bunch(ip_ranges=[]) + with mock.patch( + "manila.share.drivers.vastdata.rest.Session.get", + return_value=[vippool] + ): + with self.assertRaises( + manila_exception.VastDriverException + ) as exc: + self.rest_api.vip_pools.vips("test-vip") + self.assertIn( + "Pool test-vip has no available vips", + str(exc.exception) + ) + + def test_vips_ok(self): + vippool = driver_util.Bunch( + ip_ranges=[["15.0.0.1", "15.0.0.4"], ["10.0.0.27", "10.0.0.30"]] + ) + expected = [ + "15.0.0.1", + "15.0.0.2", + "15.0.0.3", + "15.0.0.4", + "10.0.0.27", + "10.0.0.28", + "10.0.0.29", + "10.0.0.30", + ] + with mock.patch( + "manila.share.drivers.vastdata.rest.Session.get", + return_value=[vippool] + ): + vips = self.rest_api.vip_pools.vips("test-vip") + self.assertListEqual(vips, expected) + + +class TestRestApi(unittest.TestCase): + + @mock.patch("manila.share.drivers.vastdata.rest.Session") + def test_get_sw_version(self, mock_session): + mock_session.return_value.versions.return_value = [ + mock.MagicMock(sys_version="1.0") + ] + rest_api = vast_rest.RestApi( + "host", "username", "password", True, "1.0" + ) + version = rest_api.get_sw_version() + self.assertEqual(version, "1.0") diff --git a/releasenotes/notes/add-vastdriver-5a2ca79a81bc9280.yaml b/releasenotes/notes/add-vastdriver-5a2ca79a81bc9280.yaml new file mode 100644 index 0000000000..8ebe508df7 --- /dev/null +++ b/releasenotes/notes/add-vastdriver-5a2ca79a81bc9280.yaml @@ -0,0 +1,3 @@ +--- +features: + - Added driver for VastData filesystem. \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 8c7844a508..af74589dc4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -49,3 +49,5 @@ python-cinderclient!=4.0.0,>=3.3.0 # Apache-2.0 python-novaclient>=17.2.1 # Apache-2.0 python-glanceclient>=3.2.2 # Apache-2.0 WebOb>=1.8.6 # MIT +cachetools>=5.3.3 # MIT +packaging>=24.1 # Apache-2.0