diff --git a/devstack/local.conf b/devstack/local.conf index 79bea47e..1c076bfb 100644 --- a/devstack/local.conf +++ b/devstack/local.conf @@ -126,6 +126,25 @@ SWIFT_HASH=096d08da4f8d4cce3a724c5f6c18f055 SWIFT_REPLICAS=1 SWIFT_DATA_DIR=$DEST/data/swift +## SWIFT NOTIFICATIONS ### +#Notifications must be configured properly for searchlight to process +#incremental updates. Use the following:: + +#Add the following new section in swift proxy conf file +#[filter:oslomiddleware] +#paste.filter_factory = swift.common.middleware.oslo_notifications:filter_factory +#publisher_id = swift.localhost +#Replace ,, and for your environment values +#transport_url = rabbit://:@:/ +#notification_driver = messaging +#notification_topics = searchlight_indexer + +#Add oslomiddleware to pipeline:main see example below. +#[pipeline:main] +#pipeline = catch_errors gatekeeper healthcheck ... oslomiddleware proxy-logging proxy-server +#Restart swift proxy API service (s-proxy) after making changes. + + ### DESIGNATE ### # enable_plugin designate https://git.openstack.org/openstack/designate @@ -208,3 +227,14 @@ admin_only_fields=admin_state_up,status [resource_plugin:os_neutron_port] enabled = True + +[resource_plugin:os_swift_account] +enabled = false +#Specify same value as in swift proxy config for reseller_prefix +reseller_prefix = AUTH_ + +[resource_plugin:os_swift_container] +enabled = false + +[resource_plugin:os_swift_object] +enabled = false diff --git a/devstack/plugin.sh b/devstack/plugin.sh index c881ff15..0481f9bc 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -132,6 +132,11 @@ function configure_searchlight { # Plugin config - disable designate by default since it's not typically installed iniset $SEARCHLIGHT_CONF resource_plugin:os_designate_zone enabled False iniset $SEARCHLIGHT_CONF resource_plugin:os_designate_recordset enabled False + + # Plugin config - disable swift by default since it's not typically installed + iniset $SEARCHLIGHT_CONF resource_plugin:os_swift_account enabled False + iniset $SEARCHLIGHT_CONF resource_plugin:os_swift_container enabled False + iniset $SEARCHLIGHT_CONF resource_plugin:os_swift_object enabled False } # create_searchlight_accounts - Set up common required searchlight accounts diff --git a/doc/source/plugins.rst b/doc/source/plugins.rst index d78f02bd..a6176e24 100644 --- a/doc/source/plugins.rst +++ b/doc/source/plugins.rst @@ -101,6 +101,14 @@ Please read the rest of the guide for detailed information.:: [resource_plugin:os_designate_recordset] enabled = False + [resource_plugin:os_swift_account] + enabled = False + + [resource_plugin:os_swift_container] + enabled = False + + [resource_plugin:os_swift_object] + enabled = False Common Plugin Configuration Options ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/doc/source/plugins/swift.rst b/doc/source/plugins/swift.rst new file mode 100644 index 00000000..39512c8d --- /dev/null +++ b/doc/source/plugins/swift.rst @@ -0,0 +1,181 @@ +.. + (c) Copyright 2016 Hewlett-Packard Development Company, L.P. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +****************** +Swift Plugin Guide +****************** + +WARNING: Swift plugin is currently EXPERIMENTAL as notifications aren't +fully supported. See below on enabling notifications. + +Integration is provided via a plugin. There are multiple configuration +settings required for proper indexing and incremental updates. Some of the +settings are specified in Searchlight configuration files. Others are +provided in other service configuration files. + +Searchlight Configuration +========================= + +Searchlight resource configuration options are shown below with their +configuration file and default values. + +See :ref:`searchlight-plugins` for common options with their default values, +general configuration information, and an example complete configuration. + +.. note:: + + Unless you are changing to a non-default value, you do not need to + specify any of the following configuration options. + +searchlight.conf +---------------- + +Plugin: OS::Swift::Account +^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: + + [resource_plugin:os_swift_account] + enabled = true + index_name = searchlight + #Specify same value as in swift proxy config for reseller_prefix + reseller_prefix = AUTH_ + +.. note:: + + os_swift_account is disabled by default. You need to explicitly set enabled = True as shown above + +Plugin: OS::Swift::Container +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:: + + [resource_plugin:os_swift_container] + enabled = true + index_name = searchlight + +.. note:: + + os_swift_container is disabled by default. You need to explicitly set enabled = True as shown above + +Plugin: OS::Swift::Object +^^^^^^^^^^^^^^^^^^^^^^^^^ +:: + + [resource_plugin:os_swift_object] + enabled = true + index_name = searchlight + +.. note:: + + os_swift_object is disabled by default. You need to explicitly set enabled = True as shown above + +Swift Configuration +==================== + +The Swift service currently doesn't send notifications. +Apply this patch https://review.openstack.org/#/c/249471 +for adding notification middleware to swift. + +reseller_admin_role +------------------- + +Users with the Keystone role defined in reseller_admin_role (ResellerAdmin by default) +can operate on any account. The auth system sets the request environ reseller_request +to True if a request is coming from a user with this role. + +Searchlight needs this role for its service user to access all the swift accounts +for initial indexing. The searchlight user and sevice project being referred here is the +one defined in service_credentials section of searchlight conf file. + +:: + + openstack role add --user searchlight --project service ResellerAdmin + + +proxy-server.conf +----------------- + +Notifications must be configured properly for searchlight to process +incremental updates. Use the following:: + + #Add the following new section + [filter:oslomiddleware] + paste.filter_factory = swift.common.middleware.oslo_notifications:filter_factory + publisher_id = swift.localhost + #Replace ,, and for your environment values + transport_url = rabbit://:@:/ + notification_driver = messaging + notification_topics = searchlight_indexer + + #Add oslomiddleware to pipeline:main see example below. + [pipeline:main] + pipeline = catch_errors gatekeeper healthcheck ... oslomiddleware proxy-logging proxy-server + + +.. note:: + + Restart swift proxy API service (s-proxy) after making changes. + +local.conf (devstack) +--------------------- + +The settings above may be automatically configured by ``stack.sh`` +by adding them to the following post config section in devstack. +Just place the following in local.conf and copy the above settings +underneath it.:: + + [[post-config|$SWIFT_PROXY_CONF]] + [DEFAULT] + +Release Notes +============= + +0.2.0.0 (Mitaka) +---------------- + +Swift did not generate notifications for account/container/object CRUD + +This means that search results will not include incremental updates after +the initial indexing. + +The patch (https://review.openstack.org/#/c/249471) implements this feature. + +For devstack, the easiest way to test is +cd /opt/stack/swift +git review -x 249471 + +Searchlight developers/installers should apply the above patch in Swift when +using Searchlight with the Swift Mitaka release. + +Alternatively, you may set up a cron job to re-index swift +account/container/objects periodically to get updated information. The +recommendation is to use the notifications. + +You should use the ``--no-delete`` option to prevent the index from +temporarily not containing any data (which otherwise would happen with a full +bulk indexing job):: + + searchlight-manage index sync --type OS::Swift::Account --force --no-delete + +Searchlight swift plugin resource types follow the hierarchy similar to +Swift concepts + + OS::Swift:Acccount(Parent) + -> OS:Swift::Container(Child) + -> OS::Swift::Object(Grand Child) + +which means indexing is initiated by specifying only the top parent +(OS::Swift::Account) and that will in-turn index all the child +plugins(Container and Object) + diff --git a/releasenotes/notes/bp-swift-plugin-f19776bcd35ff69d.yaml b/releasenotes/notes/bp-swift-plugin-f19776bcd35ff69d.yaml new file mode 100644 index 00000000..1d5f66b6 --- /dev/null +++ b/releasenotes/notes/bp-swift-plugin-f19776bcd35ff69d.yaml @@ -0,0 +1,14 @@ +--- +prelude: > + Swift plugin for searchlight +features: + - Three resource types are introduced for swift + plugin. + + OS::Swift:Account + ->OS::Swift::Container + -->OS::Swift::Object +issues: + - The Swift service currently doesn't send notifications. + Follow the swift plugin documentation for current + solutions. \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 242ff880..f8bcd257 100644 --- a/requirements.txt +++ b/requirements.txt @@ -48,3 +48,4 @@ python-glanceclient>=2.0.0 # Apache-2.0 python-novaclient!=2.33.0,>=2.29.0 # Apache-2.0 python-neutronclient!=4.1.0,>=2.6.0 # Apache-2.0 python-cinderclient>=1.3.1 # Apache-2.0 +python-swiftclient>=2.2.0 # Apache-2.0 diff --git a/searchlight/elasticsearch/plugins/base.py b/searchlight/elasticsearch/plugins/base.py index 3ecd3837..8961a667 100644 --- a/searchlight/elasticsearch/plugins/base.py +++ b/searchlight/elasticsearch/plugins/base.py @@ -474,6 +474,16 @@ class IndexBase(plugin.Plugin): def requires_role_separation(self): return len(self.admin_only_fields) > 0 + @classmethod + def is_plugin_enabled_by_default(cls): + ''' + Each plugin can overwrite the default value of whether a + plugin should be enabled if the value is not explicitly + set in the configuration + + ''' + return True + @classmethod def get_plugin_opts(cls): """Options that can be overridden per plugin. Note that @@ -482,7 +492,7 @@ class IndexBase(plugin.Plugin): sets of types (https://bugs.launchpad.net/searchlight/+bug/1558240) """ opts = [ - cfg.BoolOpt("enabled", default=True), + cfg.BoolOpt("enabled", default=cls.is_plugin_enabled_by_default()), cfg.StrOpt("admin_only_fields"), cfg.BoolOpt('mapping_use_doc_values') ] diff --git a/searchlight/elasticsearch/plugins/openstack_clients.py b/searchlight/elasticsearch/plugins/openstack_clients.py index c75fe706..fb0d95c8 100644 --- a/searchlight/elasticsearch/plugins/openstack_clients.py +++ b/searchlight/elasticsearch/plugins/openstack_clients.py @@ -19,8 +19,10 @@ from designateclient.v2 import client as designateclient from glanceclient.v2 import client as glance from keystoneclient import auth as ks_auth from keystoneclient import session as ks_session +from keystoneclient.v2_0 import client as ks_client import neutronclient.v2_0.client import novaclient.client +import swiftclient from oslo_config import cfg @@ -100,3 +102,84 @@ def get_cinderclient(): session=session, region_name=cfg.CONF.service_credentials.os_region_name, ) + +# Swift still needs special handling because it doesn't support +# keystone sessions. Rather than maintain two codepaths, we'll do this +_swiftclient = None + + +def clear_cached_swiftclient_on_unauthorized(fn): + def wrapper(*args, **kwargs): + global _session + global _swiftclient + try: + return fn(*args, **kwargs) + except swiftclient.exceptions.ClientException: + _session = None + _swiftclient = None + return fn(*args, **kwargs) + return wrapper + + +def get_swiftclient(): + + global _swiftclient + if _swiftclient: + return _swiftclient + _get_session() + + service_type = 'object-store' + + os_options = { + 'service_type': service_type, + 'region_name': cfg.CONF.service_credentials.os_region_name, + 'endpoint_type': cfg.CONF.service_credentials.os_endpoint_type, + } + + # When swiftclient supports session, use session instead of + # preauthtoken param below + _swiftclient = swiftclient.client.Connection( + auth_version='2', + user=cfg.CONF.service_credentials.username, + key=cfg.CONF.service_credentials.password, + authurl=cfg.CONF.service_credentials.auth_url, + tenant_name=cfg.CONF.service_credentials.tenant_name, + os_options=os_options, + cacert=cfg.CONF.service_credentials.cafile, + insecure=cfg.CONF.service_credentials.insecure + ) + + return _swiftclient + + +# TODO(lakshmiS) See if we can cache this. +# Cached members will be equal to # of accounts at max. +def get_swiftclient_st(storageurl): + service_type = 'object-store' + _get_session() + + os_options = { + 'service_type': service_type, + 'region_name': cfg.CONF.service_credentials.os_region_name, + 'endpoint_type': cfg.CONF.service_credentials.os_endpoint_type, + } + swift_client = swiftclient.client.Connection( + auth_version='2', + user=cfg.CONF.service_credentials.username, + key=cfg.CONF.service_credentials.password, + authurl=cfg.CONF.service_credentials.auth_url, + tenant_name=cfg.CONF.service_credentials.tenant_name, + os_options=os_options, + cacert=cfg.CONF.service_credentials.cafile, + insecure=cfg.CONF.service_credentials.insecure, + preauthurl=storageurl, + ) + return swift_client + + +def get_keystoneclient(): + session = _get_session() + + return ks_client.Client( + session=session, + region_name=cfg.CONF.service_credentials.os_region_name) diff --git a/searchlight/elasticsearch/plugins/swift/__init__.py b/searchlight/elasticsearch/plugins/swift/__init__.py new file mode 100755 index 00000000..726597d7 --- /dev/null +++ b/searchlight/elasticsearch/plugins/swift/__init__.py @@ -0,0 +1,212 @@ +# Copyright (c) 2016 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import logging +import six + +from oslo_utils import timeutils + +from searchlight.elasticsearch.plugins import openstack_clients +from searchlight import i18n + +LOG = logging.getLogger(__name__) +_LE = i18n._LE +_ = i18n._ + +AUTH_PREFIX = "" +ID_SEP = "/" + +# _accounts is populated when get_swift_accounts() is called; which is +# guaranteed to be called before get_swift_containers and get_swfit_objects +# since account plugin is parent and grandparent of container and object +_accounts = [] + +dateformat = "%a, %d %b %Y %H:%M:%S %Z" + + +def serialize_swift_account(account): + metadocument = {k: account.get(k, None) for k, v in six.iteritems(account) + if k.lower().startswith("x-account-meta")} + account_fields = ('id', 'name') + document = {f: account.get(f, None) for f in account_fields} + + document['domain_id'] = account.get('x-account-project-domain-id', None) + if account.get('x-timestamp'): + timestamp = float(account.get('x-timestamp')) + document['created_at'] = \ + timeutils.isotime(datetime.datetime.fromtimestamp(timestamp)) + + # lakshmiS: swift get_account() doesn't include update datetime field(?) + if account.get('updated_at'): + document['updated_at'] = account.get('updated_at') + + document.update(metadocument) + return document + + +def serialize_swift_account_notification(account): + account['name'] = account['project_name'] + account['id'] = account['account'] + account['x-account-project-domain-id'] = account['project_domain_id'] + # Note: updated_at is included in notification payload. + # No need to map or transform the date and its format. + return serialize_swift_account(account) + + +def serialize_swift_container(container): + metadocument = {k: container.get(k, None) for k, v in + six.iteritems(container) + if k.lower().startswith("x-container-meta")} + container_fields = ('id', + 'name', + 'account', + 'account_id', + 'x-container-read' + ) + document = {f: container.get(f, None) for f in container_fields} + if container.get('x-timestamp'): + timestamp = float(container.get('x-timestamp')) + document['created_at'] = \ + timeutils.isotime(datetime.datetime.fromtimestamp(timestamp)) + + # (laskhmiS) get_container doesn't include last_modified field even + # though the swift api documentation says it returns it. Include it + # when it starts sending it. + + # Notifications for container sends this field as 'updated_at' instead + # of 'last_modified'. + if container.get('updated_at'): + document['updated_at'] = container['updated_at'] + + document.update(metadocument) + return document + + +def serialize_swift_container_notification(container): + # Account Id + container name. container['account'] from notification has + # account id value. + container['id'] = container['account'] + ID_SEP + container['container'] + container['name'] = container['container'] + container['account_id'] = container['account'] + container['account'] = container['project_name'] + return serialize_swift_container(container) + + +def serialize_swift_object(sobject): + metadocument = {k: sobject.get(k, None) for k, v in six.iteritems(sobject) + if k.lower().startswith("x-object-meta")} + object_fields = ('id', + 'name', + 'account', + 'account_id', + 'container', + 'container_id', + 'etag' + ) + document = {f: sobject.get(f, None) for f in object_fields} + document['content_type'] = sobject.get('content-type', None) + document['content_length'] = sobject.get('content-length', None) + + if sobject.get('x-timestamp'): + timestamp = float(sobject.get('x-timestamp')) + document['created_at'] = \ + timeutils.isotime(datetime.datetime.fromtimestamp(timestamp)) + if sobject.get('last-modified'): + updated_dt = datetime.datetime.strptime( + sobject['last-modified'], dateformat) + document['updated_at'] = timeutils.isotime(updated_dt) + document.update(metadocument) + return document + + +def serialize_swift_object_notification(sobj): + # Account id + container name + sobj['container_id'] = sobj['account'] + ID_SEP + sobj['container'] + # Account id + container name + object name + sobj['id'] = sobj['container_id'] + ID_SEP + sobj['object'] + sobj['name'] = sobj['object'] + sobj['account_id'] = sobj['account'] + sobj['account'] = sobj['project_name'] + return serialize_swift_object(sobj) + + +@openstack_clients.clear_cached_swiftclient_on_unauthorized +def _get_storage_url_prefix(): + # Extracts swift proxy url after removing the default account id + # from the service account. Later storage_url's will be constructed + # for each account by appending the keystone tenant id. + try: + storage_url = openstack_clients.get_swiftclient().get_auth()[0] + return storage_url[:storage_url.index(AUTH_PREFIX)] + AUTH_PREFIX + except ValueError: + LOG.error(_LE("reseller_prefix %s not found in keystone endpoint ") + % AUTH_PREFIX) + raise + + +def get_swift_accounts(auth_prefix): + global AUTH_PREFIX + # TODO(lakshmiS): Add support for SERVICE_ accounts + AUTH_PREFIX = auth_prefix + ks_client = openstack_clients.get_keystoneclient() + for tenant in ks_client.tenants.list(): + storage_url = _get_storage_url_prefix() + tenant.id + sclient = openstack_clients.get_swiftclient_st(storage_url) + # 0 index has account summary + account = sclient.get_account()[0] + account['name'] = tenant.name + account['id'] = auth_prefix + tenant.id + + # store it for later usage in retrieving containers + # and objects + account_detail = {'id': account['id'], + 'name': tenant.name, + 'storage.url': storage_url} + _accounts.append(account_detail) + + yield account + + +def get_swift_containers(): + for account in _accounts: + sclient = openstack_clients.get_swiftclient_st(account['storage.url']) + # 1 index has container list + containers = sclient.get_account()[1] + for container in containers: + ctr, obj = sclient.get_container(container['name']) + ctr['id'] = account['id'] + ID_SEP + container['name'] + ctr['name'] = container['name'] + ctr['account'] = account['name'] + ctr['account_id'] = account['id'] + yield ctr + + +def get_swift_objects(): + for account in _accounts: + sclient = openstack_clients.get_swiftclient_st(account['storage.url']) + # 1 index has container list + containers = sclient.get_account()[1] + for sctr in containers: + ctr, obj = sclient.get_container(sctr['name']) + for sobject in obj: + sobj = sclient.head_object(sctr['name'], sobject['name']) + sobj['account'] = account['name'] + sobj['account_id'] = account['id'] + sobj['container'] = sctr['name'] + sobj['container_id'] = account['id'] + ID_SEP + sctr['name'] + sobj['id'] = sobj['container_id'] + ID_SEP + sobject['name'] + sobj['name'] = sobject['name'] + yield sobj diff --git a/searchlight/elasticsearch/plugins/swift/accounts.py b/searchlight/elasticsearch/plugins/swift/accounts.py new file mode 100755 index 00000000..9e8524b9 --- /dev/null +++ b/searchlight/elasticsearch/plugins/swift/accounts.py @@ -0,0 +1,107 @@ +# Copyright (c) 2016 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from oslo_config import cfg + +from searchlight.elasticsearch.plugins import base +from searchlight.elasticsearch.plugins.swift import get_swift_accounts +from searchlight.elasticsearch.plugins.swift import serialize_swift_account +from searchlight.elasticsearch.plugins.swift import swift_notification_handler + +swift_plugin_opts = [ + cfg.StrOpt('reseller_prefix', default="AUTH_", + help="prefix used in account names for auth system."), +] + +CONF = cfg.CONF +CONF.register_opts(swift_plugin_opts, group='resource_plugin:os_swift_account') + + +class AccountIndex(base.IndexBase): + NotificationHandlerCls = swift_notification_handler.SwiftAccountHandler + + def __init__(self): + super(AccountIndex, self).__init__() + self.options = cfg.CONF[self.get_config_group_name()] + + # swift_owner_headers + ADMIN_ONLY_FIELDS = ['x-account-meta-temp-url-key', + 'x-account-meta-temp-url-key-2', + 'x-account-access-control'] + + @classmethod + def get_document_type(cls): + return 'OS::Swift::Account' + + def get_mapping(self): + return { + 'dynamic': True, + 'properties': { + 'id': {'type': 'string', 'index': 'not_analyzed'}, + 'name': { + 'type': 'string', + 'fields': { + 'raw': {'type': 'string', 'index': 'not_analyzed'} + }, + }, + # TODO(lakshmiS): Removing following field(s) since account + # notifications don't include it for subsequent updates. + # Enable when it is included in future notifications. + + # The number of objects in the account. + # 'x-account-object-count': {'type': 'long'}, + + # The total number of bytes that are stored in Object Storage + # for the account. + # 'x-account-bytes-used': {'type': 'long'}, + + # The number of containers. + # 'x-account-container-count': {'type': 'long'}, + + 'domain_id': {'type': 'string', 'index': 'not_analyzed'}, + 'created_at': {'type': 'date'}, + 'updated_at': {'type': 'date'} + }, + } + + @property + def admin_only_fields(self): + from_conf = super(AccountIndex, self).admin_only_fields + return AccountIndex.ADMIN_ONLY_FIELDS + from_conf + + @classmethod + def is_plugin_enabled_by_default(cls): + return False + + @property + def allow_admin_ignore_rbac(self): + return False + + def _get_rbac_field_filters(self, request_context): + id = self.options.reseller_prefix + request_context.owner + return [ + {"term": {"id": id}} + ] + + @property + def routing_field(self): + return "id" + + def get_objects(self): + return get_swift_accounts(self.options.reseller_prefix) + + def serialize(self, obj): + return serialize_swift_account(obj) diff --git a/searchlight/elasticsearch/plugins/swift/containers.py b/searchlight/elasticsearch/plugins/swift/containers.py new file mode 100755 index 00000000..2260c7ce --- /dev/null +++ b/searchlight/elasticsearch/plugins/swift/containers.py @@ -0,0 +1,136 @@ +# Copyright (c) 2016 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from searchlight.elasticsearch.plugins import base +from searchlight.elasticsearch.plugins.swift import get_swift_containers +from searchlight.elasticsearch.plugins.swift import serialize_swift_container +from searchlight.elasticsearch.plugins.swift import swift_notification_handler + + +class ContainerIndex(base.IndexBase): + NotificationHandlerCls = swift_notification_handler.SwiftContainerHandler + + def get_notification_handler(self): + """Override because the container handler needs a handle to object + indexer for cascade delete of container to objects. + """ + return self.NotificationHandlerCls( + self.index_helper, + self.options, + object_helper=self.child_plugins[0].index_helper) + + # swift_owner_headers + ADMIN_ONLY_FIELDS = ['x-container-write', + 'x-container-sync-key', + 'x-container-sync-to', + 'x-container-meta-temp-url-key', + 'x-container-meta-temp-url-key-2'] + + @classmethod + def parent_plugin_type(cls): + return "OS::Swift::Account" + + @classmethod + def get_document_type(cls): + return 'OS::Swift::Container' + + def get_mapping(self): + return { + 'dynamic': True, + "_source": { + "excludes": ["x-container-read"] + }, + 'properties': { + 'id': {'type': 'string', 'index': 'not_analyzed'}, + 'name': { + 'type': 'string', + 'fields': { + 'raw': {'type': 'string', 'index': 'not_analyzed'} + }, + }, + 'account': { + 'type': 'string', + 'fields': { + 'raw': {'type': 'string', 'index': 'not_analyzed'} + }, + }, + 'account_id': {'type': 'string', 'index': 'not_analyzed'}, + 'x-container-read': { + 'type': 'string', 'index': 'not_analyzed', + 'store': False + }, + + # TODO(lakshmiS): Removing following field(s) since account + # notifications don't include it for subsequent updates. + # Enable when it is included in future notifications. + # 'x-container-object-count': {'type': 'long'}, + # 'x-container-bytes-used': {'type': 'long'}, + + 'created_at': {'type': 'date'}, + 'updated_at': {'type': 'date'} + }, + "_parent": { + "type": self.parent_plugin_type() + } + } + + @property + def admin_only_fields(self): + from_conf = super(ContainerIndex, self).admin_only_fields + return ContainerIndex.ADMIN_ONLY_FIELDS + from_conf + + @classmethod + def is_plugin_enabled_by_default(cls): + return False + + @property + def allow_admin_ignore_rbac(self): + return False + + def _get_rbac_field_filters(self, request_context): + tenant_member = request_context.tenant + ":*" + single_user = request_context.tenant + ":" + request_context.user + account_id = \ + self.parent_plugin.options.reseller_prefix + request_context.owner + + return [ + { + 'or': [ + { + 'term': { + 'account_id': account_id + } + }, + { + 'terms': { + 'x-container-read': [tenant_member, single_user] + } + } + ] + } + ] + + def get_parent_id_field(self): + return 'account_id' + + @property + def routing_field(self): + return "account_id" + + def get_objects(self): + return get_swift_containers() + + def serialize(self, obj): + return serialize_swift_container(obj) diff --git a/searchlight/elasticsearch/plugins/swift/objects.py b/searchlight/elasticsearch/plugins/swift/objects.py new file mode 100755 index 00000000..3514dded --- /dev/null +++ b/searchlight/elasticsearch/plugins/swift/objects.py @@ -0,0 +1,139 @@ +# Copyright (c) 2016 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +from searchlight.elasticsearch.plugins import base +from searchlight.elasticsearch.plugins.swift import get_swift_objects +from searchlight.elasticsearch.plugins.swift import serialize_swift_object +from searchlight.elasticsearch.plugins.swift import swift_notification_handler + +swift_plugin_opts = [ + cfg.StrOpt('reseller_prefix', default="AUTH_", + help="prefix used in account names for auth system."), +] + +CONF = cfg.CONF +CONF.register_opts(swift_plugin_opts, group='resource_plugin:os_swift_account') + + +class ObjectIndex(base.IndexBase): + NotificationHandlerCls = swift_notification_handler.SwiftObjectHandler + + @classmethod + def parent_plugin_type(cls): + return "OS::Swift::Container" + + @classmethod + def get_document_type(cls): + return 'OS::Swift::Object' + + def get_mapping(self): + return { + 'dynamic': True, + 'properties': { + 'id': {'type': 'string', 'index': 'not_analyzed'}, + 'name': { + 'type': 'string', + 'fields': { + 'raw': {'type': 'string', 'index': 'not_analyzed'} + }, + }, + 'account': { + 'type': 'string', + 'fields': { + 'raw': {'type': 'string', 'index': 'not_analyzed'} + }, + }, + 'account_id': {'type': 'string', 'index': 'not_analyzed'}, + 'container': { + 'type': 'string', + 'fields': { + 'raw': {'type': 'string', 'index': 'not_analyzed'} + }, + }, + 'container_id': {'type': 'string', 'index': 'not_analyzed'}, + 'content_type': {'type': 'string', 'index': 'not_analyzed'}, + 'content_length': {'type': 'long'}, + 'etag': {'type': 'string'}, + 'created_at': {'type': 'date'}, + 'updated_at': {'type': 'date'} + }, + "_parent": { + "type": self.parent_plugin_type() + } + } + + @property + def allow_admin_ignore_rbac(self): + return False + + @classmethod + def is_plugin_enabled_by_default(cls): + return False + + def _get_rbac_field_filters(self, request_context): + + tenant_member = request_context.tenant + ":*" + single_user = request_context.tenant + ":" + request_context.user + account_plugin = self.parent_plugin.parent_plugin + account_id = \ + account_plugin.options.reseller_prefix + request_context.owner + + return [ + { + "has_parent": { + "type": self.parent_plugin_type(), + "query": { + "filtered": { + "query": {"match_all": {}}, + "filter": { + "or": [ + { + 'term': { + 'account_id': account_id + } + }, + { + 'terms': { + 'x-container-read': [ + tenant_member, + single_user] + } + }, + ] + } + } + } + } + } + ] + + def get_parent_id_field(self): + return 'container_id' + + @property + def routing_field(self): + return "account_id" + + @property + def facets_with_options(self): + return ('container', 'content_type') + + def get_objects(self): + return get_swift_objects() + + def serialize(self, obj): + return serialize_swift_object(obj) diff --git a/searchlight/elasticsearch/plugins/swift/swift_notification_handler.py b/searchlight/elasticsearch/plugins/swift/swift_notification_handler.py new file mode 100644 index 00000000..c6fdd890 --- /dev/null +++ b/searchlight/elasticsearch/plugins/swift/swift_notification_handler.py @@ -0,0 +1,176 @@ +# Copyright (c) 2016 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from searchlight.elasticsearch.plugins import base +from searchlight.elasticsearch.plugins import swift +from searchlight.elasticsearch.plugins.swift \ + import serialize_swift_account_notification +from searchlight.elasticsearch.plugins.swift \ + import serialize_swift_container_notification +from searchlight.elasticsearch.plugins.swift \ + import serialize_swift_object_notification +from searchlight import i18n + + +LOG = logging.getLogger(__name__) +_LE = i18n._LE +_ = i18n._ + + +class SwiftAccountHandler(base.NotificationBase): + + def __init__(self, *args, **kwargs): + super(SwiftAccountHandler, self).__init__(*args, **kwargs) + + @classmethod + def _get_notification_exchanges(cls): + return ['swift'] + + def get_event_handlers(self): + return { + "account.create": self.create_or_update, + "account.metadata": self.create_or_update, + "account.delete": self.delete + } + + def create_or_update(self, payload, timestamp): + payload = serialize_swift_account_notification(payload) + try: + self.index_helper.save_document( + payload, + version=self.get_version(payload, timestamp)) + except Exception as exc: + LOG.error(_LE('Error saving account %(id)s ' + 'in index. Error: %(exc)s') % + {'id': payload['id'], 'exc': exc}) + + def delete(self, payload, timestamp): + version = self.get_version(payload, timestamp) + id = payload['account'] + try: + self.index_helper.delete_document( + {'_id': id, '_version': version, + '_routing': payload['account']}) + except Exception as exc: + LOG.error(_LE('Error deleting account %(id)s ' + 'from index. Error: %(exc)s') % + {'id': id, 'exc': exc}) + + +class SwiftContainerHandler(base.NotificationBase): + + def __init__(self, *args, **kwargs): + self.object_helper = kwargs.pop('object_helper') + super(SwiftContainerHandler, self).__init__(*args, **kwargs) + + @classmethod + def _get_notification_exchanges(cls): + return ['swift'] + + def get_event_handlers(self): + return { + "container.create": self.create_or_update, + "container.metadata": self.create_or_update, + "container.delete": self.delete + } + + def create_or_update(self, payload, timestamp): + payload = serialize_swift_container_notification(payload) + try: + self.index_helper.save_document( + payload, + version=self.get_version(payload, timestamp)) + except Exception as exc: + LOG.error(_LE('Error saving container %(id)s ' + 'in index. Error: %(exc)s') % + {'id': payload['id'], 'exc': exc}) + + def delete(self, payload, timestamp): + # notification payload doesn't have any date/time fields + # so temporarily use metadata timestamp value as + # updated_at field to retrieve version + # Remove it when notifcation starts sending datetime field(s) + payload['updated_at'] = timestamp + version = self.get_version(payload, timestamp) + del payload['updated_at'] + + id = payload['account'] + swift.ID_SEP + payload['container'] + try: + self.object_helper.delete_documents_with_parent( + id, version=version) + except Exception as exc: + LOG.error(_LE('Error deleting objects in container %(id)s ' + 'from index. Error: %(exc)s') % + {'id': id, 'exc': exc}) + try: + self.index_helper.delete_document( + {'_id': id, '_version': version, + '_routing': payload['account']}) + except Exception as exc: + LOG.error(_LE('Error deleting container %(id)s ' + 'from index. Error: %(exc)s') % + {'id': id, 'exc': exc}) + + +class SwiftObjectHandler(base.NotificationBase): + + def __init__(self, *args, **kwargs): + super(SwiftObjectHandler, self).__init__(*args, **kwargs) + + @classmethod + def _get_notification_exchanges(cls): + return ['swift'] + + def get_event_handlers(self): + return { + "object.create": self.create_or_update, + "object.metadata": self.create_or_update, + "object.delete": self.delete + } + + def create_or_update(self, payload, timestamp): + payload = serialize_swift_object_notification(payload) + try: + self.index_helper.save_document( + payload, + version=self.get_version(payload, timestamp) + ) + except Exception as exc: + LOG.error(_LE('Error saving object %(id)s ' + 'in index. Error: %(exc)s') % + {'id': payload['id'], 'exc': exc}) + + def delete(self, payload, timestamp): + # notification payload doesn't have any date/time fields + # so temporarily use metadata timestamp value as + # updated_at field to retrieve version + # Remove it when notifcation starts sending datetime field(s) + payload['updated_at'] = timestamp + version = self.get_version(payload, timestamp) + del payload['updated_at'] + + id = payload['account'] + swift.ID_SEP + \ + payload['container'] + swift.ID_SEP + \ + payload['object'] + try: + self.index_helper.delete_document( + {'_id': id, '_version': version, + '_routing': payload['account']}) + except Exception as exc: + LOG.error(_LE('Error deleting object %(id)s ' + 'from index. Error: %(exc)s') % + {'id': id, 'exc': exc}) diff --git a/searchlight/tests/functional/__init__.py b/searchlight/tests/functional/__init__.py index e593f411..02daa7ab 100644 --- a/searchlight/tests/functional/__init__.py +++ b/searchlight/tests/functional/__init__.py @@ -375,13 +375,18 @@ class FunctionalTest(test_utils.BaseTestCase): 'glance': {'images': 'ImageIndex', 'metadefs': 'MetadefIndex'}, 'nova': {'servers': 'ServerIndex'}, 'neutron': {'networks': 'NetworkIndex', 'ports': 'PortIndex'}, - 'cinder': {'volumes': 'VolumeIndex', 'snapshots': 'SnapshotIndex'} + 'cinder': {'volumes': 'VolumeIndex', 'snapshots': 'SnapshotIndex'}, + 'swift': {'accounts': 'AccountIndex', + 'containers': 'ContainerIndex', + 'objects': 'ObjectIndex'} } plugins = include_plugins or ( ('glance', 'images'), ('glance', 'metadefs'), ('nova', 'servers'), ('neutron', 'networks'), ('neutron', 'ports'), - ('cinder', 'volumes'), ('cinder', 'snapshots') + ('cinder', 'volumes'), ('cinder', 'snapshots'), + ('swift', 'accounts'), ('swift', 'containers'), + ('swift', 'objects') ) plugins = filter(lambda plugin: plugin not in exclude_plugins, plugins) diff --git a/searchlight/tests/unit/test_swift_account_plugin.py b/searchlight/tests/unit/test_swift_account_plugin.py new file mode 100644 index 00000000..7762f826 --- /dev/null +++ b/searchlight/tests/unit/test_swift_account_plugin.py @@ -0,0 +1,162 @@ +# Copyright (c) 2016 Hewlett-Packard Enterprise Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import six +import time + + +from oslo_utils import timeutils + +import searchlight.elasticsearch.plugins.swift as swift_plugin +from searchlight.elasticsearch.plugins.swift import\ + accounts as accounts_plugin +import searchlight.tests.unit.utils as unit_test_utils +import searchlight.tests.utils as test_utils + + +now_epoch_time = time.time() +now_utc = datetime.datetime.fromtimestamp(now_epoch_time)\ + .strftime('%Y-%m-%dT%H:%M:%SZ') + + +USER1 = u'27f4d76b-be62-4e4e-aa33bb11cc55' +ID1 = "AUTH_488ac936-663e-4e5c-537d-986021b32c4b" +ID2 = "AUTH_7554da43-6443-acdf-deac-3425223cdada" +ID3 = "AUTH_30754354-ca43-124b-12b5-789234bcdefa'" + +AUTH_PREFIX = "AUTH_" + +X_ACCOUNT_META_KEY1 = 'x-account-meta-key1' +X_ACCOUNT_META_VALUE1 = 'x-account-meta-value1' + +X_ACCOUNT_META_KEY2 = 'x-account-meta-key2' +X_ACCOUNT_META_VALUE2 = 'x-account-meta-value2' + +TENANT1 = "15b9a454cee34dbe9933ad575a0a6930" +DOMAIN_ID = "default" + +DATETIME = datetime.datetime(2016, 2, 20, 1, 13, 24, 215337) +DATE1 = timeutils.isotime(DATETIME) + + +def _account_fixture(account_id, domain_id, name, **kwargs): + fixture = { + "id": account_id, + "name": name, + "x-account-project-domain-id": domain_id, + 'x-timestamp': now_epoch_time + } + fixture.update(kwargs) + return fixture + + +def _notification_account_fixture(account_id, **kwargs): + metadata = kwargs.pop('meta', {}) + notification = { + 'account': account_id, + 'project_name': None, + 'updated_at': DATE1, + 'project_domain_name': None, + 'x-trans-id': None, + 'project_id': None, + 'project_domain_id': None + } + for k, v in six.iteritems(kwargs): + if k in notification: + notification[k] = v + for k, v in six.iteritems(metadata): + notification[k] = v + return notification + + +class TestSwiftAccountPlugin(test_utils.BaseTestCase): + def setUp(self): + super(TestSwiftAccountPlugin, self).setUp() + self.plugin = accounts_plugin.AccountIndex() + self._create_fixtures() + + def _create_fixtures(self): + self.account1 = _account_fixture( + account_id=ID1, domain_id=DOMAIN_ID, name="test-account1") + self.account2 = _account_fixture( + account_id=ID2, domain_id=DOMAIN_ID, name="test-account1", + **{X_ACCOUNT_META_KEY1: X_ACCOUNT_META_VALUE1}) + self.account3 = _account_fixture( + account_id=ID3, domain_id=DOMAIN_ID, name="test-account1", + **{X_ACCOUNT_META_KEY1: X_ACCOUNT_META_VALUE1, + X_ACCOUNT_META_KEY2: X_ACCOUNT_META_VALUE2}) + self.accounts = [self.account1, self.account2, self.account3] + + def test_index_name(self): + self.assertEqual('searchlight', self.plugin.resource_group_name) + + def test_document_type(self): + self.assertEqual('OS::Swift::Account', + self.plugin.get_document_type()) + + def test_rbac_filter(self): + fake_request = unit_test_utils.get_fake_request( + USER1, TENANT1, '/v1/search', is_admin=False + ) + rbac_terms = self.plugin._get_rbac_field_filters(fake_request.context) + self.assertEqual( + [{"term": {"id": AUTH_PREFIX + TENANT1}}], + rbac_terms + ) + + def test_admin_only_fields(self): + admin_only_fields = self.plugin.admin_only_fields + self.assertEqual(['x-account-meta-temp-url-key', + 'x-account-meta-temp-url-key-2', + 'x-account-access-control'], admin_only_fields) + + def test_serialize(self): + serialized = self.plugin.serialize(self.account1) + + self.assertEqual(ID1, serialized['id']) + self.assertEqual('test-account1', serialized['name']) + self.assertEqual(DOMAIN_ID, serialized['domain_id']) + self.assertEqual(now_utc, serialized['created_at']) + + serialized = self.plugin.serialize(self.account2) + self.assertEqual(X_ACCOUNT_META_VALUE1, + serialized[X_ACCOUNT_META_KEY1]) + + serialized = self.plugin.serialize(self.account3) + self.assertEqual(X_ACCOUNT_META_VALUE1, + serialized[X_ACCOUNT_META_KEY1]) + self.assertEqual(X_ACCOUNT_META_VALUE2, + serialized[X_ACCOUNT_META_KEY2]) + + def test_swift_account_notification_serialize(self): + notification = _notification_account_fixture( + ID1, + project_name='admin', + project_id=ID1, + project_domain_id='default', + updated_at=DATE1, + ) + + expected = { + 'id': ID1, + 'name': 'admin', + 'updated_at': DATE1, + 'domain_id': 'default' + } + + serialized = swift_plugin.serialize_swift_account_notification( + notification) + self.assertEqual(expected, serialized) diff --git a/searchlight/tests/unit/test_swift_container_plugin.py b/searchlight/tests/unit/test_swift_container_plugin.py new file mode 100644 index 00000000..03cd1028 --- /dev/null +++ b/searchlight/tests/unit/test_swift_container_plugin.py @@ -0,0 +1,175 @@ +# Copyright (c) 2016 Hewlett-Packard Enterprise Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import six +import time + + +from oslo_utils import timeutils + +import searchlight.elasticsearch.plugins.swift as swift_plugin +from searchlight.elasticsearch.plugins.swift import\ + containers as containers_plugin +import searchlight.tests.utils as test_utils + + +now_epoch_time = time.time() +now_utc = datetime.datetime.fromtimestamp(now_epoch_time).\ + strftime('%Y-%m-%dT%H:%M:%SZ') + + +USER1 = "27f4d76b-be62-4e4e-aa33bb11cc55" + +ACCOUNT_ID1 = "488ac936-663e-4e5c-537d-986021b32c4b" +ACCOUNT_ID2 = "7554da43-6443-acdf-deac-3425223cdada" +ACCOUNT_ID3 = "30754354-ca43-124b-12b5-789234bcdefa'" + +AUTH_PREFIX = "AUTH_" + +CONTAINER1 = "Container1" +CONTAINER2 = "Container2" +CONTAINER_ID1 = ACCOUNT_ID1 + swift_plugin.ID_SEP + CONTAINER1 +CONTAINER_ID2 = ACCOUNT_ID2 + swift_plugin.ID_SEP + CONTAINER2 + +X_CONTAINER_META_KEY1 = 'x-container-meta-key1' +X_CONTAINER_META_VALUE1 = 'x-container-meta-value1' + +X_CONTAINER_META_KEY2 = 'x-container-meta-key2' +X_CONTAINER_META_VALUE2 = 'x-container-meta-value2' + +TENANT1 = "15b9a454cee34dbe9933ad575a0a6930" +TENANT2 = "a7ba963f71bb43818f631febbc9df8e6" +DOMAIN_ID = "default" + +DATETIME = datetime.datetime(2016, 2, 21, 4, 41, 33, 325314) +DATE1 = timeutils.isotime(DATETIME) + + +def _container_fixture(container_id, container_name, account, + account_id, read_acl, **kwargs): + fixture = { + "id": container_id, + "name": container_name, + "account": account, + "account_id": account_id, + "x-container-read": read_acl, + 'x-timestamp': now_epoch_time + } + fixture.update(kwargs) + return fixture + + +def _notification_container_fixture(account_id, **kwargs): + metadata = kwargs.pop('meta', {}) + notification = { + 'account': account_id, + 'project_name': None, + 'container': None, + 'updated_at': DATE1, + 'project_domain_name': None, + 'x-trans-id': None, + 'project_id': None, + 'project_domain_id': None + } + for k, v in six.iteritems(kwargs): + if k in notification: + notification[k] = v + for k, v in six.iteritems(metadata): + notification[k] = v + return notification + + +class TestSwiftContainerPlugin(test_utils.BaseTestCase): + def setUp(self): + super(TestSwiftContainerPlugin, self).setUp() + self.plugin = containers_plugin.ContainerIndex() + self._create_fixtures() + + def _create_fixtures(self): + self.container1 = _container_fixture( + container_id=CONTAINER_ID1, container_name=CONTAINER1, + account='test-account1', account_id=ACCOUNT_ID1, read_acl=None) + self.container2 = _container_fixture( + container_id=CONTAINER_ID2, container_name=CONTAINER2, + account='test-account2', account_id=ACCOUNT_ID1, + read_acl=ACCOUNT_ID2 + ":*", + **{X_CONTAINER_META_KEY1: X_CONTAINER_META_VALUE1} + ) + self.container3 = _container_fixture( + container_id=CONTAINER_ID1, container_name=CONTAINER1, + account='test-account3', account_id=ACCOUNT_ID2, + read_acl=ACCOUNT_ID3 + ":*" + USER1, + **{X_CONTAINER_META_KEY1: X_CONTAINER_META_VALUE1, + X_CONTAINER_META_KEY2: X_CONTAINER_META_VALUE2}) + self.containers = [self.container1, self.container2, self.container3] + + def test_index_name(self): + self.assertEqual('searchlight', self.plugin.resource_group_name) + + def test_document_type(self): + self.assertEqual('OS::Swift::Container', + self.plugin.get_document_type()) + + def test_admin_only_fields(self): + admin_only_fields = self.plugin.admin_only_fields + self.assertEqual(['x-container-write', + 'x-container-sync-key', + 'x-container-sync-to', + 'x-container-meta-temp-url-key', + 'x-container-meta-temp-url-key-2'], + admin_only_fields) + + def test_serialize(self): + serialized = self.plugin.serialize(self.container1) + + self.assertEqual(CONTAINER_ID1, serialized['id']) + self.assertEqual(CONTAINER1, serialized['name']) + self.assertEqual(ACCOUNT_ID1, serialized['account_id']) + self.assertEqual('test-account1', serialized['account']) + + serialized = self.plugin.serialize(self.container2) + self.assertEqual(X_CONTAINER_META_VALUE1, + serialized[X_CONTAINER_META_KEY1]) + self.assertEqual(ACCOUNT_ID2 + ":*", serialized['x-container-read']) + + serialized = self.plugin.serialize(self.container3) + self.assertEqual(X_CONTAINER_META_VALUE1, + serialized[X_CONTAINER_META_KEY1]) + self.assertEqual(X_CONTAINER_META_VALUE2, + serialized[X_CONTAINER_META_KEY2]) + + def test_swift_account_notification_serialize(self): + notification = _notification_container_fixture( + swift_plugin.AUTH_PREFIX + ACCOUNT_ID1, + container=CONTAINER1, + project_name='admin', + project_domain_id='default', + project_id=ACCOUNT_ID1, + updated_at=DATE1, + ) + + expected = { + 'id': CONTAINER_ID1, + 'name': CONTAINER1, + 'account': 'admin', + 'account_id': ACCOUNT_ID1, + 'updated_at': DATE1, + 'x-container-read': None + } + + serialized = swift_plugin.serialize_swift_container_notification( + notification) + self.assertEqual(expected, serialized) diff --git a/searchlight/tests/unit/test_swift_object_plugin.py b/searchlight/tests/unit/test_swift_object_plugin.py new file mode 100644 index 00000000..12ee6fcc --- /dev/null +++ b/searchlight/tests/unit/test_swift_object_plugin.py @@ -0,0 +1,120 @@ +# Copyright (c) 2016 Hewlett-Packard Enterprise Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import time + +import searchlight.elasticsearch.plugins.swift as swift_plugin +from searchlight.elasticsearch.plugins.swift import\ + objects as objects_plugin +import searchlight.tests.utils as test_utils + + +now_epoch_time = time.time() +created_time = now_epoch_time - 3 +updated_time = now_epoch_time +updated_time_in_fmt = datetime.datetime.fromtimestamp(updated_time).\ + strftime('%a, %d %b %Y %H:%M:%S GMT') +updated_time_out_fmt = datetime.datetime.fromtimestamp(updated_time).\ + strftime('%Y-%m-%dT%H:%M:%SZ') +created_time_utc = datetime.datetime.fromtimestamp(created_time).\ + strftime('%Y-%m-%dT%H:%M:%SZ') + +USER1 = "27f4d76b-be62-4e4e-aa33bb11cc55" + +ACCOUNT_ID1 = "488ac936-663e-4e5c-537d-986021b32c4b" +CONTAINER1 = "Container1" +CONTAINER_ID1 = ACCOUNT_ID1 + swift_plugin.ID_SEP + CONTAINER1 +OBJECT1 = "Object1" +OBJECT2 = "Object2" +OBJECT_ID1 = CONTAINER_ID1 + swift_plugin.ID_SEP + OBJECT1 +OBJECT_ID2 = CONTAINER_ID1 + swift_plugin.ID_SEP + OBJECT2 + +X_OBJECT_META_KEY1 = 'x-object-meta-key1' +X_OBJECT_META_VALUE1 = 'x-object-meta-value1' + +X_OBJECT_META_KEY2 = 'x-object-meta-key2' +X_OBJECT_META_VALUE2 = 'x-object-meta-value2' + +TENANT1 = "15b9a454cee34dbe9933ad575a0a6930" + + +def _object_fixture(object_id, object_name, container_id, container_name, + account, account_id, **kwargs): + fixture = { + "id": object_id, + "name": object_name, + "account": account, + "account_id": account_id, + "container": container_name, + "container_id": container_id, + 'x-timestamp': created_time, + 'last-modified': updated_time_in_fmt + } + fixture.update(kwargs) + return fixture + + +class TestSwiftObjectPlugin(test_utils.BaseTestCase): + def setUp(self): + super(TestSwiftObjectPlugin, self).setUp() + self.plugin = objects_plugin.ObjectIndex() + self._create_fixtures() + + def _create_fixtures(self): + self.object1 = _object_fixture( + object_id=OBJECT_ID1, object_name=OBJECT1, + container_id=CONTAINER_ID1, container_name=CONTAINER1, + account='test-account1', account_id=ACCOUNT_ID1, + **{"content-type": "text/plain", "content-length": 1050}) + self.object2 = _object_fixture( + object_id=OBJECT_ID2, object_name=OBJECT2, + container_id=CONTAINER_ID1, container_name=CONTAINER1, + account='test-account1', account_id=ACCOUNT_ID1, + **{"content-type": "text/plain", "content-length": 1050, + X_OBJECT_META_KEY1: X_OBJECT_META_VALUE1, + X_OBJECT_META_KEY2: X_OBJECT_META_VALUE2}) + self.objects = [self.object1, self.object2] + + def test_index_name(self): + self.assertEqual('searchlight', self.plugin.resource_group_name) + + def test_document_type(self): + self.assertEqual('OS::Swift::Object', + self.plugin.get_document_type()) + + def test_admin_only_fields(self): + admin_only_fields = self.plugin.admin_only_fields + self.assertEqual([], admin_only_fields) + + def test_serialize(self): + serialized = self.plugin.serialize(self.object1) + + self.assertEqual(OBJECT_ID1, serialized['id']) + self.assertEqual(OBJECT1, serialized['name']) + self.assertEqual(ACCOUNT_ID1, serialized['account_id']) + self.assertEqual('test-account1', serialized['account']) + self.assertEqual(CONTAINER_ID1, serialized['container_id']) + self.assertEqual(CONTAINER1, serialized['container']) + self.assertEqual(created_time_utc, serialized['created_at']), + self.assertEqual(updated_time_out_fmt, serialized['updated_at']) + self.assertEqual("text/plain", serialized['content_type']) + self.assertEqual(1050, serialized['content_length']) + + serialized = self.plugin.serialize(self.object2) + self.assertEqual(X_OBJECT_META_VALUE1, + serialized[X_OBJECT_META_KEY1]) + self.assertEqual(X_OBJECT_META_VALUE2, + serialized[X_OBJECT_META_KEY2]) diff --git a/searchlight/tests/unit/v1/test_search.py b/searchlight/tests/unit/v1/test_search.py index f8c37c3a..30f5bdc1 100755 --- a/searchlight/tests/unit/v1/test_search.py +++ b/searchlight/tests/unit/v1/test_search.py @@ -283,7 +283,7 @@ class TestSearchDeserializer(test_utils.BaseTestCase): 'OS::Neutron::Net', 'OS::Neutron::Port', 'OS::Cinder::Volume', - 'OS::Cinder::Snapshot', + 'OS::Cinder::Snapshot' ] self.assertEqual(['searchlight-search'], output['index']) @@ -602,7 +602,7 @@ class TestSearchDeserializer(test_utils.BaseTestCase): self.assertEqual(expected_query, output['query']) def test_rbac_admin(self): - """Test that admins have RBAC applied unless 'all_projects' is true""" + """Test that admins have RBAC applied""" request = unit_test_utils.get_fake_request(is_admin=True) request.body = six.b(jsonutils.dumps({ 'query': {'match_all': {}}, diff --git a/setup.cfg b/setup.cfg index f8ce344b..d472dba6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,6 +37,9 @@ searchlight.index_backend = os_designate_zone = searchlight.elasticsearch.plugins.designate.zones:ZoneIndex os_cinder_volume = searchlight.elasticsearch.plugins.cinder.volumes:VolumeIndex os_cinder_snapshot = searchlight.elasticsearch.plugins.cinder.snapshots:SnapshotIndex + os_swift_account = searchlight.elasticsearch.plugins.swift.accounts:AccountIndex + os_swift_container = searchlight.elasticsearch.plugins.swift.containers:ContainerIndex + os_swift_object = searchlight.elasticsearch.plugins.swift.objects:ObjectIndex [build_sphinx] all_files = 1