Swift plugin
Resource types addded for OS::Swift::Account OS::Swift::Container OS::Swift::Object To Test: cd /opt/stack/searchlight git review -d 271622 sudo python setup.py install openstack role add --user searchlight --project service ResellerAdmin searchlight-manage index sync --type OS::Swift::Account restart searchlight-api, restart searchlight-listener This doesn't enable notification updates (strictly POC level). For that, please follow the instructions here: https://review.openstack.org/#/c/249471/ Implements-blueprint: swift-plugin Change-Id: Ib9838438e84d957b526fe80371aae62466c07f3e
This commit is contained in:
parent
e4b64c00d7
commit
d445624e3c
|
@ -126,6 +126,25 @@ SWIFT_HASH=096d08da4f8d4cce3a724c5f6c18f055
|
||||||
SWIFT_REPLICAS=1
|
SWIFT_REPLICAS=1
|
||||||
SWIFT_DATA_DIR=$DEST/data/swift
|
SWIFT_DATA_DIR=$DEST/data/swift
|
||||||
|
|
||||||
|
## SWIFT NOTIFICATIONS ###
|
||||||
|
#Notifications must be configured properly for searchlight to process
|
||||||
|
#incremental updates. Use the following::
|
||||||
|
|
||||||
|
#Add the following new section in swift proxy conf file
|
||||||
|
#[filter:oslomiddleware]
|
||||||
|
#paste.filter_factory = swift.common.middleware.oslo_notifications:filter_factory
|
||||||
|
#publisher_id = swift.localhost
|
||||||
|
#Replace <user>,<password>,<rabbitip> and <rabbitport> for your environment values
|
||||||
|
#transport_url = rabbit://<user>:<password>@<rabbitip>:<rabbitport>/
|
||||||
|
#notification_driver = messaging
|
||||||
|
#notification_topics = searchlight_indexer
|
||||||
|
|
||||||
|
#Add oslomiddleware to pipeline:main see example below.
|
||||||
|
#[pipeline:main]
|
||||||
|
#pipeline = catch_errors gatekeeper healthcheck ... oslomiddleware proxy-logging proxy-server
|
||||||
|
#Restart swift proxy API service (s-proxy) after making changes.
|
||||||
|
|
||||||
|
|
||||||
### DESIGNATE ###
|
### DESIGNATE ###
|
||||||
#
|
#
|
||||||
enable_plugin designate https://git.openstack.org/openstack/designate
|
enable_plugin designate https://git.openstack.org/openstack/designate
|
||||||
|
@ -208,3 +227,14 @@ admin_only_fields=admin_state_up,status
|
||||||
|
|
||||||
[resource_plugin:os_neutron_port]
|
[resource_plugin:os_neutron_port]
|
||||||
enabled = True
|
enabled = True
|
||||||
|
|
||||||
|
[resource_plugin:os_swift_account]
|
||||||
|
enabled = false
|
||||||
|
#Specify same value as in swift proxy config for reseller_prefix
|
||||||
|
reseller_prefix = AUTH_
|
||||||
|
|
||||||
|
[resource_plugin:os_swift_container]
|
||||||
|
enabled = false
|
||||||
|
|
||||||
|
[resource_plugin:os_swift_object]
|
||||||
|
enabled = false
|
||||||
|
|
|
@ -132,6 +132,11 @@ function configure_searchlight {
|
||||||
# Plugin config - disable designate by default since it's not typically installed
|
# Plugin config - disable designate by default since it's not typically installed
|
||||||
iniset $SEARCHLIGHT_CONF resource_plugin:os_designate_zone enabled False
|
iniset $SEARCHLIGHT_CONF resource_plugin:os_designate_zone enabled False
|
||||||
iniset $SEARCHLIGHT_CONF resource_plugin:os_designate_recordset enabled False
|
iniset $SEARCHLIGHT_CONF resource_plugin:os_designate_recordset enabled False
|
||||||
|
|
||||||
|
# Plugin config - disable swift by default since it's not typically installed
|
||||||
|
iniset $SEARCHLIGHT_CONF resource_plugin:os_swift_account enabled False
|
||||||
|
iniset $SEARCHLIGHT_CONF resource_plugin:os_swift_container enabled False
|
||||||
|
iniset $SEARCHLIGHT_CONF resource_plugin:os_swift_object enabled False
|
||||||
}
|
}
|
||||||
|
|
||||||
# create_searchlight_accounts - Set up common required searchlight accounts
|
# create_searchlight_accounts - Set up common required searchlight accounts
|
||||||
|
|
|
@ -101,6 +101,14 @@ Please read the rest of the guide for detailed information.::
|
||||||
[resource_plugin:os_designate_recordset]
|
[resource_plugin:os_designate_recordset]
|
||||||
enabled = False
|
enabled = False
|
||||||
|
|
||||||
|
[resource_plugin:os_swift_account]
|
||||||
|
enabled = False
|
||||||
|
|
||||||
|
[resource_plugin:os_swift_container]
|
||||||
|
enabled = False
|
||||||
|
|
||||||
|
[resource_plugin:os_swift_object]
|
||||||
|
enabled = False
|
||||||
|
|
||||||
Common Plugin Configuration Options
|
Common Plugin Configuration Options
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
|
|
@ -0,0 +1,181 @@
|
||||||
|
..
|
||||||
|
(c) Copyright 2016 Hewlett-Packard Development Company, L.P.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
not use this file except in compliance with the License. You may obtain
|
||||||
|
a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
License for the specific language governing permissions and limitations
|
||||||
|
under the License.
|
||||||
|
|
||||||
|
******************
|
||||||
|
Swift Plugin Guide
|
||||||
|
******************
|
||||||
|
|
||||||
|
WARNING: Swift plugin is currently EXPERIMENTAL as notifications aren't
|
||||||
|
fully supported. See below on enabling notifications.
|
||||||
|
|
||||||
|
Integration is provided via a plugin. There are multiple configuration
|
||||||
|
settings required for proper indexing and incremental updates. Some of the
|
||||||
|
settings are specified in Searchlight configuration files. Others are
|
||||||
|
provided in other service configuration files.
|
||||||
|
|
||||||
|
Searchlight Configuration
|
||||||
|
=========================
|
||||||
|
|
||||||
|
Searchlight resource configuration options are shown below with their
|
||||||
|
configuration file and default values.
|
||||||
|
|
||||||
|
See :ref:`searchlight-plugins` for common options with their default values,
|
||||||
|
general configuration information, and an example complete configuration.
|
||||||
|
|
||||||
|
.. note::
|
||||||
|
|
||||||
|
Unless you are changing to a non-default value, you do not need to
|
||||||
|
specify any of the following configuration options.
|
||||||
|
|
||||||
|
searchlight.conf
|
||||||
|
----------------
|
||||||
|
|
||||||
|
Plugin: OS::Swift::Account
|
||||||
|
^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
::
|
||||||
|
|
||||||
|
[resource_plugin:os_swift_account]
|
||||||
|
enabled = true
|
||||||
|
index_name = searchlight
|
||||||
|
#Specify same value as in swift proxy config for reseller_prefix
|
||||||
|
reseller_prefix = AUTH_
|
||||||
|
|
||||||
|
.. note::
|
||||||
|
|
||||||
|
os_swift_account is disabled by default. You need to explicitly set enabled = True as shown above
|
||||||
|
|
||||||
|
Plugin: OS::Swift::Container
|
||||||
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
::
|
||||||
|
|
||||||
|
[resource_plugin:os_swift_container]
|
||||||
|
enabled = true
|
||||||
|
index_name = searchlight
|
||||||
|
|
||||||
|
.. note::
|
||||||
|
|
||||||
|
os_swift_container is disabled by default. You need to explicitly set enabled = True as shown above
|
||||||
|
|
||||||
|
Plugin: OS::Swift::Object
|
||||||
|
^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
::
|
||||||
|
|
||||||
|
[resource_plugin:os_swift_object]
|
||||||
|
enabled = true
|
||||||
|
index_name = searchlight
|
||||||
|
|
||||||
|
.. note::
|
||||||
|
|
||||||
|
os_swift_object is disabled by default. You need to explicitly set enabled = True as shown above
|
||||||
|
|
||||||
|
Swift Configuration
|
||||||
|
====================
|
||||||
|
|
||||||
|
The Swift service currently doesn't send notifications.
|
||||||
|
Apply this patch https://review.openstack.org/#/c/249471
|
||||||
|
for adding notification middleware to swift.
|
||||||
|
|
||||||
|
reseller_admin_role
|
||||||
|
-------------------
|
||||||
|
|
||||||
|
Users with the Keystone role defined in reseller_admin_role (ResellerAdmin by default)
|
||||||
|
can operate on any account. The auth system sets the request environ reseller_request
|
||||||
|
to True if a request is coming from a user with this role.
|
||||||
|
|
||||||
|
Searchlight needs this role for its service user to access all the swift accounts
|
||||||
|
for initial indexing. The searchlight user and sevice project being referred here is the
|
||||||
|
one defined in service_credentials section of searchlight conf file.
|
||||||
|
|
||||||
|
::
|
||||||
|
|
||||||
|
openstack role add --user searchlight --project service ResellerAdmin
|
||||||
|
|
||||||
|
|
||||||
|
proxy-server.conf
|
||||||
|
-----------------
|
||||||
|
|
||||||
|
Notifications must be configured properly for searchlight to process
|
||||||
|
incremental updates. Use the following::
|
||||||
|
|
||||||
|
#Add the following new section
|
||||||
|
[filter:oslomiddleware]
|
||||||
|
paste.filter_factory = swift.common.middleware.oslo_notifications:filter_factory
|
||||||
|
publisher_id = swift.localhost
|
||||||
|
#Replace <user>,<password>,<rabbitip> and <rabbitport> for your environment values
|
||||||
|
transport_url = rabbit://<user>:<password>@<rabbitip>:<rabbitport>/
|
||||||
|
notification_driver = messaging
|
||||||
|
notification_topics = searchlight_indexer
|
||||||
|
|
||||||
|
#Add oslomiddleware to pipeline:main see example below.
|
||||||
|
[pipeline:main]
|
||||||
|
pipeline = catch_errors gatekeeper healthcheck ... oslomiddleware proxy-logging proxy-server
|
||||||
|
|
||||||
|
|
||||||
|
.. note::
|
||||||
|
|
||||||
|
Restart swift proxy API service (s-proxy) after making changes.
|
||||||
|
|
||||||
|
local.conf (devstack)
|
||||||
|
---------------------
|
||||||
|
|
||||||
|
The settings above may be automatically configured by ``stack.sh``
|
||||||
|
by adding them to the following post config section in devstack.
|
||||||
|
Just place the following in local.conf and copy the above settings
|
||||||
|
underneath it.::
|
||||||
|
|
||||||
|
[[post-config|$SWIFT_PROXY_CONF]]
|
||||||
|
[DEFAULT]
|
||||||
|
|
||||||
|
Release Notes
|
||||||
|
=============
|
||||||
|
|
||||||
|
0.2.0.0 (Mitaka)
|
||||||
|
----------------
|
||||||
|
|
||||||
|
Swift did not generate notifications for account/container/object CRUD
|
||||||
|
|
||||||
|
This means that search results will not include incremental updates after
|
||||||
|
the initial indexing.
|
||||||
|
|
||||||
|
The patch (https://review.openstack.org/#/c/249471) implements this feature.
|
||||||
|
|
||||||
|
For devstack, the easiest way to test is
|
||||||
|
cd /opt/stack/swift
|
||||||
|
git review -x 249471
|
||||||
|
|
||||||
|
Searchlight developers/installers should apply the above patch in Swift when
|
||||||
|
using Searchlight with the Swift Mitaka release.
|
||||||
|
|
||||||
|
Alternatively, you may set up a cron job to re-index swift
|
||||||
|
account/container/objects periodically to get updated information. The
|
||||||
|
recommendation is to use the notifications.
|
||||||
|
|
||||||
|
You should use the ``--no-delete`` option to prevent the index from
|
||||||
|
temporarily not containing any data (which otherwise would happen with a full
|
||||||
|
bulk indexing job)::
|
||||||
|
|
||||||
|
searchlight-manage index sync --type OS::Swift::Account --force --no-delete
|
||||||
|
|
||||||
|
Searchlight swift plugin resource types follow the hierarchy similar to
|
||||||
|
Swift concepts
|
||||||
|
|
||||||
|
OS::Swift:Acccount(Parent)
|
||||||
|
-> OS:Swift::Container(Child)
|
||||||
|
-> OS::Swift::Object(Grand Child)
|
||||||
|
|
||||||
|
which means indexing is initiated by specifying only the top parent
|
||||||
|
(OS::Swift::Account) and that will in-turn index all the child
|
||||||
|
plugins(Container and Object)
|
||||||
|
|
|
@ -0,0 +1,14 @@
|
||||||
|
---
|
||||||
|
prelude: >
|
||||||
|
Swift plugin for searchlight
|
||||||
|
features:
|
||||||
|
- Three resource types are introduced for swift
|
||||||
|
plugin.
|
||||||
|
|
||||||
|
OS::Swift:Account
|
||||||
|
->OS::Swift::Container
|
||||||
|
-->OS::Swift::Object
|
||||||
|
issues:
|
||||||
|
- The Swift service currently doesn't send notifications.
|
||||||
|
Follow the swift plugin documentation for current
|
||||||
|
solutions.
|
|
@ -48,3 +48,4 @@ python-glanceclient>=2.0.0 # Apache-2.0
|
||||||
python-novaclient!=2.33.0,>=2.29.0 # Apache-2.0
|
python-novaclient!=2.33.0,>=2.29.0 # Apache-2.0
|
||||||
python-neutronclient!=4.1.0,>=2.6.0 # Apache-2.0
|
python-neutronclient!=4.1.0,>=2.6.0 # Apache-2.0
|
||||||
python-cinderclient>=1.3.1 # Apache-2.0
|
python-cinderclient>=1.3.1 # Apache-2.0
|
||||||
|
python-swiftclient>=2.2.0 # Apache-2.0
|
||||||
|
|
|
@ -474,6 +474,16 @@ class IndexBase(plugin.Plugin):
|
||||||
def requires_role_separation(self):
|
def requires_role_separation(self):
|
||||||
return len(self.admin_only_fields) > 0
|
return len(self.admin_only_fields) > 0
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def is_plugin_enabled_by_default(cls):
|
||||||
|
'''
|
||||||
|
Each plugin can overwrite the default value of whether a
|
||||||
|
plugin should be enabled if the value is not explicitly
|
||||||
|
set in the configuration
|
||||||
|
|
||||||
|
'''
|
||||||
|
return True
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_plugin_opts(cls):
|
def get_plugin_opts(cls):
|
||||||
"""Options that can be overridden per plugin. Note that
|
"""Options that can be overridden per plugin. Note that
|
||||||
|
@ -482,7 +492,7 @@ class IndexBase(plugin.Plugin):
|
||||||
sets of types (https://bugs.launchpad.net/searchlight/+bug/1558240)
|
sets of types (https://bugs.launchpad.net/searchlight/+bug/1558240)
|
||||||
"""
|
"""
|
||||||
opts = [
|
opts = [
|
||||||
cfg.BoolOpt("enabled", default=True),
|
cfg.BoolOpt("enabled", default=cls.is_plugin_enabled_by_default()),
|
||||||
cfg.StrOpt("admin_only_fields"),
|
cfg.StrOpt("admin_only_fields"),
|
||||||
cfg.BoolOpt('mapping_use_doc_values')
|
cfg.BoolOpt('mapping_use_doc_values')
|
||||||
]
|
]
|
||||||
|
|
|
@ -19,8 +19,10 @@ from designateclient.v2 import client as designateclient
|
||||||
from glanceclient.v2 import client as glance
|
from glanceclient.v2 import client as glance
|
||||||
from keystoneclient import auth as ks_auth
|
from keystoneclient import auth as ks_auth
|
||||||
from keystoneclient import session as ks_session
|
from keystoneclient import session as ks_session
|
||||||
|
from keystoneclient.v2_0 import client as ks_client
|
||||||
import neutronclient.v2_0.client
|
import neutronclient.v2_0.client
|
||||||
import novaclient.client
|
import novaclient.client
|
||||||
|
import swiftclient
|
||||||
|
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
|
|
||||||
|
@ -100,3 +102,84 @@ def get_cinderclient():
|
||||||
session=session,
|
session=session,
|
||||||
region_name=cfg.CONF.service_credentials.os_region_name,
|
region_name=cfg.CONF.service_credentials.os_region_name,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Swift still needs special handling because it doesn't support
|
||||||
|
# keystone sessions. Rather than maintain two codepaths, we'll do this
|
||||||
|
_swiftclient = None
|
||||||
|
|
||||||
|
|
||||||
|
def clear_cached_swiftclient_on_unauthorized(fn):
|
||||||
|
def wrapper(*args, **kwargs):
|
||||||
|
global _session
|
||||||
|
global _swiftclient
|
||||||
|
try:
|
||||||
|
return fn(*args, **kwargs)
|
||||||
|
except swiftclient.exceptions.ClientException:
|
||||||
|
_session = None
|
||||||
|
_swiftclient = None
|
||||||
|
return fn(*args, **kwargs)
|
||||||
|
return wrapper
|
||||||
|
|
||||||
|
|
||||||
|
def get_swiftclient():
|
||||||
|
|
||||||
|
global _swiftclient
|
||||||
|
if _swiftclient:
|
||||||
|
return _swiftclient
|
||||||
|
_get_session()
|
||||||
|
|
||||||
|
service_type = 'object-store'
|
||||||
|
|
||||||
|
os_options = {
|
||||||
|
'service_type': service_type,
|
||||||
|
'region_name': cfg.CONF.service_credentials.os_region_name,
|
||||||
|
'endpoint_type': cfg.CONF.service_credentials.os_endpoint_type,
|
||||||
|
}
|
||||||
|
|
||||||
|
# When swiftclient supports session, use session instead of
|
||||||
|
# preauthtoken param below
|
||||||
|
_swiftclient = swiftclient.client.Connection(
|
||||||
|
auth_version='2',
|
||||||
|
user=cfg.CONF.service_credentials.username,
|
||||||
|
key=cfg.CONF.service_credentials.password,
|
||||||
|
authurl=cfg.CONF.service_credentials.auth_url,
|
||||||
|
tenant_name=cfg.CONF.service_credentials.tenant_name,
|
||||||
|
os_options=os_options,
|
||||||
|
cacert=cfg.CONF.service_credentials.cafile,
|
||||||
|
insecure=cfg.CONF.service_credentials.insecure
|
||||||
|
)
|
||||||
|
|
||||||
|
return _swiftclient
|
||||||
|
|
||||||
|
|
||||||
|
# TODO(lakshmiS) See if we can cache this.
|
||||||
|
# Cached members will be equal to # of accounts at max.
|
||||||
|
def get_swiftclient_st(storageurl):
|
||||||
|
service_type = 'object-store'
|
||||||
|
_get_session()
|
||||||
|
|
||||||
|
os_options = {
|
||||||
|
'service_type': service_type,
|
||||||
|
'region_name': cfg.CONF.service_credentials.os_region_name,
|
||||||
|
'endpoint_type': cfg.CONF.service_credentials.os_endpoint_type,
|
||||||
|
}
|
||||||
|
swift_client = swiftclient.client.Connection(
|
||||||
|
auth_version='2',
|
||||||
|
user=cfg.CONF.service_credentials.username,
|
||||||
|
key=cfg.CONF.service_credentials.password,
|
||||||
|
authurl=cfg.CONF.service_credentials.auth_url,
|
||||||
|
tenant_name=cfg.CONF.service_credentials.tenant_name,
|
||||||
|
os_options=os_options,
|
||||||
|
cacert=cfg.CONF.service_credentials.cafile,
|
||||||
|
insecure=cfg.CONF.service_credentials.insecure,
|
||||||
|
preauthurl=storageurl,
|
||||||
|
)
|
||||||
|
return swift_client
|
||||||
|
|
||||||
|
|
||||||
|
def get_keystoneclient():
|
||||||
|
session = _get_session()
|
||||||
|
|
||||||
|
return ks_client.Client(
|
||||||
|
session=session,
|
||||||
|
region_name=cfg.CONF.service_credentials.os_region_name)
|
||||||
|
|
|
@ -0,0 +1,212 @@
|
||||||
|
# Copyright (c) 2016 Hewlett-Packard Development Company, L.P.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
# implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
import datetime
|
||||||
|
import logging
|
||||||
|
import six
|
||||||
|
|
||||||
|
from oslo_utils import timeutils
|
||||||
|
|
||||||
|
from searchlight.elasticsearch.plugins import openstack_clients
|
||||||
|
from searchlight import i18n
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
_LE = i18n._LE
|
||||||
|
_ = i18n._
|
||||||
|
|
||||||
|
AUTH_PREFIX = ""
|
||||||
|
ID_SEP = "/"
|
||||||
|
|
||||||
|
# _accounts is populated when get_swift_accounts() is called; which is
|
||||||
|
# guaranteed to be called before get_swift_containers and get_swfit_objects
|
||||||
|
# since account plugin is parent and grandparent of container and object
|
||||||
|
_accounts = []
|
||||||
|
|
||||||
|
dateformat = "%a, %d %b %Y %H:%M:%S %Z"
|
||||||
|
|
||||||
|
|
||||||
|
def serialize_swift_account(account):
|
||||||
|
metadocument = {k: account.get(k, None) for k, v in six.iteritems(account)
|
||||||
|
if k.lower().startswith("x-account-meta")}
|
||||||
|
account_fields = ('id', 'name')
|
||||||
|
document = {f: account.get(f, None) for f in account_fields}
|
||||||
|
|
||||||
|
document['domain_id'] = account.get('x-account-project-domain-id', None)
|
||||||
|
if account.get('x-timestamp'):
|
||||||
|
timestamp = float(account.get('x-timestamp'))
|
||||||
|
document['created_at'] = \
|
||||||
|
timeutils.isotime(datetime.datetime.fromtimestamp(timestamp))
|
||||||
|
|
||||||
|
# lakshmiS: swift get_account() doesn't include update datetime field(?)
|
||||||
|
if account.get('updated_at'):
|
||||||
|
document['updated_at'] = account.get('updated_at')
|
||||||
|
|
||||||
|
document.update(metadocument)
|
||||||
|
return document
|
||||||
|
|
||||||
|
|
||||||
|
def serialize_swift_account_notification(account):
|
||||||
|
account['name'] = account['project_name']
|
||||||
|
account['id'] = account['account']
|
||||||
|
account['x-account-project-domain-id'] = account['project_domain_id']
|
||||||
|
# Note: updated_at is included in notification payload.
|
||||||
|
# No need to map or transform the date and its format.
|
||||||
|
return serialize_swift_account(account)
|
||||||
|
|
||||||
|
|
||||||
|
def serialize_swift_container(container):
|
||||||
|
metadocument = {k: container.get(k, None) for k, v in
|
||||||
|
six.iteritems(container)
|
||||||
|
if k.lower().startswith("x-container-meta")}
|
||||||
|
container_fields = ('id',
|
||||||
|
'name',
|
||||||
|
'account',
|
||||||
|
'account_id',
|
||||||
|
'x-container-read'
|
||||||
|
)
|
||||||
|
document = {f: container.get(f, None) for f in container_fields}
|
||||||
|
if container.get('x-timestamp'):
|
||||||
|
timestamp = float(container.get('x-timestamp'))
|
||||||
|
document['created_at'] = \
|
||||||
|
timeutils.isotime(datetime.datetime.fromtimestamp(timestamp))
|
||||||
|
|
||||||
|
# (laskhmiS) get_container doesn't include last_modified field even
|
||||||
|
# though the swift api documentation says it returns it. Include it
|
||||||
|
# when it starts sending it.
|
||||||
|
|
||||||
|
# Notifications for container sends this field as 'updated_at' instead
|
||||||
|
# of 'last_modified'.
|
||||||
|
if container.get('updated_at'):
|
||||||
|
document['updated_at'] = container['updated_at']
|
||||||
|
|
||||||
|
document.update(metadocument)
|
||||||
|
return document
|
||||||
|
|
||||||
|
|
||||||
|
def serialize_swift_container_notification(container):
|
||||||
|
# Account Id + container name. container['account'] from notification has
|
||||||
|
# account id value.
|
||||||
|
container['id'] = container['account'] + ID_SEP + container['container']
|
||||||
|
container['name'] = container['container']
|
||||||
|
container['account_id'] = container['account']
|
||||||
|
container['account'] = container['project_name']
|
||||||
|
return serialize_swift_container(container)
|
||||||
|
|
||||||
|
|
||||||
|
def serialize_swift_object(sobject):
|
||||||
|
metadocument = {k: sobject.get(k, None) for k, v in six.iteritems(sobject)
|
||||||
|
if k.lower().startswith("x-object-meta")}
|
||||||
|
object_fields = ('id',
|
||||||
|
'name',
|
||||||
|
'account',
|
||||||
|
'account_id',
|
||||||
|
'container',
|
||||||
|
'container_id',
|
||||||
|
'etag'
|
||||||
|
)
|
||||||
|
document = {f: sobject.get(f, None) for f in object_fields}
|
||||||
|
document['content_type'] = sobject.get('content-type', None)
|
||||||
|
document['content_length'] = sobject.get('content-length', None)
|
||||||
|
|
||||||
|
if sobject.get('x-timestamp'):
|
||||||
|
timestamp = float(sobject.get('x-timestamp'))
|
||||||
|
document['created_at'] = \
|
||||||
|
timeutils.isotime(datetime.datetime.fromtimestamp(timestamp))
|
||||||
|
if sobject.get('last-modified'):
|
||||||
|
updated_dt = datetime.datetime.strptime(
|
||||||
|
sobject['last-modified'], dateformat)
|
||||||
|
document['updated_at'] = timeutils.isotime(updated_dt)
|
||||||
|
document.update(metadocument)
|
||||||
|
return document
|
||||||
|
|
||||||
|
|
||||||
|
def serialize_swift_object_notification(sobj):
|
||||||
|
# Account id + container name
|
||||||
|
sobj['container_id'] = sobj['account'] + ID_SEP + sobj['container']
|
||||||
|
# Account id + container name + object name
|
||||||
|
sobj['id'] = sobj['container_id'] + ID_SEP + sobj['object']
|
||||||
|
sobj['name'] = sobj['object']
|
||||||
|
sobj['account_id'] = sobj['account']
|
||||||
|
sobj['account'] = sobj['project_name']
|
||||||
|
return serialize_swift_object(sobj)
|
||||||
|
|
||||||
|
|
||||||
|
@openstack_clients.clear_cached_swiftclient_on_unauthorized
|
||||||
|
def _get_storage_url_prefix():
|
||||||
|
# Extracts swift proxy url after removing the default account id
|
||||||
|
# from the service account. Later storage_url's will be constructed
|
||||||
|
# for each account by appending the keystone tenant id.
|
||||||
|
try:
|
||||||
|
storage_url = openstack_clients.get_swiftclient().get_auth()[0]
|
||||||
|
return storage_url[:storage_url.index(AUTH_PREFIX)] + AUTH_PREFIX
|
||||||
|
except ValueError:
|
||||||
|
LOG.error(_LE("reseller_prefix %s not found in keystone endpoint ")
|
||||||
|
% AUTH_PREFIX)
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def get_swift_accounts(auth_prefix):
|
||||||
|
global AUTH_PREFIX
|
||||||
|
# TODO(lakshmiS): Add support for SERVICE_ accounts
|
||||||
|
AUTH_PREFIX = auth_prefix
|
||||||
|
ks_client = openstack_clients.get_keystoneclient()
|
||||||
|
for tenant in ks_client.tenants.list():
|
||||||
|
storage_url = _get_storage_url_prefix() + tenant.id
|
||||||
|
sclient = openstack_clients.get_swiftclient_st(storage_url)
|
||||||
|
# 0 index has account summary
|
||||||
|
account = sclient.get_account()[0]
|
||||||
|
account['name'] = tenant.name
|
||||||
|
account['id'] = auth_prefix + tenant.id
|
||||||
|
|
||||||
|
# store it for later usage in retrieving containers
|
||||||
|
# and objects
|
||||||
|
account_detail = {'id': account['id'],
|
||||||
|
'name': tenant.name,
|
||||||
|
'storage.url': storage_url}
|
||||||
|
_accounts.append(account_detail)
|
||||||
|
|
||||||
|
yield account
|
||||||
|
|
||||||
|
|
||||||
|
def get_swift_containers():
|
||||||
|
for account in _accounts:
|
||||||
|
sclient = openstack_clients.get_swiftclient_st(account['storage.url'])
|
||||||
|
# 1 index has container list
|
||||||
|
containers = sclient.get_account()[1]
|
||||||
|
for container in containers:
|
||||||
|
ctr, obj = sclient.get_container(container['name'])
|
||||||
|
ctr['id'] = account['id'] + ID_SEP + container['name']
|
||||||
|
ctr['name'] = container['name']
|
||||||
|
ctr['account'] = account['name']
|
||||||
|
ctr['account_id'] = account['id']
|
||||||
|
yield ctr
|
||||||
|
|
||||||
|
|
||||||
|
def get_swift_objects():
|
||||||
|
for account in _accounts:
|
||||||
|
sclient = openstack_clients.get_swiftclient_st(account['storage.url'])
|
||||||
|
# 1 index has container list
|
||||||
|
containers = sclient.get_account()[1]
|
||||||
|
for sctr in containers:
|
||||||
|
ctr, obj = sclient.get_container(sctr['name'])
|
||||||
|
for sobject in obj:
|
||||||
|
sobj = sclient.head_object(sctr['name'], sobject['name'])
|
||||||
|
sobj['account'] = account['name']
|
||||||
|
sobj['account_id'] = account['id']
|
||||||
|
sobj['container'] = sctr['name']
|
||||||
|
sobj['container_id'] = account['id'] + ID_SEP + sctr['name']
|
||||||
|
sobj['id'] = sobj['container_id'] + ID_SEP + sobject['name']
|
||||||
|
sobj['name'] = sobject['name']
|
||||||
|
yield sobj
|
|
@ -0,0 +1,107 @@
|
||||||
|
# Copyright (c) 2016 Hewlett-Packard Development Company, L.P.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
# implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
|
||||||
|
from oslo_config import cfg
|
||||||
|
|
||||||
|
from searchlight.elasticsearch.plugins import base
|
||||||
|
from searchlight.elasticsearch.plugins.swift import get_swift_accounts
|
||||||
|
from searchlight.elasticsearch.plugins.swift import serialize_swift_account
|
||||||
|
from searchlight.elasticsearch.plugins.swift import swift_notification_handler
|
||||||
|
|
||||||
|
swift_plugin_opts = [
|
||||||
|
cfg.StrOpt('reseller_prefix', default="AUTH_",
|
||||||
|
help="prefix used in account names for auth system."),
|
||||||
|
]
|
||||||
|
|
||||||
|
CONF = cfg.CONF
|
||||||
|
CONF.register_opts(swift_plugin_opts, group='resource_plugin:os_swift_account')
|
||||||
|
|
||||||
|
|
||||||
|
class AccountIndex(base.IndexBase):
|
||||||
|
NotificationHandlerCls = swift_notification_handler.SwiftAccountHandler
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
super(AccountIndex, self).__init__()
|
||||||
|
self.options = cfg.CONF[self.get_config_group_name()]
|
||||||
|
|
||||||
|
# swift_owner_headers
|
||||||
|
ADMIN_ONLY_FIELDS = ['x-account-meta-temp-url-key',
|
||||||
|
'x-account-meta-temp-url-key-2',
|
||||||
|
'x-account-access-control']
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_document_type(cls):
|
||||||
|
return 'OS::Swift::Account'
|
||||||
|
|
||||||
|
def get_mapping(self):
|
||||||
|
return {
|
||||||
|
'dynamic': True,
|
||||||
|
'properties': {
|
||||||
|
'id': {'type': 'string', 'index': 'not_analyzed'},
|
||||||
|
'name': {
|
||||||
|
'type': 'string',
|
||||||
|
'fields': {
|
||||||
|
'raw': {'type': 'string', 'index': 'not_analyzed'}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
# TODO(lakshmiS): Removing following field(s) since account
|
||||||
|
# notifications don't include it for subsequent updates.
|
||||||
|
# Enable when it is included in future notifications.
|
||||||
|
|
||||||
|
# The number of objects in the account.
|
||||||
|
# 'x-account-object-count': {'type': 'long'},
|
||||||
|
|
||||||
|
# The total number of bytes that are stored in Object Storage
|
||||||
|
# for the account.
|
||||||
|
# 'x-account-bytes-used': {'type': 'long'},
|
||||||
|
|
||||||
|
# The number of containers.
|
||||||
|
# 'x-account-container-count': {'type': 'long'},
|
||||||
|
|
||||||
|
'domain_id': {'type': 'string', 'index': 'not_analyzed'},
|
||||||
|
'created_at': {'type': 'date'},
|
||||||
|
'updated_at': {'type': 'date'}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
@property
|
||||||
|
def admin_only_fields(self):
|
||||||
|
from_conf = super(AccountIndex, self).admin_only_fields
|
||||||
|
return AccountIndex.ADMIN_ONLY_FIELDS + from_conf
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def is_plugin_enabled_by_default(cls):
|
||||||
|
return False
|
||||||
|
|
||||||
|
@property
|
||||||
|
def allow_admin_ignore_rbac(self):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def _get_rbac_field_filters(self, request_context):
|
||||||
|
id = self.options.reseller_prefix + request_context.owner
|
||||||
|
return [
|
||||||
|
{"term": {"id": id}}
|
||||||
|
]
|
||||||
|
|
||||||
|
@property
|
||||||
|
def routing_field(self):
|
||||||
|
return "id"
|
||||||
|
|
||||||
|
def get_objects(self):
|
||||||
|
return get_swift_accounts(self.options.reseller_prefix)
|
||||||
|
|
||||||
|
def serialize(self, obj):
|
||||||
|
return serialize_swift_account(obj)
|
|
@ -0,0 +1,136 @@
|
||||||
|
# Copyright (c) 2016 Hewlett-Packard Development Company, L.P.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
# implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
from searchlight.elasticsearch.plugins import base
|
||||||
|
from searchlight.elasticsearch.plugins.swift import get_swift_containers
|
||||||
|
from searchlight.elasticsearch.plugins.swift import serialize_swift_container
|
||||||
|
from searchlight.elasticsearch.plugins.swift import swift_notification_handler
|
||||||
|
|
||||||
|
|
||||||
|
class ContainerIndex(base.IndexBase):
|
||||||
|
NotificationHandlerCls = swift_notification_handler.SwiftContainerHandler
|
||||||
|
|
||||||
|
def get_notification_handler(self):
|
||||||
|
"""Override because the container handler needs a handle to object
|
||||||
|
indexer for cascade delete of container to objects.
|
||||||
|
"""
|
||||||
|
return self.NotificationHandlerCls(
|
||||||
|
self.index_helper,
|
||||||
|
self.options,
|
||||||
|
object_helper=self.child_plugins[0].index_helper)
|
||||||
|
|
||||||
|
# swift_owner_headers
|
||||||
|
ADMIN_ONLY_FIELDS = ['x-container-write',
|
||||||
|
'x-container-sync-key',
|
||||||
|
'x-container-sync-to',
|
||||||
|
'x-container-meta-temp-url-key',
|
||||||
|
'x-container-meta-temp-url-key-2']
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def parent_plugin_type(cls):
|
||||||
|
return "OS::Swift::Account"
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_document_type(cls):
|
||||||
|
return 'OS::Swift::Container'
|
||||||
|
|
||||||
|
def get_mapping(self):
|
||||||
|
return {
|
||||||
|
'dynamic': True,
|
||||||
|
"_source": {
|
||||||
|
"excludes": ["x-container-read"]
|
||||||
|
},
|
||||||
|
'properties': {
|
||||||
|
'id': {'type': 'string', 'index': 'not_analyzed'},
|
||||||
|
'name': {
|
||||||
|
'type': 'string',
|
||||||
|
'fields': {
|
||||||
|
'raw': {'type': 'string', 'index': 'not_analyzed'}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'account': {
|
||||||
|
'type': 'string',
|
||||||
|
'fields': {
|
||||||
|
'raw': {'type': 'string', 'index': 'not_analyzed'}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'account_id': {'type': 'string', 'index': 'not_analyzed'},
|
||||||
|
'x-container-read': {
|
||||||
|
'type': 'string', 'index': 'not_analyzed',
|
||||||
|
'store': False
|
||||||
|
},
|
||||||
|
|
||||||
|
# TODO(lakshmiS): Removing following field(s) since account
|
||||||
|
# notifications don't include it for subsequent updates.
|
||||||
|
# Enable when it is included in future notifications.
|
||||||
|
# 'x-container-object-count': {'type': 'long'},
|
||||||
|
# 'x-container-bytes-used': {'type': 'long'},
|
||||||
|
|
||||||
|
'created_at': {'type': 'date'},
|
||||||
|
'updated_at': {'type': 'date'}
|
||||||
|
},
|
||||||
|
"_parent": {
|
||||||
|
"type": self.parent_plugin_type()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@property
|
||||||
|
def admin_only_fields(self):
|
||||||
|
from_conf = super(ContainerIndex, self).admin_only_fields
|
||||||
|
return ContainerIndex.ADMIN_ONLY_FIELDS + from_conf
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def is_plugin_enabled_by_default(cls):
|
||||||
|
return False
|
||||||
|
|
||||||
|
@property
|
||||||
|
def allow_admin_ignore_rbac(self):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def _get_rbac_field_filters(self, request_context):
|
||||||
|
tenant_member = request_context.tenant + ":*"
|
||||||
|
single_user = request_context.tenant + ":" + request_context.user
|
||||||
|
account_id = \
|
||||||
|
self.parent_plugin.options.reseller_prefix + request_context.owner
|
||||||
|
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
'or': [
|
||||||
|
{
|
||||||
|
'term': {
|
||||||
|
'account_id': account_id
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'terms': {
|
||||||
|
'x-container-read': [tenant_member, single_user]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
def get_parent_id_field(self):
|
||||||
|
return 'account_id'
|
||||||
|
|
||||||
|
@property
|
||||||
|
def routing_field(self):
|
||||||
|
return "account_id"
|
||||||
|
|
||||||
|
def get_objects(self):
|
||||||
|
return get_swift_containers()
|
||||||
|
|
||||||
|
def serialize(self, obj):
|
||||||
|
return serialize_swift_container(obj)
|
|
@ -0,0 +1,139 @@
|
||||||
|
# Copyright (c) 2016 Hewlett-Packard Development Company, L.P.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
# implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
from oslo_config import cfg
|
||||||
|
|
||||||
|
from searchlight.elasticsearch.plugins import base
|
||||||
|
from searchlight.elasticsearch.plugins.swift import get_swift_objects
|
||||||
|
from searchlight.elasticsearch.plugins.swift import serialize_swift_object
|
||||||
|
from searchlight.elasticsearch.plugins.swift import swift_notification_handler
|
||||||
|
|
||||||
|
swift_plugin_opts = [
|
||||||
|
cfg.StrOpt('reseller_prefix', default="AUTH_",
|
||||||
|
help="prefix used in account names for auth system."),
|
||||||
|
]
|
||||||
|
|
||||||
|
CONF = cfg.CONF
|
||||||
|
CONF.register_opts(swift_plugin_opts, group='resource_plugin:os_swift_account')
|
||||||
|
|
||||||
|
|
||||||
|
class ObjectIndex(base.IndexBase):
|
||||||
|
NotificationHandlerCls = swift_notification_handler.SwiftObjectHandler
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def parent_plugin_type(cls):
|
||||||
|
return "OS::Swift::Container"
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_document_type(cls):
|
||||||
|
return 'OS::Swift::Object'
|
||||||
|
|
||||||
|
def get_mapping(self):
|
||||||
|
return {
|
||||||
|
'dynamic': True,
|
||||||
|
'properties': {
|
||||||
|
'id': {'type': 'string', 'index': 'not_analyzed'},
|
||||||
|
'name': {
|
||||||
|
'type': 'string',
|
||||||
|
'fields': {
|
||||||
|
'raw': {'type': 'string', 'index': 'not_analyzed'}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'account': {
|
||||||
|
'type': 'string',
|
||||||
|
'fields': {
|
||||||
|
'raw': {'type': 'string', 'index': 'not_analyzed'}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'account_id': {'type': 'string', 'index': 'not_analyzed'},
|
||||||
|
'container': {
|
||||||
|
'type': 'string',
|
||||||
|
'fields': {
|
||||||
|
'raw': {'type': 'string', 'index': 'not_analyzed'}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
'container_id': {'type': 'string', 'index': 'not_analyzed'},
|
||||||
|
'content_type': {'type': 'string', 'index': 'not_analyzed'},
|
||||||
|
'content_length': {'type': 'long'},
|
||||||
|
'etag': {'type': 'string'},
|
||||||
|
'created_at': {'type': 'date'},
|
||||||
|
'updated_at': {'type': 'date'}
|
||||||
|
},
|
||||||
|
"_parent": {
|
||||||
|
"type": self.parent_plugin_type()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@property
|
||||||
|
def allow_admin_ignore_rbac(self):
|
||||||
|
return False
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def is_plugin_enabled_by_default(cls):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def _get_rbac_field_filters(self, request_context):
|
||||||
|
|
||||||
|
tenant_member = request_context.tenant + ":*"
|
||||||
|
single_user = request_context.tenant + ":" + request_context.user
|
||||||
|
account_plugin = self.parent_plugin.parent_plugin
|
||||||
|
account_id = \
|
||||||
|
account_plugin.options.reseller_prefix + request_context.owner
|
||||||
|
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"has_parent": {
|
||||||
|
"type": self.parent_plugin_type(),
|
||||||
|
"query": {
|
||||||
|
"filtered": {
|
||||||
|
"query": {"match_all": {}},
|
||||||
|
"filter": {
|
||||||
|
"or": [
|
||||||
|
{
|
||||||
|
'term': {
|
||||||
|
'account_id': account_id
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'terms': {
|
||||||
|
'x-container-read': [
|
||||||
|
tenant_member,
|
||||||
|
single_user]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
def get_parent_id_field(self):
|
||||||
|
return 'container_id'
|
||||||
|
|
||||||
|
@property
|
||||||
|
def routing_field(self):
|
||||||
|
return "account_id"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def facets_with_options(self):
|
||||||
|
return ('container', 'content_type')
|
||||||
|
|
||||||
|
def get_objects(self):
|
||||||
|
return get_swift_objects()
|
||||||
|
|
||||||
|
def serialize(self, obj):
|
||||||
|
return serialize_swift_object(obj)
|
|
@ -0,0 +1,176 @@
|
||||||
|
# Copyright (c) 2016 Hewlett-Packard Development Company, L.P.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
# implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from searchlight.elasticsearch.plugins import base
|
||||||
|
from searchlight.elasticsearch.plugins import swift
|
||||||
|
from searchlight.elasticsearch.plugins.swift \
|
||||||
|
import serialize_swift_account_notification
|
||||||
|
from searchlight.elasticsearch.plugins.swift \
|
||||||
|
import serialize_swift_container_notification
|
||||||
|
from searchlight.elasticsearch.plugins.swift \
|
||||||
|
import serialize_swift_object_notification
|
||||||
|
from searchlight import i18n
|
||||||
|
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
_LE = i18n._LE
|
||||||
|
_ = i18n._
|
||||||
|
|
||||||
|
|
||||||
|
class SwiftAccountHandler(base.NotificationBase):
|
||||||
|
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
super(SwiftAccountHandler, self).__init__(*args, **kwargs)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _get_notification_exchanges(cls):
|
||||||
|
return ['swift']
|
||||||
|
|
||||||
|
def get_event_handlers(self):
|
||||||
|
return {
|
||||||
|
"account.create": self.create_or_update,
|
||||||
|
"account.metadata": self.create_or_update,
|
||||||
|
"account.delete": self.delete
|
||||||
|
}
|
||||||
|
|
||||||
|
def create_or_update(self, payload, timestamp):
|
||||||
|
payload = serialize_swift_account_notification(payload)
|
||||||
|
try:
|
||||||
|
self.index_helper.save_document(
|
||||||
|
payload,
|
||||||
|
version=self.get_version(payload, timestamp))
|
||||||
|
except Exception as exc:
|
||||||
|
LOG.error(_LE('Error saving account %(id)s '
|
||||||
|
'in index. Error: %(exc)s') %
|
||||||
|
{'id': payload['id'], 'exc': exc})
|
||||||
|
|
||||||
|
def delete(self, payload, timestamp):
|
||||||
|
version = self.get_version(payload, timestamp)
|
||||||
|
id = payload['account']
|
||||||
|
try:
|
||||||
|
self.index_helper.delete_document(
|
||||||
|
{'_id': id, '_version': version,
|
||||||
|
'_routing': payload['account']})
|
||||||
|
except Exception as exc:
|
||||||
|
LOG.error(_LE('Error deleting account %(id)s '
|
||||||
|
'from index. Error: %(exc)s') %
|
||||||
|
{'id': id, 'exc': exc})
|
||||||
|
|
||||||
|
|
||||||
|
class SwiftContainerHandler(base.NotificationBase):
|
||||||
|
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
self.object_helper = kwargs.pop('object_helper')
|
||||||
|
super(SwiftContainerHandler, self).__init__(*args, **kwargs)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _get_notification_exchanges(cls):
|
||||||
|
return ['swift']
|
||||||
|
|
||||||
|
def get_event_handlers(self):
|
||||||
|
return {
|
||||||
|
"container.create": self.create_or_update,
|
||||||
|
"container.metadata": self.create_or_update,
|
||||||
|
"container.delete": self.delete
|
||||||
|
}
|
||||||
|
|
||||||
|
def create_or_update(self, payload, timestamp):
|
||||||
|
payload = serialize_swift_container_notification(payload)
|
||||||
|
try:
|
||||||
|
self.index_helper.save_document(
|
||||||
|
payload,
|
||||||
|
version=self.get_version(payload, timestamp))
|
||||||
|
except Exception as exc:
|
||||||
|
LOG.error(_LE('Error saving container %(id)s '
|
||||||
|
'in index. Error: %(exc)s') %
|
||||||
|
{'id': payload['id'], 'exc': exc})
|
||||||
|
|
||||||
|
def delete(self, payload, timestamp):
|
||||||
|
# notification payload doesn't have any date/time fields
|
||||||
|
# so temporarily use metadata timestamp value as
|
||||||
|
# updated_at field to retrieve version
|
||||||
|
# Remove it when notifcation starts sending datetime field(s)
|
||||||
|
payload['updated_at'] = timestamp
|
||||||
|
version = self.get_version(payload, timestamp)
|
||||||
|
del payload['updated_at']
|
||||||
|
|
||||||
|
id = payload['account'] + swift.ID_SEP + payload['container']
|
||||||
|
try:
|
||||||
|
self.object_helper.delete_documents_with_parent(
|
||||||
|
id, version=version)
|
||||||
|
except Exception as exc:
|
||||||
|
LOG.error(_LE('Error deleting objects in container %(id)s '
|
||||||
|
'from index. Error: %(exc)s') %
|
||||||
|
{'id': id, 'exc': exc})
|
||||||
|
try:
|
||||||
|
self.index_helper.delete_document(
|
||||||
|
{'_id': id, '_version': version,
|
||||||
|
'_routing': payload['account']})
|
||||||
|
except Exception as exc:
|
||||||
|
LOG.error(_LE('Error deleting container %(id)s '
|
||||||
|
'from index. Error: %(exc)s') %
|
||||||
|
{'id': id, 'exc': exc})
|
||||||
|
|
||||||
|
|
||||||
|
class SwiftObjectHandler(base.NotificationBase):
|
||||||
|
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
super(SwiftObjectHandler, self).__init__(*args, **kwargs)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _get_notification_exchanges(cls):
|
||||||
|
return ['swift']
|
||||||
|
|
||||||
|
def get_event_handlers(self):
|
||||||
|
return {
|
||||||
|
"object.create": self.create_or_update,
|
||||||
|
"object.metadata": self.create_or_update,
|
||||||
|
"object.delete": self.delete
|
||||||
|
}
|
||||||
|
|
||||||
|
def create_or_update(self, payload, timestamp):
|
||||||
|
payload = serialize_swift_object_notification(payload)
|
||||||
|
try:
|
||||||
|
self.index_helper.save_document(
|
||||||
|
payload,
|
||||||
|
version=self.get_version(payload, timestamp)
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
LOG.error(_LE('Error saving object %(id)s '
|
||||||
|
'in index. Error: %(exc)s') %
|
||||||
|
{'id': payload['id'], 'exc': exc})
|
||||||
|
|
||||||
|
def delete(self, payload, timestamp):
|
||||||
|
# notification payload doesn't have any date/time fields
|
||||||
|
# so temporarily use metadata timestamp value as
|
||||||
|
# updated_at field to retrieve version
|
||||||
|
# Remove it when notifcation starts sending datetime field(s)
|
||||||
|
payload['updated_at'] = timestamp
|
||||||
|
version = self.get_version(payload, timestamp)
|
||||||
|
del payload['updated_at']
|
||||||
|
|
||||||
|
id = payload['account'] + swift.ID_SEP + \
|
||||||
|
payload['container'] + swift.ID_SEP + \
|
||||||
|
payload['object']
|
||||||
|
try:
|
||||||
|
self.index_helper.delete_document(
|
||||||
|
{'_id': id, '_version': version,
|
||||||
|
'_routing': payload['account']})
|
||||||
|
except Exception as exc:
|
||||||
|
LOG.error(_LE('Error deleting object %(id)s '
|
||||||
|
'from index. Error: %(exc)s') %
|
||||||
|
{'id': id, 'exc': exc})
|
|
@ -375,13 +375,18 @@ class FunctionalTest(test_utils.BaseTestCase):
|
||||||
'glance': {'images': 'ImageIndex', 'metadefs': 'MetadefIndex'},
|
'glance': {'images': 'ImageIndex', 'metadefs': 'MetadefIndex'},
|
||||||
'nova': {'servers': 'ServerIndex'},
|
'nova': {'servers': 'ServerIndex'},
|
||||||
'neutron': {'networks': 'NetworkIndex', 'ports': 'PortIndex'},
|
'neutron': {'networks': 'NetworkIndex', 'ports': 'PortIndex'},
|
||||||
'cinder': {'volumes': 'VolumeIndex', 'snapshots': 'SnapshotIndex'}
|
'cinder': {'volumes': 'VolumeIndex', 'snapshots': 'SnapshotIndex'},
|
||||||
|
'swift': {'accounts': 'AccountIndex',
|
||||||
|
'containers': 'ContainerIndex',
|
||||||
|
'objects': 'ObjectIndex'}
|
||||||
}
|
}
|
||||||
plugins = include_plugins or (
|
plugins = include_plugins or (
|
||||||
('glance', 'images'), ('glance', 'metadefs'),
|
('glance', 'images'), ('glance', 'metadefs'),
|
||||||
('nova', 'servers'),
|
('nova', 'servers'),
|
||||||
('neutron', 'networks'), ('neutron', 'ports'),
|
('neutron', 'networks'), ('neutron', 'ports'),
|
||||||
('cinder', 'volumes'), ('cinder', 'snapshots')
|
('cinder', 'volumes'), ('cinder', 'snapshots'),
|
||||||
|
('swift', 'accounts'), ('swift', 'containers'),
|
||||||
|
('swift', 'objects')
|
||||||
)
|
)
|
||||||
plugins = filter(lambda plugin: plugin not in exclude_plugins, plugins)
|
plugins = filter(lambda plugin: plugin not in exclude_plugins, plugins)
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,162 @@
|
||||||
|
# Copyright (c) 2016 Hewlett-Packard Enterprise Development Company, L.P.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
# implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
import datetime
|
||||||
|
import six
|
||||||
|
import time
|
||||||
|
|
||||||
|
|
||||||
|
from oslo_utils import timeutils
|
||||||
|
|
||||||
|
import searchlight.elasticsearch.plugins.swift as swift_plugin
|
||||||
|
from searchlight.elasticsearch.plugins.swift import\
|
||||||
|
accounts as accounts_plugin
|
||||||
|
import searchlight.tests.unit.utils as unit_test_utils
|
||||||
|
import searchlight.tests.utils as test_utils
|
||||||
|
|
||||||
|
|
||||||
|
now_epoch_time = time.time()
|
||||||
|
now_utc = datetime.datetime.fromtimestamp(now_epoch_time)\
|
||||||
|
.strftime('%Y-%m-%dT%H:%M:%SZ')
|
||||||
|
|
||||||
|
|
||||||
|
USER1 = u'27f4d76b-be62-4e4e-aa33bb11cc55'
|
||||||
|
ID1 = "AUTH_488ac936-663e-4e5c-537d-986021b32c4b"
|
||||||
|
ID2 = "AUTH_7554da43-6443-acdf-deac-3425223cdada"
|
||||||
|
ID3 = "AUTH_30754354-ca43-124b-12b5-789234bcdefa'"
|
||||||
|
|
||||||
|
AUTH_PREFIX = "AUTH_"
|
||||||
|
|
||||||
|
X_ACCOUNT_META_KEY1 = 'x-account-meta-key1'
|
||||||
|
X_ACCOUNT_META_VALUE1 = 'x-account-meta-value1'
|
||||||
|
|
||||||
|
X_ACCOUNT_META_KEY2 = 'x-account-meta-key2'
|
||||||
|
X_ACCOUNT_META_VALUE2 = 'x-account-meta-value2'
|
||||||
|
|
||||||
|
TENANT1 = "15b9a454cee34dbe9933ad575a0a6930"
|
||||||
|
DOMAIN_ID = "default"
|
||||||
|
|
||||||
|
DATETIME = datetime.datetime(2016, 2, 20, 1, 13, 24, 215337)
|
||||||
|
DATE1 = timeutils.isotime(DATETIME)
|
||||||
|
|
||||||
|
|
||||||
|
def _account_fixture(account_id, domain_id, name, **kwargs):
|
||||||
|
fixture = {
|
||||||
|
"id": account_id,
|
||||||
|
"name": name,
|
||||||
|
"x-account-project-domain-id": domain_id,
|
||||||
|
'x-timestamp': now_epoch_time
|
||||||
|
}
|
||||||
|
fixture.update(kwargs)
|
||||||
|
return fixture
|
||||||
|
|
||||||
|
|
||||||
|
def _notification_account_fixture(account_id, **kwargs):
|
||||||
|
metadata = kwargs.pop('meta', {})
|
||||||
|
notification = {
|
||||||
|
'account': account_id,
|
||||||
|
'project_name': None,
|
||||||
|
'updated_at': DATE1,
|
||||||
|
'project_domain_name': None,
|
||||||
|
'x-trans-id': None,
|
||||||
|
'project_id': None,
|
||||||
|
'project_domain_id': None
|
||||||
|
}
|
||||||
|
for k, v in six.iteritems(kwargs):
|
||||||
|
if k in notification:
|
||||||
|
notification[k] = v
|
||||||
|
for k, v in six.iteritems(metadata):
|
||||||
|
notification[k] = v
|
||||||
|
return notification
|
||||||
|
|
||||||
|
|
||||||
|
class TestSwiftAccountPlugin(test_utils.BaseTestCase):
|
||||||
|
def setUp(self):
|
||||||
|
super(TestSwiftAccountPlugin, self).setUp()
|
||||||
|
self.plugin = accounts_plugin.AccountIndex()
|
||||||
|
self._create_fixtures()
|
||||||
|
|
||||||
|
def _create_fixtures(self):
|
||||||
|
self.account1 = _account_fixture(
|
||||||
|
account_id=ID1, domain_id=DOMAIN_ID, name="test-account1")
|
||||||
|
self.account2 = _account_fixture(
|
||||||
|
account_id=ID2, domain_id=DOMAIN_ID, name="test-account1",
|
||||||
|
**{X_ACCOUNT_META_KEY1: X_ACCOUNT_META_VALUE1})
|
||||||
|
self.account3 = _account_fixture(
|
||||||
|
account_id=ID3, domain_id=DOMAIN_ID, name="test-account1",
|
||||||
|
**{X_ACCOUNT_META_KEY1: X_ACCOUNT_META_VALUE1,
|
||||||
|
X_ACCOUNT_META_KEY2: X_ACCOUNT_META_VALUE2})
|
||||||
|
self.accounts = [self.account1, self.account2, self.account3]
|
||||||
|
|
||||||
|
def test_index_name(self):
|
||||||
|
self.assertEqual('searchlight', self.plugin.resource_group_name)
|
||||||
|
|
||||||
|
def test_document_type(self):
|
||||||
|
self.assertEqual('OS::Swift::Account',
|
||||||
|
self.plugin.get_document_type())
|
||||||
|
|
||||||
|
def test_rbac_filter(self):
|
||||||
|
fake_request = unit_test_utils.get_fake_request(
|
||||||
|
USER1, TENANT1, '/v1/search', is_admin=False
|
||||||
|
)
|
||||||
|
rbac_terms = self.plugin._get_rbac_field_filters(fake_request.context)
|
||||||
|
self.assertEqual(
|
||||||
|
[{"term": {"id": AUTH_PREFIX + TENANT1}}],
|
||||||
|
rbac_terms
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_admin_only_fields(self):
|
||||||
|
admin_only_fields = self.plugin.admin_only_fields
|
||||||
|
self.assertEqual(['x-account-meta-temp-url-key',
|
||||||
|
'x-account-meta-temp-url-key-2',
|
||||||
|
'x-account-access-control'], admin_only_fields)
|
||||||
|
|
||||||
|
def test_serialize(self):
|
||||||
|
serialized = self.plugin.serialize(self.account1)
|
||||||
|
|
||||||
|
self.assertEqual(ID1, serialized['id'])
|
||||||
|
self.assertEqual('test-account1', serialized['name'])
|
||||||
|
self.assertEqual(DOMAIN_ID, serialized['domain_id'])
|
||||||
|
self.assertEqual(now_utc, serialized['created_at'])
|
||||||
|
|
||||||
|
serialized = self.plugin.serialize(self.account2)
|
||||||
|
self.assertEqual(X_ACCOUNT_META_VALUE1,
|
||||||
|
serialized[X_ACCOUNT_META_KEY1])
|
||||||
|
|
||||||
|
serialized = self.plugin.serialize(self.account3)
|
||||||
|
self.assertEqual(X_ACCOUNT_META_VALUE1,
|
||||||
|
serialized[X_ACCOUNT_META_KEY1])
|
||||||
|
self.assertEqual(X_ACCOUNT_META_VALUE2,
|
||||||
|
serialized[X_ACCOUNT_META_KEY2])
|
||||||
|
|
||||||
|
def test_swift_account_notification_serialize(self):
|
||||||
|
notification = _notification_account_fixture(
|
||||||
|
ID1,
|
||||||
|
project_name='admin',
|
||||||
|
project_id=ID1,
|
||||||
|
project_domain_id='default',
|
||||||
|
updated_at=DATE1,
|
||||||
|
)
|
||||||
|
|
||||||
|
expected = {
|
||||||
|
'id': ID1,
|
||||||
|
'name': 'admin',
|
||||||
|
'updated_at': DATE1,
|
||||||
|
'domain_id': 'default'
|
||||||
|
}
|
||||||
|
|
||||||
|
serialized = swift_plugin.serialize_swift_account_notification(
|
||||||
|
notification)
|
||||||
|
self.assertEqual(expected, serialized)
|
|
@ -0,0 +1,175 @@
|
||||||
|
# Copyright (c) 2016 Hewlett-Packard Enterprise Development Company, L.P.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
# implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
import datetime
|
||||||
|
import six
|
||||||
|
import time
|
||||||
|
|
||||||
|
|
||||||
|
from oslo_utils import timeutils
|
||||||
|
|
||||||
|
import searchlight.elasticsearch.plugins.swift as swift_plugin
|
||||||
|
from searchlight.elasticsearch.plugins.swift import\
|
||||||
|
containers as containers_plugin
|
||||||
|
import searchlight.tests.utils as test_utils
|
||||||
|
|
||||||
|
|
||||||
|
now_epoch_time = time.time()
|
||||||
|
now_utc = datetime.datetime.fromtimestamp(now_epoch_time).\
|
||||||
|
strftime('%Y-%m-%dT%H:%M:%SZ')
|
||||||
|
|
||||||
|
|
||||||
|
USER1 = "27f4d76b-be62-4e4e-aa33bb11cc55"
|
||||||
|
|
||||||
|
ACCOUNT_ID1 = "488ac936-663e-4e5c-537d-986021b32c4b"
|
||||||
|
ACCOUNT_ID2 = "7554da43-6443-acdf-deac-3425223cdada"
|
||||||
|
ACCOUNT_ID3 = "30754354-ca43-124b-12b5-789234bcdefa'"
|
||||||
|
|
||||||
|
AUTH_PREFIX = "AUTH_"
|
||||||
|
|
||||||
|
CONTAINER1 = "Container1"
|
||||||
|
CONTAINER2 = "Container2"
|
||||||
|
CONTAINER_ID1 = ACCOUNT_ID1 + swift_plugin.ID_SEP + CONTAINER1
|
||||||
|
CONTAINER_ID2 = ACCOUNT_ID2 + swift_plugin.ID_SEP + CONTAINER2
|
||||||
|
|
||||||
|
X_CONTAINER_META_KEY1 = 'x-container-meta-key1'
|
||||||
|
X_CONTAINER_META_VALUE1 = 'x-container-meta-value1'
|
||||||
|
|
||||||
|
X_CONTAINER_META_KEY2 = 'x-container-meta-key2'
|
||||||
|
X_CONTAINER_META_VALUE2 = 'x-container-meta-value2'
|
||||||
|
|
||||||
|
TENANT1 = "15b9a454cee34dbe9933ad575a0a6930"
|
||||||
|
TENANT2 = "a7ba963f71bb43818f631febbc9df8e6"
|
||||||
|
DOMAIN_ID = "default"
|
||||||
|
|
||||||
|
DATETIME = datetime.datetime(2016, 2, 21, 4, 41, 33, 325314)
|
||||||
|
DATE1 = timeutils.isotime(DATETIME)
|
||||||
|
|
||||||
|
|
||||||
|
def _container_fixture(container_id, container_name, account,
|
||||||
|
account_id, read_acl, **kwargs):
|
||||||
|
fixture = {
|
||||||
|
"id": container_id,
|
||||||
|
"name": container_name,
|
||||||
|
"account": account,
|
||||||
|
"account_id": account_id,
|
||||||
|
"x-container-read": read_acl,
|
||||||
|
'x-timestamp': now_epoch_time
|
||||||
|
}
|
||||||
|
fixture.update(kwargs)
|
||||||
|
return fixture
|
||||||
|
|
||||||
|
|
||||||
|
def _notification_container_fixture(account_id, **kwargs):
|
||||||
|
metadata = kwargs.pop('meta', {})
|
||||||
|
notification = {
|
||||||
|
'account': account_id,
|
||||||
|
'project_name': None,
|
||||||
|
'container': None,
|
||||||
|
'updated_at': DATE1,
|
||||||
|
'project_domain_name': None,
|
||||||
|
'x-trans-id': None,
|
||||||
|
'project_id': None,
|
||||||
|
'project_domain_id': None
|
||||||
|
}
|
||||||
|
for k, v in six.iteritems(kwargs):
|
||||||
|
if k in notification:
|
||||||
|
notification[k] = v
|
||||||
|
for k, v in six.iteritems(metadata):
|
||||||
|
notification[k] = v
|
||||||
|
return notification
|
||||||
|
|
||||||
|
|
||||||
|
class TestSwiftContainerPlugin(test_utils.BaseTestCase):
|
||||||
|
def setUp(self):
|
||||||
|
super(TestSwiftContainerPlugin, self).setUp()
|
||||||
|
self.plugin = containers_plugin.ContainerIndex()
|
||||||
|
self._create_fixtures()
|
||||||
|
|
||||||
|
def _create_fixtures(self):
|
||||||
|
self.container1 = _container_fixture(
|
||||||
|
container_id=CONTAINER_ID1, container_name=CONTAINER1,
|
||||||
|
account='test-account1', account_id=ACCOUNT_ID1, read_acl=None)
|
||||||
|
self.container2 = _container_fixture(
|
||||||
|
container_id=CONTAINER_ID2, container_name=CONTAINER2,
|
||||||
|
account='test-account2', account_id=ACCOUNT_ID1,
|
||||||
|
read_acl=ACCOUNT_ID2 + ":*",
|
||||||
|
**{X_CONTAINER_META_KEY1: X_CONTAINER_META_VALUE1}
|
||||||
|
)
|
||||||
|
self.container3 = _container_fixture(
|
||||||
|
container_id=CONTAINER_ID1, container_name=CONTAINER1,
|
||||||
|
account='test-account3', account_id=ACCOUNT_ID2,
|
||||||
|
read_acl=ACCOUNT_ID3 + ":*" + USER1,
|
||||||
|
**{X_CONTAINER_META_KEY1: X_CONTAINER_META_VALUE1,
|
||||||
|
X_CONTAINER_META_KEY2: X_CONTAINER_META_VALUE2})
|
||||||
|
self.containers = [self.container1, self.container2, self.container3]
|
||||||
|
|
||||||
|
def test_index_name(self):
|
||||||
|
self.assertEqual('searchlight', self.plugin.resource_group_name)
|
||||||
|
|
||||||
|
def test_document_type(self):
|
||||||
|
self.assertEqual('OS::Swift::Container',
|
||||||
|
self.plugin.get_document_type())
|
||||||
|
|
||||||
|
def test_admin_only_fields(self):
|
||||||
|
admin_only_fields = self.plugin.admin_only_fields
|
||||||
|
self.assertEqual(['x-container-write',
|
||||||
|
'x-container-sync-key',
|
||||||
|
'x-container-sync-to',
|
||||||
|
'x-container-meta-temp-url-key',
|
||||||
|
'x-container-meta-temp-url-key-2'],
|
||||||
|
admin_only_fields)
|
||||||
|
|
||||||
|
def test_serialize(self):
|
||||||
|
serialized = self.plugin.serialize(self.container1)
|
||||||
|
|
||||||
|
self.assertEqual(CONTAINER_ID1, serialized['id'])
|
||||||
|
self.assertEqual(CONTAINER1, serialized['name'])
|
||||||
|
self.assertEqual(ACCOUNT_ID1, serialized['account_id'])
|
||||||
|
self.assertEqual('test-account1', serialized['account'])
|
||||||
|
|
||||||
|
serialized = self.plugin.serialize(self.container2)
|
||||||
|
self.assertEqual(X_CONTAINER_META_VALUE1,
|
||||||
|
serialized[X_CONTAINER_META_KEY1])
|
||||||
|
self.assertEqual(ACCOUNT_ID2 + ":*", serialized['x-container-read'])
|
||||||
|
|
||||||
|
serialized = self.plugin.serialize(self.container3)
|
||||||
|
self.assertEqual(X_CONTAINER_META_VALUE1,
|
||||||
|
serialized[X_CONTAINER_META_KEY1])
|
||||||
|
self.assertEqual(X_CONTAINER_META_VALUE2,
|
||||||
|
serialized[X_CONTAINER_META_KEY2])
|
||||||
|
|
||||||
|
def test_swift_account_notification_serialize(self):
|
||||||
|
notification = _notification_container_fixture(
|
||||||
|
swift_plugin.AUTH_PREFIX + ACCOUNT_ID1,
|
||||||
|
container=CONTAINER1,
|
||||||
|
project_name='admin',
|
||||||
|
project_domain_id='default',
|
||||||
|
project_id=ACCOUNT_ID1,
|
||||||
|
updated_at=DATE1,
|
||||||
|
)
|
||||||
|
|
||||||
|
expected = {
|
||||||
|
'id': CONTAINER_ID1,
|
||||||
|
'name': CONTAINER1,
|
||||||
|
'account': 'admin',
|
||||||
|
'account_id': ACCOUNT_ID1,
|
||||||
|
'updated_at': DATE1,
|
||||||
|
'x-container-read': None
|
||||||
|
}
|
||||||
|
|
||||||
|
serialized = swift_plugin.serialize_swift_container_notification(
|
||||||
|
notification)
|
||||||
|
self.assertEqual(expected, serialized)
|
|
@ -0,0 +1,120 @@
|
||||||
|
# Copyright (c) 2016 Hewlett-Packard Enterprise Development Company, L.P.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
# implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
import datetime
|
||||||
|
import time
|
||||||
|
|
||||||
|
import searchlight.elasticsearch.plugins.swift as swift_plugin
|
||||||
|
from searchlight.elasticsearch.plugins.swift import\
|
||||||
|
objects as objects_plugin
|
||||||
|
import searchlight.tests.utils as test_utils
|
||||||
|
|
||||||
|
|
||||||
|
now_epoch_time = time.time()
|
||||||
|
created_time = now_epoch_time - 3
|
||||||
|
updated_time = now_epoch_time
|
||||||
|
updated_time_in_fmt = datetime.datetime.fromtimestamp(updated_time).\
|
||||||
|
strftime('%a, %d %b %Y %H:%M:%S GMT')
|
||||||
|
updated_time_out_fmt = datetime.datetime.fromtimestamp(updated_time).\
|
||||||
|
strftime('%Y-%m-%dT%H:%M:%SZ')
|
||||||
|
created_time_utc = datetime.datetime.fromtimestamp(created_time).\
|
||||||
|
strftime('%Y-%m-%dT%H:%M:%SZ')
|
||||||
|
|
||||||
|
USER1 = "27f4d76b-be62-4e4e-aa33bb11cc55"
|
||||||
|
|
||||||
|
ACCOUNT_ID1 = "488ac936-663e-4e5c-537d-986021b32c4b"
|
||||||
|
CONTAINER1 = "Container1"
|
||||||
|
CONTAINER_ID1 = ACCOUNT_ID1 + swift_plugin.ID_SEP + CONTAINER1
|
||||||
|
OBJECT1 = "Object1"
|
||||||
|
OBJECT2 = "Object2"
|
||||||
|
OBJECT_ID1 = CONTAINER_ID1 + swift_plugin.ID_SEP + OBJECT1
|
||||||
|
OBJECT_ID2 = CONTAINER_ID1 + swift_plugin.ID_SEP + OBJECT2
|
||||||
|
|
||||||
|
X_OBJECT_META_KEY1 = 'x-object-meta-key1'
|
||||||
|
X_OBJECT_META_VALUE1 = 'x-object-meta-value1'
|
||||||
|
|
||||||
|
X_OBJECT_META_KEY2 = 'x-object-meta-key2'
|
||||||
|
X_OBJECT_META_VALUE2 = 'x-object-meta-value2'
|
||||||
|
|
||||||
|
TENANT1 = "15b9a454cee34dbe9933ad575a0a6930"
|
||||||
|
|
||||||
|
|
||||||
|
def _object_fixture(object_id, object_name, container_id, container_name,
|
||||||
|
account, account_id, **kwargs):
|
||||||
|
fixture = {
|
||||||
|
"id": object_id,
|
||||||
|
"name": object_name,
|
||||||
|
"account": account,
|
||||||
|
"account_id": account_id,
|
||||||
|
"container": container_name,
|
||||||
|
"container_id": container_id,
|
||||||
|
'x-timestamp': created_time,
|
||||||
|
'last-modified': updated_time_in_fmt
|
||||||
|
}
|
||||||
|
fixture.update(kwargs)
|
||||||
|
return fixture
|
||||||
|
|
||||||
|
|
||||||
|
class TestSwiftObjectPlugin(test_utils.BaseTestCase):
|
||||||
|
def setUp(self):
|
||||||
|
super(TestSwiftObjectPlugin, self).setUp()
|
||||||
|
self.plugin = objects_plugin.ObjectIndex()
|
||||||
|
self._create_fixtures()
|
||||||
|
|
||||||
|
def _create_fixtures(self):
|
||||||
|
self.object1 = _object_fixture(
|
||||||
|
object_id=OBJECT_ID1, object_name=OBJECT1,
|
||||||
|
container_id=CONTAINER_ID1, container_name=CONTAINER1,
|
||||||
|
account='test-account1', account_id=ACCOUNT_ID1,
|
||||||
|
**{"content-type": "text/plain", "content-length": 1050})
|
||||||
|
self.object2 = _object_fixture(
|
||||||
|
object_id=OBJECT_ID2, object_name=OBJECT2,
|
||||||
|
container_id=CONTAINER_ID1, container_name=CONTAINER1,
|
||||||
|
account='test-account1', account_id=ACCOUNT_ID1,
|
||||||
|
**{"content-type": "text/plain", "content-length": 1050,
|
||||||
|
X_OBJECT_META_KEY1: X_OBJECT_META_VALUE1,
|
||||||
|
X_OBJECT_META_KEY2: X_OBJECT_META_VALUE2})
|
||||||
|
self.objects = [self.object1, self.object2]
|
||||||
|
|
||||||
|
def test_index_name(self):
|
||||||
|
self.assertEqual('searchlight', self.plugin.resource_group_name)
|
||||||
|
|
||||||
|
def test_document_type(self):
|
||||||
|
self.assertEqual('OS::Swift::Object',
|
||||||
|
self.plugin.get_document_type())
|
||||||
|
|
||||||
|
def test_admin_only_fields(self):
|
||||||
|
admin_only_fields = self.plugin.admin_only_fields
|
||||||
|
self.assertEqual([], admin_only_fields)
|
||||||
|
|
||||||
|
def test_serialize(self):
|
||||||
|
serialized = self.plugin.serialize(self.object1)
|
||||||
|
|
||||||
|
self.assertEqual(OBJECT_ID1, serialized['id'])
|
||||||
|
self.assertEqual(OBJECT1, serialized['name'])
|
||||||
|
self.assertEqual(ACCOUNT_ID1, serialized['account_id'])
|
||||||
|
self.assertEqual('test-account1', serialized['account'])
|
||||||
|
self.assertEqual(CONTAINER_ID1, serialized['container_id'])
|
||||||
|
self.assertEqual(CONTAINER1, serialized['container'])
|
||||||
|
self.assertEqual(created_time_utc, serialized['created_at']),
|
||||||
|
self.assertEqual(updated_time_out_fmt, serialized['updated_at'])
|
||||||
|
self.assertEqual("text/plain", serialized['content_type'])
|
||||||
|
self.assertEqual(1050, serialized['content_length'])
|
||||||
|
|
||||||
|
serialized = self.plugin.serialize(self.object2)
|
||||||
|
self.assertEqual(X_OBJECT_META_VALUE1,
|
||||||
|
serialized[X_OBJECT_META_KEY1])
|
||||||
|
self.assertEqual(X_OBJECT_META_VALUE2,
|
||||||
|
serialized[X_OBJECT_META_KEY2])
|
|
@ -283,7 +283,7 @@ class TestSearchDeserializer(test_utils.BaseTestCase):
|
||||||
'OS::Neutron::Net',
|
'OS::Neutron::Net',
|
||||||
'OS::Neutron::Port',
|
'OS::Neutron::Port',
|
||||||
'OS::Cinder::Volume',
|
'OS::Cinder::Volume',
|
||||||
'OS::Cinder::Snapshot',
|
'OS::Cinder::Snapshot'
|
||||||
]
|
]
|
||||||
|
|
||||||
self.assertEqual(['searchlight-search'], output['index'])
|
self.assertEqual(['searchlight-search'], output['index'])
|
||||||
|
@ -602,7 +602,7 @@ class TestSearchDeserializer(test_utils.BaseTestCase):
|
||||||
self.assertEqual(expected_query, output['query'])
|
self.assertEqual(expected_query, output['query'])
|
||||||
|
|
||||||
def test_rbac_admin(self):
|
def test_rbac_admin(self):
|
||||||
"""Test that admins have RBAC applied unless 'all_projects' is true"""
|
"""Test that admins have RBAC applied"""
|
||||||
request = unit_test_utils.get_fake_request(is_admin=True)
|
request = unit_test_utils.get_fake_request(is_admin=True)
|
||||||
request.body = six.b(jsonutils.dumps({
|
request.body = six.b(jsonutils.dumps({
|
||||||
'query': {'match_all': {}},
|
'query': {'match_all': {}},
|
||||||
|
|
|
@ -37,6 +37,9 @@ searchlight.index_backend =
|
||||||
os_designate_zone = searchlight.elasticsearch.plugins.designate.zones:ZoneIndex
|
os_designate_zone = searchlight.elasticsearch.plugins.designate.zones:ZoneIndex
|
||||||
os_cinder_volume = searchlight.elasticsearch.plugins.cinder.volumes:VolumeIndex
|
os_cinder_volume = searchlight.elasticsearch.plugins.cinder.volumes:VolumeIndex
|
||||||
os_cinder_snapshot = searchlight.elasticsearch.plugins.cinder.snapshots:SnapshotIndex
|
os_cinder_snapshot = searchlight.elasticsearch.plugins.cinder.snapshots:SnapshotIndex
|
||||||
|
os_swift_account = searchlight.elasticsearch.plugins.swift.accounts:AccountIndex
|
||||||
|
os_swift_container = searchlight.elasticsearch.plugins.swift.containers:ContainerIndex
|
||||||
|
os_swift_object = searchlight.elasticsearch.plugins.swift.objects:ObjectIndex
|
||||||
|
|
||||||
[build_sphinx]
|
[build_sphinx]
|
||||||
all_files = 1
|
all_files = 1
|
||||||
|
|
Loading…
Reference in New Issue