Swift plugin
Resource types addded for OS::Swift::Account OS::Swift::Container OS::Swift::Object To Test: cd /opt/stack/searchlight git review -d 271622 sudo python setup.py install openstack role add --user searchlight --project service ResellerAdmin searchlight-manage index sync --type OS::Swift::Account restart searchlight-api, restart searchlight-listener This doesn't enable notification updates (strictly POC level). For that, please follow the instructions here: https://review.openstack.org/#/c/249471/ Implements-blueprint: swift-plugin Change-Id: Ib9838438e84d957b526fe80371aae62466c07f3e
This commit is contained in:
Lakshmi N Sampath
parent
e4b64c00d7
commit
d445624e3c
|
|
@ -126,6 +126,25 @@ SWIFT_HASH=096d08da4f8d4cce3a724c5f6c18f055
|
|||
SWIFT_REPLICAS=1
|
||||
SWIFT_DATA_DIR=$DEST/data/swift
|
||||
|
||||
## SWIFT NOTIFICATIONS ###
|
||||
#Notifications must be configured properly for searchlight to process
|
||||
#incremental updates. Use the following::
|
||||
|
||||
#Add the following new section in swift proxy conf file
|
||||
#[filter:oslomiddleware]
|
||||
#paste.filter_factory = swift.common.middleware.oslo_notifications:filter_factory
|
||||
#publisher_id = swift.localhost
|
||||
#Replace <user>,<password>,<rabbitip> and <rabbitport> for your environment values
|
||||
#transport_url = rabbit://<user>:<password>@<rabbitip>:<rabbitport>/
|
||||
#notification_driver = messaging
|
||||
#notification_topics = searchlight_indexer
|
||||
|
||||
#Add oslomiddleware to pipeline:main see example below.
|
||||
#[pipeline:main]
|
||||
#pipeline = catch_errors gatekeeper healthcheck ... oslomiddleware proxy-logging proxy-server
|
||||
#Restart swift proxy API service (s-proxy) after making changes.
|
||||
|
||||
|
||||
### DESIGNATE ###
|
||||
#
|
||||
enable_plugin designate https://git.openstack.org/openstack/designate
|
||||
|
|
@ -208,3 +227,14 @@ admin_only_fields=admin_state_up,status
|
|||
|
||||
[resource_plugin:os_neutron_port]
|
||||
enabled = True
|
||||
|
||||
[resource_plugin:os_swift_account]
|
||||
enabled = false
|
||||
#Specify same value as in swift proxy config for reseller_prefix
|
||||
reseller_prefix = AUTH_
|
||||
|
||||
[resource_plugin:os_swift_container]
|
||||
enabled = false
|
||||
|
||||
[resource_plugin:os_swift_object]
|
||||
enabled = false
|
||||
|
|
|
|||
|
|
@ -132,6 +132,11 @@ function configure_searchlight {
|
|||
# Plugin config - disable designate by default since it's not typically installed
|
||||
iniset $SEARCHLIGHT_CONF resource_plugin:os_designate_zone enabled False
|
||||
iniset $SEARCHLIGHT_CONF resource_plugin:os_designate_recordset enabled False
|
||||
|
||||
# Plugin config - disable swift by default since it's not typically installed
|
||||
iniset $SEARCHLIGHT_CONF resource_plugin:os_swift_account enabled False
|
||||
iniset $SEARCHLIGHT_CONF resource_plugin:os_swift_container enabled False
|
||||
iniset $SEARCHLIGHT_CONF resource_plugin:os_swift_object enabled False
|
||||
}
|
||||
|
||||
# create_searchlight_accounts - Set up common required searchlight accounts
|
||||
|
|
|
|||
|
|
@ -101,6 +101,14 @@ Please read the rest of the guide for detailed information.::
|
|||
[resource_plugin:os_designate_recordset]
|
||||
enabled = False
|
||||
|
||||
[resource_plugin:os_swift_account]
|
||||
enabled = False
|
||||
|
||||
[resource_plugin:os_swift_container]
|
||||
enabled = False
|
||||
|
||||
[resource_plugin:os_swift_object]
|
||||
enabled = False
|
||||
|
||||
Common Plugin Configuration Options
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
|
|
|||
|
|
@ -0,0 +1,181 @@
|
|||
..
|
||||
(c) Copyright 2016 Hewlett-Packard Development Company, L.P.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
not use this file except in compliance with the License. You may obtain
|
||||
a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
License for the specific language governing permissions and limitations
|
||||
under the License.
|
||||
|
||||
******************
|
||||
Swift Plugin Guide
|
||||
******************
|
||||
|
||||
WARNING: Swift plugin is currently EXPERIMENTAL as notifications aren't
|
||||
fully supported. See below on enabling notifications.
|
||||
|
||||
Integration is provided via a plugin. There are multiple configuration
|
||||
settings required for proper indexing and incremental updates. Some of the
|
||||
settings are specified in Searchlight configuration files. Others are
|
||||
provided in other service configuration files.
|
||||
|
||||
Searchlight Configuration
|
||||
=========================
|
||||
|
||||
Searchlight resource configuration options are shown below with their
|
||||
configuration file and default values.
|
||||
|
||||
See :ref:`searchlight-plugins` for common options with their default values,
|
||||
general configuration information, and an example complete configuration.
|
||||
|
||||
.. note::
|
||||
|
||||
Unless you are changing to a non-default value, you do not need to
|
||||
specify any of the following configuration options.
|
||||
|
||||
searchlight.conf
|
||||
----------------
|
||||
|
||||
Plugin: OS::Swift::Account
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
::
|
||||
|
||||
[resource_plugin:os_swift_account]
|
||||
enabled = true
|
||||
index_name = searchlight
|
||||
#Specify same value as in swift proxy config for reseller_prefix
|
||||
reseller_prefix = AUTH_
|
||||
|
||||
.. note::
|
||||
|
||||
os_swift_account is disabled by default. You need to explicitly set enabled = True as shown above
|
||||
|
||||
Plugin: OS::Swift::Container
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
::
|
||||
|
||||
[resource_plugin:os_swift_container]
|
||||
enabled = true
|
||||
index_name = searchlight
|
||||
|
||||
.. note::
|
||||
|
||||
os_swift_container is disabled by default. You need to explicitly set enabled = True as shown above
|
||||
|
||||
Plugin: OS::Swift::Object
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
::
|
||||
|
||||
[resource_plugin:os_swift_object]
|
||||
enabled = true
|
||||
index_name = searchlight
|
||||
|
||||
.. note::
|
||||
|
||||
os_swift_object is disabled by default. You need to explicitly set enabled = True as shown above
|
||||
|
||||
Swift Configuration
|
||||
====================
|
||||
|
||||
The Swift service currently doesn't send notifications.
|
||||
Apply this patch https://review.openstack.org/#/c/249471
|
||||
for adding notification middleware to swift.
|
||||
|
||||
reseller_admin_role
|
||||
-------------------
|
||||
|
||||
Users with the Keystone role defined in reseller_admin_role (ResellerAdmin by default)
|
||||
can operate on any account. The auth system sets the request environ reseller_request
|
||||
to True if a request is coming from a user with this role.
|
||||
|
||||
Searchlight needs this role for its service user to access all the swift accounts
|
||||
for initial indexing. The searchlight user and sevice project being referred here is the
|
||||
one defined in service_credentials section of searchlight conf file.
|
||||
|
||||
::
|
||||
|
||||
openstack role add --user searchlight --project service ResellerAdmin
|
||||
|
||||
|
||||
proxy-server.conf
|
||||
-----------------
|
||||
|
||||
Notifications must be configured properly for searchlight to process
|
||||
incremental updates. Use the following::
|
||||
|
||||
#Add the following new section
|
||||
[filter:oslomiddleware]
|
||||
paste.filter_factory = swift.common.middleware.oslo_notifications:filter_factory
|
||||
publisher_id = swift.localhost
|
||||
#Replace <user>,<password>,<rabbitip> and <rabbitport> for your environment values
|
||||
transport_url = rabbit://<user>:<password>@<rabbitip>:<rabbitport>/
|
||||
notification_driver = messaging
|
||||
notification_topics = searchlight_indexer
|
||||
|
||||
#Add oslomiddleware to pipeline:main see example below.
|
||||
[pipeline:main]
|
||||
pipeline = catch_errors gatekeeper healthcheck ... oslomiddleware proxy-logging proxy-server
|
||||
|
||||
|
||||
.. note::
|
||||
|
||||
Restart swift proxy API service (s-proxy) after making changes.
|
||||
|
||||
local.conf (devstack)
|
||||
---------------------
|
||||
|
||||
The settings above may be automatically configured by ``stack.sh``
|
||||
by adding them to the following post config section in devstack.
|
||||
Just place the following in local.conf and copy the above settings
|
||||
underneath it.::
|
||||
|
||||
[[post-config|$SWIFT_PROXY_CONF]]
|
||||
[DEFAULT]
|
||||
|
||||
Release Notes
|
||||
=============
|
||||
|
||||
0.2.0.0 (Mitaka)
|
||||
----------------
|
||||
|
||||
Swift did not generate notifications for account/container/object CRUD
|
||||
|
||||
This means that search results will not include incremental updates after
|
||||
the initial indexing.
|
||||
|
||||
The patch (https://review.openstack.org/#/c/249471) implements this feature.
|
||||
|
||||
For devstack, the easiest way to test is
|
||||
cd /opt/stack/swift
|
||||
git review -x 249471
|
||||
|
||||
Searchlight developers/installers should apply the above patch in Swift when
|
||||
using Searchlight with the Swift Mitaka release.
|
||||
|
||||
Alternatively, you may set up a cron job to re-index swift
|
||||
account/container/objects periodically to get updated information. The
|
||||
recommendation is to use the notifications.
|
||||
|
||||
You should use the ``--no-delete`` option to prevent the index from
|
||||
temporarily not containing any data (which otherwise would happen with a full
|
||||
bulk indexing job)::
|
||||
|
||||
searchlight-manage index sync --type OS::Swift::Account --force --no-delete
|
||||
|
||||
Searchlight swift plugin resource types follow the hierarchy similar to
|
||||
Swift concepts
|
||||
|
||||
OS::Swift:Acccount(Parent)
|
||||
-> OS:Swift::Container(Child)
|
||||
-> OS::Swift::Object(Grand Child)
|
||||
|
||||
which means indexing is initiated by specifying only the top parent
|
||||
(OS::Swift::Account) and that will in-turn index all the child
|
||||
plugins(Container and Object)
|
||||
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
---
|
||||
prelude: >
|
||||
Swift plugin for searchlight
|
||||
features:
|
||||
- Three resource types are introduced for swift
|
||||
plugin.
|
||||
|
||||
OS::Swift:Account
|
||||
->OS::Swift::Container
|
||||
-->OS::Swift::Object
|
||||
issues:
|
||||
- The Swift service currently doesn't send notifications.
|
||||
Follow the swift plugin documentation for current
|
||||
solutions.
|
||||
|
|
@ -48,3 +48,4 @@ python-glanceclient>=2.0.0 # Apache-2.0
|
|||
python-novaclient!=2.33.0,>=2.29.0 # Apache-2.0
|
||||
python-neutronclient!=4.1.0,>=2.6.0 # Apache-2.0
|
||||
python-cinderclient>=1.3.1 # Apache-2.0
|
||||
python-swiftclient>=2.2.0 # Apache-2.0
|
||||
|
|
|
|||
|
|
@ -474,6 +474,16 @@ class IndexBase(plugin.Plugin):
|
|||
def requires_role_separation(self):
|
||||
return len(self.admin_only_fields) > 0
|
||||
|
||||
@classmethod
|
||||
def is_plugin_enabled_by_default(cls):
|
||||
'''
|
||||
Each plugin can overwrite the default value of whether a
|
||||
plugin should be enabled if the value is not explicitly
|
||||
set in the configuration
|
||||
|
||||
'''
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def get_plugin_opts(cls):
|
||||
"""Options that can be overridden per plugin. Note that
|
||||
|
|
@ -482,7 +492,7 @@ class IndexBase(plugin.Plugin):
|
|||
sets of types (https://bugs.launchpad.net/searchlight/+bug/1558240)
|
||||
"""
|
||||
opts = [
|
||||
cfg.BoolOpt("enabled", default=True),
|
||||
cfg.BoolOpt("enabled", default=cls.is_plugin_enabled_by_default()),
|
||||
cfg.StrOpt("admin_only_fields"),
|
||||
cfg.BoolOpt('mapping_use_doc_values')
|
||||
]
|
||||
|
|
|
|||
|
|
@ -19,8 +19,10 @@ from designateclient.v2 import client as designateclient
|
|||
from glanceclient.v2 import client as glance
|
||||
from keystoneclient import auth as ks_auth
|
||||
from keystoneclient import session as ks_session
|
||||
from keystoneclient.v2_0 import client as ks_client
|
||||
import neutronclient.v2_0.client
|
||||
import novaclient.client
|
||||
import swiftclient
|
||||
|
||||
from oslo_config import cfg
|
||||
|
||||
|
|
@ -100,3 +102,84 @@ def get_cinderclient():
|
|||
session=session,
|
||||
region_name=cfg.CONF.service_credentials.os_region_name,
|
||||
)
|
||||
|
||||
# Swift still needs special handling because it doesn't support
|
||||
# keystone sessions. Rather than maintain two codepaths, we'll do this
|
||||
_swiftclient = None
|
||||
|
||||
|
||||
def clear_cached_swiftclient_on_unauthorized(fn):
|
||||
def wrapper(*args, **kwargs):
|
||||
global _session
|
||||
global _swiftclient
|
||||
try:
|
||||
return fn(*args, **kwargs)
|
||||
except swiftclient.exceptions.ClientException:
|
||||
_session = None
|
||||
_swiftclient = None
|
||||
return fn(*args, **kwargs)
|
||||
return wrapper
|
||||
|
||||
|
||||
def get_swiftclient():
|
||||
|
||||
global _swiftclient
|
||||
if _swiftclient:
|
||||
return _swiftclient
|
||||
_get_session()
|
||||
|
||||
service_type = 'object-store'
|
||||
|
||||
os_options = {
|
||||
'service_type': service_type,
|
||||
'region_name': cfg.CONF.service_credentials.os_region_name,
|
||||
'endpoint_type': cfg.CONF.service_credentials.os_endpoint_type,
|
||||
}
|
||||
|
||||
# When swiftclient supports session, use session instead of
|
||||
# preauthtoken param below
|
||||
_swiftclient = swiftclient.client.Connection(
|
||||
auth_version='2',
|
||||
user=cfg.CONF.service_credentials.username,
|
||||
key=cfg.CONF.service_credentials.password,
|
||||
authurl=cfg.CONF.service_credentials.auth_url,
|
||||
tenant_name=cfg.CONF.service_credentials.tenant_name,
|
||||
os_options=os_options,
|
||||
cacert=cfg.CONF.service_credentials.cafile,
|
||||
insecure=cfg.CONF.service_credentials.insecure
|
||||
)
|
||||
|
||||
return _swiftclient
|
||||
|
||||
|
||||
# TODO(lakshmiS) See if we can cache this.
|
||||
# Cached members will be equal to # of accounts at max.
|
||||
def get_swiftclient_st(storageurl):
|
||||
service_type = 'object-store'
|
||||
_get_session()
|
||||
|
||||
os_options = {
|
||||
'service_type': service_type,
|
||||
'region_name': cfg.CONF.service_credentials.os_region_name,
|
||||
'endpoint_type': cfg.CONF.service_credentials.os_endpoint_type,
|
||||
}
|
||||
swift_client = swiftclient.client.Connection(
|
||||
auth_version='2',
|
||||
user=cfg.CONF.service_credentials.username,
|
||||
key=cfg.CONF.service_credentials.password,
|
||||
authurl=cfg.CONF.service_credentials.auth_url,
|
||||
tenant_name=cfg.CONF.service_credentials.tenant_name,
|
||||
os_options=os_options,
|
||||
cacert=cfg.CONF.service_credentials.cafile,
|
||||
insecure=cfg.CONF.service_credentials.insecure,
|
||||
preauthurl=storageurl,
|
||||
)
|
||||
return swift_client
|
||||
|
||||
|
||||
def get_keystoneclient():
|
||||
session = _get_session()
|
||||
|
||||
return ks_client.Client(
|
||||
session=session,
|
||||
region_name=cfg.CONF.service_credentials.os_region_name)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,212 @@
|
|||
# Copyright (c) 2016 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import datetime
|
||||
import logging
|
||||
import six
|
||||
|
||||
from oslo_utils import timeutils
|
||||
|
||||
from searchlight.elasticsearch.plugins import openstack_clients
|
||||
from searchlight import i18n
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
_LE = i18n._LE
|
||||
_ = i18n._
|
||||
|
||||
AUTH_PREFIX = ""
|
||||
ID_SEP = "/"
|
||||
|
||||
# _accounts is populated when get_swift_accounts() is called; which is
|
||||
# guaranteed to be called before get_swift_containers and get_swfit_objects
|
||||
# since account plugin is parent and grandparent of container and object
|
||||
_accounts = []
|
||||
|
||||
dateformat = "%a, %d %b %Y %H:%M:%S %Z"
|
||||
|
||||
|
||||
def serialize_swift_account(account):
|
||||
metadocument = {k: account.get(k, None) for k, v in six.iteritems(account)
|
||||
if k.lower().startswith("x-account-meta")}
|
||||
account_fields = ('id', 'name')
|
||||
document = {f: account.get(f, None) for f in account_fields}
|
||||
|
||||
document['domain_id'] = account.get('x-account-project-domain-id', None)
|
||||
if account.get('x-timestamp'):
|
||||
timestamp = float(account.get('x-timestamp'))
|
||||
document['created_at'] = \
|
||||
timeutils.isotime(datetime.datetime.fromtimestamp(timestamp))
|
||||
|
||||
# lakshmiS: swift get_account() doesn't include update datetime field(?)
|
||||
if account.get('updated_at'):
|
||||
document['updated_at'] = account.get('updated_at')
|
||||
|
||||
document.update(metadocument)
|
||||
return document
|
||||
|
||||
|
||||
def serialize_swift_account_notification(account):
|
||||
account['name'] = account['project_name']
|
||||
account['id'] = account['account']
|
||||
account['x-account-project-domain-id'] = account['project_domain_id']
|
||||
# Note: updated_at is included in notification payload.
|
||||
# No need to map or transform the date and its format.
|
||||
return serialize_swift_account(account)
|
||||
|
||||
|
||||
def serialize_swift_container(container):
|
||||
metadocument = {k: container.get(k, None) for k, v in
|
||||
six.iteritems(container)
|
||||
if k.lower().startswith("x-container-meta")}
|
||||
container_fields = ('id',
|
||||
'name',
|
||||
'account',
|
||||
'account_id',
|
||||
'x-container-read'
|
||||
)
|
||||
document = {f: container.get(f, None) for f in container_fields}
|
||||
if container.get('x-timestamp'):
|
||||
timestamp = float(container.get('x-timestamp'))
|
||||
document['created_at'] = \
|
||||
timeutils.isotime(datetime.datetime.fromtimestamp(timestamp))
|
||||
|
||||
# (laskhmiS) get_container doesn't include last_modified field even
|
||||
# though the swift api documentation says it returns it. Include it
|
||||
# when it starts sending it.
|
||||
|
||||
# Notifications for container sends this field as 'updated_at' instead
|
||||
# of 'last_modified'.
|
||||
if container.get('updated_at'):
|
||||
document['updated_at'] = container['updated_at']
|
||||
|
||||
document.update(metadocument)
|
||||
return document
|
||||
|
||||
|
||||
def serialize_swift_container_notification(container):
|
||||
# Account Id + container name. container['account'] from notification has
|
||||
# account id value.
|
||||
container['id'] = container['account'] + ID_SEP + container['container']
|
||||
container['name'] = container['container']
|
||||
container['account_id'] = container['account']
|
||||
container['account'] = container['project_name']
|
||||
return serialize_swift_container(container)
|
||||
|
||||
|
||||
def serialize_swift_object(sobject):
|
||||
metadocument = {k: sobject.get(k, None) for k, v in six.iteritems(sobject)
|
||||
if k.lower().startswith("x-object-meta")}
|
||||
object_fields = ('id',
|
||||
'name',
|
||||
'account',
|
||||
'account_id',
|
||||
'container',
|
||||
'container_id',
|
||||
'etag'
|
||||
)
|
||||
document = {f: sobject.get(f, None) for f in object_fields}
|
||||
document['content_type'] = sobject.get('content-type', None)
|
||||
document['content_length'] = sobject.get('content-length', None)
|
||||
|
||||
if sobject.get('x-timestamp'):
|
||||
timestamp = float(sobject.get('x-timestamp'))
|
||||
document['created_at'] = \
|
||||
timeutils.isotime(datetime.datetime.fromtimestamp(timestamp))
|
||||
if sobject.get('last-modified'):
|
||||
updated_dt = datetime.datetime.strptime(
|
||||
sobject['last-modified'], dateformat)
|
||||
document['updated_at'] = timeutils.isotime(updated_dt)
|
||||
document.update(metadocument)
|
||||
return document
|
||||
|
||||
|
||||
def serialize_swift_object_notification(sobj):
|
||||
# Account id + container name
|
||||
sobj['container_id'] = sobj['account'] + ID_SEP + sobj['container']
|
||||
# Account id + container name + object name
|
||||
sobj['id'] = sobj['container_id'] + ID_SEP + sobj['object']
|
||||
sobj['name'] = sobj['object']
|
||||
sobj['account_id'] = sobj['account']
|
||||
sobj['account'] = sobj['project_name']
|
||||
return serialize_swift_object(sobj)
|
||||
|
||||
|
||||
@openstack_clients.clear_cached_swiftclient_on_unauthorized
|
||||
def _get_storage_url_prefix():
|
||||
# Extracts swift proxy url after removing the default account id
|
||||
# from the service account. Later storage_url's will be constructed
|
||||
# for each account by appending the keystone tenant id.
|
||||
try:
|
||||
storage_url = openstack_clients.get_swiftclient().get_auth()[0]
|
||||
return storage_url[:storage_url.index(AUTH_PREFIX)] + AUTH_PREFIX
|
||||
except ValueError:
|
||||
LOG.error(_LE("reseller_prefix %s not found in keystone endpoint ")
|
||||
% AUTH_PREFIX)
|
||||
raise
|
||||
|
||||
|
||||
def get_swift_accounts(auth_prefix):
|
||||
global AUTH_PREFIX
|
||||
# TODO(lakshmiS): Add support for SERVICE_ accounts
|
||||
AUTH_PREFIX = auth_prefix
|
||||
ks_client = openstack_clients.get_keystoneclient()
|
||||
for tenant in ks_client.tenants.list():
|
||||
storage_url = _get_storage_url_prefix() + tenant.id
|
||||
sclient = openstack_clients.get_swiftclient_st(storage_url)
|
||||
# 0 index has account summary
|
||||
account = sclient.get_account()[0]
|
||||
account['name'] = tenant.name
|
||||
account['id'] = auth_prefix + tenant.id
|
||||
|
||||
# store it for later usage in retrieving containers
|
||||
# and objects
|
||||
account_detail = {'id': account['id'],
|
||||
'name': tenant.name,
|
||||
'storage.url': storage_url}
|
||||
_accounts.append(account_detail)
|
||||
|
||||
yield account
|
||||
|
||||
|
||||
def get_swift_containers():
|
||||
for account in _accounts:
|
||||
sclient = openstack_clients.get_swiftclient_st(account['storage.url'])
|
||||
# 1 index has container list
|
||||
containers = sclient.get_account()[1]
|
||||
for container in containers:
|
||||
ctr, obj = sclient.get_container(container['name'])
|
||||
ctr['id'] = account['id'] + ID_SEP + container['name']
|
||||
ctr['name'] = container['name']
|
||||
ctr['account'] = account['name']
|
||||
ctr['account_id'] = account['id']
|
||||
yield ctr
|
||||
|
||||
|
||||
def get_swift_objects():
|
||||
for account in _accounts:
|
||||
sclient = openstack_clients.get_swiftclient_st(account['storage.url'])
|
||||
# 1 index has container list
|
||||
containers = sclient.get_account()[1]
|
||||
for sctr in containers:
|
||||
ctr, obj = sclient.get_container(sctr['name'])
|
||||
for sobject in obj:
|
||||
sobj = sclient.head_object(sctr['name'], sobject['name'])
|
||||
sobj['account'] = account['name']
|
||||
sobj['account_id'] = account['id']
|
||||
sobj['container'] = sctr['name']
|
||||
sobj['container_id'] = account['id'] + ID_SEP + sctr['name']
|
||||
sobj['id'] = sobj['container_id'] + ID_SEP + sobject['name']
|
||||
sobj['name'] = sobject['name']
|
||||
yield sobj
|
||||
|
|
@ -0,0 +1,107 @@
|
|||
# Copyright (c) 2016 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
from oslo_config import cfg
|
||||
|
||||
from searchlight.elasticsearch.plugins import base
|
||||
from searchlight.elasticsearch.plugins.swift import get_swift_accounts
|
||||
from searchlight.elasticsearch.plugins.swift import serialize_swift_account
|
||||
from searchlight.elasticsearch.plugins.swift import swift_notification_handler
|
||||
|
||||
swift_plugin_opts = [
|
||||
cfg.StrOpt('reseller_prefix', default="AUTH_",
|
||||
help="prefix used in account names for auth system."),
|
||||
]
|
||||
|
||||
CONF = cfg.CONF
|
||||
CONF.register_opts(swift_plugin_opts, group='resource_plugin:os_swift_account')
|
||||
|
||||
|
||||
class AccountIndex(base.IndexBase):
|
||||
NotificationHandlerCls = swift_notification_handler.SwiftAccountHandler
|
||||
|
||||
def __init__(self):
|
||||
super(AccountIndex, self).__init__()
|
||||
self.options = cfg.CONF[self.get_config_group_name()]
|
||||
|
||||
# swift_owner_headers
|
||||
ADMIN_ONLY_FIELDS = ['x-account-meta-temp-url-key',
|
||||
'x-account-meta-temp-url-key-2',
|
||||
'x-account-access-control']
|
||||
|
||||
@classmethod
|
||||
def get_document_type(cls):
|
||||
return 'OS::Swift::Account'
|
||||
|
||||
def get_mapping(self):
|
||||
return {
|
||||
'dynamic': True,
|
||||
'properties': {
|
||||
'id': {'type': 'string', 'index': 'not_analyzed'},
|
||||
'name': {
|
||||
'type': 'string',
|
||||
'fields': {
|
||||
'raw': {'type': 'string', 'index': 'not_analyzed'}
|
||||
},
|
||||
},
|
||||
# TODO(lakshmiS): Removing following field(s) since account
|
||||
# notifications don't include it for subsequent updates.
|
||||
# Enable when it is included in future notifications.
|
||||
|
||||
# The number of objects in the account.
|
||||
# 'x-account-object-count': {'type': 'long'},
|
||||
|
||||
# The total number of bytes that are stored in Object Storage
|
||||
# for the account.
|
||||
# 'x-account-bytes-used': {'type': 'long'},
|
||||
|
||||
# The number of containers.
|
||||
# 'x-account-container-count': {'type': 'long'},
|
||||
|
||||
'domain_id': {'type': 'string', 'index': 'not_analyzed'},
|
||||
'created_at': {'type': 'date'},
|
||||
'updated_at': {'type': 'date'}
|
||||
},
|
||||
}
|
||||
|
||||
@property
|
||||
def admin_only_fields(self):
|
||||
from_conf = super(AccountIndex, self).admin_only_fields
|
||||
return AccountIndex.ADMIN_ONLY_FIELDS + from_conf
|
||||
|
||||
@classmethod
|
||||
def is_plugin_enabled_by_default(cls):
|
||||
return False
|
||||
|
||||
@property
|
||||
def allow_admin_ignore_rbac(self):
|
||||
return False
|
||||
|
||||
def _get_rbac_field_filters(self, request_context):
|
||||
id = self.options.reseller_prefix + request_context.owner
|
||||
return [
|
||||
{"term": {"id": id}}
|
||||
]
|
||||
|
||||
@property
|
||||
def routing_field(self):
|
||||
return "id"
|
||||
|
||||
def get_objects(self):
|
||||
return get_swift_accounts(self.options.reseller_prefix)
|
||||
|
||||
def serialize(self, obj):
|
||||
return serialize_swift_account(obj)
|
||||
|
|
@ -0,0 +1,136 @@
|
|||
# Copyright (c) 2016 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from searchlight.elasticsearch.plugins import base
|
||||
from searchlight.elasticsearch.plugins.swift import get_swift_containers
|
||||
from searchlight.elasticsearch.plugins.swift import serialize_swift_container
|
||||
from searchlight.elasticsearch.plugins.swift import swift_notification_handler
|
||||
|
||||
|
||||
class ContainerIndex(base.IndexBase):
|
||||
NotificationHandlerCls = swift_notification_handler.SwiftContainerHandler
|
||||
|
||||
def get_notification_handler(self):
|
||||
"""Override because the container handler needs a handle to object
|
||||
indexer for cascade delete of container to objects.
|
||||
"""
|
||||
return self.NotificationHandlerCls(
|
||||
self.index_helper,
|
||||
self.options,
|
||||
object_helper=self.child_plugins[0].index_helper)
|
||||
|
||||
# swift_owner_headers
|
||||
ADMIN_ONLY_FIELDS = ['x-container-write',
|
||||
'x-container-sync-key',
|
||||
'x-container-sync-to',
|
||||
'x-container-meta-temp-url-key',
|
||||
'x-container-meta-temp-url-key-2']
|
||||
|
||||
@classmethod
|
||||
def parent_plugin_type(cls):
|
||||
return "OS::Swift::Account"
|
||||
|
||||
@classmethod
|
||||
def get_document_type(cls):
|
||||
return 'OS::Swift::Container'
|
||||
|
||||
def get_mapping(self):
|
||||
return {
|
||||
'dynamic': True,
|
||||
"_source": {
|
||||
"excludes": ["x-container-read"]
|
||||
},
|
||||
'properties': {
|
||||
'id': {'type': 'string', 'index': 'not_analyzed'},
|
||||
'name': {
|
||||
'type': 'string',
|
||||
'fields': {
|
||||
'raw': {'type': 'string', 'index': 'not_analyzed'}
|
||||
},
|
||||
},
|
||||
'account': {
|
||||
'type': 'string',
|
||||
'fields': {
|
||||
'raw': {'type': 'string', 'index': 'not_analyzed'}
|
||||
},
|
||||
},
|
||||
'account_id': {'type': 'string', 'index': 'not_analyzed'},
|
||||
'x-container-read': {
|
||||
'type': 'string', 'index': 'not_analyzed',
|
||||
'store': False
|
||||
},
|
||||
|
||||
# TODO(lakshmiS): Removing following field(s) since account
|
||||
# notifications don't include it for subsequent updates.
|
||||
# Enable when it is included in future notifications.
|
||||
# 'x-container-object-count': {'type': 'long'},
|
||||
# 'x-container-bytes-used': {'type': 'long'},
|
||||
|
||||
'created_at': {'type': 'date'},
|
||||
'updated_at': {'type': 'date'}
|
||||
},
|
||||
"_parent": {
|
||||
"type": self.parent_plugin_type()
|
||||
}
|
||||
}
|
||||
|
||||
@property
|
||||
def admin_only_fields(self):
|
||||
from_conf = super(ContainerIndex, self).admin_only_fields
|
||||
return ContainerIndex.ADMIN_ONLY_FIELDS + from_conf
|
||||
|
||||
@classmethod
|
||||
def is_plugin_enabled_by_default(cls):
|
||||
return False
|
||||
|
||||
@property
|
||||
def allow_admin_ignore_rbac(self):
|
||||
return False
|
||||
|
||||
def _get_rbac_field_filters(self, request_context):
|
||||
tenant_member = request_context.tenant + ":*"
|
||||
single_user = request_context.tenant + ":" + request_context.user
|
||||
account_id = \
|
||||
self.parent_plugin.options.reseller_prefix + request_context.owner
|
||||
|
||||
return [
|
||||
{
|
||||
'or': [
|
||||
{
|
||||
'term': {
|
||||
'account_id': account_id
|
||||
}
|
||||
},
|
||||
{
|
||||
'terms': {
|
||||
'x-container-read': [tenant_member, single_user]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
def get_parent_id_field(self):
|
||||
return 'account_id'
|
||||
|
||||
@property
|
||||
def routing_field(self):
|
||||
return "account_id"
|
||||
|
||||
def get_objects(self):
|
||||
return get_swift_containers()
|
||||
|
||||
def serialize(self, obj):
|
||||
return serialize_swift_container(obj)
|
||||
|
|
@ -0,0 +1,139 @@
|
|||
# Copyright (c) 2016 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from oslo_config import cfg
|
||||
|
||||
from searchlight.elasticsearch.plugins import base
|
||||
from searchlight.elasticsearch.plugins.swift import get_swift_objects
|
||||
from searchlight.elasticsearch.plugins.swift import serialize_swift_object
|
||||
from searchlight.elasticsearch.plugins.swift import swift_notification_handler
|
||||
|
||||
swift_plugin_opts = [
|
||||
cfg.StrOpt('reseller_prefix', default="AUTH_",
|
||||
help="prefix used in account names for auth system."),
|
||||
]
|
||||
|
||||
CONF = cfg.CONF
|
||||
CONF.register_opts(swift_plugin_opts, group='resource_plugin:os_swift_account')
|
||||
|
||||
|
||||
class ObjectIndex(base.IndexBase):
|
||||
NotificationHandlerCls = swift_notification_handler.SwiftObjectHandler
|
||||
|
||||
@classmethod
|
||||
def parent_plugin_type(cls):
|
||||
return "OS::Swift::Container"
|
||||
|
||||
@classmethod
|
||||
def get_document_type(cls):
|
||||
return 'OS::Swift::Object'
|
||||
|
||||
def get_mapping(self):
|
||||
return {
|
||||
'dynamic': True,
|
||||
'properties': {
|
||||
'id': {'type': 'string', 'index': 'not_analyzed'},
|
||||
'name': {
|
||||
'type': 'string',
|
||||
'fields': {
|
||||
'raw': {'type': 'string', 'index': 'not_analyzed'}
|
||||
},
|
||||
},
|
||||
'account': {
|
||||
'type': 'string',
|
||||
'fields': {
|
||||
'raw': {'type': 'string', 'index': 'not_analyzed'}
|
||||
},
|
||||
},
|
||||
'account_id': {'type': 'string', 'index': 'not_analyzed'},
|
||||
'container': {
|
||||
'type': 'string',
|
||||
'fields': {
|
||||
'raw': {'type': 'string', 'index': 'not_analyzed'}
|
||||
},
|
||||
},
|
||||
'container_id': {'type': 'string', 'index': 'not_analyzed'},
|
||||
'content_type': {'type': 'string', 'index': 'not_analyzed'},
|
||||
'content_length': {'type': 'long'},
|
||||
'etag': {'type': 'string'},
|
||||
'created_at': {'type': 'date'},
|
||||
'updated_at': {'type': 'date'}
|
||||
},
|
||||
"_parent": {
|
||||
"type": self.parent_plugin_type()
|
||||
}
|
||||
}
|
||||
|
||||
@property
|
||||
def allow_admin_ignore_rbac(self):
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def is_plugin_enabled_by_default(cls):
|
||||
return False
|
||||
|
||||
def _get_rbac_field_filters(self, request_context):
|
||||
|
||||
tenant_member = request_context.tenant + ":*"
|
||||
single_user = request_context.tenant + ":" + request_context.user
|
||||
account_plugin = self.parent_plugin.parent_plugin
|
||||
account_id = \
|
||||
account_plugin.options.reseller_prefix + request_context.owner
|
||||
|
||||
return [
|
||||
{
|
||||
"has_parent": {
|
||||
"type": self.parent_plugin_type(),
|
||||
"query": {
|
||||
"filtered": {
|
||||
"query": {"match_all": {}},
|
||||
"filter": {
|
||||
"or": [
|
||||
{
|
||||
'term': {
|
||||
'account_id': account_id
|
||||
}
|
||||
},
|
||||
{
|
||||
'terms': {
|
||||
'x-container-read': [
|
||||
tenant_member,
|
||||
single_user]
|
||||
}
|
||||
},
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
def get_parent_id_field(self):
|
||||
return 'container_id'
|
||||
|
||||
@property
|
||||
def routing_field(self):
|
||||
return "account_id"
|
||||
|
||||
@property
|
||||
def facets_with_options(self):
|
||||
return ('container', 'content_type')
|
||||
|
||||
def get_objects(self):
|
||||
return get_swift_objects()
|
||||
|
||||
def serialize(self, obj):
|
||||
return serialize_swift_object(obj)
|
||||
|
|
@ -0,0 +1,176 @@
|
|||
# Copyright (c) 2016 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import logging
|
||||
|
||||
from searchlight.elasticsearch.plugins import base
|
||||
from searchlight.elasticsearch.plugins import swift
|
||||
from searchlight.elasticsearch.plugins.swift \
|
||||
import serialize_swift_account_notification
|
||||
from searchlight.elasticsearch.plugins.swift \
|
||||
import serialize_swift_container_notification
|
||||
from searchlight.elasticsearch.plugins.swift \
|
||||
import serialize_swift_object_notification
|
||||
from searchlight import i18n
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
_LE = i18n._LE
|
||||
_ = i18n._
|
||||
|
||||
|
||||
class SwiftAccountHandler(base.NotificationBase):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(SwiftAccountHandler, self).__init__(*args, **kwargs)
|
||||
|
||||
@classmethod
|
||||
def _get_notification_exchanges(cls):
|
||||
return ['swift']
|
||||
|
||||
def get_event_handlers(self):
|
||||
return {
|
||||
"account.create": self.create_or_update,
|
||||
"account.metadata": self.create_or_update,
|
||||
"account.delete": self.delete
|
||||
}
|
||||
|
||||
def create_or_update(self, payload, timestamp):
|
||||
payload = serialize_swift_account_notification(payload)
|
||||
try:
|
||||
self.index_helper.save_document(
|
||||
payload,
|
||||
version=self.get_version(payload, timestamp))
|
||||
except Exception as exc:
|
||||
LOG.error(_LE('Error saving account %(id)s '
|
||||
'in index. Error: %(exc)s') %
|
||||
{'id': payload['id'], 'exc': exc})
|
||||
|
||||
def delete(self, payload, timestamp):
|
||||
version = self.get_version(payload, timestamp)
|
||||
id = payload['account']
|
||||
try:
|
||||
self.index_helper.delete_document(
|
||||
{'_id': id, '_version': version,
|
||||
'_routing': payload['account']})
|
||||
except Exception as exc:
|
||||
LOG.error(_LE('Error deleting account %(id)s '
|
||||
'from index. Error: %(exc)s') %
|
||||
{'id': id, 'exc': exc})
|
||||
|
||||
|
||||
class SwiftContainerHandler(base.NotificationBase):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.object_helper = kwargs.pop('object_helper')
|
||||
super(SwiftContainerHandler, self).__init__(*args, **kwargs)
|
||||
|
||||
@classmethod
|
||||
def _get_notification_exchanges(cls):
|
||||
return ['swift']
|
||||
|
||||
def get_event_handlers(self):
|
||||
return {
|
||||
"container.create": self.create_or_update,
|
||||
"container.metadata": self.create_or_update,
|
||||
"container.delete": self.delete
|
||||
}
|
||||
|
||||
def create_or_update(self, payload, timestamp):
|
||||
payload = serialize_swift_container_notification(payload)
|
||||
try:
|
||||
self.index_helper.save_document(
|
||||
payload,
|
||||
version=self.get_version(payload, timestamp))
|
||||
except Exception as exc:
|
||||
LOG.error(_LE('Error saving container %(id)s '
|
||||
'in index. Error: %(exc)s') %
|
||||
{'id': payload['id'], 'exc': exc})
|
||||
|
||||
def delete(self, payload, timestamp):
|
||||
# notification payload doesn't have any date/time fields
|
||||
# so temporarily use metadata timestamp value as
|
||||
# updated_at field to retrieve version
|
||||
# Remove it when notifcation starts sending datetime field(s)
|
||||
payload['updated_at'] = timestamp
|
||||
version = self.get_version(payload, timestamp)
|
||||
del payload['updated_at']
|
||||
|
||||
id = payload['account'] + swift.ID_SEP + payload['container']
|
||||
try:
|
||||
self.object_helper.delete_documents_with_parent(
|
||||
id, version=version)
|
||||
except Exception as exc:
|
||||
LOG.error(_LE('Error deleting objects in container %(id)s '
|
||||
'from index. Error: %(exc)s') %
|
||||
{'id': id, 'exc': exc})
|
||||
try:
|
||||
self.index_helper.delete_document(
|
||||
{'_id': id, '_version': version,
|
||||
'_routing': payload['account']})
|
||||
except Exception as exc:
|
||||
LOG.error(_LE('Error deleting container %(id)s '
|
||||
'from index. Error: %(exc)s') %
|
||||
{'id': id, 'exc': exc})
|
||||
|
||||
|
||||
class SwiftObjectHandler(base.NotificationBase):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(SwiftObjectHandler, self).__init__(*args, **kwargs)
|
||||
|
||||
@classmethod
|
||||
def _get_notification_exchanges(cls):
|
||||
return ['swift']
|
||||
|
||||
def get_event_handlers(self):
|
||||
return {
|
||||
"object.create": self.create_or_update,
|
||||
"object.metadata": self.create_or_update,
|
||||
"object.delete": self.delete
|
||||
}
|
||||
|
||||
def create_or_update(self, payload, timestamp):
|
||||
payload = serialize_swift_object_notification(payload)
|
||||
try:
|
||||
self.index_helper.save_document(
|
||||
payload,
|
||||
version=self.get_version(payload, timestamp)
|
||||
)
|
||||
except Exception as exc:
|
||||
LOG.error(_LE('Error saving object %(id)s '
|
||||
'in index. Error: %(exc)s') %
|
||||
{'id': payload['id'], 'exc': exc})
|
||||
|
||||
def delete(self, payload, timestamp):
|
||||
# notification payload doesn't have any date/time fields
|
||||
# so temporarily use metadata timestamp value as
|
||||
# updated_at field to retrieve version
|
||||
# Remove it when notifcation starts sending datetime field(s)
|
||||
payload['updated_at'] = timestamp
|
||||
version = self.get_version(payload, timestamp)
|
||||
del payload['updated_at']
|
||||
|
||||
id = payload['account'] + swift.ID_SEP + \
|
||||
payload['container'] + swift.ID_SEP + \
|
||||
payload['object']
|
||||
try:
|
||||
self.index_helper.delete_document(
|
||||
{'_id': id, '_version': version,
|
||||
'_routing': payload['account']})
|
||||
except Exception as exc:
|
||||
LOG.error(_LE('Error deleting object %(id)s '
|
||||
'from index. Error: %(exc)s') %
|
||||
{'id': id, 'exc': exc})
|
||||
|
|
@ -375,13 +375,18 @@ class FunctionalTest(test_utils.BaseTestCase):
|
|||
'glance': {'images': 'ImageIndex', 'metadefs': 'MetadefIndex'},
|
||||
'nova': {'servers': 'ServerIndex'},
|
||||
'neutron': {'networks': 'NetworkIndex', 'ports': 'PortIndex'},
|
||||
'cinder': {'volumes': 'VolumeIndex', 'snapshots': 'SnapshotIndex'}
|
||||
'cinder': {'volumes': 'VolumeIndex', 'snapshots': 'SnapshotIndex'},
|
||||
'swift': {'accounts': 'AccountIndex',
|
||||
'containers': 'ContainerIndex',
|
||||
'objects': 'ObjectIndex'}
|
||||
}
|
||||
plugins = include_plugins or (
|
||||
('glance', 'images'), ('glance', 'metadefs'),
|
||||
('nova', 'servers'),
|
||||
('neutron', 'networks'), ('neutron', 'ports'),
|
||||
('cinder', 'volumes'), ('cinder', 'snapshots')
|
||||
('cinder', 'volumes'), ('cinder', 'snapshots'),
|
||||
('swift', 'accounts'), ('swift', 'containers'),
|
||||
('swift', 'objects')
|
||||
)
|
||||
plugins = filter(lambda plugin: plugin not in exclude_plugins, plugins)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,162 @@
|
|||
# Copyright (c) 2016 Hewlett-Packard Enterprise Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import datetime
|
||||
import six
|
||||
import time
|
||||
|
||||
|
||||
from oslo_utils import timeutils
|
||||
|
||||
import searchlight.elasticsearch.plugins.swift as swift_plugin
|
||||
from searchlight.elasticsearch.plugins.swift import\
|
||||
accounts as accounts_plugin
|
||||
import searchlight.tests.unit.utils as unit_test_utils
|
||||
import searchlight.tests.utils as test_utils
|
||||
|
||||
|
||||
now_epoch_time = time.time()
|
||||
now_utc = datetime.datetime.fromtimestamp(now_epoch_time)\
|
||||
.strftime('%Y-%m-%dT%H:%M:%SZ')
|
||||
|
||||
|
||||
USER1 = u'27f4d76b-be62-4e4e-aa33bb11cc55'
|
||||
ID1 = "AUTH_488ac936-663e-4e5c-537d-986021b32c4b"
|
||||
ID2 = "AUTH_7554da43-6443-acdf-deac-3425223cdada"
|
||||
ID3 = "AUTH_30754354-ca43-124b-12b5-789234bcdefa'"
|
||||
|
||||
AUTH_PREFIX = "AUTH_"
|
||||
|
||||
X_ACCOUNT_META_KEY1 = 'x-account-meta-key1'
|
||||
X_ACCOUNT_META_VALUE1 = 'x-account-meta-value1'
|
||||
|
||||
X_ACCOUNT_META_KEY2 = 'x-account-meta-key2'
|
||||
X_ACCOUNT_META_VALUE2 = 'x-account-meta-value2'
|
||||
|
||||
TENANT1 = "15b9a454cee34dbe9933ad575a0a6930"
|
||||
DOMAIN_ID = "default"
|
||||
|
||||
DATETIME = datetime.datetime(2016, 2, 20, 1, 13, 24, 215337)
|
||||
DATE1 = timeutils.isotime(DATETIME)
|
||||
|
||||
|
||||
def _account_fixture(account_id, domain_id, name, **kwargs):
|
||||
fixture = {
|
||||
"id": account_id,
|
||||
"name": name,
|
||||
"x-account-project-domain-id": domain_id,
|
||||
'x-timestamp': now_epoch_time
|
||||
}
|
||||
fixture.update(kwargs)
|
||||
return fixture
|
||||
|
||||
|
||||
def _notification_account_fixture(account_id, **kwargs):
|
||||
metadata = kwargs.pop('meta', {})
|
||||
notification = {
|
||||
'account': account_id,
|
||||
'project_name': None,
|
||||
'updated_at': DATE1,
|
||||
'project_domain_name': None,
|
||||
'x-trans-id': None,
|
||||
'project_id': None,
|
||||
'project_domain_id': None
|
||||
}
|
||||
for k, v in six.iteritems(kwargs):
|
||||
if k in notification:
|
||||
notification[k] = v
|
||||
for k, v in six.iteritems(metadata):
|
||||
notification[k] = v
|
||||
return notification
|
||||
|
||||
|
||||
class TestSwiftAccountPlugin(test_utils.BaseTestCase):
|
||||
def setUp(self):
|
||||
super(TestSwiftAccountPlugin, self).setUp()
|
||||
self.plugin = accounts_plugin.AccountIndex()
|
||||
self._create_fixtures()
|
||||
|
||||
def _create_fixtures(self):
|
||||
self.account1 = _account_fixture(
|
||||
account_id=ID1, domain_id=DOMAIN_ID, name="test-account1")
|
||||
self.account2 = _account_fixture(
|
||||
account_id=ID2, domain_id=DOMAIN_ID, name="test-account1",
|
||||
**{X_ACCOUNT_META_KEY1: X_ACCOUNT_META_VALUE1})
|
||||
self.account3 = _account_fixture(
|
||||
account_id=ID3, domain_id=DOMAIN_ID, name="test-account1",
|
||||
**{X_ACCOUNT_META_KEY1: X_ACCOUNT_META_VALUE1,
|
||||
X_ACCOUNT_META_KEY2: X_ACCOUNT_META_VALUE2})
|
||||
self.accounts = [self.account1, self.account2, self.account3]
|
||||
|
||||
def test_index_name(self):
|
||||
self.assertEqual('searchlight', self.plugin.resource_group_name)
|
||||
|
||||
def test_document_type(self):
|
||||
self.assertEqual('OS::Swift::Account',
|
||||
self.plugin.get_document_type())
|
||||
|
||||
def test_rbac_filter(self):
|
||||
fake_request = unit_test_utils.get_fake_request(
|
||||
USER1, TENANT1, '/v1/search', is_admin=False
|
||||
)
|
||||
rbac_terms = self.plugin._get_rbac_field_filters(fake_request.context)
|
||||
self.assertEqual(
|
||||
[{"term": {"id": AUTH_PREFIX + TENANT1}}],
|
||||
rbac_terms
|
||||
)
|
||||
|
||||
def test_admin_only_fields(self):
|
||||
admin_only_fields = self.plugin.admin_only_fields
|
||||
self.assertEqual(['x-account-meta-temp-url-key',
|
||||
'x-account-meta-temp-url-key-2',
|
||||
'x-account-access-control'], admin_only_fields)
|
||||
|
||||
def test_serialize(self):
|
||||
serialized = self.plugin.serialize(self.account1)
|
||||
|
||||
self.assertEqual(ID1, serialized['id'])
|
||||
self.assertEqual('test-account1', serialized['name'])
|
||||
self.assertEqual(DOMAIN_ID, serialized['domain_id'])
|
||||
self.assertEqual(now_utc, serialized['created_at'])
|
||||
|
||||
serialized = self.plugin.serialize(self.account2)
|
||||
self.assertEqual(X_ACCOUNT_META_VALUE1,
|
||||
serialized[X_ACCOUNT_META_KEY1])
|
||||
|
||||
serialized = self.plugin.serialize(self.account3)
|
||||
self.assertEqual(X_ACCOUNT_META_VALUE1,
|
||||
serialized[X_ACCOUNT_META_KEY1])
|
||||
self.assertEqual(X_ACCOUNT_META_VALUE2,
|
||||
serialized[X_ACCOUNT_META_KEY2])
|
||||
|
||||
def test_swift_account_notification_serialize(self):
|
||||
notification = _notification_account_fixture(
|
||||
ID1,
|
||||
project_name='admin',
|
||||
project_id=ID1,
|
||||
project_domain_id='default',
|
||||
updated_at=DATE1,
|
||||
)
|
||||
|
||||
expected = {
|
||||
'id': ID1,
|
||||
'name': 'admin',
|
||||
'updated_at': DATE1,
|
||||
'domain_id': 'default'
|
||||
}
|
||||
|
||||
serialized = swift_plugin.serialize_swift_account_notification(
|
||||
notification)
|
||||
self.assertEqual(expected, serialized)
|
||||
|
|
@ -0,0 +1,175 @@
|
|||
# Copyright (c) 2016 Hewlett-Packard Enterprise Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import datetime
|
||||
import six
|
||||
import time
|
||||
|
||||
|
||||
from oslo_utils import timeutils
|
||||
|
||||
import searchlight.elasticsearch.plugins.swift as swift_plugin
|
||||
from searchlight.elasticsearch.plugins.swift import\
|
||||
containers as containers_plugin
|
||||
import searchlight.tests.utils as test_utils
|
||||
|
||||
|
||||
now_epoch_time = time.time()
|
||||
now_utc = datetime.datetime.fromtimestamp(now_epoch_time).\
|
||||
strftime('%Y-%m-%dT%H:%M:%SZ')
|
||||
|
||||
|
||||
USER1 = "27f4d76b-be62-4e4e-aa33bb11cc55"
|
||||
|
||||
ACCOUNT_ID1 = "488ac936-663e-4e5c-537d-986021b32c4b"
|
||||
ACCOUNT_ID2 = "7554da43-6443-acdf-deac-3425223cdada"
|
||||
ACCOUNT_ID3 = "30754354-ca43-124b-12b5-789234bcdefa'"
|
||||
|
||||
AUTH_PREFIX = "AUTH_"
|
||||
|
||||
CONTAINER1 = "Container1"
|
||||
CONTAINER2 = "Container2"
|
||||
CONTAINER_ID1 = ACCOUNT_ID1 + swift_plugin.ID_SEP + CONTAINER1
|
||||
CONTAINER_ID2 = ACCOUNT_ID2 + swift_plugin.ID_SEP + CONTAINER2
|
||||
|
||||
X_CONTAINER_META_KEY1 = 'x-container-meta-key1'
|
||||
X_CONTAINER_META_VALUE1 = 'x-container-meta-value1'
|
||||
|
||||
X_CONTAINER_META_KEY2 = 'x-container-meta-key2'
|
||||
X_CONTAINER_META_VALUE2 = 'x-container-meta-value2'
|
||||
|
||||
TENANT1 = "15b9a454cee34dbe9933ad575a0a6930"
|
||||
TENANT2 = "a7ba963f71bb43818f631febbc9df8e6"
|
||||
DOMAIN_ID = "default"
|
||||
|
||||
DATETIME = datetime.datetime(2016, 2, 21, 4, 41, 33, 325314)
|
||||
DATE1 = timeutils.isotime(DATETIME)
|
||||
|
||||
|
||||
def _container_fixture(container_id, container_name, account,
|
||||
account_id, read_acl, **kwargs):
|
||||
fixture = {
|
||||
"id": container_id,
|
||||
"name": container_name,
|
||||
"account": account,
|
||||
"account_id": account_id,
|
||||
"x-container-read": read_acl,
|
||||
'x-timestamp': now_epoch_time
|
||||
}
|
||||
fixture.update(kwargs)
|
||||
return fixture
|
||||
|
||||
|
||||
def _notification_container_fixture(account_id, **kwargs):
|
||||
metadata = kwargs.pop('meta', {})
|
||||
notification = {
|
||||
'account': account_id,
|
||||
'project_name': None,
|
||||
'container': None,
|
||||
'updated_at': DATE1,
|
||||
'project_domain_name': None,
|
||||
'x-trans-id': None,
|
||||
'project_id': None,
|
||||
'project_domain_id': None
|
||||
}
|
||||
for k, v in six.iteritems(kwargs):
|
||||
if k in notification:
|
||||
notification[k] = v
|
||||
for k, v in six.iteritems(metadata):
|
||||
notification[k] = v
|
||||
return notification
|
||||
|
||||
|
||||
class TestSwiftContainerPlugin(test_utils.BaseTestCase):
|
||||
def setUp(self):
|
||||
super(TestSwiftContainerPlugin, self).setUp()
|
||||
self.plugin = containers_plugin.ContainerIndex()
|
||||
self._create_fixtures()
|
||||
|
||||
def _create_fixtures(self):
|
||||
self.container1 = _container_fixture(
|
||||
container_id=CONTAINER_ID1, container_name=CONTAINER1,
|
||||
account='test-account1', account_id=ACCOUNT_ID1, read_acl=None)
|
||||
self.container2 = _container_fixture(
|
||||
container_id=CONTAINER_ID2, container_name=CONTAINER2,
|
||||
account='test-account2', account_id=ACCOUNT_ID1,
|
||||
read_acl=ACCOUNT_ID2 + ":*",
|
||||
**{X_CONTAINER_META_KEY1: X_CONTAINER_META_VALUE1}
|
||||
)
|
||||
self.container3 = _container_fixture(
|
||||
container_id=CONTAINER_ID1, container_name=CONTAINER1,
|
||||
account='test-account3', account_id=ACCOUNT_ID2,
|
||||
read_acl=ACCOUNT_ID3 + ":*" + USER1,
|
||||
**{X_CONTAINER_META_KEY1: X_CONTAINER_META_VALUE1,
|
||||
X_CONTAINER_META_KEY2: X_CONTAINER_META_VALUE2})
|
||||
self.containers = [self.container1, self.container2, self.container3]
|
||||
|
||||
def test_index_name(self):
|
||||
self.assertEqual('searchlight', self.plugin.resource_group_name)
|
||||
|
||||
def test_document_type(self):
|
||||
self.assertEqual('OS::Swift::Container',
|
||||
self.plugin.get_document_type())
|
||||
|
||||
def test_admin_only_fields(self):
|
||||
admin_only_fields = self.plugin.admin_only_fields
|
||||
self.assertEqual(['x-container-write',
|
||||
'x-container-sync-key',
|
||||
'x-container-sync-to',
|
||||
'x-container-meta-temp-url-key',
|
||||
'x-container-meta-temp-url-key-2'],
|
||||
admin_only_fields)
|
||||
|
||||
def test_serialize(self):
|
||||
serialized = self.plugin.serialize(self.container1)
|
||||
|
||||
self.assertEqual(CONTAINER_ID1, serialized['id'])
|
||||
self.assertEqual(CONTAINER1, serialized['name'])
|
||||
self.assertEqual(ACCOUNT_ID1, serialized['account_id'])
|
||||
self.assertEqual('test-account1', serialized['account'])
|
||||
|
||||
serialized = self.plugin.serialize(self.container2)
|
||||
self.assertEqual(X_CONTAINER_META_VALUE1,
|
||||
serialized[X_CONTAINER_META_KEY1])
|
||||
self.assertEqual(ACCOUNT_ID2 + ":*", serialized['x-container-read'])
|
||||
|
||||
serialized = self.plugin.serialize(self.container3)
|
||||
self.assertEqual(X_CONTAINER_META_VALUE1,
|
||||
serialized[X_CONTAINER_META_KEY1])
|
||||
self.assertEqual(X_CONTAINER_META_VALUE2,
|
||||
serialized[X_CONTAINER_META_KEY2])
|
||||
|
||||
def test_swift_account_notification_serialize(self):
|
||||
notification = _notification_container_fixture(
|
||||
swift_plugin.AUTH_PREFIX + ACCOUNT_ID1,
|
||||
container=CONTAINER1,
|
||||
project_name='admin',
|
||||
project_domain_id='default',
|
||||
project_id=ACCOUNT_ID1,
|
||||
updated_at=DATE1,
|
||||
)
|
||||
|
||||
expected = {
|
||||
'id': CONTAINER_ID1,
|
||||
'name': CONTAINER1,
|
||||
'account': 'admin',
|
||||
'account_id': ACCOUNT_ID1,
|
||||
'updated_at': DATE1,
|
||||
'x-container-read': None
|
||||
}
|
||||
|
||||
serialized = swift_plugin.serialize_swift_container_notification(
|
||||
notification)
|
||||
self.assertEqual(expected, serialized)
|
||||
|
|
@ -0,0 +1,120 @@
|
|||
# Copyright (c) 2016 Hewlett-Packard Enterprise Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import datetime
|
||||
import time
|
||||
|
||||
import searchlight.elasticsearch.plugins.swift as swift_plugin
|
||||
from searchlight.elasticsearch.plugins.swift import\
|
||||
objects as objects_plugin
|
||||
import searchlight.tests.utils as test_utils
|
||||
|
||||
|
||||
now_epoch_time = time.time()
|
||||
created_time = now_epoch_time - 3
|
||||
updated_time = now_epoch_time
|
||||
updated_time_in_fmt = datetime.datetime.fromtimestamp(updated_time).\
|
||||
strftime('%a, %d %b %Y %H:%M:%S GMT')
|
||||
updated_time_out_fmt = datetime.datetime.fromtimestamp(updated_time).\
|
||||
strftime('%Y-%m-%dT%H:%M:%SZ')
|
||||
created_time_utc = datetime.datetime.fromtimestamp(created_time).\
|
||||
strftime('%Y-%m-%dT%H:%M:%SZ')
|
||||
|
||||
USER1 = "27f4d76b-be62-4e4e-aa33bb11cc55"
|
||||
|
||||
ACCOUNT_ID1 = "488ac936-663e-4e5c-537d-986021b32c4b"
|
||||
CONTAINER1 = "Container1"
|
||||
CONTAINER_ID1 = ACCOUNT_ID1 + swift_plugin.ID_SEP + CONTAINER1
|
||||
OBJECT1 = "Object1"
|
||||
OBJECT2 = "Object2"
|
||||
OBJECT_ID1 = CONTAINER_ID1 + swift_plugin.ID_SEP + OBJECT1
|
||||
OBJECT_ID2 = CONTAINER_ID1 + swift_plugin.ID_SEP + OBJECT2
|
||||
|
||||
X_OBJECT_META_KEY1 = 'x-object-meta-key1'
|
||||
X_OBJECT_META_VALUE1 = 'x-object-meta-value1'
|
||||
|
||||
X_OBJECT_META_KEY2 = 'x-object-meta-key2'
|
||||
X_OBJECT_META_VALUE2 = 'x-object-meta-value2'
|
||||
|
||||
TENANT1 = "15b9a454cee34dbe9933ad575a0a6930"
|
||||
|
||||
|
||||
def _object_fixture(object_id, object_name, container_id, container_name,
|
||||
account, account_id, **kwargs):
|
||||
fixture = {
|
||||
"id": object_id,
|
||||
"name": object_name,
|
||||
"account": account,
|
||||
"account_id": account_id,
|
||||
"container": container_name,
|
||||
"container_id": container_id,
|
||||
'x-timestamp': created_time,
|
||||
'last-modified': updated_time_in_fmt
|
||||
}
|
||||
fixture.update(kwargs)
|
||||
return fixture
|
||||
|
||||
|
||||
class TestSwiftObjectPlugin(test_utils.BaseTestCase):
|
||||
def setUp(self):
|
||||
super(TestSwiftObjectPlugin, self).setUp()
|
||||
self.plugin = objects_plugin.ObjectIndex()
|
||||
self._create_fixtures()
|
||||
|
||||
def _create_fixtures(self):
|
||||
self.object1 = _object_fixture(
|
||||
object_id=OBJECT_ID1, object_name=OBJECT1,
|
||||
container_id=CONTAINER_ID1, container_name=CONTAINER1,
|
||||
account='test-account1', account_id=ACCOUNT_ID1,
|
||||
**{"content-type": "text/plain", "content-length": 1050})
|
||||
self.object2 = _object_fixture(
|
||||
object_id=OBJECT_ID2, object_name=OBJECT2,
|
||||
container_id=CONTAINER_ID1, container_name=CONTAINER1,
|
||||
account='test-account1', account_id=ACCOUNT_ID1,
|
||||
**{"content-type": "text/plain", "content-length": 1050,
|
||||
X_OBJECT_META_KEY1: X_OBJECT_META_VALUE1,
|
||||
X_OBJECT_META_KEY2: X_OBJECT_META_VALUE2})
|
||||
self.objects = [self.object1, self.object2]
|
||||
|
||||
def test_index_name(self):
|
||||
self.assertEqual('searchlight', self.plugin.resource_group_name)
|
||||
|
||||
def test_document_type(self):
|
||||
self.assertEqual('OS::Swift::Object',
|
||||
self.plugin.get_document_type())
|
||||
|
||||
def test_admin_only_fields(self):
|
||||
admin_only_fields = self.plugin.admin_only_fields
|
||||
self.assertEqual([], admin_only_fields)
|
||||
|
||||
def test_serialize(self):
|
||||
serialized = self.plugin.serialize(self.object1)
|
||||
|
||||
self.assertEqual(OBJECT_ID1, serialized['id'])
|
||||
self.assertEqual(OBJECT1, serialized['name'])
|
||||
self.assertEqual(ACCOUNT_ID1, serialized['account_id'])
|
||||
self.assertEqual('test-account1', serialized['account'])
|
||||
self.assertEqual(CONTAINER_ID1, serialized['container_id'])
|
||||
self.assertEqual(CONTAINER1, serialized['container'])
|
||||
self.assertEqual(created_time_utc, serialized['created_at']),
|
||||
self.assertEqual(updated_time_out_fmt, serialized['updated_at'])
|
||||
self.assertEqual("text/plain", serialized['content_type'])
|
||||
self.assertEqual(1050, serialized['content_length'])
|
||||
|
||||
serialized = self.plugin.serialize(self.object2)
|
||||
self.assertEqual(X_OBJECT_META_VALUE1,
|
||||
serialized[X_OBJECT_META_KEY1])
|
||||
self.assertEqual(X_OBJECT_META_VALUE2,
|
||||
serialized[X_OBJECT_META_KEY2])
|
||||
|
|
@ -283,7 +283,7 @@ class TestSearchDeserializer(test_utils.BaseTestCase):
|
|||
'OS::Neutron::Net',
|
||||
'OS::Neutron::Port',
|
||||
'OS::Cinder::Volume',
|
||||
'OS::Cinder::Snapshot',
|
||||
'OS::Cinder::Snapshot'
|
||||
]
|
||||
|
||||
self.assertEqual(['searchlight-search'], output['index'])
|
||||
|
|
@ -602,7 +602,7 @@ class TestSearchDeserializer(test_utils.BaseTestCase):
|
|||
self.assertEqual(expected_query, output['query'])
|
||||
|
||||
def test_rbac_admin(self):
|
||||
"""Test that admins have RBAC applied unless 'all_projects' is true"""
|
||||
"""Test that admins have RBAC applied"""
|
||||
request = unit_test_utils.get_fake_request(is_admin=True)
|
||||
request.body = six.b(jsonutils.dumps({
|
||||
'query': {'match_all': {}},
|
||||
|
|
|
|||
|
|
@ -37,6 +37,9 @@ searchlight.index_backend =
|
|||
os_designate_zone = searchlight.elasticsearch.plugins.designate.zones:ZoneIndex
|
||||
os_cinder_volume = searchlight.elasticsearch.plugins.cinder.volumes:VolumeIndex
|
||||
os_cinder_snapshot = searchlight.elasticsearch.plugins.cinder.snapshots:SnapshotIndex
|
||||
os_swift_account = searchlight.elasticsearch.plugins.swift.accounts:AccountIndex
|
||||
os_swift_container = searchlight.elasticsearch.plugins.swift.containers:ContainerIndex
|
||||
os_swift_object = searchlight.elasticsearch.plugins.swift.objects:ObjectIndex
|
||||
|
||||
[build_sphinx]
|
||||
all_files = 1
|
||||
|
|
|
|||
Loading…
Reference in New Issue