Add support for ElasticSearch 5.x

Searchlight works with ElasticSearch 5.x but devstack plugin
and the test setup does not provide any option to use it.

Change-Id: Ia597f52dd14f41d90d8e44b1fea24ad8e781879f
Signed-off-by: Trinh Nguyen <dangtrinhnt@gmail.com>
Story: #1648255
Task: #25265
Task: #26181
This commit is contained in:
Trinh Nguyen 2018-09-06 10:20:31 +09:00
parent c4cf0c1eaa
commit 2d4043eb0d
9 changed files with 218 additions and 63 deletions

View File

@ -64,6 +64,9 @@ IMAGE_URLS+=",https://cloud-images.ubuntu.com/releases/xenial/release/ubuntu-16.
# Fedora 21 cloud image (e.g. for AWS LoadBalancer resource in Heat)
#IMAGE_URLS+=",http://download.fedoraproject.org/pub/fedora/linux/releases/21/Cloud/Images/x86_64/Fedora-Cloud-Base-20141203-21.x86_64.qcow2"
# ElasticSearch version
# Currently, only ElasticSearch 2.x and 5.x are supported
ELASTICSEARCH_VERSION=5.6.11
### CONFIGURE INSTALLED SERVICES ###
#

View File

@ -48,9 +48,10 @@ SEARCHLIGHT_SERVICE_PORT=${SEARCHLIGHT_SERVICE_PORT:-9393}
SEARCHLIGHT_SERVICE_PORT_INT=${SEARCHLIGHT_SERVICE_PORT_INT:-19393}
ELASTICSEARCH_VERSION=${ELASTICSEARCH_VERSION:-2.3.4}
ELASTICSEARCH_BASEURL=${ELASTICSEARCH_BASEURL:-https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution}
ELASTICSEARCH_BASEURL_DEB=${ELASTICSEARCH_BASEURL}/deb/elasticsearch
ELASTICSEARCH_BASEURL_RPM=${ELASTICSEARCH_BASEURL}/rpm/elasticsearch
# Base URL for ElasticSearch 5.x and 6.x
ELASTICSEARCH_BASEURL=https://artifacts.elastic.co/downloads/elasticsearch
# Base URL for ElasticSearch 2.x
ELASTICSEARCH_BASEURL_LEGACY=https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution
# Helper Functions
# ----------------
@ -191,10 +192,26 @@ function init_searchlight {
$SEARCHLIGHT_BIN_DIR/searchlight-manage --config-file $SEARCHLIGHT_CONF index sync --force
}
# Install Searchlight's requirements
# See https://elasticsearch-py.readthedocs.io/en/master/#compatibility
function _setup_searchlight_dev {
setup_develop $SEARCHLIGHT_DIR
if [[ $ELASTICSEARCH_VERSION =~ ^5 ]]; then
echo "Installing python elasticsearch for ES 5.x"
$REQUIREMENTS_DIR/.venv/bin/edit-constraints $REQUIREMENTS_DIR/upper-constraints.txt elasticsearch
pip_install -U -r $SEARCHLIGHT_DIR/elasticsearch5.txt
elif [[ $ELASTICSEARCH_VERSION =~ ^6 ]]; then
echo "WARNING - Searchlight is not tested with ES 6.x!!!"
# echo "Installing python elasticsearch for ES 6.x"
# $REQUIREMENTS_DIR/.venv/bin/edit-constraints $REQUIREMENTS_DIR/upper-constraints.txt eleasticsearch
# pip install -c $REQUIREMENTS_DIR/upper-constraints.txt -U -r $SEARCHLIGHT_DIR/elasticsearch6.txt
fi
}
# install_searchlight - Collect source and prepare
function install_searchlight {
git_clone $SEARCHLIGHT_REPO $SEARCHLIGHT_DIR $SEARCHLIGHT_BRANCH
setup_develop $SEARCHLIGHT_DIR
_setup_searchlight_dev
_download_elasticsearch
_install_elasticsearch
pip_install uwsgi
@ -256,12 +273,25 @@ function _wget_elasticsearch {
}
function _download_elasticsearch {
echo "Downloading elasticsearch"
if is_ubuntu; then
_wget_elasticsearch $ELASTICSEARCH_BASEURL_DEB/${ELASTICSEARCH_VERSION} elasticsearch-${ELASTICSEARCH_VERSION}.deb
arch="deb"
elif is_fedora; then
_wget_elasticsearch $ELASTICSEARCH_BASEURL_RPM/${ELASTICSEARCH_VERSION} elasticsearch-${ELASTICSEARCH_VERSION}.rpm
arch="rpm"
else
echo "Unknown architecture; can't download ElasticSearch"
fi
ELASTICSEARCH_FILENAME=elasticsearch-${ELASTICSEARCH_VERSION}.${arch}
if [[ $ELASTICSEARCH_VERSION =~ ^2 ]]; then
ELASTICSEARCH_URL=${ELASTICSEARCH_BASEURL_LEGACY}/${arch}/elasticsearch/${ELASTICSEARCH_VERSION}
elif [[ $ELASTICSEARCH_VERSION =~ ^5 ]]; then
ELASTICSEARCH_URL=${ELASTICSEARCH_BASEURL}
else
echo "Current Searchlight only supports ElasticSearch 2.x and 5.x"
fi
echo "Downloading ElasticSearch $ELASTICSEARCH_VERSION"
echo "ElasticSearch URL is $ELASTICSEARCH_URL"
_wget_elasticsearch $ELASTICSEARCH_URL $ELASTICSEARCH_FILENAME
}
function _check_elasticsearch_ready {
@ -280,7 +310,7 @@ function _start_elasticsearch {
sudo /bin/systemctl start elasticsearch.service
_check_elasticsearch_ready
else
echo "Unsupported architecture...can not start elasticsearch."
echo "Unsupported architecture... Can not start elasticsearch."
fi
}
@ -296,15 +326,14 @@ function _stop_elasticsearch {
}
function _install_elasticsearch {
echo "Installing elasticsearch"
pip_install_gr elasticsearch
# echo "Installing elasticsearch"
# pip_install_gr elasticsearch
if is_package_installed elasticsearch; then
echo "Note: elasticsearch was already installed."
return
fi
if is_ubuntu; then
is_package_installed default-jdk-headless || install_package default-jdk-headless
sudo dpkg -i ${FILES}/elasticsearch-${ELASTICSEARCH_VERSION}.deb
sudo update-rc.d elasticsearch defaults 95 10
elif is_fedora; then

View File

@ -17,56 +17,59 @@
Installing and Configuring Elasticsearch
========================================
The Searchlight indexing service is responsible for indexing data in
`Elasticsearch <http://www.elastic.co>`_;
Elasticsearch has very good documentation on installation but some pointers
are provided here.
.. IMPORTANT:: We *strongly* recommend using Elasticsearch 2.x and the
accompanying python client version. Searchlight has not been tested
with v5.
The Searchlight indexing service is responsible for indexing data in
Elasticsearch [1]_; Elasticsearch has very good documentation on
installation but some pointers are provided here.
.. IMPORTANT:: We *strongly* recommend using ElasticSearch 5.x and the
accompanying python client version [2]_. ElasticSearch 2.x is still
supported and will be removed after Stein cycle.
Installation
~~~~~~~~~~~~
Elasticsearch requires a Java Runtime Environment (or Java Development Kit). OpenJDK
and Oracle's Java are supported. Information on the current recommended version can
be found in the `installation instructions <http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/setup.html>`_.
be found in the installation instructions [3]_.
Installing from packages
########################
Installing from a download package
##################################
See the `latest Elasticsearch instructions <https://www.elastic.co/guide/en/elasticsearch/reference/2.3/setup-repositories.html>`_
for instructions about installing in Debian/Ubuntu and Red Hat/Fedora.
Installing from a package has the advantage of including scripts to run
`Elasticsearch as a service <https://www.elastic.co/guide/en/elasticsearch/reference/2.3/setup-service.html>`_.
Installing from a download
##########################
Links to various formats and also older versions of Elasticsearch can be found
on the `download page <http://www.elasticsearch.org/download>`_. Once
downloaded and extracted, you can start Elasticsearch with::
on the download page [4]_. Once downloaded and extracted, you can start
Elasticsearch with:
$ bin/elasticsearch
.. code-block:: console
For more details see the `installation instructions <http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/setup.html>`_.
bin/elasticsearch
Quick command line example with 2.3.4:
For more details see the installation instructions [5]_
Quick command line example with 5.6.11:
.. note::
Do the following commands as "root" or via sudo <command>
Download the ES package::
Download the ES package:
$ cd ~
$ wget https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/2.3.4/elasticsearch-2.3.4.deb
$ sudo dpkg -i elasticsearch-2.3.4.deb
$ sudo update-rc.d elasticsearch defaults 95 10
$ sudo /etc/init.d/elasticsearch start
.. code-block:: console
cd ~
wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.6.11.deb
sudo dpkg -i elasticsearch-5.6.11.deb
sudo update-rc.d elasticsearch defaults 95 10
sudo /etc/init.d/elasticsearch start
.. note::
If you install ElasticSearch 5.x, You should install
python-elasticsearch>=5.0.0. global-requirements currently constrains it to v2.x
Configuration
~~~~~~~~~~~~~
Elasticsearch comes with very a very sensible default configuration that
allows for clustering and high performance out of the box. There are some
settings, both general and specific to Searchlight's indexing service, that might
@ -78,6 +81,7 @@ a superset of JSON.
Indices
#######
Elasticsearch (and Lucene) store information in indices. Within an index can
be one or more document types. Searchlight's indexing service uses an index
per service that has a plugin available, and each plugin generally will have
@ -86,7 +90,9 @@ its own document type. For instance, the glance plugin has *glance.image* and
Elasticsearch it may make sense to change the default sharing and replication
mechanism. We also recommend disabling implicit index creation, though if you
are sharing an Elasticsearch installation this may be inadvisable. The
following options control indexing behavior::
following options control indexing behavior:
.. code-block:: yaml
# Number of shards for each index (performance)
index.number_of_shards: 5
@ -99,10 +105,13 @@ following options control indexing behavior::
Index settings
**************
In addition to server-wide index settings it's possible to configure
Searchlight to apply settings to indices it creates with
``searchlight-manage``. Index settings can be specified as follows in
``searchlight.conf``::
``searchlight.conf``:
.. code-block:: yaml
[elasticsearch]
index_settings = refresh_interval:2s,number_of_replicas:1
@ -115,30 +124,32 @@ Index settings are applied at creation time and so are not limited to the
are created. If you wish to update settings for an existing index, you
should use the Elasticsearch API to do so or reindex.
See also:
* http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-index\_.html
* http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/indices-update-settings.html
See also [9]_, [10]_.
Scripts
#######
The scripting module allows to use scripts in order to evaluate custom expressions.
Scripting is turned off by default in elasticsearch latest versions.
Searchlight doesn't allow scripts in the search api but requires scripts to sync Index updates
from notifications. For security purpose index updates are allowed only for admin role::
The scripting module allows to use scripts in order to evaluate custom
expressions. Scripting is turned off by default in elasticsearch latest
versions. Searchlight doesn't allow scripts in the search api but requires
scripts to sync Index updates from notifications. For security purpose index
updates are allowed only for admin role.
.. code-block:: yaml
script.engine.groovy.inline.update: on
See also:
* https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-scripting.html#modules-scripting
See also [8]_.
Development
###########
For development, Elasticsearch's default configuration is overkill. It's
possible to run Elasticsearch with a much lower memory footprint than by
default, and you may wish to disable clustering behavior.
.. code-block:: yaml
# Configures elasticsearch as a single node (no discovery)
node.local: true
@ -148,9 +159,12 @@ default, and you may wish to disable clustering behavior.
JVM settings
************
Setting the ES_HEAP_SIZE environment variable will restrict how much memory
Elasticsearch uses, equivalent to setting -Xmx and -Xms to the same value for
the Java runtime. For development you can set it as low as a few tens of MB::
the Java runtime. For development you can set it as low as a few tens of MB:
.. code-block:: console
export ES_HEAP_SIZE=40m
@ -159,7 +173,10 @@ requires memory on top of that.
Production
##########
Some settings you may wish to change for production::
Some settings you may wish to change for production:
.. code-block:: yaml
# Cluster name is used by cluster discovery; it's important to ensure
# this is set across all nodes you wish to be in the cluster
@ -187,23 +204,43 @@ Some settings you may wish to change for production::
# ES_HEAP_SIZE appropriately (see below). Linux only.
bootstrap.mlockall: true
For more details see Elasticsearch's `configuration information <http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/setup-configuration.html>`_.
For more details see Elasticsearch's configuration information [6]_.
Specifying nodes in a cluster
*****************************
Elasticsearch's default discovery relies on multicast requests. If this isn't
a good fit, you can use unicast discovery::
a good fit, you can use unicast discovery:
.. code-block:: yaml
discovery.zen.ping.multicast.enabled: false
discovery.zen.ping.unicast.hosts: ['w.x.y.z', 'w.x.y.z']
See `<http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/modules-discovery-zen.html>`_
for more details.
See [7]_ for more details.
JVM settings
************
For production, Elasticsearch recommends setting the ES_HEAP_SIZE environment
variable to a value around 60% of a dedicated machine's memory::
variable to a value around 60% of a dedicated machine's memory:
.. code-block:: yaml
export ES_HEAP_SIZE=2g
References
~~~~~~~~~~
.. [1] https://www.elastic.co/
.. [2] https://elasticsearch-py.readthedocs.io/en/master/#compatibility
.. [3] https://www.elastic.co/guide/en/elasticsearch/reference/current/setup.html
.. [4] https://www.elastic.co/downloads/elasticsearch
.. [5] https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html
.. [6] https://www.elastic.co/guide/en/elasticsearch/reference/current/settings.html
.. [7] https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-discovery-zen.html
.. [8] https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-scripting.html#modules-scripting
.. [9] https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index\_.html
.. [10] https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-update-settings.html

View File

@ -16,7 +16,7 @@
Running Searchlight API using uWSGI
========================================
===================================
The recommended way to deploy Searchlight is have a web server such as Apache
or nginx to handle http requests and proxy these requests to Searchlight WSGI
app running in uWSGI. Searchlight comes with some configuration templates on

2
elasticsearch5.txt Normal file
View File

@ -0,0 +1,2 @@
# Elasticsearch 5.x
elasticsearch>=5.0.0,<6.0.0

2
elasticsearch6.txt Normal file
View File

@ -0,0 +1,2 @@
# Elasticsearch 6.x
elasticsearch>=6.0.0,<7.0.0

View File

@ -0,0 +1,65 @@
---
prelude: >
Replace this text with content to appear at the top of the section for this
release. All of the prelude content is merged together and then rendered
separately from the items listed in other parts of the file, so the text
needs to be worded so that both the prelude and the other items make sense
when read independently. This may mean repeating some details. Not every
release note requires a prelude. Usually only notes describing major
features or adding release theme details should have a prelude.
features:
- |
List new features here, or remove this section. All of the list items in
this section are combined when the release notes are rendered, so the text
needs to be worded so that it does not depend on any information only
available in another section, such as the prelude. This may mean repeating
some details.
issues:
- |
List known issues here, or remove this section. All of the list items in
this section are combined when the release notes are rendered, so the text
needs to be worded so that it does not depend on any information only
available in another section, such as the prelude. This may mean repeating
some details.
upgrade:
- |
List upgrade notes here, or remove this section. All of the list items in
this section are combined when the release notes are rendered, so the text
needs to be worded so that it does not depend on any information only
available in another section, such as the prelude. This may mean repeating
some details.
deprecations:
- |
List deprecations notes here, or remove this section. All of the list
items in this section are combined when the release notes are rendered, so
the text needs to be worded so that it does not depend on any information
only available in another section, such as the prelude. This may mean
repeating some details.
critical:
- |
Add critical notes here, or remove this section. All of the list items in
this section are combined when the release notes are rendered, so the text
needs to be worded so that it does not depend on any information only
available in another section, such as the prelude. This may mean repeating
some details.
security:
- |
Add security notes here, or remove this section. All of the list items in
this section are combined when the release notes are rendered, so the text
needs to be worded so that it does not depend on any information only
available in another section, such as the prelude. This may mean repeating
some details.
fixes:
- |
Add normal bug fixes here, or remove this section. All of the list items
in this section are combined when the release notes are rendered, so the
text needs to be worded so that it does not depend on any information only
available in another section, such as the prelude. This may mean repeating
some details.
other:
- |
Add other notes here, or remove this section. All of the list items in
this section are combined when the release notes are rendered, so the text
needs to be worded so that it does not depend on any information only
available in another section, such as the prelude. This may mean repeating
some details.

View File

@ -39,7 +39,15 @@ oslo.serialization!=2.19.1,>=2.18.0 # Apache-2.0
osprofiler>=1.4.0 # Apache-2.0
elasticsearch<=3.0.0,>=2.0.0 # Apache-2.0
# ElasticSearch 6.x
# elasticsearch>=6.0.0,<7.0.0
# ElasticSearch 5.x
# elasticsearch>=5.0.0,<6.0.0
# ElasticSearch 2.x
# elasticsearch>=2.0.0,<3.0.0
# See:
# https://elasticsearch-py.readthedocs.io/en/master/#compatibility
elasticsearch<3.0.0,>=2.0.0 # Apache-2.0
# Clients required for indexing plugins
python-designateclient>=2.7.0 # Apache-2.0

View File

@ -13,7 +13,16 @@ if [[ $VERSION_CODENAME = bionic ]]; then
else
sudo apt-get install -y default-jre
fi
wget https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/2.3.4/elasticsearch-2.3.4.deb
sudo dpkg -i elasticsearch-2.3.4.deb
ELASTICSEARCH_MAJOR_VERSION=${ELASTICSEARCH_MAJOR_VERSION:"2"}
if [[ "$ELASTICSEARCH_MAJOR_VERSION" == "5" ]]; then
echo "Downloading and installing elasticsearch 5.6.11"
wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.6.11.deb
sudo dpkg -i elasticsearch-5.6.11.deb
else
echo "Downloading and installing elasticsearch 2.3.4"
wget https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/2.3.4/elasticsearch-2.3.4.deb
sudo dpkg -i elasticsearch-2.3.4.deb
fi
# Make 'elasticsearch' binary callable from within functional tests
sudo ln -s /usr/share/elasticsearch/bin/elasticsearch /usr/local/bin/elasticsearch