From 2d4043eb0d654ae5cc7fe834beecd417dde8f2ce Mon Sep 17 00:00:00 2001 From: Trinh Nguyen Date: Thu, 6 Sep 2018 10:20:31 +0900 Subject: [PATCH] Add support for ElasticSearch 5.x Searchlight works with ElasticSearch 5.x but devstack plugin and the test setup does not provide any option to use it. Change-Id: Ia597f52dd14f41d90d8e44b1fea24ad8e781879f Signed-off-by: Trinh Nguyen Story: #1648255 Task: #25265 Task: #26181 --- devstack/{local.conf => local.conf.example} | 3 + devstack/plugin.sh | 51 +++++-- doc/source/install/elasticsearch.rst | 133 +++++++++++------- doc/source/install/uwsgi.rst | 2 +- elasticsearch5.txt | 2 + elasticsearch6.txt | 2 + ...ack-elasticsearch-5x-48934b931a31c823.yaml | 65 +++++++++ requirements.txt | 10 +- tools/test-setup.sh | 13 +- 9 files changed, 218 insertions(+), 63 deletions(-) rename devstack/{local.conf => local.conf.example} (98%) create mode 100644 elasticsearch5.txt create mode 100644 elasticsearch6.txt create mode 100644 releasenotes/notes/add-devstack-elasticsearch-5x-48934b931a31c823.yaml diff --git a/devstack/local.conf b/devstack/local.conf.example similarity index 98% rename from devstack/local.conf rename to devstack/local.conf.example index f66bf1aa..90f94acc 100644 --- a/devstack/local.conf +++ b/devstack/local.conf.example @@ -64,6 +64,9 @@ IMAGE_URLS+=",https://cloud-images.ubuntu.com/releases/xenial/release/ubuntu-16. # Fedora 21 cloud image (e.g. for AWS LoadBalancer resource in Heat) #IMAGE_URLS+=",http://download.fedoraproject.org/pub/fedora/linux/releases/21/Cloud/Images/x86_64/Fedora-Cloud-Base-20141203-21.x86_64.qcow2" +# ElasticSearch version +# Currently, only ElasticSearch 2.x and 5.x are supported +ELASTICSEARCH_VERSION=5.6.11 ### CONFIGURE INSTALLED SERVICES ### # diff --git a/devstack/plugin.sh b/devstack/plugin.sh index f1a2d804..baffb531 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -48,9 +48,10 @@ SEARCHLIGHT_SERVICE_PORT=${SEARCHLIGHT_SERVICE_PORT:-9393} SEARCHLIGHT_SERVICE_PORT_INT=${SEARCHLIGHT_SERVICE_PORT_INT:-19393} ELASTICSEARCH_VERSION=${ELASTICSEARCH_VERSION:-2.3.4} -ELASTICSEARCH_BASEURL=${ELASTICSEARCH_BASEURL:-https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution} -ELASTICSEARCH_BASEURL_DEB=${ELASTICSEARCH_BASEURL}/deb/elasticsearch -ELASTICSEARCH_BASEURL_RPM=${ELASTICSEARCH_BASEURL}/rpm/elasticsearch +# Base URL for ElasticSearch 5.x and 6.x +ELASTICSEARCH_BASEURL=https://artifacts.elastic.co/downloads/elasticsearch +# Base URL for ElasticSearch 2.x +ELASTICSEARCH_BASEURL_LEGACY=https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution # Helper Functions # ---------------- @@ -191,10 +192,26 @@ function init_searchlight { $SEARCHLIGHT_BIN_DIR/searchlight-manage --config-file $SEARCHLIGHT_CONF index sync --force } +# Install Searchlight's requirements +# See https://elasticsearch-py.readthedocs.io/en/master/#compatibility +function _setup_searchlight_dev { + setup_develop $SEARCHLIGHT_DIR + if [[ $ELASTICSEARCH_VERSION =~ ^5 ]]; then + echo "Installing python elasticsearch for ES 5.x" + $REQUIREMENTS_DIR/.venv/bin/edit-constraints $REQUIREMENTS_DIR/upper-constraints.txt elasticsearch + pip_install -U -r $SEARCHLIGHT_DIR/elasticsearch5.txt + elif [[ $ELASTICSEARCH_VERSION =~ ^6 ]]; then + echo "WARNING - Searchlight is not tested with ES 6.x!!!" + # echo "Installing python elasticsearch for ES 6.x" + # $REQUIREMENTS_DIR/.venv/bin/edit-constraints $REQUIREMENTS_DIR/upper-constraints.txt eleasticsearch + # pip install -c $REQUIREMENTS_DIR/upper-constraints.txt -U -r $SEARCHLIGHT_DIR/elasticsearch6.txt + fi +} + # install_searchlight - Collect source and prepare function install_searchlight { git_clone $SEARCHLIGHT_REPO $SEARCHLIGHT_DIR $SEARCHLIGHT_BRANCH - setup_develop $SEARCHLIGHT_DIR + _setup_searchlight_dev _download_elasticsearch _install_elasticsearch pip_install uwsgi @@ -256,12 +273,25 @@ function _wget_elasticsearch { } function _download_elasticsearch { - echo "Downloading elasticsearch" if is_ubuntu; then - _wget_elasticsearch $ELASTICSEARCH_BASEURL_DEB/${ELASTICSEARCH_VERSION} elasticsearch-${ELASTICSEARCH_VERSION}.deb + arch="deb" elif is_fedora; then - _wget_elasticsearch $ELASTICSEARCH_BASEURL_RPM/${ELASTICSEARCH_VERSION} elasticsearch-${ELASTICSEARCH_VERSION}.rpm + arch="rpm" + else + echo "Unknown architecture; can't download ElasticSearch" fi + ELASTICSEARCH_FILENAME=elasticsearch-${ELASTICSEARCH_VERSION}.${arch} + + if [[ $ELASTICSEARCH_VERSION =~ ^2 ]]; then + ELASTICSEARCH_URL=${ELASTICSEARCH_BASEURL_LEGACY}/${arch}/elasticsearch/${ELASTICSEARCH_VERSION} + elif [[ $ELASTICSEARCH_VERSION =~ ^5 ]]; then + ELASTICSEARCH_URL=${ELASTICSEARCH_BASEURL} + else + echo "Current Searchlight only supports ElasticSearch 2.x and 5.x" + fi + echo "Downloading ElasticSearch $ELASTICSEARCH_VERSION" + echo "ElasticSearch URL is $ELASTICSEARCH_URL" + _wget_elasticsearch $ELASTICSEARCH_URL $ELASTICSEARCH_FILENAME } function _check_elasticsearch_ready { @@ -280,7 +310,7 @@ function _start_elasticsearch { sudo /bin/systemctl start elasticsearch.service _check_elasticsearch_ready else - echo "Unsupported architecture...can not start elasticsearch." + echo "Unsupported architecture... Can not start elasticsearch." fi } @@ -296,15 +326,14 @@ function _stop_elasticsearch { } function _install_elasticsearch { - echo "Installing elasticsearch" - pip_install_gr elasticsearch + # echo "Installing elasticsearch" + # pip_install_gr elasticsearch if is_package_installed elasticsearch; then echo "Note: elasticsearch was already installed." return fi if is_ubuntu; then is_package_installed default-jdk-headless || install_package default-jdk-headless - sudo dpkg -i ${FILES}/elasticsearch-${ELASTICSEARCH_VERSION}.deb sudo update-rc.d elasticsearch defaults 95 10 elif is_fedora; then diff --git a/doc/source/install/elasticsearch.rst b/doc/source/install/elasticsearch.rst index e22cbee5..aef4813a 100644 --- a/doc/source/install/elasticsearch.rst +++ b/doc/source/install/elasticsearch.rst @@ -17,56 +17,59 @@ Installing and Configuring Elasticsearch ======================================== -The Searchlight indexing service is responsible for indexing data in -`Elasticsearch `_; -Elasticsearch has very good documentation on installation but some pointers -are provided here. -.. IMPORTANT:: We *strongly* recommend using Elasticsearch 2.x and the - accompanying python client version. Searchlight has not been tested - with v5. +The Searchlight indexing service is responsible for indexing data in +Elasticsearch [1]_; Elasticsearch has very good documentation on +installation but some pointers are provided here. + +.. IMPORTANT:: We *strongly* recommend using ElasticSearch 5.x and the + accompanying python client version [2]_. ElasticSearch 2.x is still + supported and will be removed after Stein cycle. Installation ~~~~~~~~~~~~ Elasticsearch requires a Java Runtime Environment (or Java Development Kit). OpenJDK and Oracle's Java are supported. Information on the current recommended version can -be found in the `installation instructions `_. +be found in the installation instructions [3]_. -Installing from packages -######################## +Installing from a download package +################################## -See the `latest Elasticsearch instructions `_ -for instructions about installing in Debian/Ubuntu and Red Hat/Fedora. -Installing from a package has the advantage of including scripts to run -`Elasticsearch as a service `_. - -Installing from a download -########################## Links to various formats and also older versions of Elasticsearch can be found -on the `download page `_. Once -downloaded and extracted, you can start Elasticsearch with:: +on the download page [4]_. Once downloaded and extracted, you can start +Elasticsearch with: - $ bin/elasticsearch +.. code-block:: console -For more details see the `installation instructions `_. + bin/elasticsearch -Quick command line example with 2.3.4: +For more details see the installation instructions [5]_ + +Quick command line example with 5.6.11: .. note:: Do the following commands as "root" or via sudo -Download the ES package:: +Download the ES package: - $ cd ~ - $ wget https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/2.3.4/elasticsearch-2.3.4.deb - $ sudo dpkg -i elasticsearch-2.3.4.deb - $ sudo update-rc.d elasticsearch defaults 95 10 - $ sudo /etc/init.d/elasticsearch start +.. code-block:: console + + cd ~ + wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.6.11.deb + sudo dpkg -i elasticsearch-5.6.11.deb + sudo update-rc.d elasticsearch defaults 95 10 + sudo /etc/init.d/elasticsearch start + +.. note:: + + If you install ElasticSearch 5.x, You should install + python-elasticsearch>=5.0.0. global-requirements currently constrains it to v2.x Configuration ~~~~~~~~~~~~~ + Elasticsearch comes with very a very sensible default configuration that allows for clustering and high performance out of the box. There are some settings, both general and specific to Searchlight's indexing service, that might @@ -78,6 +81,7 @@ a superset of JSON. Indices ####### + Elasticsearch (and Lucene) store information in indices. Within an index can be one or more document types. Searchlight's indexing service uses an index per service that has a plugin available, and each plugin generally will have @@ -86,7 +90,9 @@ its own document type. For instance, the glance plugin has *glance.image* and Elasticsearch it may make sense to change the default sharing and replication mechanism. We also recommend disabling implicit index creation, though if you are sharing an Elasticsearch installation this may be inadvisable. The -following options control indexing behavior:: +following options control indexing behavior: + +.. code-block:: yaml # Number of shards for each index (performance) index.number_of_shards: 5 @@ -99,10 +105,13 @@ following options control indexing behavior:: Index settings ************** + In addition to server-wide index settings it's possible to configure Searchlight to apply settings to indices it creates with ``searchlight-manage``. Index settings can be specified as follows in -``searchlight.conf``:: +``searchlight.conf``: + +.. code-block:: yaml [elasticsearch] index_settings = refresh_interval:2s,number_of_replicas:1 @@ -115,30 +124,32 @@ Index settings are applied at creation time and so are not limited to the are created. If you wish to update settings for an existing index, you should use the Elasticsearch API to do so or reindex. -See also: - -* http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-index\_.html -* http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/indices-update-settings.html +See also [9]_, [10]_. Scripts ####### -The scripting module allows to use scripts in order to evaluate custom expressions. -Scripting is turned off by default in elasticsearch latest versions. -Searchlight doesn't allow scripts in the search api but requires scripts to sync Index updates -from notifications. For security purpose index updates are allowed only for admin role:: + +The scripting module allows to use scripts in order to evaluate custom +expressions. Scripting is turned off by default in elasticsearch latest +versions. Searchlight doesn't allow scripts in the search api but requires +scripts to sync Index updates from notifications. For security purpose index +updates are allowed only for admin role. + +.. code-block:: yaml script.engine.groovy.inline.update: on -See also: - -* https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-scripting.html#modules-scripting +See also [8]_. Development ########### + For development, Elasticsearch's default configuration is overkill. It's possible to run Elasticsearch with a much lower memory footprint than by default, and you may wish to disable clustering behavior. +.. code-block:: yaml + # Configures elasticsearch as a single node (no discovery) node.local: true @@ -148,9 +159,12 @@ default, and you may wish to disable clustering behavior. JVM settings ************ + Setting the ES_HEAP_SIZE environment variable will restrict how much memory Elasticsearch uses, equivalent to setting -Xmx and -Xms to the same value for -the Java runtime. For development you can set it as low as a few tens of MB:: +the Java runtime. For development you can set it as low as a few tens of MB: + +.. code-block:: console export ES_HEAP_SIZE=40m @@ -159,7 +173,10 @@ requires memory on top of that. Production ########## -Some settings you may wish to change for production:: + +Some settings you may wish to change for production: + +.. code-block:: yaml # Cluster name is used by cluster discovery; it's important to ensure # this is set across all nodes you wish to be in the cluster @@ -187,23 +204,43 @@ Some settings you may wish to change for production:: # ES_HEAP_SIZE appropriately (see below). Linux only. bootstrap.mlockall: true -For more details see Elasticsearch's `configuration information `_. +For more details see Elasticsearch's configuration information [6]_. Specifying nodes in a cluster ***************************** + Elasticsearch's default discovery relies on multicast requests. If this isn't -a good fit, you can use unicast discovery:: +a good fit, you can use unicast discovery: + +.. code-block:: yaml discovery.zen.ping.multicast.enabled: false discovery.zen.ping.unicast.hosts: ['w.x.y.z', 'w.x.y.z'] -See ``_ -for more details. +See [7]_ for more details. JVM settings ************ + For production, Elasticsearch recommends setting the ES_HEAP_SIZE environment -variable to a value around 60% of a dedicated machine's memory:: +variable to a value around 60% of a dedicated machine's memory: + +.. code-block:: yaml export ES_HEAP_SIZE=2g + + +References +~~~~~~~~~~ + +.. [1] https://www.elastic.co/ +.. [2] https://elasticsearch-py.readthedocs.io/en/master/#compatibility +.. [3] https://www.elastic.co/guide/en/elasticsearch/reference/current/setup.html +.. [4] https://www.elastic.co/downloads/elasticsearch +.. [5] https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html +.. [6] https://www.elastic.co/guide/en/elasticsearch/reference/current/settings.html +.. [7] https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-discovery-zen.html +.. [8] https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-scripting.html#modules-scripting +.. [9] https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index\_.html +.. [10] https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-update-settings.html diff --git a/doc/source/install/uwsgi.rst b/doc/source/install/uwsgi.rst index ff9a8914..2c9b9281 100644 --- a/doc/source/install/uwsgi.rst +++ b/doc/source/install/uwsgi.rst @@ -16,7 +16,7 @@ Running Searchlight API using uWSGI -======================================== +=================================== The recommended way to deploy Searchlight is have a web server such as Apache or nginx to handle http requests and proxy these requests to Searchlight WSGI app running in uWSGI. Searchlight comes with some configuration templates on diff --git a/elasticsearch5.txt b/elasticsearch5.txt new file mode 100644 index 00000000..3725739f --- /dev/null +++ b/elasticsearch5.txt @@ -0,0 +1,2 @@ +# Elasticsearch 5.x +elasticsearch>=5.0.0,<6.0.0 diff --git a/elasticsearch6.txt b/elasticsearch6.txt new file mode 100644 index 00000000..cc46dba1 --- /dev/null +++ b/elasticsearch6.txt @@ -0,0 +1,2 @@ +# Elasticsearch 6.x +elasticsearch>=6.0.0,<7.0.0 diff --git a/releasenotes/notes/add-devstack-elasticsearch-5x-48934b931a31c823.yaml b/releasenotes/notes/add-devstack-elasticsearch-5x-48934b931a31c823.yaml new file mode 100644 index 00000000..5e93448a --- /dev/null +++ b/releasenotes/notes/add-devstack-elasticsearch-5x-48934b931a31c823.yaml @@ -0,0 +1,65 @@ +--- +prelude: > + Replace this text with content to appear at the top of the section for this + release. All of the prelude content is merged together and then rendered + separately from the items listed in other parts of the file, so the text + needs to be worded so that both the prelude and the other items make sense + when read independently. This may mean repeating some details. Not every + release note requires a prelude. Usually only notes describing major + features or adding release theme details should have a prelude. +features: + - | + List new features here, or remove this section. All of the list items in + this section are combined when the release notes are rendered, so the text + needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. +issues: + - | + List known issues here, or remove this section. All of the list items in + this section are combined when the release notes are rendered, so the text + needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. +upgrade: + - | + List upgrade notes here, or remove this section. All of the list items in + this section are combined when the release notes are rendered, so the text + needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. +deprecations: + - | + List deprecations notes here, or remove this section. All of the list + items in this section are combined when the release notes are rendered, so + the text needs to be worded so that it does not depend on any information + only available in another section, such as the prelude. This may mean + repeating some details. +critical: + - | + Add critical notes here, or remove this section. All of the list items in + this section are combined when the release notes are rendered, so the text + needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. +security: + - | + Add security notes here, or remove this section. All of the list items in + this section are combined when the release notes are rendered, so the text + needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. +fixes: + - | + Add normal bug fixes here, or remove this section. All of the list items + in this section are combined when the release notes are rendered, so the + text needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. +other: + - | + Add other notes here, or remove this section. All of the list items in + this section are combined when the release notes are rendered, so the text + needs to be worded so that it does not depend on any information only + available in another section, such as the prelude. This may mean repeating + some details. diff --git a/requirements.txt b/requirements.txt index 212fde2e..19021ba2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -39,7 +39,15 @@ oslo.serialization!=2.19.1,>=2.18.0 # Apache-2.0 osprofiler>=1.4.0 # Apache-2.0 -elasticsearch<=3.0.0,>=2.0.0 # Apache-2.0 +# ElasticSearch 6.x +# elasticsearch>=6.0.0,<7.0.0 +# ElasticSearch 5.x +# elasticsearch>=5.0.0,<6.0.0 +# ElasticSearch 2.x +# elasticsearch>=2.0.0,<3.0.0 +# See: +# https://elasticsearch-py.readthedocs.io/en/master/#compatibility +elasticsearch<3.0.0,>=2.0.0 # Apache-2.0 # Clients required for indexing plugins python-designateclient>=2.7.0 # Apache-2.0 diff --git a/tools/test-setup.sh b/tools/test-setup.sh index 1a910673..140a50d3 100755 --- a/tools/test-setup.sh +++ b/tools/test-setup.sh @@ -13,7 +13,16 @@ if [[ $VERSION_CODENAME = bionic ]]; then else sudo apt-get install -y default-jre fi -wget https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/2.3.4/elasticsearch-2.3.4.deb -sudo dpkg -i elasticsearch-2.3.4.deb + +ELASTICSEARCH_MAJOR_VERSION=${ELASTICSEARCH_MAJOR_VERSION:"2"} +if [[ "$ELASTICSEARCH_MAJOR_VERSION" == "5" ]]; then + echo "Downloading and installing elasticsearch 5.6.11" + wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.6.11.deb + sudo dpkg -i elasticsearch-5.6.11.deb +else + echo "Downloading and installing elasticsearch 2.3.4" + wget https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/2.3.4/elasticsearch-2.3.4.deb + sudo dpkg -i elasticsearch-2.3.4.deb +fi # Make 'elasticsearch' binary callable from within functional tests sudo ln -s /usr/share/elasticsearch/bin/elasticsearch /usr/local/bin/elasticsearch