Force Elasticsearch indexing

Aded a new option to the searchlight-maange index
command. This will force all indexing to occur
from Elasticsearch. No plugin APIs will be used.

Change-Id: I1c18b2ee4e916430de0cdfd162494d6cab4babb1
Implements: blueprint allow-reindexing-into-new-mapping

doc/source/indexingservice.rst

@@ -209,6 +209,27 @@ setting can be added to the Searchlight configuration file::
 
     logging_default_format_string = %(asctime)s.%(msecs)03d %(process)d %(thread)d %(threadName)s %(levelname)s %(name)s [-] %(instance)s%(message)s
 
+Force Elasticsearch indexing
+----------------------------
+The Newton Searchlight release introduced the ability to reindex
+from Elasticsearch only, bypassing the plugin APIs altogether.
+This option is useful if there has been a change to the mapping
+definitions or the index settings. This functionality is enabled
+with the option ``--apply-mapping-changes`` for the ``index`` command.
+
+A sample usage would be::
+
+    $ searchlight-manage index aliases --apply-mapping-changes
+
+The ``--type`` option is not compatible with the ``--apply-mapping-changes``
+option. Specifying both options on the command line will result in an error.
+
+.. warning::
+
+    The resource group cannot be changed when using this option.
+    If you do change the resource group, the underlying index will
+    be changed and will result in an empty index.
+
 Elasticsearch Index Cleanup
 ===========================
 

releasenotes/notes/bp-allow-reindexing-into-new-mapping-bd78e94842fd2705.yaml

@@ -0,0 +1,11 @@
+---
+prelude: >
+    Add new functionality to searchlight-manage
+    to force a re-indexing from Elasticsearch only.
+    This option allows new mappings to be applied to
+    the Elasticsearch indices without the overhead of
+    retrieving data from the plugin APIs.
+features:
+  - Add a new option to the searchlight-manage index
+    command to force a re-indexing from Elasticsearch
+    only.  The plugin APIs will not be used with option.

searchlight/cmd/manage.py

@@ -118,7 +118,9 @@ class IndexCommands(object):
           help='Index only this type (or a comma separated list)')
     @args('--force', dest='force', action='store_true',
           help="Don't prompt (answer 'y')")
-    def sync(self, group=None, _type=None, force=False):
+    @args('--apply-mapping-changes', dest='force_es', action='store_true',
+          help="Use existing indexed data but apply mappings and settings")
+    def sync(self, group=None, _type=None, force=False, force_es=False):
         def wait_for_threads():
             """Patiently wait for all running threads to complete.
             """
@@ -156,6 +158,13 @@ class IndexCommands(object):
 
             sys.exit(0)
 
+        if force_es and _type:
+            # The user cannot specify both of these options simultaneously.
+            print("\nInvalid set of options.")
+            print("Cannot specify both '--type' and '--apply-mapping-changes "
+                  "simultaneously.\n")
+            sys.exit(1)
+
         try:
             max_workers = cfg.CONF.manage.workers
         except cfg.ConfigFileValueError as e:
@@ -242,12 +251,18 @@ class IndexCommands(object):
         # As an optimization, if any types are explicitly requested, we
         # will index them from their service APIs. The rest will be
         # indexed from an existing ES index, if one exists.
+        #
+        # Also, if force_es is set the user wishes to use ES exclusively
+        # as the source for all data. This implies everything in the
+        # es_reindex list and nothing in the plugins_to_index list.
         es_reindex = []
         plugins_to_index = copy.copy(plugins_list)
-        if _type:
+        if _type or force_es:
             for resource_type, ext in plugins_list:
                 doc_type = ext.obj.get_document_type()
 
+                # If force_es is set, then "_type" is None. Always do this.
+                # If force_es is None, then "_type" is set. Adjust as needed.
                 if doc_type not in _type:
                     es_reindex.append(doc_type)
                     # Don't reindex this type