Introduce gerrit-index module

gerrit-index module is meant to keep common parts for Lucene and
Elasticsearch indexes implementation.

Change-Id: Ie265dd9a262bdfec4b15e8abda885ffae7bb1aa4
Signed-off-by: Dariusz Luksza <dluksza@collab.net>

gerrit-elasticsearch/BUCK

@@ -4,11 +4,11 @@ java_library(
   deps = [
     '//gerrit-antlr:query_exception',
     '//gerrit-extension-api:api',
-    '//gerrit-lucene:query_builder',
-    '//gerrit-lucene:lucene',
+    '//gerrit-lucene:lucene', # only for LuceneAccountIndex
     '//gerrit-reviewdb:client',
     '//gerrit-reviewdb:server',
     '//gerrit-server:server',
+    '//gerrit-index:index',
     '//lib:gson',
     '//lib:guava',
     '//lib:gwtorm',

gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/AbstractElasticIndex.java

@@ -16,11 +16,11 @@ package com.google.gerrit.elasticsearch;
 
 import static com.google.common.base.MoreObjects.firstNonNull;
 import static com.google.common.base.Preconditions.checkState;
-import static com.google.gerrit.lucene.AbstractLuceneIndex.setReady;
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
 
 import com.google.common.base.Strings;
 import com.google.common.collect.Iterables;
+import com.google.gerrit.index.IndexUtils;
 import com.google.gerrit.server.config.GerritServerConfig;
 import com.google.gerrit.server.config.SitePaths;
 import com.google.gerrit.server.index.FieldDef.FillArgs;
@@ -110,7 +110,7 @@ abstract class AbstractElasticIndex<K, V> implements Index<K, V> {
 
   @Override
   public void markReady(boolean ready) throws IOException {
-    setReady(sitePaths, indexName, schema.getVersion(), ready);
+    IndexUtils.setReady(sitePaths, indexName, schema.getVersion(), ready);
   }
 
   @Override

gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticChangeIndex.java

@@ -24,7 +24,7 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
-import com.google.gerrit.lucene.LuceneChangeIndex;
+import com.google.gerrit.index.IndexUtils;
 import com.google.gerrit.reviewdb.client.Account;
 import com.google.gerrit.reviewdb.client.Change;
 import com.google.gerrit.reviewdb.client.Change.Id;
@@ -228,7 +228,7 @@ class ElasticChangeIndex extends AbstractElasticIndex<Change.Id, ChangeData>
         sort.setIgnoreUnmapped();
       }
       QueryBuilder qb = queryBuilder.toQueryBuilder(p);
-      fields = LuceneChangeIndex.fields(opts);
+      fields = IndexUtils.fields(opts);
       SearchSourceBuilder searchSource = new SearchSourceBuilder()
           .query(qb)
           .from(opts.start())

gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticIndexModule.java

@@ -14,9 +14,9 @@
 
 package com.google.gerrit.elasticsearch;
 
+import com.google.gerrit.index.SingleVersionModule;
 import com.google.gerrit.lifecycle.LifecycleModule;
 import com.google.gerrit.lucene.LuceneAccountIndex;
-import com.google.gerrit.lucene.LuceneIndexModule.SingleVersionModule;
 import com.google.gerrit.server.config.GerritServerConfig;
 import com.google.gerrit.server.index.IndexConfig;
 import com.google.gerrit.server.index.IndexModule;

gerrit-index/BUCK

@@ -0,0 +1,13 @@
+java_library(
+  name = 'index',
+  srcs = glob(['src/main/java/**/*.java']),
+  deps = [
+    '//gerrit-extension-api:api',
+    '//gerrit-server:server',
+    '//gerrit-patch-jgit:server',
+    '//lib/guice:guice',
+    '//lib/jgit/org.eclipse.jgit:jgit',
+    '//lib:guava',
+  ],
+  visibility = ['PUBLIC'],
+)

gerrit-index/src/main/java/com/google/gerrit/index/GerritIndexStatus.java

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package com.google.gerrit.lucene;
+package com.google.gerrit.index;
 
 import com.google.common.primitives.Ints;
 import com.google.gerrit.server.config.SitePaths;
@@ -24,13 +24,13 @@ import org.eclipse.jgit.util.FS;
 
 import java.io.IOException;
 
-class GerritIndexStatus {
+public class GerritIndexStatus {
   private static final String SECTION = "index";
   private static final String KEY_READY = "ready";
 
   private final FileBasedConfig cfg;
 
-  GerritIndexStatus(SitePaths sitePaths)
+  public GerritIndexStatus(SitePaths sitePaths)
       throws ConfigInvalidException, IOException {
     cfg = new FileBasedConfig(
         sitePaths.index_dir.resolve("gerrit_index.config").toFile(),
@@ -39,16 +39,16 @@ class GerritIndexStatus {
     convertLegacyConfig();
   }
 
-  void setReady(String indexName, int version, boolean ready) {
+  public void setReady(String indexName, int version, boolean ready) {
     cfg.setBoolean(SECTION, indexDirName(indexName, version), KEY_READY, ready);
   }
 
-  boolean getReady(String indexName, int version) {
+  public boolean getReady(String indexName, int version) {
     return cfg.getBoolean(SECTION, indexDirName(indexName, version), KEY_READY,
         false);
   }
 
-  void save() throws IOException {
+  public void save() throws IOException {
     cfg.save();
   }
 
@@ -62,8 +62,8 @@ class GerritIndexStatus {
         if (ready != null) {
           dirty = false;
           cfg.unset(SECTION, subsection, KEY_READY);
-          cfg.setString(SECTION,
-              indexDirName(ChangeSchemaDefinitions.NAME, v), KEY_READY, ready);
+          cfg.setString(SECTION, indexDirName(ChangeSchemaDefinitions.NAME, v),
+              KEY_READY, ready);
         }
       }
     }

gerrit-index/src/main/java/com/google/gerrit/index/IndexUtils.java

@@ -0,0 +1,67 @@
+// Copyright (C) 2013 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.index;
+
+import static com.google.gerrit.server.index.change.ChangeField.CHANGE;
+import static com.google.gerrit.server.index.change.ChangeField.LEGACY_ID;
+import static com.google.gerrit.server.index.change.ChangeField.PROJECT;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import com.google.gerrit.server.config.SitePaths;
+import com.google.gerrit.server.index.QueryOptions;
+
+import org.eclipse.jgit.errors.ConfigInvalidException;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+
+public final class IndexUtils {
+  public static final Map<String, String> CUSTOM_CHAR_MAPPING =
+      ImmutableMap.of("_", " ", ".", " ");
+
+  public static void setReady(SitePaths sitePaths, String name, int version,
+      boolean ready) throws IOException {
+    try {
+      GerritIndexStatus cfg = new GerritIndexStatus(sitePaths);
+      cfg.setReady(name, version, ready);
+      cfg.save();
+    } catch (ConfigInvalidException e) {
+      throw new IOException(e);
+    }
+  }
+
+  public static Set<String> fields(QueryOptions opts) {
+    // Ensure we request enough fields to construct a ChangeData. We need both
+    // change ID and project, which can either come via the Change field or
+    // separate fields.
+    Set<String> fs = opts.fields();
+    if (fs.contains(CHANGE.getName())) {
+      // A Change is always sufficient.
+      return fs;
+    }
+    if (fs.contains(PROJECT.getName()) && fs.contains(LEGACY_ID.getName())) {
+      return fs;
+    }
+    return Sets.union(fs,
+        ImmutableSet.of(LEGACY_ID.getName(), PROJECT.getName()));
+  }
+
+  private IndexUtils() {
+    // hide default constructor
+  }
+}

gerrit-index/src/main/java/com/google/gerrit/index/SingleVersionModule.java

@@ -0,0 +1,106 @@
+// Copyright (C) 2013 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.index;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.gerrit.extensions.events.LifecycleListener;
+import com.google.gerrit.lifecycle.LifecycleModule;
+import com.google.gerrit.server.config.GerritServerConfig;
+import com.google.gerrit.server.index.Index;
+import com.google.gerrit.server.index.IndexDefinition;
+import com.google.gerrit.server.index.Schema;
+import com.google.inject.Inject;
+import com.google.inject.ProvisionException;
+import com.google.inject.Singleton;
+import com.google.inject.TypeLiteral;
+import com.google.inject.name.Named;
+import com.google.inject.name.Names;
+
+import org.eclipse.jgit.lib.Config;
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
+
+@Singleton
+public class SingleVersionModule extends LifecycleModule {
+  static final String SINGLE_VERSIONS = "LuceneIndexModule/SingleVersions";
+
+  private final Map<String, Integer> singleVersions;
+
+  public SingleVersionModule(Map<String, Integer> singleVersions) {
+    this.singleVersions = singleVersions;
+  }
+
+  @Override
+  public void configure() {
+    listener().to(SingleVersionListener.class);
+    bind(new TypeLiteral<Map<String, Integer>>() {})
+        .annotatedWith(Names.named(SINGLE_VERSIONS))
+        .toInstance(singleVersions);
+  }
+
+  @Singleton
+  static class SingleVersionListener implements LifecycleListener {
+    private final Set<String> disabled;
+    private final Collection<IndexDefinition<?, ?, ?>> defs;
+    private final Map<String, Integer> singleVersions;
+
+    @Inject
+    SingleVersionListener(
+        @GerritServerConfig Config cfg,
+        Collection<IndexDefinition<?, ?, ?>> defs,
+        @Named(SINGLE_VERSIONS) Map<String, Integer> singleVersions) {
+      this.defs = defs;
+      this.singleVersions = singleVersions;
+
+      disabled = ImmutableSet.copyOf(
+          cfg.getStringList("index", null, "testDisable"));
+    }
+
+    @Override
+    public void start() {
+      for (IndexDefinition<?, ?, ?> def : defs) {
+        start(def);
+      }
+    }
+
+    private <K, V, I extends Index<K, V>> void start(
+        IndexDefinition<K, V, I> def) {
+      if (disabled.contains(def.getName())) {
+        return;
+      }
+      Schema<V> schema;
+      Integer v = singleVersions.get(def.getName());
+      if (v == null) {
+        schema = def.getLatest();
+      } else {
+        schema = def.getSchemas().get(v);
+        if (schema == null) {
+          throw new ProvisionException(String.format(
+                "Unrecognized %s schema version: %s", def.getName(), v));
+        }
+      }
+      I index = def.getIndexFactory().create(schema);
+      def.getIndexCollection().setSearchIndex(index);
+      def.getIndexCollection().addWriteIndex(index);
+    }
+
+    @Override
+    public void stop() {
+      // Do nothing; indexes are closed by IndexCollection.
+    }
+  }
+}

gerrit-lucene/BUCK

@@ -27,6 +27,7 @@ java_library(
     '//gerrit-extension-api:api',
     '//gerrit-reviewdb:server',
     '//gerrit-server:server',
+    '//gerrit-index:index',
     '//lib:guava',
     '//lib:gwtorm',
     '//lib/guice:guice',

gerrit-lucene/src/main/java/com/google/gerrit/lucene/AbstractLuceneIndex.java

@@ -25,6 +25,7 @@ import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.ListeningExecutorService;
 import com.google.common.util.concurrent.MoreExecutors;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.google.gerrit.index.IndexUtils;
 import com.google.gerrit.server.config.SitePaths;
 import com.google.gerrit.server.index.FieldDef;
 import com.google.gerrit.server.index.FieldDef.FillArgs;
@@ -51,7 +52,6 @@ import org.apache.lucene.search.ReferenceManager.RefreshListener;
 import org.apache.lucene.search.SearcherFactory;
 import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
-import org.eclipse.jgit.errors.ConfigInvalidException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -75,17 +75,6 @@ public abstract class AbstractLuceneIndex<K, V> implements Index<K, V> {
     return f.getName() + "_SORT";
   }
 
-  public static void setReady(SitePaths sitePaths, String name, int version,
-      boolean ready) throws IOException {
-    try {
-      GerritIndexStatus cfg = new GerritIndexStatus(sitePaths);
-      cfg.setReady(name, version, ready);
-      cfg.save();
-    } catch (ConfigInvalidException e) {
-      throw new IOException(e);
-    }
-  }
-
   private final Schema<V> schema;
   private final SitePaths sitePaths;
   private final Directory dir;
@@ -198,7 +187,7 @@ public abstract class AbstractLuceneIndex<K, V> implements Index<K, V> {
 
   @Override
   public void markReady(boolean ready) throws IOException {
-    setReady(sitePaths, name, schema.getVersion(), ready);
+    IndexUtils.setReady(sitePaths, name, schema.getVersion(), ready);
   }
 
   @Override

gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java

@@ -18,7 +18,6 @@ import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.gerrit.lucene.AbstractLuceneIndex.sortFieldName;
 import static com.google.gerrit.lucene.LuceneVersionManager.CHANGES_PREFIX;
 import static com.google.gerrit.server.git.QueueProvider.QueueType.INTERACTIVE;
-import static com.google.gerrit.server.index.change.ChangeField.CHANGE;
 import static com.google.gerrit.server.index.change.ChangeField.LEGACY_ID;
 import static com.google.gerrit.server.index.change.ChangeField.PROJECT;
 import static com.google.gerrit.server.index.change.ChangeIndexRewriter.CLOSED_STATUSES;
@@ -27,8 +26,6 @@ import static com.google.gerrit.server.index.change.ChangeIndexRewriter.OPEN_STA
 import com.google.common.base.Throwables;
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.FluentIterable;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Multimap;
@@ -36,6 +33,7 @@ import com.google.common.collect.Sets;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.gerrit.index.IndexUtils;
 import com.google.gerrit.reviewdb.client.Account;
 import com.google.gerrit.reviewdb.client.Change;
 import com.google.gerrit.reviewdb.client.PatchSet;
@@ -92,7 +90,6 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
@@ -112,8 +109,6 @@ public class LuceneChangeIndex implements ChangeIndex {
 
   public static final String CHANGES_OPEN = "open";
   public static final String CHANGES_CLOSED = "closed";
-  public static final Map<String, String> CUSTOM_CHAR_MAPPING = ImmutableMap.of(
-      "_", " ", ".", " ");
 
   static final String UPDATED_SORT_FIELD =
       sortFieldName(ChangeField.UPDATED);
@@ -321,7 +316,7 @@ public class LuceneChangeIndex implements ChangeIndex {
         throw new OrmException("interrupted");
       }
 
-      final Set<String> fields = fields(opts);
+      final Set<String> fields = IndexUtils.fields(opts);
       return new ChangeDataResults(
           executor.submit(new Callable<List<Document>>() {
             @Override
@@ -409,22 +404,6 @@ public class LuceneChangeIndex implements ChangeIndex {
     }
   }
 
-  public static Set<String> fields(QueryOptions opts) {
-    // Ensure we request enough fields to construct a ChangeData. We need both
-    // change ID and project, which can either come via the Change field or
-    // separate fields.
-    Set<String> fs = opts.fields();
-    if (fs.contains(CHANGE.getName())) {
-      // A Change is always sufficient.
-      return fs;
-    }
-    if (fs.contains(PROJECT.getName()) && fs.contains(LEGACY_ID.getName())) {
-      return fs;
-    }
-    return Sets.union(fs,
-        ImmutableSet.of(LEGACY_ID.getName(), PROJECT.getName()));
-  }
-
   private static Multimap<String, IndexableField> fields(Document doc,
       Set<String> fields) {
     Multimap<String, IndexableField> stored =

gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java

@@ -15,37 +15,23 @@
 package com.google.gerrit.lucene;
 
 import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-import com.google.gerrit.extensions.events.LifecycleListener;
+import com.google.gerrit.index.SingleVersionModule;
 import com.google.gerrit.lifecycle.LifecycleModule;
 import com.google.gerrit.server.config.GerritServerConfig;
-import com.google.gerrit.server.index.Index;
 import com.google.gerrit.server.index.IndexConfig;
-import com.google.gerrit.server.index.IndexDefinition;
 import com.google.gerrit.server.index.IndexModule;
-import com.google.gerrit.server.index.Schema;
 import com.google.gerrit.server.index.account.AccountIndex;
 import com.google.gerrit.server.index.change.ChangeIndex;
-import com.google.inject.Inject;
 import com.google.inject.Provides;
-import com.google.inject.ProvisionException;
 import com.google.inject.Singleton;
-import com.google.inject.TypeLiteral;
 import com.google.inject.assistedinject.FactoryModuleBuilder;
-import com.google.inject.name.Named;
-import com.google.inject.name.Names;
 
 import org.apache.lucene.search.BooleanQuery;
 import org.eclipse.jgit.lib.Config;
 
-import java.util.Collection;
 import java.util.Map;
-import java.util.Set;
 
 public class LuceneIndexModule extends LifecycleModule {
-  private static final String SINGLE_VERSIONS =
-      "LuceneIndexModule/SingleVersions";
-
   public static LuceneIndexModule singleVersionAllLatest(int threads) {
     return new LuceneIndexModule(ImmutableMap.<String, Integer> of(), threads);
   }
@@ -104,72 +90,4 @@ public class LuceneIndexModule extends LifecycleModule {
       listener().to(LuceneVersionManager.class);
     }
   }
-
-  public static class SingleVersionModule extends LifecycleModule {
-    private final Map<String, Integer> singleVersions;
-
-    public SingleVersionModule(Map<String, Integer> singleVersions) {
-      this.singleVersions = singleVersions;
-    }
-
-    @Override
-    public void configure() {
-      listener().to(SingleVersionListener.class);
-      bind(new TypeLiteral<Map<String, Integer>>() {})
-          .annotatedWith(Names.named(SINGLE_VERSIONS))
-          .toInstance(singleVersions);
-    }
-  }
-
-  @Singleton
-  private static class SingleVersionListener implements LifecycleListener {
-    private final Set<String> disabled;
-    private final Collection<IndexDefinition<?, ?, ?>> defs;
-    private final Map<String, Integer> singleVersions;
-
-    @Inject
-    SingleVersionListener(
-        @GerritServerConfig Config cfg,
-        Collection<IndexDefinition<?, ?, ?>> defs,
-        @Named(SINGLE_VERSIONS) Map<String, Integer> singleVersions) {
-      this.defs = defs;
-      this.singleVersions = singleVersions;
-
-      disabled = ImmutableSet.copyOf(
-          cfg.getStringList("index", null, "testDisable"));
-    }
-
-    @Override
-    public void start() {
-      for (IndexDefinition<?, ?, ?> def : defs) {
-        start(def);
-      }
-    }
-
-    private <K, V, I extends Index<K, V>> void start(
-        IndexDefinition<K, V, I> def) {
-      if (disabled.contains(def.getName())) {
-        return;
-      }
-      Schema<V> schema;
-      Integer v = singleVersions.get(def.getName());
-      if (v == null) {
-        schema = def.getLatest();
-      } else {
-        schema = def.getSchemas().get(v);
-        if (schema == null) {
-          throw new ProvisionException(String.format(
-                "Unrecognized %s schema version: %s", def.getName(), v));
-        }
-      }
-      I index = def.getIndexFactory().create(schema);
-      def.getIndexCollection().setSearchIndex(index);
-      def.getIndexCollection().addWriteIndex(index);
-    }
-
-    @Override
-    public void stop() {
-      // Do nothing; indexes are closed by IndexCollection.
-    }
-  }
 }

gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneVersionManager.java

@@ -20,6 +20,7 @@ import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.primitives.Ints;
 import com.google.gerrit.extensions.events.LifecycleListener;
+import com.google.gerrit.index.GerritIndexStatus;
 import com.google.gerrit.server.config.GerritServerConfig;
 import com.google.gerrit.server.config.SitePaths;
 import com.google.gerrit.server.index.Index;

gerrit-pgm/BUCK

@@ -47,7 +47,7 @@ java_library(
     ':init-api',
     ':util',
     '//gerrit-common:annotations',
-    '//gerrit-lucene:lucene',
+    '//gerrit-index:index',
     '//lib:args4j',
     '//lib:derby',
     '//lib:gwtjsonrpc',

gerrit-pgm/src/main/java/com/google/gerrit/pgm/init/InitIndex.java

@@ -16,7 +16,7 @@ package com.google.gerrit.pgm.init;
 
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Sets;
-import com.google.gerrit.lucene.AbstractLuceneIndex;
+import com.google.gerrit.index.IndexUtils;
 import com.google.gerrit.pgm.init.api.ConsoleUI;
 import com.google.gerrit.pgm.init.api.InitFlags;
 import com.google.gerrit.pgm.init.api.InitStep;
@@ -72,7 +72,7 @@ class InitIndex implements InitStep {
 
     if ((site.isNew || isEmptySite()) && type == IndexType.LUCENE) {
       for (SchemaDefinitions<?> def : IndexModule.ALL_SCHEMA_DEFS) {
-        AbstractLuceneIndex.setReady(
+        IndexUtils.setReady(
             site, def.getName(), def.getLatest().getVersion(), true);
       }
     } else {