Implement Account index for ElasticSearch

Initial implementation of the account index for ElasticSearch.

TODO: Define schema mappings when Gerrit starts up
TODO: Add support for online reindex

Change-Id: I21034daa71c36a1e462e3107b0c9a7ba6392f379
Signed-off-by: Dariusz Luksza <dluksza@collab.net>
Signed-off-by: David Pursehouse <dpursehouse@collab.net>

Documentation/config-gerrit.txt

@@ -2618,8 +2618,8 @@ Sample Lucene index configuration:
 
 ==== Elasticsearch configuration
 
-WARNING: ElasticSearch implementation is incomplete. Right now it is
+WARNING: The Elasticsearch support is incomplete. Online reindexing
-still using parts of Lucene index.
+is not implemented yet.
 
 Open and closed changes are indexed in a single index, separated
 into types 'open_changes' and 'closed_changes' respectively.

gerrit-elasticsearch/BUCK

@@ -4,7 +4,6 @@ java_library(
   deps = [
     '//gerrit-antlr:query_exception',
     '//gerrit-extension-api:api',
-    '//gerrit-lucene:lucene', # only for LuceneAccountIndex
     '//gerrit-reviewdb:client',
     '//gerrit-reviewdb:server',
     '//gerrit-server:server',

gerrit-elasticsearch/BUILD

@@ -4,7 +4,6 @@ java_library(
   deps = [
     '//gerrit-antlr:query_exception',
     '//gerrit-extension-api:api',
-    '//gerrit-lucene:lucene', # only for LuceneAccountIndex
     '//gerrit-reviewdb:client',
     '//gerrit-reviewdb:server',
     '//gerrit-server:server',

gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticAccountIndex.java

@@ -0,0 +1,234 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.elasticsearch;
+
+import static com.google.gerrit.server.index.account.AccountField.ID;
+import static com.google.gson.FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
+import com.google.gerrit.reviewdb.client.Account;
+import com.google.gerrit.reviewdb.client.Account.Id;
+import com.google.gerrit.server.account.AccountCache;
+import com.google.gerrit.server.account.AccountState;
+import com.google.gerrit.server.config.GerritServerConfig;
+import com.google.gerrit.server.config.SitePaths;
+import com.google.gerrit.server.index.FieldDef.FillArgs;
+import com.google.gerrit.server.index.IndexUtils;
+import com.google.gerrit.server.index.QueryOptions;
+import com.google.gerrit.server.index.Schema;
+import com.google.gerrit.server.index.account.AccountField;
+import com.google.gerrit.server.index.account.AccountIndex;
+import com.google.gerrit.server.query.DataSource;
+import com.google.gerrit.server.query.Predicate;
+import com.google.gerrit.server.query.QueryParseException;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gwtorm.server.OrmException;
+import com.google.gwtorm.server.ResultSet;
+import com.google.inject.assistedinject.Assisted;
+import com.google.inject.assistedinject.AssistedInject;
+
+import org.eclipse.jgit.lib.Config;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import io.searchbox.client.JestResult;
+import io.searchbox.core.Bulk;
+import io.searchbox.core.Bulk.Builder;
+import io.searchbox.core.Search;
+import io.searchbox.core.search.sort.Sort;
+import io.searchbox.core.search.sort.Sort.Sorting;
+
+class ElasticAccountIndex extends AbstractElasticIndex<Account.Id, AccountState>
+    implements AccountIndex {
+  static class AccountMapping {
+    MappingProperties accounts;
+
+    AccountMapping(Schema<AccountState> schema) {
+      this.accounts = ElasticMapping.createMapping(schema);
+    }
+  }
+
+  static final String ACCOUNTS = "accounts";
+  static final String ACCOUNTS_PREFIX = ACCOUNTS + "_";
+
+  private static final Logger log =
+      LoggerFactory.getLogger(ElasticAccountIndex.class);
+
+  private final Gson gson;
+  private final AccountMapping mapping;
+  private final AccountCache accountCache;
+  private final ElasticQueryBuilder queryBuilder;
+
+  @AssistedInject
+  ElasticAccountIndex(
+      @GerritServerConfig Config cfg,
+      FillArgs fillArgs,
+      SitePaths sitePaths,
+      AccountCache accountCache,
+      @Assisted Schema<AccountState> schema) {
+    super(cfg, fillArgs, sitePaths, schema, ACCOUNTS_PREFIX);
+    this.accountCache = accountCache;
+    this.mapping = new AccountMapping(schema);
+    this.queryBuilder = new ElasticQueryBuilder();
+    this.gson = new GsonBuilder()
+        .setFieldNamingPolicy(LOWER_CASE_WITH_UNDERSCORES).create();
+  }
+
+  @Override
+  public void replace(AccountState as) throws IOException {
+    Bulk bulk = new Bulk.Builder()
+        .defaultIndex(indexName)
+        .defaultType(ACCOUNTS)
+        .addAction(insert(ACCOUNTS, as))
+        .refresh(refresh)
+        .build();
+      JestResult result = client.execute(bulk);
+    if (!result.isSucceeded()) {
+      throw new IOException(
+          String.format("Failed to replace account %s in index %s: %s",
+              as.getAccount().getId(), indexName, result.getErrorMessage()));
+    }
+  }
+
+  @Override
+  public DataSource<AccountState> getSource(Predicate<AccountState> p,
+      QueryOptions opts) throws QueryParseException {
+    return new QuerySource(p, opts);
+  }
+
+  @Override
+  protected Builder addActions(Builder builder, Id c) {
+    return builder.addAction(delete(ACCOUNTS, c));
+  }
+
+  @Override
+  protected String getMappings() {
+    ImmutableMap<String, AccountMapping> mappings =
+        ImmutableMap.of("mappings", mapping);
+    return gson.toJson(mappings);
+  }
+
+  @Override
+  protected String getId(AccountState as) {
+    return as.getAccount().getId().toString();
+  }
+
+  private class QuerySource implements DataSource<AccountState> {
+    private final Search search;
+    private final Set<String> fields;
+
+    QuerySource(Predicate<AccountState> p, QueryOptions opts)
+        throws QueryParseException {
+      QueryBuilder qb = queryBuilder.toQueryBuilder(p);
+      fields = IndexUtils.accountFields(opts);
+      SearchSourceBuilder searchSource = new SearchSourceBuilder()
+          .query(qb)
+          .from(opts.start())
+          .size(opts.limit())
+          .fields(Lists.newArrayList(fields));
+
+      Sort sort = new Sort(AccountField.ID.getName(), Sorting.ASC);
+      sort.setIgnoreUnmapped();
+
+      search = new Search.Builder(searchSource.toString())
+          .addType(ACCOUNTS)
+          .addIndex(indexName)
+          .addSort(ImmutableList.of(sort))
+          .build();
+    }
+
+    @Override
+    public int getCardinality() {
+      return 10;
+    }
+
+    @Override
+    public ResultSet<AccountState> read() throws OrmException {
+      try {
+        List<AccountState> results = Collections.emptyList();
+        JestResult result = client.execute(search);
+        if (result.isSucceeded()) {
+          JsonObject obj = result.getJsonObject().getAsJsonObject("hits");
+          if (obj.get("hits") != null) {
+            JsonArray json = obj.getAsJsonArray("hits");
+            results = Lists.newArrayListWithCapacity(json.size());
+            for (int i = 0; i < json.size(); i++) {
+              results.add(toChangeData(json.get(i)));
+            }
+          }
+        } else {
+          log.error(result.getErrorMessage());
+        }
+        final List<AccountState> r = Collections.unmodifiableList(results);
+        return new ResultSet<AccountState>() {
+          @Override
+          public Iterator<AccountState> iterator() {
+            return r.iterator();
+          }
+
+          @Override
+          public List<AccountState> toList() {
+            return r;
+          }
+
+          @Override
+          public void close() {
+            // Do nothing.
+          }
+        };
+      } catch (IOException e) {
+        throw new OrmException(e);
+      }
+    }
+
+    @Override
+    public String toString() {
+      return search.toString();
+    }
+
+    private AccountState toChangeData(JsonElement json) {
+      JsonElement source = json.getAsJsonObject().get("_source");
+      if (source == null) {
+        source = json.getAsJsonObject().get("fields");
+      }
+      return toAccountState(source);
+    }
+
+    private AccountState toAccountState(JsonElement element) {
+      Account.Id id = new Account.Id(
+          element.getAsJsonObject().get(ID.getName()).getAsInt());
+      // Use the AccountCache rather than depending on any stored fields in the
+      // document (of which there shouldn't be any. The most expensive part to
+      // compute anyway is the effective group IDs, and we don't have a good way
+      // to reindex when those change.
+      return accountCache.get(id);
+    }
+  }
+}

gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticIndexModule.java

@@ -15,7 +15,6 @@
 package com.google.gerrit.elasticsearch;
 
 import com.google.gerrit.lifecycle.LifecycleModule;
-import com.google.gerrit.lucene.LuceneAccountIndex;
 import com.google.gerrit.server.config.GerritServerConfig;
 import com.google.gerrit.server.index.IndexConfig;
 import com.google.gerrit.server.index.IndexModule;
@@ -56,9 +55,7 @@ public class ElasticIndexModule extends LifecycleModule {
             .build(ChangeIndex.Factory.class));
     install(
         new FactoryModuleBuilder()
-            // until we implement Elasticsearch index for accounts we need to
+            .implement(AccountIndex.class, ElasticAccountIndex.class)
-            // use Lucene to make all tests green and Gerrit server to work
-            .implement(AccountIndex.class, LuceneAccountIndex.class)
             .build(AccountIndex.Factory.class));
 
     install(new IndexModule(threads));

gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticQueryAccountsTest.java

@@ -0,0 +1,90 @@
+// Copyright (C) 2016 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.elasticsearch;
+
+import static com.google.gerrit.elasticsearch.ElasticAccountIndex.ACCOUNTS_PREFIX;
+
+import com.google.gerrit.elasticsearch.ElasticAccountIndex.AccountMapping;
+import com.google.gerrit.elasticsearch.ElasticTestUtils.ElasticNodeInfo;
+import com.google.gerrit.server.index.account.AccountSchemaDefinitions;
+import com.google.gerrit.server.query.account.AbstractQueryAccountsTest;
+import com.google.gerrit.testutil.InMemoryModule;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+
+import org.eclipse.jgit.lib.Config;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.util.concurrent.ExecutionException;
+
+public class ElasticQueryAccountsTest extends AbstractQueryAccountsTest {
+  private static final String INDEX_NAME =
+      String.format("%s%04d", ACCOUNTS_PREFIX,
+          AccountSchemaDefinitions.INSTANCE.getLatest().getVersion());
+  private static ElasticNodeInfo nodeInfo;
+
+  @BeforeClass
+  public static void startIndexService()
+      throws InterruptedException, ExecutionException {
+    if (nodeInfo != null) {
+      // do not start Elasticsearch twice
+      return;
+    }
+    nodeInfo = ElasticTestUtils.startElasticsearchNode();
+    createIndexes();
+  }
+
+  private static void createIndexes() {
+    AccountMapping accountMapping =
+        new AccountMapping(AccountSchemaDefinitions.INSTANCE.getLatest());
+    nodeInfo.node
+        .client()
+        .admin()
+        .indices()
+        .prepareCreate(INDEX_NAME)
+        .addMapping(ElasticAccountIndex.ACCOUNTS,
+            ElasticTestUtils.gson.toJson(accountMapping))
+        .execute()
+        .actionGet();
+  }
+
+  @AfterClass
+  public static void stopElasticsearchServer() {
+    if (nodeInfo != null) {
+      nodeInfo.node.close();
+      nodeInfo.elasticDir.delete();
+      nodeInfo = null;
+    }
+  }
+
+  @After
+  public void cleanupIndex() {
+    if (nodeInfo != null) {
+      ElasticTestUtils.deleteIndexes(nodeInfo.node, INDEX_NAME);
+      createIndexes();
+    }
+  }
+
+  @Override
+  protected Injector createInjector() {
+    Config elasticsearchConfig = new Config(config);
+    InMemoryModule.setDefaults(elasticsearchConfig);
+    ElasticTestUtils.configure(elasticsearchConfig, nodeInfo.port);
+    return Guice.createInjector(
+        new InMemoryModule(elasticsearchConfig, notesMigration));
+  }
+}

gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneAccountIndex.java

@@ -16,13 +16,12 @@ package com.google.gerrit.lucene;
 
 import static com.google.gerrit.server.index.account.AccountField.ID;
 
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Sets;
 import com.google.gerrit.reviewdb.client.Account;
 import com.google.gerrit.server.account.AccountCache;
 import com.google.gerrit.server.account.AccountState;
 import com.google.gerrit.server.config.GerritServerConfig;
 import com.google.gerrit.server.config.SitePaths;
+import com.google.gerrit.server.index.IndexUtils;
 import com.google.gerrit.server.index.QueryOptions;
 import com.google.gerrit.server.index.Schema;
 import com.google.gerrit.server.index.account.AccountIndex;
@@ -56,7 +55,6 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Set;
 import java.util.concurrent.ExecutionException;
 
 public class LuceneAccountIndex
@@ -164,7 +162,7 @@ public class LuceneAccountIndex
         List<AccountState> result = new ArrayList<>(docs.scoreDocs.length);
         for (int i = opts.start(); i < docs.scoreDocs.length; i++) {
           ScoreDoc sd = docs.scoreDocs[i];
-          Document doc = searcher.doc(sd.doc, fields(opts));
+          Document doc = searcher.doc(sd.doc, IndexUtils.accountFields(opts));
           result.add(toAccountState(doc));
         }
         final List<AccountState> r = Collections.unmodifiableList(result);
@@ -198,13 +196,6 @@ public class LuceneAccountIndex
     }
   }
 
-  private Set<String> fields(QueryOptions opts) {
-    Set<String> fs = opts.fields();
-    return fs.contains(ID.getName())
-        ? fs
-        : Sets.union(fs, ImmutableSet.of(ID.getName()));
-  }
-
   private AccountState toAccountState(Document doc) {
     Account.Id id =
         new Account.Id(doc.getField(ID.getName()).numericValue().intValue());

gerrit-server/src/main/java/com/google/gerrit/server/index/IndexUtils.java

@@ -14,6 +14,7 @@
 
 package com.google.gerrit.server.index;
 
+import static com.google.gerrit.server.index.account.AccountField.ID;
 import static com.google.gerrit.server.index.change.ChangeField.CHANGE;
 import static com.google.gerrit.server.index.change.ChangeField.LEGACY_ID;
 import static com.google.gerrit.server.index.change.ChangeField.PROJECT;
@@ -44,6 +45,13 @@ public final class IndexUtils {
     }
   }
 
+  public static Set<String> accountFields(QueryOptions opts) {
+    Set<String> fs = opts.fields();
+    return fs.contains(ID.getName())
+        ? fs
+        : Sets.union(fs, ImmutableSet.of(ID.getName()));
+  }
+
   public static Set<String> changeFields(QueryOptions opts) {
     // Ensure we request enough fields to construct a ChangeData. We need both
     // change ID and project, which can either come via the Change field or