Merge changes from topic 'group-index'

* changes:
  Index groups that are created during init
  Implement group index for ElasticSearch
  Add REST endpoint to reindex a single group
  Support default query for groups
  Add query operator that matches groups that are visible to all users
  Add group query operator to match groups by owner
  Add group query operator to match groups by description
  Support querying groups by name and name part
  Set '_more_groups' on last group of query result
  Add tests for group index
  Groups API: Add query methods
  Support arbitrary group queries via REST
  Add new optional interface for REST collections to get request params
  Add GroupQueryProcessor
  IsVisibleToPredicate: Move describe method into base class
  Index group on creation and whenever a group is evicted from cache
  Initial implementation of group index in Lucene
  Add initial GroupQueryBuilder and define key predicate for group index
  Add schema and index definitions for groups index

gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticGroupIndex.java

@@ -0,0 +1,228 @@
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.elasticsearch;
+
+import static com.google.gson.FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.gerrit.elasticsearch.ElasticMapping.MappingProperties;
+import com.google.gerrit.reviewdb.client.AccountGroup;
+import com.google.gerrit.server.account.GroupCache;
+import com.google.gerrit.server.config.GerritServerConfig;
+import com.google.gerrit.server.config.SitePaths;
+import com.google.gerrit.server.index.FieldDef.FillArgs;
+import com.google.gerrit.server.index.IndexUtils;
+import com.google.gerrit.server.index.QueryOptions;
+import com.google.gerrit.server.index.Schema;
+import com.google.gerrit.server.index.group.GroupField;
+import com.google.gerrit.server.index.group.GroupIndex;
+import com.google.gerrit.server.query.DataSource;
+import com.google.gerrit.server.query.Predicate;
+import com.google.gerrit.server.query.QueryParseException;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gwtorm.server.OrmException;
+import com.google.gwtorm.server.ResultSet;
+import com.google.inject.Provider;
+import com.google.inject.assistedinject.Assisted;
+import com.google.inject.assistedinject.AssistedInject;
+
+import org.eclipse.jgit.lib.Config;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import io.searchbox.client.JestResult;
+import io.searchbox.core.Bulk;
+import io.searchbox.core.Bulk.Builder;
+import io.searchbox.core.Search;
+import io.searchbox.core.search.sort.Sort;
+import io.searchbox.core.search.sort.Sort.Sorting;
+
+public class ElasticGroupIndex
+    extends AbstractElasticIndex<AccountGroup.UUID, AccountGroup>
+    implements GroupIndex {
+  static class GroupMapping {
+    MappingProperties groups;
+
+    GroupMapping(Schema<AccountGroup> schema) {
+      this.groups = ElasticMapping.createMapping(schema);
+    }
+  }
+
+  static final String GROUPS = "groups";
+  static final String GROUPS_PREFIX = GROUPS + "_";
+
+  private static final Logger log =
+      LoggerFactory.getLogger(ElasticGroupIndex.class);
+
+  private final Gson gson;
+  private final GroupMapping mapping;
+  private final Provider<GroupCache> groupCache;
+  private final ElasticQueryBuilder queryBuilder;
+
+  @AssistedInject
+  ElasticGroupIndex(
+      @GerritServerConfig Config cfg,
+      FillArgs fillArgs,
+      SitePaths sitePaths,
+      Provider<GroupCache> groupCache,
+      @Assisted Schema<AccountGroup> schema) {
+    super(cfg, fillArgs, sitePaths, schema, GROUPS_PREFIX);
+    this.groupCache = groupCache;
+    this.mapping = new GroupMapping(schema);
+    this.queryBuilder = new ElasticQueryBuilder();
+    this.gson = new GsonBuilder()
+        .setFieldNamingPolicy(LOWER_CASE_WITH_UNDERSCORES).create();
+  }
+
+  @Override
+  public void replace(AccountGroup group) throws IOException {
+    Bulk bulk = new Bulk.Builder()
+        .defaultIndex(indexName)
+        .defaultType(GROUPS)
+        .addAction(insert(GROUPS, group))
+        .refresh(refresh)
+        .build();
+    JestResult result = client.execute(bulk);
+    if (!result.isSucceeded()) {
+      throw new IOException(
+          String.format("Failed to replace group %s in index %s: %s",
+              group.getGroupUUID().get(), indexName, result.getErrorMessage()));
+    }
+  }
+
+  @Override
+  public DataSource<AccountGroup> getSource(Predicate<AccountGroup> p,
+      QueryOptions opts) throws QueryParseException {
+    return new QuerySource(p, opts);
+  }
+
+  @Override
+  protected Builder addActions(Builder builder, AccountGroup.UUID c) {
+    return builder.addAction(delete(GROUPS, c));
+  }
+
+  @Override
+  protected String getMappings() {
+    ImmutableMap<String, GroupMapping> mappings =
+        ImmutableMap.of("mappings", mapping);
+    return gson.toJson(mappings);
+  }
+
+  @Override
+  protected String getId(AccountGroup group) {
+    return group.getGroupUUID().get();
+  }
+
+  private class QuerySource implements DataSource<AccountGroup> {
+    private final Search search;
+    private final Set<String> fields;
+
+    QuerySource(Predicate<AccountGroup> p, QueryOptions opts)
+        throws QueryParseException {
+      QueryBuilder qb = queryBuilder.toQueryBuilder(p);
+      fields = IndexUtils.groupFields(opts);
+      SearchSourceBuilder searchSource = new SearchSourceBuilder()
+          .query(qb)
+          .from(opts.start())
+          .size(opts.limit())
+          .fields(Lists.newArrayList(fields));
+
+      Sort sort = new Sort(GroupField.UUID.getName(), Sorting.ASC);
+      sort.setIgnoreUnmapped();
+
+      search = new Search.Builder(searchSource.toString())
+          .addType(GROUPS)
+          .addIndex(indexName)
+          .addSort(ImmutableList.of(sort))
+          .build();
+    }
+
+    @Override
+    public int getCardinality() {
+      return 10;
+    }
+
+    @Override
+    public ResultSet<AccountGroup> read() throws OrmException {
+      try {
+        List<AccountGroup> results = Collections.emptyList();
+        JestResult result = client.execute(search);
+        if (result.isSucceeded()) {
+          JsonObject obj = result.getJsonObject().getAsJsonObject("hits");
+          if (obj.get("hits") != null) {
+            JsonArray json = obj.getAsJsonArray("hits");
+            results = Lists.newArrayListWithCapacity(json.size());
+            for (int i = 0; i < json.size(); i++) {
+              results.add(toAccountGroup(json.get(i)));
+            }
+          }
+        } else {
+          log.error(result.getErrorMessage());
+        }
+        final List<AccountGroup> r = Collections.unmodifiableList(results);
+        return new ResultSet<AccountGroup>() {
+          @Override
+          public Iterator<AccountGroup> iterator() {
+            return r.iterator();
+          }
+
+          @Override
+          public List<AccountGroup> toList() {
+            return r;
+          }
+
+          @Override
+          public void close() {
+            // Do nothing.
+          }
+        };
+      } catch (IOException e) {
+        throw new OrmException(e);
+      }
+    }
+
+    @Override
+    public String toString() {
+      return search.toString();
+    }
+
+    private AccountGroup toAccountGroup(JsonElement json) {
+      JsonElement source = json.getAsJsonObject().get("_source");
+      if (source == null) {
+        source = json.getAsJsonObject().get("fields");
+      }
+
+      AccountGroup.UUID uuid = new AccountGroup.UUID(
+          source.getAsJsonObject().get(GroupField.UUID.getName()).getAsString());
+      // Use the GroupCache rather than depending on any stored fields in the
+      // document (of which there shouldn't be any).
+      return groupCache.get().get(uuid);
+    }
+  }
+}

gerrit-elasticsearch/src/main/java/com/google/gerrit/elasticsearch/ElasticIndexModule.java

@@ -21,6 +21,7 @@ import com.google.gerrit.server.index.IndexModule;
 import com.google.gerrit.server.index.SingleVersionModule;
 import com.google.gerrit.server.index.account.AccountIndex;
 import com.google.gerrit.server.index.change.ChangeIndex;
+import com.google.gerrit.server.index.group.GroupIndex;
 import com.google.inject.Provides;
 import com.google.inject.Singleton;
 import com.google.inject.assistedinject.FactoryModuleBuilder;
@@ -49,14 +50,18 @@ public class ElasticIndexModule extends LifecycleModule {
 
   @Override
   protected void configure() {
+    install(
+        new FactoryModuleBuilder()
+        .implement(AccountIndex.class, ElasticAccountIndex.class)
+        .build(AccountIndex.Factory.class));
     install(
         new FactoryModuleBuilder()
             .implement(ChangeIndex.class, ElasticChangeIndex.class)
             .build(ChangeIndex.Factory.class));
     install(
         new FactoryModuleBuilder()
-            .implement(AccountIndex.class, ElasticAccountIndex.class)
-            .build(AccountIndex.Factory.class));
+        .implement(GroupIndex.class, ElasticGroupIndex.class)
+        .build(GroupIndex.Factory.class));
 
     install(new IndexModule(threads));
     install(new SingleVersionModule(singleVersions));

gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticQueryGroupsTest.java

@@ -0,0 +1,69 @@
+// Copyright (C) 2017 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.elasticsearch;
+
+import com.google.gerrit.elasticsearch.ElasticTestUtils.ElasticNodeInfo;
+import com.google.gerrit.server.query.group.AbstractQueryGroupsTest;
+import com.google.gerrit.testutil.InMemoryModule;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+
+import org.eclipse.jgit.lib.Config;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.util.concurrent.ExecutionException;
+
+public class ElasticQueryGroupsTest extends AbstractQueryGroupsTest {
+  private static ElasticNodeInfo nodeInfo;
+
+  @BeforeClass
+  public static void startIndexService()
+      throws InterruptedException, ExecutionException {
+    if (nodeInfo != null) {
+      // do not start Elasticsearch twice
+      return;
+    }
+    nodeInfo = ElasticTestUtils.startElasticsearchNode();
+    ElasticTestUtils.createAllIndexes(nodeInfo);
+  }
+
+  @AfterClass
+  public static void stopElasticsearchServer() {
+    if (nodeInfo != null) {
+      nodeInfo.node.close();
+      nodeInfo.elasticDir.delete();
+      nodeInfo = null;
+    }
+  }
+
+  @After
+  public void cleanupIndex() {
+    if (nodeInfo != null) {
+      ElasticTestUtils.deleteAllIndexes(nodeInfo);
+      ElasticTestUtils.createAllIndexes(nodeInfo);
+    }
+  }
+
+  @Override
+  protected Injector createInjector() {
+    Config elasticsearchConfig = new Config(config);
+    InMemoryModule.setDefaults(elasticsearchConfig);
+    ElasticTestUtils.configure(elasticsearchConfig, nodeInfo.port);
+    return Guice.createInjector(
+        new InMemoryModule(elasticsearchConfig, notesMigration));
+  }
+}

gerrit-elasticsearch/src/test/java/com/google/gerrit/elasticsearch/ElasticTestUtils.java

@@ -19,16 +19,20 @@ import static com.google.gerrit.elasticsearch.ElasticAccountIndex.ACCOUNTS_PREFI
 import static com.google.gerrit.elasticsearch.ElasticChangeIndex.CHANGES_PREFIX;
 import static com.google.gerrit.elasticsearch.ElasticChangeIndex.CLOSED_CHANGES;
 import static com.google.gerrit.elasticsearch.ElasticChangeIndex.OPEN_CHANGES;
+import static com.google.gerrit.elasticsearch.ElasticGroupIndex.GROUPS_PREFIX;
 
 import com.google.common.base.Strings;
 import com.google.common.io.Files;
 import com.google.gerrit.elasticsearch.ElasticAccountIndex.AccountMapping;
 import com.google.gerrit.elasticsearch.ElasticChangeIndex.ChangeMapping;
+import com.google.gerrit.elasticsearch.ElasticGroupIndex.GroupMapping;
+import com.google.gerrit.reviewdb.client.AccountGroup;
 import com.google.gerrit.server.account.AccountState;
 import com.google.gerrit.server.index.IndexModule.IndexType;
 import com.google.gerrit.server.index.Schema;
 import com.google.gerrit.server.index.account.AccountSchemaDefinitions;
 import com.google.gerrit.server.index.change.ChangeSchemaDefinitions;
+import com.google.gerrit.server.index.group.GroupSchemaDefinitions;
 import com.google.gerrit.server.query.change.ChangeData;
 import com.google.gson.FieldNamingPolicy;
 import com.google.gson.Gson;
@@ -154,6 +158,20 @@ final class ElasticTestUtils {
         .addMapping(ElasticAccountIndex.ACCOUNTS, gson.toJson(accountMapping))
         .execute()
         .actionGet();
+
+    Schema<AccountGroup> groupSchema =
+        GroupSchemaDefinitions.INSTANCE.getLatest();
+    GroupMapping groupMapping = new GroupMapping(groupSchema);
+    nodeInfo.node
+        .client()
+        .admin()
+        .indices()
+        .prepareCreate(
+            String.format(
+                "%s%04d", GROUPS_PREFIX, groupSchema.getVersion()))
+        .addMapping(ElasticGroupIndex.GROUPS, gson.toJson(groupMapping))
+        .execute()
+        .actionGet();
   }
 
   private static String getHttpPort(Node node)