Store hashtags in change index

When changes are queried the hashtags are retured with each change.
These hashtags should be read from the change index rather than
loading them from NoteDb. Loading the change notes for each change in
the result set leads to poor query performance when NoteDb is enabled.

The existing HASHTAG field in the change index contains the lower case
hashtags to support case-insensitive queries by hashtag. To be able to
retrieve the hashtags with the original case from the index we must
add a new field that stores the hashtags with the original case. This
new field is not used for serving queries, hence it is of type
STORED_ONLY.

Change-Id: I0a9eb0ad0ac849d8b7b3ac6965dc8b5f38d04a32
Signed-off-by: Edwin Kempin <ekempin@google.com>

gerrit-acceptance-tests/src/test/java/com/google/gerrit/acceptance/rest/change/HashtagsIT.java

@@ -237,6 +237,14 @@ public class HashtagsIT extends AbstractDaemonTest {
     assertMessage(r, "Hashtag removed: tag3");
   }
 
+  @Test
+  public void testHashtagWithMixedCase() throws Exception {
+    PushOneCommit.Result r = createChange();
+    addHashtags(r, "MyHashtag");
+    assertThatGet(r).containsExactly("MyHashtag");
+    assertMessage(r, "Hashtag added: MyHashtag");
+  }
+
   private IterableSubject<
         ? extends IterableSubject<?, String, Iterable<String>>,
         String, Iterable<String>>

gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java

@@ -114,6 +114,8 @@ public class LuceneChangeIndex implements ChangeIndex {
   private static final String PATCH_SET_FIELD = ChangeField.PATCH_SET.getName();
   private static final String REVIEWEDBY_FIELD =
       ChangeField.REVIEWEDBY.getName();
+  private static final String HASHTAG_FIELD =
+      ChangeField.HASHTAG_CASE_AWARE.getName();
   private static final String STARREDBY_FIELD = ChangeField.STARREDBY.getName();
 
   static Term idTerm(ChangeData cd) {
@@ -414,6 +416,9 @@ public class LuceneChangeIndex implements ChangeIndex {
     if (fields.contains(REVIEWEDBY_FIELD)) {
       decodeReviewedBy(doc, cd);
     }
+    if (fields.contains(HASHTAG_FIELD)) {
+      decodeHashtags(doc, cd);
+    }
     if (fields.contains(STARREDBY_FIELD)) {
       decodeStarredBy(doc, cd);
     }
@@ -470,6 +475,15 @@ public class LuceneChangeIndex implements ChangeIndex {
     }
   }
 
+  private void decodeHashtags(Document doc, ChangeData cd) {
+    IndexableField[] hashtag = doc.getFields(HASHTAG_FIELD);
+    Set<String> hashtags = Sets.newHashSetWithExpectedSize(hashtag.length);
+    for (IndexableField r : hashtag) {
+      hashtags.add(r.binaryValue().utf8ToString());
+    }
+    cd.setHashtags(hashtags);
+  }
+
   private void decodeStarredBy(Document doc, ChangeData cd) {
     IndexableField[] starredBy = doc.getFields(STARREDBY_FIELD);
     Set<Account.Id> accounts =

gerrit-server/src/main/java/com/google/gerrit/server/change/ChangeJson.java

@@ -394,7 +394,7 @@ public class ChangeJson {
     out.project = in.getProject().get();
     out.branch = in.getDest().getShortName();
     out.topic = in.getTopic();
-    out.hashtags = ctl.getNotes().load().getHashtags();
+    out.hashtags = cd.hashtags();
     out.changeId = in.getKey().get();
     if (in.getStatus() != Change.Status.MERGED) {
       SubmitTypeRecord str = cd.submitTypeRecord();

gerrit-server/src/main/java/com/google/gerrit/server/index/change/ChangeField.java

@@ -15,6 +15,7 @@
 package com.google.gerrit.server.index.change;
 
 import static com.google.common.base.MoreObjects.firstNonNull;
+import static java.nio.charset.StandardCharsets.UTF_8;
 
 import com.google.common.base.Function;
 import com.google.common.base.Splitter;
@@ -237,14 +238,29 @@ public class ChangeField {
         @Override
         public Iterable<String> get(ChangeData input, FillArgs args)
             throws OrmException {
-          return ImmutableSet.copyOf(Iterables.transform(input.notes().load()
+          return ImmutableSet.copyOf(Iterables.transform(input.hashtags(),
-              .getHashtags(), new Function<String, String>() {
+              new Function<String, String>() {
-
             @Override
             public String apply(String input) {
               return input.toLowerCase();
             }
+          }));
+        }
+      };
 
+  /** Hashtags with original case. */
+  public static final FieldDef<ChangeData, Iterable<byte[]>> HASHTAG_CASE_AWARE =
+      new FieldDef.Repeatable<ChangeData, byte[]>(
+          "_hashtag", FieldType.STORED_ONLY, true) {
+        @Override
+        public Iterable<byte[]> get(ChangeData input, FillArgs args)
+            throws OrmException {
+          return ImmutableSet.copyOf(Iterables.transform(input.hashtags(),
+              new Function<String, byte[]>() {
+            @Override
+            public byte[] apply(String hashtag) {
+              return hashtag.getBytes(UTF_8);
+            }
           }));
         }
       };

gerrit-server/src/main/java/com/google/gerrit/server/index/change/ChangeSchemaDefinitions.java

@@ -64,8 +64,12 @@ public class ChangeSchemaDefinitions extends SchemaDefinitions<ChangeData> {
   @Deprecated
   static final Schema<ChangeData> V27 = schema(V26.getFields().values());
 
+  @Deprecated
   static final Schema<ChangeData> V28 = schema(V27, ChangeField.STARREDBY);
 
+  static final Schema<ChangeData> V29 =
+      schema(V28, ChangeField.HASHTAG_CASE_AWARE);
+
   public static final ChangeSchemaDefinitions INSTANCE =
       new ChangeSchemaDefinitions();
 

gerrit-server/src/main/java/com/google/gerrit/server/query/change/ChangeData.java

@@ -337,6 +337,7 @@ public class ChangeData {
   private ChangedLines changedLines;
   private SubmitTypeRecord submitTypeRecord;
   private Boolean mergeable;
+  private Set<String> hashtags;
   private Set<Account.Id> editsByUser;
   private Set<Account.Id> reviewedBy;
   private Set<Account.Id> draftsByUser;
@@ -1018,6 +1019,17 @@ public class ChangeData {
     this.reviewedBy = reviewedBy;
   }
 
+  public Set<String> hashtags() throws OrmException {
+    if (hashtags == null) {
+      hashtags = notes().getHashtags();
+    }
+    return hashtags;
+  }
+
+  public void setHashtags(Set<String> hashtags) {
+    this.hashtags = hashtags;
+  }
+
   public Set<Account.Id> starredBy() throws OrmException {
     if (starredByUser == null) {
       starredByUser = starredChangesUtil.byChange(legacyId);