opendev/gerrit
Code Issues Proposed changes

Rewrite schema versioning in preparation for multiple versions

Browse Source 
Store indexes in versioned directories, and don't consider open/closed
to have separate versions.

Extract a simple base Schema class with a version and a list of
fields, which is analogous to the DB's SchemaVersion. Construct
indexes using specific versions.

Change-Id: I11e8e3a78cd62b1a01aa122bfd747ed5295fe057
This commit is contained in:
Dave Borowitz
2013-06-24 11:56:53 -06:00
parent e12f8b0314
commit 0bd69febcb
8 changed files with 185 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
gerrit-lucene/src/main/java/com/google/gerrit/lucene/IndexVersionCheck.java
View File

@@ -15,13 +15,12 @@
package com.google.gerrit.lucene;
import static com.google.gerrit.lucene.LuceneChangeIndex.LUCENE_VERSION;
import static org.apache.lucene.util.Version.LUCENE_CURRENT;
import com.google.common.collect.ImmutableMap;
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.ChangeField;
import com.google.gerrit.server.index.ChangeSchemas;
import com.google.inject.Inject;
import com.google.inject.ProvisionException;
@@ -36,8 +35,8 @@ import java.util.Map;
public class IndexVersionCheck implements LifecycleListener {
public static final Map<String, Integer> SCHEMA_VERSIONS = ImmutableMap.of(
LuceneChangeIndex.CHANGES_OPEN, ChangeField.SCHEMA_VERSION,
LuceneChangeIndex.CHANGES_CLOSED, ChangeField.SCHEMA_VERSION);
LuceneChangeIndex.CHANGES_OPEN, ChangeSchemas.getLatestRelease().getVersion(),
LuceneChangeIndex.CHANGES_CLOSED, ChangeSchemas.getLatestRelease().getVersion());
public static File gerritIndexConfig(SitePaths sitePaths) {
return new File(sitePaths.index_dir, "gerrit_index.config");

17
gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneChangeIndex.java
View File

@@ -35,6 +35,7 @@ import com.google.gerrit.server.index.FieldDef;
import com.google.gerrit.server.index.FieldDef.FillArgs;
import com.google.gerrit.server.index.FieldType;
import com.google.gerrit.server.index.IndexRewriteImpl;
import com.google.gerrit.server.index.Schema;
import com.google.gerrit.server.query.Predicate;
import com.google.gerrit.server.query.QueryParseException;
import com.google.gerrit.server.query.change.ChangeData;
@@ -93,8 +94,8 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
LoggerFactory.getLogger(LuceneChangeIndex.class);
public static final Version LUCENE_VERSION = Version.LUCENE_43;
public static final String CHANGES_OPEN = "changes_open";
public static final String CHANGES_CLOSED = "changes_closed";
public static final String CHANGES_OPEN = "open";
public static final String CHANGES_CLOSED = "closed";
private static final String ID_FIELD = ChangeField.LEGACY_ID.getName();
private static IndexWriterConfig getIndexWriterConfig(Config cfg, String name) {
@@ -113,19 +114,23 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
private final FillArgs fillArgs;
private final ExecutorService executor;
private final boolean readOnly;
private final Schema<ChangeData> fields;
private final SubIndex openIndex;
private final SubIndex closedIndex;
LuceneChangeIndex(Config cfg, SitePaths sitePaths,
ListeningScheduledExecutorService executor, FillArgs fillArgs,
boolean readOnly) throws IOException {
Schema<ChangeData> fields, boolean readOnly) throws IOException {
this.sitePaths = sitePaths;
this.fillArgs = fillArgs;
this.executor = executor;
this.readOnly = readOnly;
openIndex = new SubIndex(new File(sitePaths.index_dir, CHANGES_OPEN),
this.fields = fields;
File dir = new File(sitePaths.index_dir, "changes_" + fields.getVersion());
openIndex = new SubIndex(new File(dir, CHANGES_OPEN),
getIndexWriterConfig(cfg, "changes_open"));
closedIndex = new SubIndex(new File(sitePaths.index_dir, CHANGES_CLOSED),
closedIndex = new SubIndex(new File(dir, CHANGES_CLOSED),
getIndexWriterConfig(cfg, "changes_closed"));
}
@@ -323,7 +328,7 @@ public class LuceneChangeIndex implements ChangeIndex, LifecycleListener {
private Document toDocument(ChangeData cd) throws IOException {
try {
Document result = new Document();
for (FieldDef<ChangeData, ?> f : ChangeField.ALL.values()) {
for (FieldDef<ChangeData, ?> f : fields.getFields().values()) {
if (f.isRepeatable()) {
add(result, f, (Iterable<?>) f.get(cd, fillArgs));
} else {

4
gerrit-lucene/src/main/java/com/google/gerrit/lucene/LuceneIndexModule.java
View File

@@ -19,6 +19,7 @@ import com.google.gerrit.lifecycle.LifecycleModule;
import com.google.gerrit.server.config.GerritServerConfig;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.ChangeIndex;
import com.google.gerrit.server.index.ChangeSchemas;
import com.google.gerrit.server.index.FieldDef.FillArgs;
import com.google.gerrit.server.index.IndexExecutor;
import com.google.gerrit.server.index.IndexModule;
@@ -61,6 +62,7 @@ public class LuceneIndexModule extends LifecycleModule {
SitePaths sitePaths,
@IndexExecutor ListeningScheduledExecutorService executor,
FillArgs fillArgs) throws IOException {
return new LuceneChangeIndex(cfg, sitePaths, executor, fillArgs, readOnly);
return new LuceneChangeIndex(cfg, sitePaths, executor, fillArgs,
ChangeSchemas.getLatestRelease(), readOnly);
}
}

43
gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeField.java
View File

@@ -14,8 +14,6 @@
package com.google.gerrit.server.index;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.gerrit.reviewdb.client.Account;
import com.google.gerrit.reviewdb.client.ChangeMessage;
@@ -30,11 +28,7 @@ import com.google.gerrit.server.query.change.ChangeStatusPredicate;
import com.google.gwtorm.server.OrmException;
import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.lang.reflect.ParameterizedType;
import java.sql.Timestamp;
import java.util.Map;
import java.util.Set;
/**
@@ -44,13 +38,8 @@ import java.util.Set;
* {@link ChangeQueryBuilder} for querying that field, and a method on
* {@link ChangeData} used for populating the corresponding document fields in
* the secondary index.
* <p>
* Used to generate a schema for index implementations that require one.
*/
public class ChangeField {
/** Increment whenever making schema changes. */
public static final int SCHEMA_VERSION = 15;
/** Legacy change ID. */
public static final FieldDef<ChangeData, Integer> LEGACY_ID =
new FieldDef.Single<ChangeData, Integer>("_id",
@@ -284,36 +273,4 @@ public class ChangeField {
return r;
}
};
public static final ImmutableMap<String, FieldDef<ChangeData, ?>> ALL;
static {
Map<String, FieldDef<ChangeData, ?>> fields = Maps.newHashMap();
for (Field f : ChangeField.class.getFields()) {
if (Modifier.isPublic(f.getModifiers())
&& Modifier.isStatic(f.getModifiers())
&& Modifier.isFinal(f.getModifiers())
&& FieldDef.class.isAssignableFrom(f.getType())) {
ParameterizedType t = (ParameterizedType) f.getGenericType();
if (t.getActualTypeArguments()[0] == ChangeData.class) {
try {
@SuppressWarnings("unchecked")
FieldDef<ChangeData, ?> fd = (FieldDef<ChangeData, ?>) f.get(null);
fields.put(fd.getName(), fd);
} catch (IllegalArgumentException e) {
throw new ExceptionInInitializerError(e);
} catch (IllegalAccessException e) {
throw new ExceptionInInitializerError(e);
}
} else {
throw new ExceptionInInitializerError(
"non-ChangeData ChangeField: " + f);
}
}
}
if (fields.isEmpty()) {
throw new ExceptionInInitializerError("no ChangeFields found");
}
ALL = ImmutableMap.copyOf(fields);
}
}

110
gerrit-server/src/main/java/com/google/gerrit/server/index/ChangeSchemas.java Normal file
View File

@@ -0,0 +1,110 @@
// Copyright (C) 2013 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.package com.google.gerrit.server.git;
package com.google.gerrit.server.index;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import com.google.gerrit.server.query.change.ChangeData;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.lang.reflect.ParameterizedType;
import java.util.Arrays;
import java.util.Map;
/** Secondary index schemas for changes. */
public class ChangeSchemas {
@SuppressWarnings("unchecked")
static final Schema<ChangeData> V1 = release(
ChangeField.LEGACY_ID,
ChangeField.ID,
ChangeField.STATUS,
ChangeField.PROJECT,
ChangeField.REF,
ChangeField.TOPIC,
ChangeField.UPDATED,
ChangeField.SORTKEY,
ChangeField.FILE,
ChangeField.OWNER,
ChangeField.REVIEWER,
ChangeField.COMMIT,
ChangeField.TR,
ChangeField.LABEL,
ChangeField.REVIEWED,
ChangeField.COMMIT_MESSAGE,
ChangeField.COMMENT);
private static Schema<ChangeData> release(FieldDef<ChangeData, ?>... fields) {
return new Schema<ChangeData>(true, Arrays.asList(fields));
}
@SuppressWarnings("unused")
private static Schema<ChangeData> developer(FieldDef<ChangeData, ?>... fields) {
return new Schema<ChangeData>(false, Arrays.asList(fields));
}
private static final ImmutableMap<Integer, Schema<ChangeData>> ALL;
public Schema<ChangeData> get(int version) {
Schema<ChangeData> schema = ALL.get(version);
checkArgument(schema != null, "Unrecognized schema version: %s", version);
return schema;
}
public static Schema<ChangeData> getLatestRelease() {
Schema<ChangeData> latest = null;
for (Schema<ChangeData> schema : ALL.values()) {
if (schema.isRelease()) {
latest = schema;
}
}
checkState(latest != null, "No released schema versions found");
return latest;
}
static {
Map<Integer, Schema<ChangeData>> all = Maps.newTreeMap();
for (Field f : ChangeSchemas.class.getDeclaredFields()) {
if (Modifier.isStatic(f.getModifiers())
&& Modifier.isFinal(f.getModifiers())
&& Schema.class.isAssignableFrom(f.getType())) {
ParameterizedType t = (ParameterizedType) f.getGenericType();
if (t.getActualTypeArguments()[0] == ChangeData.class) {
try {
@SuppressWarnings("unchecked")
Schema<ChangeData> schema = (Schema<ChangeData>) f.get(null);
checkArgument(f.getName().startsWith("V"));
schema.setVersion(Integer.parseInt(f.getName().substring(1)));
all.put(schema.getVersion(), schema);
} catch (IllegalArgumentException e) {
throw new ExceptionInInitializerError(e);
} catch (IllegalAccessException e) {
throw new ExceptionInInitializerError(e);
}
} else {
throw new ExceptionInInitializerError(
"non-ChangeData schema: " + f);
}
}
}
if (all.isEmpty()) {
throw new ExceptionInInitializerError("no ChangeSchemas found");
}
ALL = ImmutableMap.copyOf(all);
}
}

49
gerrit-server/src/main/java/com/google/gerrit/server/index/Schema.java Normal file
View File

@@ -0,0 +1,49 @@
// Copyright (C) 2013 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.package com.google.gerrit.server.git;
package com.google.gerrit.server.index;
import com.google.common.collect.ImmutableMap;
/** Specific version of a secondary index schema. */
public class Schema<T> {
private final boolean release;
private final ImmutableMap<String, FieldDef<T, ?>> fields;
private int version;
protected Schema(boolean release, Iterable<FieldDef<T, ?>> fields) {
this.release = release;
ImmutableMap.Builder<String, FieldDef<T, ?>> b = ImmutableMap.builder();
for (FieldDef<T, ?> f : fields) {
b.put(f.getName(), f);
}
this.fields = b.build();
}
public final boolean isRelease() {
return release;
}
public final int getVersion() {
return version;
}
public final ImmutableMap<String, FieldDef<T, ?>> getFields() {
return fields;
}
void setVersion(int version) {
this.version = version;
}
}

6
gerrit-solr/src/main/java/com/google/gerrit/solr/IndexVersionCheck.java
View File

@@ -17,7 +17,7 @@ package com.google.gerrit.solr;
import com.google.common.collect.ImmutableMap;
import com.google.gerrit.extensions.events.LifecycleListener;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.ChangeField;
import com.google.gerrit.server.index.ChangeSchemas;
import com.google.inject.Inject;
import com.google.inject.ProvisionException;
@@ -31,8 +31,8 @@ import java.util.Map;
class IndexVersionCheck implements LifecycleListener {
public static final Map<String, Integer> SCHEMA_VERSIONS = ImmutableMap.of(
SolrChangeIndex.CHANGES_OPEN, ChangeField.SCHEMA_VERSION,
SolrChangeIndex.CHANGES_CLOSED, ChangeField.SCHEMA_VERSION);
SolrChangeIndex.CHANGES_OPEN, ChangeSchemas.getLatestRelease().getVersion(),
SolrChangeIndex.CHANGES_CLOSED, ChangeSchemas.getLatestRelease().getVersion());
public static File solrIndexConfig(SitePaths sitePaths) {
return new File(sitePaths.index_dir, "gerrit_index.config");

10
gerrit-solr/src/main/java/com/google/gerrit/solr/SolrChangeIndex.java
View File

@@ -35,13 +35,13 @@ import com.google.gerrit.server.index.FieldDef;
import com.google.gerrit.server.index.FieldDef.FillArgs;
import com.google.gerrit.server.index.FieldType;
import com.google.gerrit.server.index.IndexRewriteImpl;
import com.google.gerrit.server.index.Schema;
import com.google.gerrit.server.query.Predicate;
import com.google.gerrit.server.query.QueryParseException;
import com.google.gerrit.server.query.change.ChangeData;
import com.google.gerrit.server.query.change.ChangeDataSource;
import com.google.gwtorm.server.OrmException;
import com.google.gwtorm.server.ResultSet;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.apache.lucene.search.Query;
@@ -76,14 +76,16 @@ class SolrChangeIndex implements ChangeIndex, LifecycleListener {
private final SitePaths sitePaths;
private final CloudSolrServer openIndex;
private final CloudSolrServer closedIndex;
private final Schema<ChangeData> schema;
@Inject
SolrChangeIndex(
@GerritServerConfig Config cfg,
FillArgs fillArgs,
SitePaths sitePaths) throws IOException {
SitePaths sitePaths,
Schema<ChangeData> schema) throws IOException {
this.fillArgs = fillArgs;
this.sitePaths = sitePaths;
this.schema = schema;
String url = cfg.getString("index", "solr", "url");
if (Strings.isNullOrEmpty(url)) {
@@ -270,7 +272,7 @@ class SolrChangeIndex implements ChangeIndex, LifecycleListener {
private SolrInputDocument toDocument(ChangeData cd) throws IOException {
try {
SolrInputDocument result = new SolrInputDocument();
for (FieldDef<ChangeData, ?> f : ChangeField.ALL.values()) {
for (FieldDef<ChangeData, ?> f : schema.getFields().values()) {
if (f.isRepeatable()) {
add(result, f, (Iterable<?>) f.get(cd, fillArgs));
} else {