opendev/gerrit
Code Issues Proposed changes

Rewrite upload archive tests as real integration tests

Browse Source 
Start gerrit server using StandaloneSiteTest and configure git client
connection using git-core client and SSH protocol.

Expand the tests for fetching tar archive and all supported compression
algorithms: gzip and bz2 in addition to xz.

Test Plan:

  $ bazel test //javatests/com/google/gerrit/integration/git:upload-archive

Change-Id: I007fd3d736c2be5fdcae6f354752a3663c739ea9
This commit is contained in:
David Ostrovsky
2020-05-10 16:52:58 +02:00
parent c17859c52c
commit edf6214513
6 changed files with 244 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Documentation/dev-bazel.txt
View File

@@ -353,6 +353,7 @@ The following values are currently supported for the group name:
* edit
* elastic
* git
* git-upload-archive
* git-protocol-v2
* notedb
* pgm

16
java/com/google/gerrit/acceptance/SshSession.java
View File

@@ -47,11 +47,10 @@ public class SshSession {
}
@SuppressWarnings("resource")
public String exec(String command, InputStream opt) throws Exception {
public String exec(String command) throws Exception {
ChannelExec channel = (ChannelExec) getSession().openChannel("exec");
try {
channel.setCommand(command);
channel.setInputStream(opt);
InputStream in = channel.getInputStream();
InputStream err = channel.getErrStream();
channel.connect();
@@ -66,19 +65,6 @@ public class SshSession {
}
}
public InputStream exec2(String command, InputStream opt) throws Exception {
ChannelExec channel = (ChannelExec) getSession().openChannel("exec");
channel.setCommand(command);
channel.setInputStream(opt);
InputStream in = channel.getInputStream();
channel.connect();
return in;
}
public String exec(String command) throws Exception {
return exec(command, null);
}
private boolean hasError() {
return error != null;
}

17
java/com/google/gerrit/acceptance/StandaloneSiteTest.java
View File

@@ -42,6 +42,7 @@ import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InterruptedIOException;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Collections;
import org.eclipse.jgit.lib.Config;
@@ -205,8 +206,22 @@ public abstract class StandaloneSiteTest {
protected static String execute(
ImmutableList<String> cmd, File dir, ImmutableMap<String, String> env) throws IOException {
return execute(cmd, dir, env, null);
}
protected static String execute(
ImmutableList<String> cmd,
File dir,
ImmutableMap<String, String> env,
@Nullable Path outputPath)
throws IOException {
ProcessBuilder pb = new ProcessBuilder(cmd);
pb.directory(dir).redirectErrorStream(true);
pb.directory(dir);
if (outputPath != null) {
pb.redirectOutput(outputPath.toFile());
} else {
pb.redirectErrorStream(true);
}
pb.environment().putAll(env);
Process p = pb.start();
byte[] out;

165
javatests/com/google/gerrit/acceptance/ssh/UploadArchiveIT.java
View File

@@ -1,165 +0,0 @@
// Copyright (C) 2015 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.acceptance.ssh;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth.assertWithMessage;
import com.google.common.base.Splitter;
import com.google.gerrit.acceptance.AbstractDaemonTest;
import com.google.gerrit.acceptance.NoHttpd;
import com.google.gerrit.acceptance.PushOneCommit;
import com.google.gerrit.acceptance.PushOneCommit.Result;
import com.google.gerrit.acceptance.UseSsh;
import com.google.gerrit.acceptance.config.GerritConfig;
import com.google.gerrit.git.ObjectIds;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.util.Set;
import java.util.TreeSet;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
import org.eclipse.jgit.transport.PacketLineIn;
import org.eclipse.jgit.transport.PacketLineOut;
import org.eclipse.jgit.util.IO;
import org.junit.Test;
@NoHttpd
@UseSsh
public class UploadArchiveIT extends AbstractDaemonTest {
@Test
@GerritConfig(name = "download.archive", value = "off")
public void archiveFeatureOff() throws Exception {
assertArchiveNotPermitted();
}
@Test
@GerritConfig(
name = "download.archive",
values = {"tar", "tbz2", "tgz", "txz"})
public void zipFormatDisabled() throws Exception {
assertArchiveNotPermitted();
}
@Test
public void zipFormat() throws Exception {
PushOneCommit.Result r = createChange();
String abbreviated = abbreviateName(r);
String c = command(r, "zip", abbreviated);
InputStream out =
adminSshSession.exec2("git-upload-archive " + project.get(), argumentsToInputStream(c));
// Wrap with PacketLineIn to read ACK bytes from output stream
PacketLineIn in = new PacketLineIn(out);
String tmp = in.readString();
assertThat(tmp).isEqualTo("ACK");
in.readString();
// Skip length (4 bytes) + 1 byte
// to position the output stream to the raw zip stream
byte[] buffer = new byte[5];
IO.readFully(out, buffer, 0, 5);
Set<String> entryNames = new TreeSet<>();
try (ZipArchiveInputStream zip = new ZipArchiveInputStream(out)) {
ZipArchiveEntry zipEntry = zip.getNextZipEntry();
while (zipEntry != null) {
String name = zipEntry.getName();
entryNames.add(name);
zipEntry = zip.getNextZipEntry();
}
}
assertThat(entryNames)
.containsExactly(
String.format("%s/", abbreviated),
String.format("%s/%s", abbreviated, PushOneCommit.FILE_NAME))
.inOrder();
}
// Make sure we have coverage for the dependency on xz.
@Test
public void txzFormat() throws Exception {
PushOneCommit.Result r = createChange();
String abbreviated = abbreviateName(r);
String c = command(r, "tar.xz", abbreviated);
try (InputStream out =
adminSshSession.exec2("git-upload-archive " + project.get(), argumentsToInputStream(c))) {
// Wrap with PacketLineIn to read ACK bytes from output stream
PacketLineIn in = new PacketLineIn(out);
String packet = in.readString();
assertThat(packet).isEqualTo("ACK");
// Discard first bit of data, which should be empty.
packet = in.readString();
assertThat(packet).isEmpty();
// Make sure the next one is not on the error channel
packet = in.readString();
// 1 = DATA. It would be nicer to parse the OutputStream with SideBandInputStream from JGit,
// but
// that is currently not public.
char channel = packet.charAt(0);
if (channel != 1) {
assertWithMessage("got packet on channel " + (int) channel, packet).fail();
}
}
}
private String command(PushOneCommit.Result r, String format, String abbreviated) {
String c =
String.format(
"-f=%s --prefix=%s/ %s %s",
format, abbreviated, r.getCommit().name(), PushOneCommit.FILE_NAME);
return c;
}
private void assertArchiveNotPermitted() throws Exception {
PushOneCommit.Result r = createChange();
String abbreviated = abbreviateName(r);
String c = command(r, "zip", abbreviated);
InputStream out =
adminSshSession.exec2("git-upload-archive " + project.get(), argumentsToInputStream(c));
// Wrap with PacketLineIn to read ACK bytes from output stream
PacketLineIn in = new PacketLineIn(out);
String tmp = in.readString();
assertThat(tmp).isEqualTo("ACK");
in.readString();
tmp = in.readString();
tmp = tmp.substring(1);
assertThat(tmp).isEqualTo("fatal: upload-archive not permitted for format zip");
}
private String abbreviateName(Result r) throws Exception {
return ObjectIds.abbreviateName(r.getCommit(), 8, testRepo.getRevWalk().getObjectReader());
}
private InputStream argumentsToInputStream(String c) throws Exception {
ByteArrayOutputStream out = new ByteArrayOutputStream();
PacketLineOut pctOut = new PacketLineOut(out);
for (String arg : Splitter.on(' ').split(c)) {
pctOut.writeString("argument " + arg);
}
pctOut.end();
return new ByteArrayInputStream(out.toByteArray());
}
}

10
javatests/com/google/gerrit/integration/git/BUILD
View File

@@ -1,7 +1,13 @@
load("//javatests/com/google/gerrit/acceptance:tests.bzl", "acceptance_tests")
acceptance_tests(
srcs = glob(["*IT.java"]),
group = "git",
srcs = ["GitProtocolV2IT.java"],
group = "protocol-v2",
labels = ["git-protocol-v2"],
)
acceptance_tests(
srcs = ["UploadArchiveIT.java"],
group = "upload-archive",
labels = ["git-upload-archive"],
)

218
javatests/com/google/gerrit/integration/git/UploadArchiveIT.java Normal file
View File

@@ -0,0 +1,218 @@
// Copyright (C) 2020 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.gerrit.integration.git;
import static com.google.common.truth.Truth.assertThat;
import static com.google.gerrit.acceptance.PushOneCommit.FILE_CONTENT;
import static com.google.gerrit.acceptance.PushOneCommit.FILE_NAME;
import static com.google.gerrit.testing.GerritJUnit.assertThrows;
import static java.nio.charset.StandardCharsets.UTF_8;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.gerrit.acceptance.GerritServer.TestSshServerAddress;
import com.google.gerrit.acceptance.NoHttpd;
import com.google.gerrit.acceptance.StandaloneSiteTest;
import com.google.gerrit.acceptance.UseSsh;
import com.google.gerrit.acceptance.config.GerritConfig;
import com.google.gerrit.common.RawInputUtil;
import com.google.gerrit.entities.Project;
import com.google.gerrit.extensions.api.GerritApi;
import com.google.gerrit.extensions.common.ChangeInput;
import com.google.gerrit.extensions.common.CommitInfo;
import com.google.gerrit.extensions.restapi.RestApiException;
import com.google.inject.Inject;
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.InetSocketAddress;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.Set;
import java.util.TreeSet;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.ArchiveInputStream;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
import org.junit.Test;
@NoHttpd
@UseSsh
public class UploadArchiveIT extends StandaloneSiteTest {
private static final String[] SSH_KEYGEN_CMD =
new String[] {"ssh-keygen", "-t", "rsa", "-q", "-P", "", "-f"};
private static final String GIT_SSH_COMMAND =
"ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o 'IdentitiesOnly yes' -i";
private static final String ARCHIVE = "archive";
@Inject private GerritApi gApi;
@Inject private @TestSshServerAddress InetSocketAddress sshAddress;
private String sshDestination;
private String identityPath;
private Project.NameKey project;
private CommitInfo commit;
@Test
@GerritConfig(name = "download.archive", value = "off")
public void archiveFeatureOff() throws Exception {
try (ServerContext ctx = startServer()) {
setUpTestHarness(ctx);
assertArchiveNotPermitted();
}
}
@Test
@GerritConfig(
name = "download.archive",
values = {"tar", "tbz2", "tgz", "txz"})
public void zipFormatDisabled() throws Exception {
try (ServerContext ctx = startServer()) {
setUpTestHarness(ctx);
assertArchiveNotPermitted();
}
}
@Test
public void verifyUploadArchiveFormats() throws Exception {
try (ServerContext ctx = startServer()) {
setUpTestHarness(ctx);
setUpChange();
for (String f : Arrays.asList("zip", "tar", "tar.gz", "tar.bz2", "tar.xz")) {
verifyUploadArchive(f);
}
}
}
private void verifyUploadArchive(String format) throws Exception {
Path outputPath = sitePaths.data_dir.resolve(ARCHIVE);
execute(
cmd(format, commit.commit),
sitePaths.data_dir.toFile(),
ImmutableMap.of("GIT_SSH_COMMAND", GIT_SSH_COMMAND + identityPath),
outputPath);
try (InputStream fi = Files.newInputStream(outputPath);
InputStream bi = new BufferedInputStream(fi);
ArchiveInputStream archive = archiveStreamForFormat(bi, format)) {
assertEntries(archive);
}
}
private ArchiveInputStream archiveStreamForFormat(InputStream bi, String format)
throws IOException {
switch (format) {
case "zip":
return new ZipArchiveInputStream(bi);
case "tar":
return new TarArchiveInputStream(bi);
case "tar.gz":
return new TarArchiveInputStream(new GzipCompressorInputStream(bi));
case "tar.bz2":
return new TarArchiveInputStream(new BZip2CompressorInputStream(bi));
case "tar.xz":
return new TarArchiveInputStream(new XZCompressorInputStream(bi));
default:
throw new IllegalArgumentException("Unknown archive format: " + format);
}
}
private void setUpTestHarness(ServerContext ctx) throws RestApiException, Exception {
ctx.getInjector().injectMembers(this);
project = Project.nameKey("upload-archive-project-test");
gApi.projects().create(project.get());
setUpAuthentication();
sshDestination =
String.format(
"ssh://%s@%s:%s/%s",
admin.username(), sshAddress.getHostName(), sshAddress.getPort(), project.get());
identityPath =
sitePaths.data_dir.resolve(String.format("id_rsa_%s", admin.username())).toString();
}
private void setUpAuthentication() throws Exception {
execute(
ImmutableList.<String>builder()
.add(SSH_KEYGEN_CMD)
.add(String.format("id_rsa_%s", admin.username()))
.build());
gApi.accounts()
.id(admin.username())
.addSshKey(
new String(
java.nio.file.Files.readAllBytes(
sitePaths.data_dir.resolve(String.format("id_rsa_%s.pub", admin.username()))),
UTF_8));
}
private ImmutableList<String> cmd(String format, String commit) {
return ImmutableList.<String>builder()
.add("git")
.add("archive")
.add("-f=" + format)
.add("--prefix=" + commit + "/")
.add("--remote=" + sshDestination)
.add(commit)
.add(FILE_NAME)
.build();
}
private String execute(ImmutableList<String> cmd) throws Exception {
return execute(cmd, sitePaths.data_dir.toFile(), ImmutableMap.of());
}
private void assertArchiveNotPermitted() {
IOException exception =
assertThrows(
IOException.class,
() ->
execute(
cmd("zip", "master"),
sitePaths.data_dir.toFile(),
ImmutableMap.of("GIT_SSH_COMMAND", GIT_SSH_COMMAND + identityPath)));
assertThat(exception)
.hasMessageThat()
.contains("fatal: upload-archive not permitted for format zip");
}
private void setUpChange() throws Exception {
ChangeInput in = new ChangeInput(project.get(), "master", "Test change");
in.newBranch = true;
String changeId = gApi.changes().create(in).info().changeId;
gApi.changes().id(changeId).edit().modifyFile(FILE_NAME, RawInputUtil.create(FILE_CONTENT));
gApi.changes().id(changeId).edit().publish();
commit = gApi.changes().id(changeId).current().commit(false);
}
private void assertEntries(ArchiveInputStream o) throws IOException {
Set<String> entryNames = new TreeSet<>();
ArchiveEntry e;
while ((e = o.getNextEntry()) != null) {
entryNames.add(e.getName());
}
assertThat(entryNames)
.containsExactly(
String.format("%s/", commit.commit), String.format("%s/%s", commit.commit, FILE_NAME))
.inOrder();
}
}