diff --git a/Documentation/dev-e2e-tests.txt b/Documentation/dev-e2e-tests.txt new file mode 100644 index 0000000000..7329a43669 --- /dev/null +++ b/Documentation/dev-e2e-tests.txt @@ -0,0 +1,92 @@ += Gerrit Code Review - End to end load tests + +This document provides a description of a Gerrit load test scenario implemented using the link:http://gatling.io[`Gatling`] framework. + +Similar scenarios have been successfully used to compare performance of different Gerrit versions or study the Gerrit response +under different load profiles. + +== What is Gatling? + +Gatling is a load testing tool which provides out of the box support for the HTTP protocol. Documentation on how to write an +HTTP load test can be found link:https://gatling.io/docs/current/http/http_protocol/[`here`]. + +However, in the scenario we are proposing, we are leveraging the link:https://github.com/GerritForge/gatling-git[`Gatling Git extension`] +to run tests at Git protocol level. + +Gatling is written in Scala, but the abstraction provided by the Gatling DSL makes the scenarios implementation easy even without any Scala knowledge. + +Examples of scenarios can be found in the `e2e-tests` directory. + +=== How to run the load tests + +==== Prerequisites + +* link:https://www.scala-lang.org/download/[`Scala 2.12`] + +==== How to build + +---- +sbt compile +---- + +==== Setup + +If you are running SSH commands the private keys of the users used for testing need to go in `/tmp/ssh-keys`. +The keys need to be generated this way (JSch won't validate them [otherwise](https://stackoverflow.com/questions/53134212/invalid-privatekey-when-using-jsch): + +---- +ssh-keygen -m PEM -t rsa -C "test@mail.com" -f /tmp/ssh-keys/id_rsa +---- + +*NOTE*: Don't forget to add the public keys for the testing user(s) to your git server + +==== Input file + +The ReplayRecordsScenario is fed by the data coming from the [src/test/resources/data/requests.json](/src/test/resources/data/requests.json) file. +Such file contains the commands and repo used during the load test. +Below an example: + +---- +[ + { + "url": "ssh://admin@localhost:29418/loadtest-repo.git", + "cmd": "clone" + }, + { + "url": "http://localhost:8080/loadtest-repo.git", + "cmd": "fetch" + } +] +---- + +Valid commands are: +* fetch +* pull +* push +* clone + +==== How to use the framework + +Run all tests: +---- +sbt "gatling:test" +---- + +Run a single test: +---- +sbt "gatling:testOnly com.google.gerrit.scenarios.ReplayRecordsFromFeederScenario" +---- + +Generate the last report: +---- +sbt "gatling:lastReport" +---- + +GERRIT +------ +Part of link:index.html[Gerrit Code Review] + +SEARCHBOX +--------- + +[scala]: diff --git a/e2e-tests/load-tests/.gitignore b/e2e-tests/load-tests/.gitignore new file mode 100644 index 0000000000..052f424b27 --- /dev/null +++ b/e2e-tests/load-tests/.gitignore @@ -0,0 +1,16 @@ +.idea/ + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +### Scala ### +*.class +*.log +target +project/target diff --git a/e2e-tests/load-tests/build.sbt b/e2e-tests/load-tests/build.sbt new file mode 100644 index 0000000000..46a3202eb1 --- /dev/null +++ b/e2e-tests/load-tests/build.sbt @@ -0,0 +1,18 @@ +import Dependencies._ + +enablePlugins(GatlingPlugin) + +lazy val gatlingGitExtension = RootProject(uri("git://github.com/GerritForge/gatling-git.git")) +lazy val root = (project in file(".")) + .settings( + inThisBuild(List( + organization := "com.google.gerrit", + scalaVersion := "2.12.8", + version := "0.1.0-SNAPSHOT" + )), + name := "gerrit", + libraryDependencies ++= + gatling ++ + Seq("io.gatling" % "gatling-core" % "3.1.1" ) ++ + Seq("io.gatling" % "gatling-app" % "3.1.1" ) + ) dependsOn(gatlingGitExtension) diff --git a/e2e-tests/load-tests/project/Dependencies.scala b/e2e-tests/load-tests/project/Dependencies.scala new file mode 100644 index 0000000000..72d2ac25d9 --- /dev/null +++ b/e2e-tests/load-tests/project/Dependencies.scala @@ -0,0 +1,8 @@ +import sbt._ + +object Dependencies { + lazy val gatling = Seq( + "io.gatling.highcharts" % "gatling-charts-highcharts", + "io.gatling" % "gatling-test-framework", + ).map(_ % "3.1.1" % Test) +} diff --git a/e2e-tests/load-tests/project/build.properties b/e2e-tests/load-tests/project/build.properties new file mode 100644 index 0000000000..0cd8b07982 --- /dev/null +++ b/e2e-tests/load-tests/project/build.properties @@ -0,0 +1 @@ +sbt.version=1.2.3 diff --git a/e2e-tests/load-tests/project/plugins.sbt b/e2e-tests/load-tests/project/plugins.sbt new file mode 100644 index 0000000000..36cd201542 --- /dev/null +++ b/e2e-tests/load-tests/project/plugins.sbt @@ -0,0 +1 @@ +addSbtPlugin("io.gatling" % "gatling-sbt" % "3.0.0") diff --git a/e2e-tests/load-tests/src/test/resources/application.conf b/e2e-tests/load-tests/src/test/resources/application.conf new file mode 100644 index 0000000000..33da75d93d --- /dev/null +++ b/e2e-tests/load-tests/src/test/resources/application.conf @@ -0,0 +1,30 @@ +http { + username: "default_username", + username: ${?GIT_HTTP_USERNAME}, + + password: "default_password", + password: ${?GIT_HTTP_PASSWORD}, +} + +ssh { + private_key_path: "/tmp/ssh-keys/id_rsa", + private_key_path: ${?GIT_SSH_PRIVATE_KEY_PATH}, +} + +tmpFiles { + basePath: "/tmp" + basePath: ${?TMP_BASE_PATH} +} + +commands { + push { + numFiles: 4 + numFiles: ${?NUM_FILES} + minContentLength: 100 + minContentLength: ${?MIN_CONTENT_LEGTH} + maxContentLength: 10000 + maxContentLength: ${?MAX_CONTENT_LEGTH} + commitPrefix: "" + commitPrefix: ${?COMMIT_PREFIX} + } +} diff --git a/e2e-tests/load-tests/src/test/resources/data/requests.json b/e2e-tests/load-tests/src/test/resources/data/requests.json new file mode 100644 index 0000000000..86f9bf135b --- /dev/null +++ b/e2e-tests/load-tests/src/test/resources/data/requests.json @@ -0,0 +1,26 @@ +[ + { + "url": "ssh://admin@localhost:29418/loadtest-repo", + "cmd": "clone" + }, + { + "url": "ssh://admin@localhost:29418/loadtest-repo", + "cmd": "pull" + }, + { + "url": "ssh://admin@localhost:29418/loadtest-repo", + "cmd": "push" + }, + { + "url": "http://localhost:8080/loadtest-repo", + "cmd": "clone" + }, + { + "url": "http://localhost:8080/loadtest-repo", + "cmd": "pull" + }, + { + "url": "http://localhost:8080/loadtest-repo", + "cmd": "push" + } +] diff --git a/e2e-tests/load-tests/src/test/resources/gatling.conf b/e2e-tests/load-tests/src/test/resources/gatling.conf new file mode 100644 index 0000000000..94c371bbba --- /dev/null +++ b/e2e-tests/load-tests/src/test/resources/gatling.conf @@ -0,0 +1,128 @@ +######################### +# Gatling Configuration # +######################### + +# This file contains all the settings configurable for Gatling with their default values + +gatling { + core { + #outputDirectoryBaseName = "" # The prefix for each simulation result folder (then suffixed by the report generation timestamp) + #runDescription = "" # The description for this simulation run, displayed in each report + #encoding = "utf-8" # Encoding to use throughout Gatling for file and string manipulation + #simulationClass = "" # The FQCN of the simulation to run (when used in conjunction with noReports, the simulation for which assertions will be validated) + #elFileBodiesCacheMaxCapacity = 200 # Cache size for request body EL templates, set to 0 to disable + #rawFileBodiesCacheMaxCapacity = 200 # Cache size for request body Raw templates, set to 0 to disable + #rawFileBodiesInMemoryMaxSize = 1000 # Below this limit, raw file bodies will be cached in memory + #pebbleFileBodiesCacheMaxCapacity = 200 # Cache size for request body Peeble templates, set to 0 to disable + #shutdownTimeout = 5000 # Milliseconds to wait for the actor system to shutdown + extract { + regex { + #cacheMaxCapacity = 200 # Cache size for the compiled regexes, set to 0 to disable caching + } + xpath { + #cacheMaxCapacity = 200 # Cache size for the compiled XPath queries, set to 0 to disable caching + } + jsonPath { + #cacheMaxCapacity = 200 # Cache size for the compiled jsonPath queries, set to 0 to disable caching + #preferJackson = false # When set to true, prefer Jackson over Boon for JSON-related operations + } + css { + #cacheMaxCapacity = 200 # Cache size for the compiled CSS selectors queries, set to 0 to disable caching + } + } + directory { + simulations = "./src/test/scala" + #simulations = user-files/simulations # Directory where simulation classes are located (for bundle packaging only) + resources = "./src/test/resources/data" # Directory where resources, such as feeder files and request bodies are located (for bundle packaging only) + #reportsOnly = "" # If set, name of report folder to look for in order to generate its report + binaries = "./target/scala-2.12/classes" # If set, name of the folder where compiles classes are located: Defaults to GATLING_HOME/target. + #results = results # Name of the folder where all reports folder are located + } + } + charting { + #noReports = false # When set to true, don't generate HTML reports + #maxPlotPerSeries = 1000 # Number of points per graph in Gatling reports + #useGroupDurationMetric = false # Switch group timings from cumulated response time to group duration. + indicators { + #lowerBound = 800 # Lower bound for the requests' response time to track in the reports and the console summary + #higherBound = 1200 # Higher bound for the requests' response time to track in the reports and the console summary + #percentile1 = 50 # Value for the 1st percentile to track in the reports, the console summary and Graphite + #percentile2 = 75 # Value for the 2nd percentile to track in the reports, the console summary and Graphite + #percentile3 = 95 # Value for the 3rd percentile to track in the reports, the console summary and Graphite + #percentile4 = 99 # Value for the 4th percentile to track in the reports, the console summary and Graphite + } + } + http { + #fetchedCssCacheMaxCapacity = 200 # Cache size for CSS parsed content, set to 0 to disable + #fetchedHtmlCacheMaxCapacity = 200 # Cache size for HTML parsed content, set to 0 to disable + #perUserCacheMaxCapacity = 200 # Per virtual user cache size, set to 0 to disable + #warmUpUrl = "https://gatling.io" # The URL to use to warm-up the HTTP stack (blank means disabled) + #enableGA = true # Very light Google Analytics, please support + ssl { + keyStore { + #type = "" # Type of SSLContext's KeyManagers store + #file = "" # Location of SSLContext's KeyManagers store + #password = "" # Password for SSLContext's KeyManagers store + #algorithm = "" # Algorithm used SSLContext's KeyManagers store + } + trustStore { + #type = "" # Type of SSLContext's TrustManagers store + #file = "" # Location of SSLContext's TrustManagers store + #password = "" # Password for SSLContext's TrustManagers store + #algorithm = "" # Algorithm used by SSLContext's TrustManagers store + } + } + ahc { + #connectTimeout = 10000 # Timeout in millis for establishing a TCP socket + #handshakeTimeout = 10000 # Timeout in millis for performing TLS handshake + #pooledConnectionIdleTimeout = 60000 # Timeout in millis for a connection to stay idle in the pool + #maxRetry = 2 # Number of times that a request should be tried again + #requestTimeout = 60000 # Timeout in millis for performing an HTTP request + #enableSni = true # When set to true, enable Server Name indication (SNI) + #enableHostnameVerification = false # When set to true, enable hostname verification: SSLEngine.setHttpsEndpointIdentificationAlgorithm("HTTPS") + #useInsecureTrustManager = true # Use an insecure TrustManager that trusts all server certificates + #filterInsecureCipherSuites = true # Turn to false to not filter out insecure and weak cipher suites + #sslEnabledProtocols = [TLSv1.2, TLSv1.1, TLSv1] # Array of enabled protocols for HTTPS, if empty use the JDK defaults + #sslEnabledCipherSuites = [] # Array of enabled cipher suites for HTTPS, if empty use the AHC defaults + #sslSessionCacheSize = 0 # SSLSession cache size, set to 0 to use JDK's default + #sslSessionTimeout = 0 # SSLSession timeout in seconds, set to 0 to use JDK's default (24h) + #disableSslSessionResumption = false # if true, SSLSessions won't be resumed + #useOpenSsl = true # if OpenSSL should be used instead of JSSE + #useNativeTransport = false # if native transport should be used instead of Java NIO (requires netty-transport-native-epoll, currently Linux only) + #enableZeroCopy = true # if zero-copy upload should be used if possible + #tcpNoDelay = true + #soReuseAddress = false + #allocator = "pooled" # switch to unpooled for unpooled ByteBufAllocator + #maxThreadLocalCharBufferSize = 200000 # Netty's default is 16k + } + dns { + #queryTimeout = 5000 # Timeout in millis of each DNS query in millis + #maxQueriesPerResolve = 6 # Maximum allowed number of DNS queries for a given name resolution + } + } + jms { + #replyTimeoutScanPeriod = 1000 # scan period for timedout reply messages + } + data { + #writers = [console, file] # The list of DataWriters to which Gatling write simulation data (currently supported : console, file, graphite, jdbc) + console { + #light = false # When set to true, displays a light version without detailed request stats + #writePeriod = 5 # Write interval, in seconds + } + file { + #bufferSize = 8192 # FileDataWriter's internal data buffer size, in bytes + } + leak { + #noActivityTimeout = 30 # Period, in seconds, for which Gatling may have no activity before considering a leak may be happening + } + graphite { + #light = false # only send the all* stats + #host = "localhost" # The host where the Carbon server is located + #port = 2003 # The port to which the Carbon server listens to (2003 is default for plaintext, 2004 is default for pickle) + #protocol = "tcp" # The protocol used to send data to Carbon (currently supported : "tcp", "udp") + #rootPathPrefix = "gatling" # The common prefix of all metrics sent to Graphite + #bufferSize = 8192 # Internal data buffer size, in bytes + #writePeriod = 1 # Write period, in seconds + } + } +} diff --git a/e2e-tests/load-tests/src/test/resources/hooks/commit-msg b/e2e-tests/load-tests/src/test/resources/hooks/commit-msg new file mode 100644 index 0000000000..b05a6717b0 --- /dev/null +++ b/e2e-tests/load-tests/src/test/resources/hooks/commit-msg @@ -0,0 +1,43 @@ +#!/bin/sh +# +# Part of Gerrit Code Review (https://www.gerritcodereview.com/) +# +# Copyright (C) 2009 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# avoid [[ which is not POSIX sh. +if test "$#" != 1 ; then + echo "$0 requires an argument." + exit 1 +fi + +if test ! -f "$1" ; then + echo "file does not exist: $1" + exit 1 +fi + +if test ! -s "$1" ; then + echo "file is empty: $1" + exit 1 +fi + +# $RANDOM will be undefined if not using bash, so don't use set -u +random=$( (whoami ; hostname ; date; cat $1 ; echo $RANDOM) | git hash-object --stdin) +dest="$1.tmp.${random}" + +# Avoid the --in-place option which only appeared in Git 2.8 +# Avoid the --if-exists option which only appeared in Git 2.15 +cat "$1" \ +| git -c trailer.ifexists=doNothing interpret-trailers --trailer "Change-Id: I${random}" > "${dest}" \ +&& mv "${dest}" "$1" diff --git a/e2e-tests/load-tests/src/test/scala/com/google/gerrit/scenarios/ReplayRecordsFromFeeder.scala b/e2e-tests/load-tests/src/test/scala/com/google/gerrit/scenarios/ReplayRecordsFromFeeder.scala new file mode 100644 index 0000000000..c0eab392d9 --- /dev/null +++ b/e2e-tests/load-tests/src/test/scala/com/google/gerrit/scenarios/ReplayRecordsFromFeeder.scala @@ -0,0 +1,72 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.gerrit.scenarios + +import com.github.barbasa.gatling.git.protocol.GitProtocol +import com.github.barbasa.gatling.git.request.builder.GitRequestBuilder +import io.gatling.core.Predef._ +import io.gatling.core.structure.ScenarioBuilder +import java.io._ + +import com.github.barbasa.gatling.git.{ + GatlingGitConfiguration, + GitRequestSession +} +import org.apache.commons.io.FileUtils + +import scala.concurrent.duration._ +import org.eclipse.jgit.hooks._ + +class ReplayRecordsFromFeederScenario extends Simulation { + + val gitProtocol = GitProtocol() + implicit val conf = GatlingGitConfiguration() + implicit val postMessageHook: Option[String] = Some( + s"hooks/${CommitMsgHook.NAME}") + + val feeder = jsonFile("data/requests.json").circular + + val replayCallsScenario: ScenarioBuilder = + scenario("Git commands") + .repeat(10000) { + feed(feeder) + .exec(new GitRequestBuilder(GitRequestSession("${cmd}", "${url}"))) + } + + setUp( + replayCallsScenario.inject( + nothingFor(4 seconds), + atOnceUsers(10), + rampUsers(10) during (5 seconds), + constantUsersPerSec(20) during (15 seconds), + constantUsersPerSec(20) during (15 seconds) randomized + )) + .protocols(gitProtocol) + .maxDuration(60 seconds) + + after { + try { + //After is often called too early. Some retries should be implemented. + Thread.sleep(5000) + FileUtils.deleteDirectory(new File(conf.tmpBasePath)) + } catch { + case e: IOException => { + System.err.println( + "Unable to delete temporary directory: " + conf.tmpBasePath) + e.printStackTrace + } + } + } +}