E2E load tests example scenarios

Introduce examples of scenarios to show how to load test Gerrit
using Gatling.

This initial examples only include tests over the git protocol, however
it is possible to expand the scenarios to mix different protocols (Git, SSH, Http).

Feature: Issue 10900
Change-Id: I00927b3b83f45ca852305780f5e67a4272d1ab22

Documentation/dev-e2e-tests.txt

@@ -0,0 +1,92 @@
+= Gerrit Code Review - End to end load tests
+
+This document provides a description of a Gerrit load test scenario implemented using the link:http://gatling.io[`Gatling`] framework.
+
+Similar scenarios have been successfully used to compare performance of different Gerrit versions or study the Gerrit response
+under different load profiles.
+
+== What is Gatling?
+
+Gatling is a load testing tool which provides out of the box support for the HTTP protocol. Documentation on how to write an
+HTTP load test can be found link:https://gatling.io/docs/current/http/http_protocol/[`here`].
+
+However, in the scenario we are proposing, we are leveraging the link:https://github.com/GerritForge/gatling-git[`Gatling Git extension`]
+to run tests at Git protocol level.
+
+Gatling is written in Scala, but the abstraction provided by the Gatling DSL makes the scenarios implementation easy even without any Scala knowledge.
+
+Examples of scenarios can be found in the `e2e-tests` directory.
+
+=== How to run the load tests
+
+==== Prerequisites
+
+* link:https://www.scala-lang.org/download/[`Scala 2.12`]
+
+==== How to build
+
+----
+sbt compile
+----
+
+==== Setup
+
+If you are running SSH commands the private keys of the users used for testing need to go in `/tmp/ssh-keys`.
+The keys need to be generated this way (JSch won't validate them [otherwise](https://stackoverflow.com/questions/53134212/invalid-privatekey-when-using-jsch):
+
+----
+ssh-keygen -m PEM -t rsa -C "test@mail.com" -f /tmp/ssh-keys/id_rsa
+----
+
+*NOTE*: Don't forget to add the public keys for the testing user(s) to your git server
+
+==== Input file
+
+The ReplayRecordsScenario is fed by the data coming from the [src/test/resources/data/requests.json](/src/test/resources/data/requests.json) file.
+Such file contains the commands and repo used during the load test.
+Below an example:
+
+----
+[
+  {
+    "url": "ssh://admin@localhost:29418/loadtest-repo.git",
+    "cmd": "clone"
+  },
+  {
+    "url": "http://localhost:8080/loadtest-repo.git",
+    "cmd": "fetch"
+  }
+]
+----
+
+Valid commands are:
+* fetch
+* pull
+* push
+* clone
+
+==== How to use the framework
+
+Run all tests:
+----
+sbt "gatling:test"
+----
+
+Run a single test:
+----
+sbt "gatling:testOnly com.google.gerrit.scenarios.ReplayRecordsFromFeederScenario"
+----
+
+Generate the last report:
+----
+sbt "gatling:lastReport"
+----
+
+GERRIT
+------
+Part of link:index.html[Gerrit Code Review]
+
+SEARCHBOX
+---------
+
+[scala]:

e2e-tests/load-tests/.gitignore

@@ -0,0 +1,16 @@
+.idea/
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+### Scala ###
+*.class
+*.log
+target
+project/target

e2e-tests/load-tests/build.sbt

@@ -0,0 +1,18 @@
+import Dependencies._
+
+enablePlugins(GatlingPlugin)
+
+lazy val gatlingGitExtension = RootProject(uri("git://github.com/GerritForge/gatling-git.git"))
+lazy val root = (project in file("."))
+  .settings(
+    inThisBuild(List(
+      organization := "com.google.gerrit",
+      scalaVersion := "2.12.8",
+      version := "0.1.0-SNAPSHOT"
+    )),
+    name := "gerrit",
+    libraryDependencies ++=
+      gatling ++
+        Seq("io.gatling" % "gatling-core" % "3.1.1" ) ++
+        Seq("io.gatling" % "gatling-app" % "3.1.1" )
+  ) dependsOn(gatlingGitExtension)

e2e-tests/load-tests/project/Dependencies.scala

@@ -0,0 +1,8 @@
+import sbt._
+
+object Dependencies {
+  lazy val gatling = Seq(
+    "io.gatling.highcharts" % "gatling-charts-highcharts",
+    "io.gatling" % "gatling-test-framework",
+  ).map(_ % "3.1.1" % Test)
+}

e2e-tests/load-tests/project/build.properties

@@ -0,0 +1 @@
+sbt.version=1.2.3

e2e-tests/load-tests/project/plugins.sbt

@@ -0,0 +1 @@
+addSbtPlugin("io.gatling" % "gatling-sbt" % "3.0.0")

e2e-tests/load-tests/src/test/resources/application.conf

@@ -0,0 +1,30 @@
+http {
+  username: "default_username",
+  username: ${?GIT_HTTP_USERNAME},
+
+  password: "default_password",
+  password: ${?GIT_HTTP_PASSWORD},
+}
+
+ssh {
+  private_key_path: "/tmp/ssh-keys/id_rsa",
+  private_key_path: ${?GIT_SSH_PRIVATE_KEY_PATH},
+}
+
+tmpFiles {
+  basePath: "/tmp"
+  basePath: ${?TMP_BASE_PATH}
+}
+
+commands {
+  push {
+    numFiles: 4
+    numFiles: ${?NUM_FILES}
+    minContentLength: 100
+    minContentLength: ${?MIN_CONTENT_LEGTH}
+    maxContentLength: 10000
+    maxContentLength: ${?MAX_CONTENT_LEGTH}
+    commitPrefix: ""
+    commitPrefix: ${?COMMIT_PREFIX}
+  }
+}

e2e-tests/load-tests/src/test/resources/data/requests.json

@@ -0,0 +1,26 @@
+[
+  {
+    "url": "ssh://admin@localhost:29418/loadtest-repo",
+    "cmd": "clone"
+  },
+  {
+    "url": "ssh://admin@localhost:29418/loadtest-repo",
+    "cmd": "pull"
+  },
+  {
+    "url": "ssh://admin@localhost:29418/loadtest-repo",
+    "cmd": "push"
+  },
+  {
+    "url": "http://localhost:8080/loadtest-repo",
+    "cmd": "clone"
+  },
+  {
+    "url": "http://localhost:8080/loadtest-repo",
+    "cmd": "pull"
+  },
+  {
+    "url": "http://localhost:8080/loadtest-repo",
+    "cmd": "push"
+  }
+]

e2e-tests/load-tests/src/test/resources/gatling.conf

@@ -0,0 +1,128 @@
+#########################
+# Gatling Configuration #
+#########################
+
+# This file contains all the settings configurable for Gatling with their default values
+
+gatling {
+  core {
+    #outputDirectoryBaseName = "" # The prefix for each simulation result folder (then suffixed by the report generation timestamp)
+    #runDescription = ""          # The description for this simulation run, displayed in each report
+    #encoding = "utf-8"           # Encoding to use throughout Gatling for file and string manipulation
+    #simulationClass = ""         # The FQCN of the simulation to run (when used in conjunction with noReports, the simulation for which assertions will be validated)
+    #elFileBodiesCacheMaxCapacity = 200        # Cache size for request body EL templates, set to 0 to disable
+    #rawFileBodiesCacheMaxCapacity = 200       # Cache size for request body Raw templates, set to 0 to disable
+    #rawFileBodiesInMemoryMaxSize = 1000       # Below this limit, raw file bodies will be cached in memory
+    #pebbleFileBodiesCacheMaxCapacity = 200    # Cache size for request body Peeble templates, set to 0 to disable
+    #shutdownTimeout = 5000                    # Milliseconds to wait for the actor system to shutdown
+    extract {
+      regex {
+        #cacheMaxCapacity = 200 # Cache size for the compiled regexes, set to 0 to disable caching
+      }
+      xpath {
+        #cacheMaxCapacity = 200 # Cache size for the compiled XPath queries,  set to 0 to disable caching
+      }
+      jsonPath {
+        #cacheMaxCapacity = 200 # Cache size for the compiled jsonPath queries, set to 0 to disable caching
+        #preferJackson = false  # When set to true, prefer Jackson over Boon for JSON-related operations
+      }
+      css {
+        #cacheMaxCapacity = 200 # Cache size for the compiled CSS selectors queries,  set to 0 to disable caching
+      }
+    }
+    directory {
+      simulations = "./src/test/scala"
+      #simulations = user-files/simulations # Directory where simulation classes are located (for bundle packaging only)
+      resources = "./src/test/resources/data"     # Directory where resources, such as feeder files and request bodies are located (for bundle packaging only)
+      #reportsOnly = ""                     # If set, name of report folder to look for in order to generate its report
+      binaries = "./target/scala-2.12/classes"                        # If set, name of the folder where compiles classes are located: Defaults to GATLING_HOME/target.
+      #results = results                    # Name of the folder where all reports folder are located
+    }
+  }
+  charting {
+    #noReports = false       # When set to true, don't generate HTML reports
+    #maxPlotPerSeries = 1000 # Number of points per graph in Gatling reports
+    #useGroupDurationMetric = false  # Switch group timings from cumulated response time to group duration.
+    indicators {
+      #lowerBound = 800      # Lower bound for the requests' response time to track in the reports and the console summary
+      #higherBound = 1200    # Higher bound for the requests' response time to track in the reports and the console summary
+      #percentile1 = 50      # Value for the 1st percentile to track in the reports, the console summary and Graphite
+      #percentile2 = 75      # Value for the 2nd percentile to track in the reports, the console summary and Graphite
+      #percentile3 = 95      # Value for the 3rd percentile to track in the reports, the console summary and Graphite
+      #percentile4 = 99      # Value for the 4th percentile to track in the reports, the console summary and Graphite
+    }
+  }
+  http {
+    #fetchedCssCacheMaxCapacity = 200          # Cache size for CSS parsed content, set to 0 to disable
+    #fetchedHtmlCacheMaxCapacity = 200         # Cache size for HTML parsed content, set to 0 to disable
+    #perUserCacheMaxCapacity = 200             # Per virtual user cache size, set to 0 to disable
+    #warmUpUrl = "https://gatling.io"           # The URL to use to warm-up the HTTP stack (blank means disabled)
+    #enableGA = true                           # Very light Google Analytics, please support
+    ssl {
+      keyStore {
+        #type = ""      # Type of SSLContext's KeyManagers store
+        #file = ""      # Location of SSLContext's KeyManagers store
+        #password = ""  # Password for SSLContext's KeyManagers store
+        #algorithm = "" # Algorithm used SSLContext's KeyManagers store
+      }
+      trustStore {
+        #type = ""      # Type of SSLContext's TrustManagers store
+        #file = ""      # Location of SSLContext's TrustManagers store
+        #password = ""  # Password for SSLContext's TrustManagers store
+        #algorithm = "" # Algorithm used by SSLContext's TrustManagers store
+      }
+    }
+    ahc {
+      #connectTimeout = 10000                              # Timeout in millis for establishing a TCP socket
+      #handshakeTimeout = 10000                            # Timeout in millis for performing TLS handshake
+      #pooledConnectionIdleTimeout = 60000                 # Timeout in millis for a connection to stay idle in the pool
+      #maxRetry = 2                                        # Number of times that a request should be tried again
+      #requestTimeout = 60000                              # Timeout in millis for performing an HTTP request
+      #enableSni = true                                    # When set to true, enable Server Name indication (SNI)
+      #enableHostnameVerification = false                  # When set to true, enable hostname verification: SSLEngine.setHttpsEndpointIdentificationAlgorithm("HTTPS")
+      #useInsecureTrustManager = true                      # Use an insecure TrustManager that trusts all server certificates
+      #filterInsecureCipherSuites = true                   # Turn to false to not filter out insecure and weak cipher suites
+      #sslEnabledProtocols = [TLSv1.2, TLSv1.1, TLSv1]     # Array of enabled protocols for HTTPS, if empty use the JDK defaults
+      #sslEnabledCipherSuites = []                         # Array of enabled cipher suites for HTTPS, if empty use the AHC defaults
+      #sslSessionCacheSize = 0                             # SSLSession cache size, set to 0 to use JDK's default
+      #sslSessionTimeout = 0                               # SSLSession timeout in seconds, set to 0 to use JDK's default (24h)
+      #disableSslSessionResumption = false                 # if true, SSLSessions won't be resumed
+      #useOpenSsl = true                                   # if OpenSSL should be used instead of JSSE
+      #useNativeTransport = false                          # if native transport should be used instead of Java NIO (requires netty-transport-native-epoll, currently Linux only)
+      #enableZeroCopy = true                               # if zero-copy upload should be used if possible
+      #tcpNoDelay = true
+      #soReuseAddress = false
+      #allocator = "pooled"                            # switch to unpooled for unpooled ByteBufAllocator
+      #maxThreadLocalCharBufferSize = 200000           # Netty's default is 16k
+    }
+    dns {
+      #queryTimeout = 5000                             # Timeout in millis of each DNS query in millis
+      #maxQueriesPerResolve = 6                        # Maximum allowed number of DNS queries for a given name resolution
+    }
+  }
+  jms {
+    #replyTimeoutScanPeriod = 1000  # scan period for timedout reply messages
+  }
+  data {
+    #writers = [console, file]      # The list of DataWriters to which Gatling write simulation data (currently supported : console, file, graphite, jdbc)
+    console {
+      #light = false                # When set to true, displays a light version without detailed request stats
+      #writePeriod = 5              # Write interval, in seconds
+    }
+    file {
+      #bufferSize = 8192            # FileDataWriter's internal data buffer size, in bytes
+    }
+    leak {
+      #noActivityTimeout = 30  # Period, in seconds, for which Gatling may have no activity before considering a leak may be happening
+    }
+    graphite {
+      #light = false              # only send the all* stats
+      #host = "localhost"         # The host where the Carbon server is located
+      #port = 2003                # The port to which the Carbon server listens to (2003 is default for plaintext, 2004 is default for pickle)
+      #protocol = "tcp"           # The protocol used to send data to Carbon (currently supported : "tcp", "udp")
+      #rootPathPrefix = "gatling" # The common prefix of all metrics sent to Graphite
+      #bufferSize = 8192          # Internal data buffer size, in bytes
+      #writePeriod = 1            # Write period, in seconds
+    }
+  }
+}

e2e-tests/load-tests/src/test/resources/hooks/commit-msg

@@ -0,0 +1,43 @@
+#!/bin/sh
+#
+# Part of Gerrit Code Review (https://www.gerritcodereview.com/)
+#
+# Copyright (C) 2009 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# avoid [[ which is not POSIX sh.
+if test "$#" != 1 ; then
+  echo "$0 requires an argument."
+  exit 1
+fi
+
+if test ! -f "$1" ; then
+  echo "file does not exist: $1"
+  exit 1
+fi
+
+if test ! -s "$1" ; then
+  echo "file is empty: $1"
+  exit 1
+fi
+
+# $RANDOM will be undefined if not using bash, so don't use set -u
+random=$( (whoami ; hostname ; date; cat $1 ; echo $RANDOM) | git hash-object --stdin)
+dest="$1.tmp.${random}"
+
+# Avoid the --in-place option which only appeared in Git 2.8
+# Avoid the --if-exists option which only appeared in Git 2.15
+cat "$1" \
+| git -c trailer.ifexists=doNothing interpret-trailers --trailer "Change-Id: I${random}" > "${dest}" \
+&& mv "${dest}" "$1"

e2e-tests/load-tests/src/test/scala/com/google/gerrit/scenarios/ReplayRecordsFromFeeder.scala

@@ -0,0 +1,72 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.gerrit.scenarios
+
+import com.github.barbasa.gatling.git.protocol.GitProtocol
+import com.github.barbasa.gatling.git.request.builder.GitRequestBuilder
+import io.gatling.core.Predef._
+import io.gatling.core.structure.ScenarioBuilder
+import java.io._
+
+import com.github.barbasa.gatling.git.{
+  GatlingGitConfiguration,
+  GitRequestSession
+}
+import org.apache.commons.io.FileUtils
+
+import scala.concurrent.duration._
+import org.eclipse.jgit.hooks._
+
+class ReplayRecordsFromFeederScenario extends Simulation {
+
+  val gitProtocol = GitProtocol()
+  implicit val conf = GatlingGitConfiguration()
+  implicit val postMessageHook: Option[String] = Some(
+    s"hooks/${CommitMsgHook.NAME}")
+
+  val feeder = jsonFile("data/requests.json").circular
+
+  val replayCallsScenario: ScenarioBuilder =
+    scenario("Git commands")
+      .repeat(10000) {
+        feed(feeder)
+          .exec(new GitRequestBuilder(GitRequestSession("${cmd}", "${url}")))
+      }
+
+  setUp(
+    replayCallsScenario.inject(
+      nothingFor(4 seconds),
+      atOnceUsers(10),
+      rampUsers(10) during (5 seconds),
+      constantUsersPerSec(20) during (15 seconds),
+      constantUsersPerSec(20) during (15 seconds) randomized
+    ))
+    .protocols(gitProtocol)
+    .maxDuration(60 seconds)
+
+  after {
+    try {
+      //After is often called too early. Some retries should be implemented.
+      Thread.sleep(5000)
+      FileUtils.deleteDirectory(new File(conf.tmpBasePath))
+    } catch {
+      case e: IOException => {
+        System.err.println(
+          "Unable to delete temporary directory: " + conf.tmpBasePath)
+        e.printStackTrace
+      }
+    }
+  }
+}