Add Gerrit 3.6 -> 3.7 upgrade test job

This updates the gerrit upgrade testing job to upgrade from 3.6 to 3.7.

This upgrade requires an offline reindex which is new for us since we've
been on Gerrit 3.x. In order to support this offline reindex requirement
the gerrit role is modified to trigger an offline reindex in the role's
start tasks if the flag to do so is set. I expect this will really only
be used in testing, but it allows us to reuse most everything else in
testing and in production which is nice.

Change-Id: Ibe68176970394cbe71c3126ff3fe7a1b0601b09a

playbooks/roles/gerrit/defaults/main.yaml

@@ -5,6 +5,7 @@ gerrit_site_dir: "{{ gerrit_home_dir }}/review_site"
 gerrit_run_compose_up: false
 gerrit_run_init: false
 gerrit_run_init_dev_mode: false
+gerrit_run_reindex: false
 gerrit_packed_git_open_files: 4096
 gerrit_container_image: docker.io/opendevorg/gerrit:3.6
 gerrit_container_volumes:

playbooks/roles/gerrit/tasks/start.yaml

@@ -6,6 +6,14 @@
     chdir: /etc/gerrit-compose/
   when: gerrit_run_init | bool
 
+- name: Run gerrit reindex
+  shell:
+    cmd: >
+      docker-compose run shell
+      java -jar /var/gerrit/bin/gerrit.war reindex -d /var/gerrit
+    chdir: /etc/gerrit-compose/
+  when: gerrit_run_reindex | bool
+
 - name: Run docker-compose up
   shell:
     cmd: docker-compose up -d

playbooks/zuul/upgrade-review.yaml

@@ -7,7 +7,7 @@
 - name: Ensure initial gerrit state
   import_playbook: ../service-review.yaml
   vars:
-    gerrit_container_image: docker.io/opendevorg/gerrit:3.5
+    gerrit_container_image: docker.io/opendevorg/gerrit:3.6
 
 - hosts: "review:!disabled"
   name: "Wait for gerrit to be up and running"
@@ -23,12 +23,6 @@
 - hosts: "review:!disabled"
   name: "Prepare Gerrit for Upgrade"
   tasks:
-    - name: Run gerrit sticky approvals migration command
-      shell:
-        cmd: |
-          ssh -i /root/.ssh/id_25519 -p 29418 admin@localhost \
-            gerrit copy-approvals -v
-
     - name: Stop gerrit before we upgrade
       shell:
         cmd: docker-compose down
@@ -62,8 +56,10 @@
 - name: Perform gerrit upgrade
   import_playbook: ../service-review.yaml
   vars:
-    gerrit_container_image: docker.io/opendevorg/gerrit:3.6
+    gerrit_container_image: docker.io/opendevorg/gerrit:3.7
     gerrit_run_init: true
+    # Gerrit 3.6 -> 3.7 upgrade requires an offline reindex
+    gerrit_run_reindex: true
 
 - hosts: "review:!disabled"
   name: "Post upgrade config check"

zuul.d/project.yaml

@@ -75,13 +75,13 @@
               - name: opendev-buildset-registry
               - name: system-config-build-image-gerrit-3.7
                 soft: true
-        #- system-config-upgrade-review:
-        #    dependencies:
-        #      - name: opendev-buildset-registry
-        #      - name: system-config-build-image-gerrit-3.6
-        #        soft: true
-        #      - name: system-config-build-image-gerrit-3.7
-        #        soft: true
+        - system-config-upgrade-review:
+            dependencies:
+              - name: opendev-buildset-registry
+              - name: system-config-build-image-gerrit-3.6
+                soft: true
+              - name: system-config-build-image-gerrit-3.7
+                soft: true
         - system-config-build-image-refstack
         - system-config-run-refstack:
             dependencies:

zuul.d/system-config-run.yaml

@@ -1124,20 +1124,20 @@
     vars:
       zuul_test_gerrit_version: '3.7'
 
-#- job:
-#    name: system-config-upgrade-review
-#    parent: system-config-run-review-base
-#    description: |
-#      Test we can upgrade a gerrit 3.6 to 3.7
-#    requires:
-#      - gerrit-3.6-container-image
-#      - gerrit-3.7-container-image
-#    vars:
-#      run_playbooks:
-#        - playbooks/letsencrypt.yaml
-#        - playbooks/zuul/upgrade-review.yaml
-#      run_test_playbook: playbooks/zuul/test-review.yaml
-#      zuul_test_gerrit_version: '3.6'
+- job:
+    name: system-config-upgrade-review
+    parent: system-config-run-review-base
+    description: |
+      Test we can upgrade a gerrit 3.6 to 3.7
+    requires:
+      - gerrit-3.6-container-image
+      - gerrit-3.7-container-image
+    vars:
+      run_playbooks:
+        - playbooks/letsencrypt.yaml
+        - playbooks/zuul/upgrade-review.yaml
+      run_test_playbook: playbooks/zuul/test-review.yaml
+      zuul_test_gerrit_version: '3.6'
 
 - job:
     name: system-config-run-static