Add Gerrit 3.5 to 3.6 upgrade testing

This adds upgrade testing from our current Gerrit version (3.5) to the
likely future version of our next upgrade (3.6).

To do so we have to refactor the gerrit testing becase the 3.5 to 3.6
upgrade requires we run a command against 3.5. The previous upgrade
system assumed the old version could be left alone and jumped straight
into the upgrade finally testing the end state. Now we have split up the
gerrit bootstrapping and gerrit testing so that normal gerrit testing
and upgrade testing can run these different tasks at different points in
the gerrit deployment process.

Now the upgrade tests use the bootstrapping playbook to create users,
projects, and changes on the old version of gerrit before running the
copy-approvals command. Then after the upgrade we run the test assertion
portion of the job.

Change-Id: Id58b27e6f717f794a8ef7a048eec7fbb3bc52af6

playbooks/zuul/bootstrap-and-test-review.yaml

@@ -0,0 +1,5 @@
+- name: Bootstrap gerrit users, projects, and changes
+  import_playbook: ./bootstrap-test-review.yaml
+
+- name: Test gerrit is functional
+  import_playbook: ./test-review.yaml

playbooks/zuul/bootstrap-test-review.yaml

@@ -1,6 +1,5 @@
 - hosts: "review"
   tasks:
-
     - name: Wait for Gerrit to be up
       uri:
         url: http://localhost:8081/a/accounts/admin/sshkeys
@@ -266,42 +265,3 @@
         - [ PUT, 201 ]
         - [ DELETE, 204 ]
         - [ PUT, 201 ]
-
-    - name: Run selenium container
-      include_role:
-        name: run-selenium
-
-- name: Run rename playbook
-  import_playbook: rename_repos.yaml
-  vars:
-    repolist: /home/zuul/src/opendev.org/opendev/system-config/playbooks/zuul/test_gerrit_renames.yaml
-
-- hosts: "review"
-  tasks:
-    - name: Get group UUID of renamed group
-      uri:
-        url: 'http://localhost:8081/a/groups/CI-tools-updated'
-        method: GET
-        user: admin
-        password: secret
-        status_code: 200
-        return_content: yes
-      register: group_renamed_raw
-
-    - name: Debug serialized json
-      debug:
-        var: group_renamed_raw
-
-    - name: Deserialize returned data to internal variable.
-      set_fact:
-        # The first 4 bytes of the returned data are )]}' which is
-        # invalid json.
-        group_renamed_json: '{{ group_renamed_raw.content | regex_replace("^....", "") | from_json }}'
-
-    - name: Debug deserialized json
-      debug:
-        var: group_renamed_json
-
-    - name: Check renamed UUID is consistent
-      assert:
-        that: group_renamed_json['id'] == group_json['id']

playbooks/zuul/test-review.yaml

@@ -0,0 +1,66 @@
+- hosts: "review"
+  tasks:
+    - name: Run selenium container
+      include_role:
+        name: run-selenium
+
+    # Get the group info so that we can check group UUIDs don't change when
+    # groups are renamed.
+    - name: Get group UUID
+      uri:
+        url: 'http://localhost:8081/a/groups/CI-tools'
+        method: GET
+        user: admin
+        password: secret
+        status_code: 200
+        return_content: yes
+      register: group_raw
+
+    - name: Debug serialized json
+      debug:
+        var: group_raw
+
+    - name: Deserialize returned data to internal variable.
+      set_fact:
+        # The first 4 bytes of the returned data are )]}' which is
+        # invalid json.
+        group_json: '{{ group_raw.content | regex_replace("^....", "") | from_json }}'
+
+    - name: Debug deserialized json
+      debug:
+        var: group_json
+
+- name: Run rename playbook
+  import_playbook: ../rename_repos.yaml
+  vars:
+    repolist: /home/zuul/src/opendev.org/opendev/system-config/playbooks/zuul/test_gerrit_renames.yaml
+
+- hosts: "review"
+  tasks:
+    - name: Get group UUID of renamed group
+      uri:
+        url: 'http://localhost:8081/a/groups/CI-tools-updated'
+        method: GET
+        user: admin
+        password: secret
+        status_code: 200
+        return_content: yes
+      register: group_renamed_raw
+
+    - name: Debug serialized json
+      debug:
+        var: group_renamed_raw
+
+    - name: Deserialize returned data to internal variable.
+      set_fact:
+        # The first 4 bytes of the returned data are )]}' which is
+        # invalid json.
+        group_renamed_json: '{{ group_renamed_raw.content | regex_replace("^....", "") | from_json }}'
+
+    - name: Debug deserialized json
+      debug:
+        var: group_renamed_json
+
+    - name: Check renamed UUID is consistent
+      assert:
+        that: group_renamed_json['id'] == group_json['id']

playbooks/zuul/upgrade-review.yaml

@@ -1,15 +1,34 @@
+# Note this playbook is in the zuul/ dir because it is very test specific
+# currently. We could potentially rewrite things so that this can be used
+# in production but it isn't currently ready for that.
+#
+# In particular it bootstraps users and test changes assuming a test env.
+
 - name: Ensure initial gerrit state
   import_playbook: ../service-review.yaml
   vars:
-    gerrit_container_image: docker.io/opendevorg/gerrit:3.4
+    gerrit_container_image: docker.io/opendevorg/gerrit:3.5
 
 - hosts: "review:!disabled"
-  name: "Prepare Gerrit for Upgrade"
+  name: "Wait for gerrit to be up and running"
   tasks:
     - name: Pause for a few seconds to give gerrit time to start
       wait_for:
         timeout: 30
 
+- name: Bootstrap gerrit to be semi useable
+  # This is necessary to perform actions on the old side pre upgrade
+  import_playbook: ./bootstrap-test-review.yaml
+
+- hosts: "review:!disabled"
+  name: "Prepare Gerrit for Upgrade"
+  tasks:
+    - name: Run gerrit sticky approvals migration command
+      shell:
+        cmd: |
+          ssh -i /root/.ssh/id_25519 -p 29418 admin@localhost \
+            gerrit copy-approvals -v
+
     - name: Stop gerrit before we upgrade
       shell:
         cmd: docker-compose down
@@ -33,7 +52,7 @@
 - name: Perform gerrit upgrade
   import_playbook: ../service-review.yaml
   vars:
-    gerrit_container_image: docker.io/opendevorg/gerrit:3.5
+    gerrit_container_image: docker.io/opendevorg/gerrit:3.6
     gerrit_run_init: true
 
 - hosts: "review:!disabled"

zuul.d/project.yaml

@@ -70,15 +70,13 @@
               - name: opendev-buildset-registry
               - name: system-config-build-image-gerrit-3.6
                 soft: true
-        # Temporarily disabled until we have 3.6 images to perform
+        - system-config-upgrade-review:
-        # a 3.5 -> 3.6 upgrade.
+            dependencies:
-        #- system-config-upgrade-review:
+              - name: opendev-buildset-registry
-        #    dependencies:
+              - name: system-config-build-image-gerrit-3.5
-        #      - name: opendev-buildset-registry
+                soft: true
-        #      - name: system-config-build-image-gerrit-3.4
+              - name: system-config-build-image-gerrit-3.6
-        #        soft: true
+                soft: true
-        #      - name: system-config-build-image-gerrit-3.5
-        #        soft: true
         - system-config-build-image-refstack
         - system-config-run-refstack:
             dependencies:

zuul.d/system-config-run.yaml

@@ -936,7 +936,7 @@
       run_playbooks:
         - playbooks/letsencrypt.yaml
         - playbooks/service-review.yaml
-      run_test_playbook: playbooks/test-review.yaml
+      run_test_playbook: playbooks/zuul/bootstrap-and-test-review.yaml
     host-vars:
       review99.opendev.org:
         host_copy_output:
@@ -984,15 +984,16 @@
     name: system-config-upgrade-review
     parent: system-config-run-review-base
     description: |
-      Test we can upgrade a gerrit 3.4 to 3.5
+      Test we can upgrade a gerrit 3.5 to 3.6
     requires:
-      - gerrit-3.4-container-image
       - gerrit-3.5-container-image
+      - gerrit-3.6-container-image
     vars:
       run_playbooks:
         - playbooks/letsencrypt.yaml
         - playbooks/zuul/upgrade-review.yaml
-      zuul_test_gerrit_version: '3.4'
+      run_test_playbook: playbooks/zuul/test-review.yaml
+      zuul_test_gerrit_version: '3.5'
 
 - job:
     name: system-config-run-static