Merge "Expand gerrit testing to multiple changes"

playbooks/test-review.yaml

@@ -12,6 +12,17 @@
       delay: 1
       retries: 120
 
+    - name: Create CI group
+      uri:
+        url: 'http://localhost:8081/a/groups/CI-tools'
+        method: PUT
+        user: admin
+        password: secret
+        body_format: json
+        body:
+          description: 'Continuous Integration Tools'
+        status_code: 201
+
     - name: Create Zuul user
       uri:
         url: http://localhost:8081/a/accounts/zuul
@@ -25,6 +36,75 @@
           http_password: 'secret'
         status_code: 201
 
+    - name: Add Zuul to group
+      uri:
+        url: 'http://localhost:8081/a/groups/CI-tools/members/zuul'
+        method: PUT
+        user: admin
+        password: secret
+        status_code: 201
+
+    - name: Get group UUID
+      uri:
+        url: 'http://localhost:8081/a/groups/CI-tools'
+        method: GET
+        user: admin
+        password: secret
+        status_code: 200
+        return_content: yes
+      register: group_raw
+
+    - name: Debug serialized json
+      debug:
+        var: group_raw
+
+    - name: Deserialize returned data to internal variable.
+      set_fact:
+        # The first 4 bytes of the returned data are )]}' which is
+        # invalid json.
+        group_json: '{{ group_raw.content | regex_replace("^....", "") | from_json }}'
+
+    - name: Debug deserialized json
+      debug:
+        var: group_json
+
+    - name: Setup key and initial gerrit config
+      shell:
+        executable: /bin/bash
+        cmd: |
+          set -xe
+
+          ssh-keygen -t ed25519 -f /root/.ssh/id_25519 -P ""
+          curl -X POST --user "admin:secret" -H "Content-Type: text/plain" -d@/root/.ssh/id_25519.pub http://localhost:8081/a/accounts/admin/sshkeys
+          ssh-keyscan -p 29418 localhost >> /root/.ssh/known_hosts
+
+          git config --global user.name "Admin"
+          git config --global user.email "admin@example.com"
+
+          git clone http://admin:secret@localhost:8081/a/All-Projects
+
+          pushd All-Projects
+
+          echo "{{ group_json.id }}	CI-tools" >> groups # noqa 203
+          git commit -a -m "Add CI-tools group"
+          git push origin HEAD:refs/meta/config
+
+          cat >> project.config << EOF
+          [label "Verified"]
+                  function = MaxWithBlock
+                  value = -2 Fails
+                  value = -1 Doesn't seem to work
+                  value = 0 No score
+                  value = +1 Works for me
+                  value = +2 Verified
+          [access "refs/heads/*"]
+                  label-Verified = -2..+2 group CI-tools
+          EOF
+          git commit -a -m 'Update verify tags'
+          git push origin HEAD:refs/meta/config
+
+          popd
+
     - name: Create temp dir for project creation
       shell: mktemp -d
       register: project_tmp
@@ -43,9 +123,8 @@
         chdir: "{{ project_tmp.stdout }}"
         cmd: |
           set -x
+
           git init .
-          git config user.name "Admin"
-          git config user.email "admin@example.com"
           cat >.gitreview <<EOF
           [gerrit]
           host=localhost
@@ -60,23 +139,77 @@
           curl -Lo .git/hooks/commit-msg http://localhost:8081/tools/hooks/commit-msg
           chmod u+x .git/hooks/commit-msg
 
-          cat >file.txt <<EOF
+          cat >file-1.txt <<EOF
           Hello, this is a file
           EOF
-          git add file.txt
-          git commit -m "Sample review"
+          git add file-1.txt
+          git commit -m "Sample review 1"
           git push gerrit HEAD:refs/for/master
 
-    - name: Post a sample review
+          cat >file-2.txt <<EOF
+          Hello, this is a file
+          EOF
+          git add file-2.txt
+          git commit -m "Sample review 2"
+          git push gerrit HEAD:refs/for/master
+
+          cat >file-3.txt <<EOF
+          Hello, this is a file
+          EOF
+          git add file-3.txt
+          git commit -m "Sample review 3"
+          git push gerrit HEAD:refs/for/master
+
+    - name: Post a sample failure review
       uri:
-        url: http://localhost:8081/a/changes/1/revisions/1/review
+        url: 'http://localhost:8081/a/changes/{{ item }}/revisions/1/review'
         method: POST
         user: zuul
         password: secret
         body_format: json
         body:
           tag: 'autogenerated:zuul:check'
-          message: "Patch Set 8: Verified-1\n\nBuild failed (check pipeline).  For information on how to proceed, see\nhttps://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing\n\n\n- system-config-puppet-apply-4-ubuntu-xenial https://zuul.opendev.org/t/openstack/build/00b1ce7d5d994339a54dd8142a6813a6 : SUCCESS in 10m 22s\n- system-config-legacy-logstash-filters https://zuul.opendev.org/t/openstack/build/20efbb1b6983453884103e9628a70942 : SUCCESS in 8m 24s\n- opendev-tox-docs https://zuul.opendev.org/t/openstack/build/43a00a7e190b4ce2b3e20a86166c011a : SUCCESS in 3m 40s\n- opendev-buildset-registry https://zuul.opendev.org/t/openstack/build/bc4a4559645b4088a9f8586c043c4764 : SUCCESS in 46m 19s\n- tox-linters https://zuul.opendev.org/t/openstack/build/cb469750b250430b95a02697931d6030 : FAILURE in 19m 14s\n- system-config-run-base https://zuul.opendev.org/t/openstack/build/0897fafc9652480794652fc49e3c9baf : SUCCESS in 12m 48s\n- system-config-run-base-ansible-devel https://zuul.opendev.org/t/openstack/build/57c4eb5174554cbfbb06fd9118eb1ae4 : SUCCESS in 16m 37s (non-voting)\n- system-config-run-gitea https://zuul.opendev.org/t/openstack/build/30afb96669fe4f19b6215e79615ecd90 : SUCCESS in 37m 58s\n- system-config-run-review https://zuul.opendev.org/t/openstack/build/76c69b8062284d959a85b55bc460f822 : FAILURE in 15m 27s"
+          labels:
+            'Verified': -2
+          message: |
+            Build failed (check pipeline).  For information on how to proceed, see
+            https://docs.opendev.org/opendev/infra-manual/latest/developers.html#automated-testing
+
+
+            - test-success https://zuul.opendev.org/t/openstack/build/00b1ce7d5d994339a54dd8142a6813a6 : SUCCESS in {{ range(1, 59) | random }}m 22s
+            - test-failure https://zuul.opendev.org/t/openstack/build/00b1ce7d5d994339a54dd8142a6813a6 : FAILURE in {{ range(1, 59) | random }}m 22s
+            - test-retry-limit https://zuul.opendev.org/t/openstack/build/20efbb1b6983453884103e9628a70942 : RETRY_LIMIT in {{ range(1, 59) | random }}m 24s
+            - test-skipped https://zuul.opendev.org/t/openstack/build/43a00a7e190b4ce2b3e20a86166c011a : SKIPPED in {{ range(1, 59) | random }}m 40s
+            - test-aborted https://zuul.opendev.org/t/openstack/build/bc4a4559645b4088a9f8586c043c4764 : ABORTED in {{ range(1, 59) | random }}m 19s
+            - test-merger-failure https://zuul.opendev.org/t/openstack/build/cb469750b250430b95a02697931d6030 : MERGER_FAILURE in {{ range(1, 59) | random }}m 14s
+            - test-node-failure https://zuul.opendev.org/t/openstack/build/0897fafc9652480794652fc49e3c9baf : NODE_FAILURE in {{ range(1, 59) | random }}m 48s
+            - test-timed-out https://zuul.opendev.org/t/openstack/build/57c4eb5174554cbfbb06fd9118eb1ae4 : TIMED_OUT in {{ range(1, 59) | random }}m 37s (non-voting)
+            - test-post-failure https://zuul.opendev.org/t/openstack/build/30afb96669fe4f19b6215e79615ecd90 : POST_FAILURE in {{ range(1, 59) | random }}m 58s
+            - test-config-error https://zuul.opendev.org/t/openstack/build/76c69b8062284d959a85b55bc460f822 : CONFIG_ERROR in {{ range(1, 59) | random }}m 27s
+            - test-disk-full https://zuul.opendev.org/t/openstack/build/0897fafc9743580794652fc49e3c9baf : DISK_FULL in {{ range(1, 59) | random }}m 48s
+      loop:
+        - 1
+        - 2
+        - 3
+
+    - name: Post a sample good review in another pipeline
+      uri:
+        url: 'http://localhost:8081/a/changes/{{ item }}/revisions/1/review'
+        method: POST
+        user: zuul
+        password: secret
+        body_format: json
+        body:
+          tag: 'autogenerated:zuul:check-secondary'
+          message: |
+            Build succeeded (Secondary pipeline).
+
+
+            - test-second-ci https://zuul.opendev.org/t/openstack/build/f9c14856bf4e485089ca8fc8f00f324b : SUCCESS in {{ range(1, 59) | random}}m 11s
+      loop:
+        - 1
+        - 2
+        - 3
 
     - name: Run selenium container
       include_role:

testinfra/test_gerrit.py

@@ -35,20 +35,37 @@ def test_gerrit_screenshot(host):
         driver.get("http://localhost:8081")
         WebDriverWait(driver, 30).until(lambda driver: driver.execute_script(
             'return document.readyState') == 'complete')
+        # NOTE(ianw): The page doesn't really seem to be complete at
+        # this point, not sure what to listen for...
+        time.sleep(5)
         driver.save_screenshot("/var/log/screenshots/gerrit-main-page.png")
 
-        driver.get("http://localhost:8081/c/test-project/+/1")
-        time.sleep(5)
-        driver.execute_script(
-            "document.querySelector('gr-app').shadowRoot"
-            ".querySelector('gr-app-element').shadowRoot"
-            ".querySelector('main')"
-            ".querySelector('gr-change-view').shadowRoot"
-            ".querySelector('paper-tab[data-name=\"change-view-tab-header-zuul-results-summary\"]')"
-            ".click()"
-        )
-        time.sleep(5)
-        driver.save_screenshot("/var/log/screenshots/gerrit-change-page.png")
+        for change in (1, 2):
+            driver.get("http://localhost:8081/c/test-project/+/%s" % change)
+            time.sleep(5)
+            driver.execute_script(
+                "document.querySelector('gr-app').shadowRoot"
+                ".querySelector('gr-app-element').shadowRoot"
+                ".querySelector('main')"
+                ".querySelector('gr-change-view').shadowRoot"
+                ".querySelector('paper-tab[data-name=\"change-view-tab-header-zuul-results-summary\"]')"
+                ".click()"
+            )
+            time.sleep(5)
+
+            original_size = driver.get_window_size()
+            required_width = driver.execute_script(
+                'return document.body.parentNode.scrollWidth')
+            required_height = driver.execute_script(
+                'return document.body.parentNode.scrollHeight') + 100
+            driver.set_window_size(required_width, required_height)
+
+            driver.find_element_by_tag_name('body'). \
+                screenshot("/var/log/screenshots/gerrit-change-page-%s.png" % change)
+
+            driver.set_window_size(
+                original_size['width'], original_size['height'])
+
     except TimeoutException as e:
         raise e
     finally:

zuul.d/docker-images/gerrit.yaml

@@ -102,6 +102,8 @@
       - docker/gerrit/base/
       - docker/gerrit/bazel/
       - playbooks/zuul/gerrit/
+      - playbooks/test-review.yaml
+      - testinfra/test_gerrit.py
       - jeepyb/
 
 - job: