Add integration testing with jeepyb and gerrit

This adds integration testing to gerritlib with jeepyb and gerrit.

Depends-On: https://review.opendev.org/704660
Change-Id: I1815e63693656d52c10a9cb9bf60f999e036dfed

.zuul.yaml

@@ -0,0 +1,16 @@
+- job:
+    name: gerritlib-jeepyb-integration
+    description: Run manage_projects with gerritlib from source against Gerrit
+    run: playbooks/jeepyb-integration/run.yaml
+    post-run: playbooks/jeepyb-integration/post.yaml
+    required-projects:
+      - opendev/gerritlib
+      - opendev/jeepyb
+
+- project:
+    check:
+      jobs:
+        - gerritlib-jeepyb-integration
+    gate:
+      jobs:
+        - gerritlib-jeepyb-integration

MANIFEST.in

@@ -4,5 +4,6 @@ include ChangeLog
 
 exclude .gitignore
 exclude .gitreview
+exclude tools/sshkey
 
 global-exclude *.pyc

playbooks/jeepyb-integration/post.yaml

@@ -0,0 +1,18 @@
+- hosts: all
+  tasks:
+    - name: Create logs directory
+      file:
+        state: directory
+        path: "{{ ansible_user_dir }}/logs"
+        mode: 0755
+    - name: Save docker log files
+      become: true
+      shell: |
+        docker logs tools_gerrit_1 > {{ ansible_user_dir }}/logs/gerrit.log 2>&1
+        docker logs tools_gerritconfig_1 > {{ ansible_user_dir }}/logs/gerritconfig.log 2>&1
+    - name: Copy docker log files to executor
+      synchronize:
+        mode: pull
+        src: "{{ ansible_user_dir }}/logs/"
+        dest: "{{ zuul.executor.log_root }}/container_logs/"
+        verify_host: true

playbooks/jeepyb-integration/run.yaml

@@ -0,0 +1,86 @@
+- hosts: all
+  roles:
+    - name: install-docker
+  tasks:
+    # Configure Jeepyb and Gerritlib
+    - name: Create jeepyb git directory
+      file:
+        state: directory
+        path: "/tmp/jeepyb-git"
+    - name: Create jeepyb cache directory
+      file:
+        state: directory
+        path: "/tmp/jeepyb-cache"
+    # TODO(clarkb) add python3 version of job
+    - name: Install jeepyb
+      command: python2 -m pip install src/opendev.org/opendev/jeepyb
+      become: true
+    - name: Install gerritlib
+      # Install after Jeepyb so that we don't use gerritlib from pypi
+      command: python2 -m pip install src/opendev.org/opendev/gerritlib
+      become: true
+
+    # Run a gerrit
+    - name: Install docker-compose
+      package:
+        name:
+          - docker-compose
+        state: present
+      become: true
+    - name: Run docker-compose up
+      shell:
+        cmd: docker-compose up -d
+        chdir: src/opendev.org/opendev/gerritlib/tools/
+    - name: Wait for Gerrit Admin user to be created
+      uri:
+        url: http://localhost:8080/a/accounts/admin/sshkeys
+        method: GET
+        user: admin
+        password: secret
+      register: result
+      until: result.status == 200 and result.redirected == false
+      delay: 1
+      retries: 120
+    - name: fetch ssh host keys from gerrit
+      shell: ssh-keyscan -p 29418 localhost >> ~/.ssh/known_hosts
+
+    - name: Set perms on new ssh private key
+      become: true
+      file:
+        path: src/opendev.org/opendev/gerritlib/tools/sshkey/admin
+        owner: zuul
+        group: zuul
+    - name: Set perms on new ssh public key
+      become: true
+      file:
+        path: src/opendev.org/opendev/gerritlib/tools/sshkey/admin.pub
+        owner: zuul
+        group: zuul
+
+    # Configure git
+    - name: Set git user name
+      command: git config --global user.name Zuul
+    - name: Set git user email
+      command: git config --global user.email admin@example.com
+
+    # Test jeepyb and gerritlib
+    - name: Manage single gerrit project
+      shell:
+        cmd: PROJECTS_INI=tools/projects.ini PROJECTS_YAML=tools/projects.yaml manage-projects -v test/test-repo-1
+        chdir: src/opendev.org/opendev/gerritlib
+    - name: Manage all gerrit projects
+      shell:
+        cmd: PROJECTS_INI=tools/projects.ini PROJECTS_YAML=tools/projects.yaml manage-projects -v
+        chdir: src/opendev.org/opendev/gerritlib
+    - name: Check test-repo-1 exists
+      shell:
+        cmd: ssh -i tools/sshkey/admin -p 29418 admin@localhost gerrit ls-projects | grep test-repo-1
+        chdir: src/opendev.org/opendev/gerritlib
+    - name: Check test-repo-2 exists
+      shell:
+        cmd: ssh -i tools/sshkey/admin -p 29418 admin@localhost gerrit ls-projects | grep test-repo-2
+        chdir: src/opendev.org/opendev/gerritlib
+    - name: Check test-repo-3 does not exist
+      shell:
+        cmd: ssh -i tools/sshkey/admin -p 29418 admin@localhost gerrit ls-projects | grep -v test-repo-3
+        chdir: src/opendev.org/opendev/gerritlib

tools/acls/test/retired.config

@@ -0,0 +1,2 @@
+[project]
+state = read only

tools/acls/test/test.config

@@ -0,0 +1,10 @@
+[access "refs/heads/*"]
+abandon = group Registered Users
+label-Code-Review = -2..+2 group Registered Users
+label-Workflow = -1..+1 group Registered Users
+
+[receive]
+requireChangeId = true
+
+[submit]
+mergeContent = true

tools/docker-compose.yaml

@@ -0,0 +1,32 @@
+# Version 2 is the latest that is supported by docker-compose in
+# Ubuntu Xenial.
+version: '2'
+
+services:
+  gerrit:
+    image: gerritcodereview/gerrit
+    ports:
+      - "8080:8080"
+      - "29418:29418"
+    command: |
+      /bin/sh -c '\
+        git config -f /var/gerrit/etc/gerrit.config gerrit.canonicalWebUrl http://localhost:8080/ && \
+        git config -f /var/gerrit/etc/gerrit.config gerrit.ui POLYGERRIT && \
+        git config -f /var/gerrit/etc/gerrit.config sendemail.enable false && \
+        git config -f /var/gerrit/etc/gerrit.config noteDb.changes.autoMigrate true && \
+        /var/gerrit/bin/gerrit.sh run'
+  gerritconfig:
+    # TODO(clarkb) use an ansible specific image?
+    image: zuul/zuul-executor
+    environment:
+      - http_proxy
+      - https_proxy
+      - no_proxy=${no_proxy},gerrit
+    depends_on:
+      - gerrit
+    volumes:
+      - "./sshkey:/var/ssh:z"
+      - "./playbooks/:/var/playbooks/:z"
+    # NOTE(pabelanger): Be sure to update this line each time we change the
+    # default version of ansible for Zuul.
+    command: "/usr/local/lib/zuul/ansible/2.8/bin/ansible-playbook /var/playbooks/setup.yaml"

tools/playbooks/project.config

@@ -0,0 +1,84 @@
+[project]
+	description = Access inherited by all other projects.
+[receive]
+	requireContributorAgreement = false
+	requireSignedOffBy = false
+	requireChangeId = true
+	enableSignedPush = false
+[submit]
+	mergeContent = true
+[capability]
+	administrateServer = group Administrators
+	priority = batch group Non-Interactive Users
+	streamEvents = group Non-Interactive Users
+[access "refs/*"]
+	read = group Administrators
+	read = group Anonymous Users
+[access "refs/for/*"]
+	addPatchSet = group Registered Users
+	forgeAuthor = group Registered Users
+	forgeCommitter = group Registered Users
+[access "refs/for/refs/*"]
+	push = group Registered Users
+	pushMerge = group Registered Users
+[access "refs/heads/*"]
+	create = group Administrators
+	create = group Project Owners
+	forgeAuthor = group Registered Users
+	forgeCommitter = group Administrators
+	forgeCommitter = group Project Owners
+	forgeCommitter = group Registered Users
+	push = +force group Administrators
+	push = group Project Owners
+	label-Code-Review = -2..+2 group Registered Users
+	label-Verified = -2..+2 group Administrators
+	label-Verified = -2..+2 group Non-Interactive Users
+	label-Workflow = -1..+1 group Registered Users
+	submit = group Administrators
+	submit = group Project Owners
+	submit = group Non-Interactive Users
+	editTopicName = +force group Administrators
+	editTopicName = +force group Project Owners
+[access "refs/meta/config"]
+	exclusiveGroupPermissions = read
+	read = group Administrators
+	read = group Project Owners
+	create = group Administrators
+	create = group Project Owners
+	push = group Administrators
+	push = group Project Owners
+	label-Code-Review = -2..+2 group Administrators
+	label-Code-Review = -2..+2 group Project Owners
+	submit = group Administrators
+	submit = group Project Owners
+[access "refs/tags/*"]
+	create = group Administrators
+	create = group Project Owners
+	createTag = group Administrators
+	createTag = group Project Owners
+	createSignedTag = group Administrators
+	createSignedTag = group Project Owners
+[label "Code-Review"]
+	function = MaxWithBlock
+	defaultValue = 0
+	copyMinScore = true
+	copyAllScoresOnTrivialRebase = true
+	value = -2 This shall not be merged
+	value = -1 I would prefer this is not merged as is
+	value =  0 No score
+	value = +1 Looks good to me, but someone else must approve
+	value = +2 Looks good to me, approved
+[label "Verified"]
+	function = MaxWithBlock
+	value = -2 Fails
+	value = -1 Does not seem to work
+	value =  0 No score
+	value = +1 Works for me
+	value = +2 Verified
+	defaultValue = 0
+[label "Workflow"]
+	function = MaxWithBlock
+	value = -1 Work in progress
+	value =  0 Ready for reviews
+	value = +1 Approved
+	defaultValue = 0

tools/playbooks/setup.yaml

@@ -0,0 +1,55 @@
+- hosts: localhost
+  gather_facts: false
+  tasks:
+    - name: Wait for Gerrit to start
+      wait_for:
+        host: gerrit
+        port: 29418
+    - name: Generate admin SSH key for Gerrit
+      command: ssh-keygen -f /var/ssh/admin -N ''
+      args:
+        creates: /var/ssh/admin.pub
+    # The Gerrit container puts up a helpful info page the first time
+    # you hit the web server; get past that.
+    - name: Get Gerrit first login screen
+      uri:
+        url: http://gerrit:8080/a/accounts/self/sshkeys
+        method: GET
+        user: admin
+        password: secret
+      ignore_errors: true
+    - name: Add admin SSH key to Gerrit
+      uri:
+        url: http://gerrit:8080/a/accounts/self/sshkeys
+        method: POST
+        user: admin
+        password: secret
+        body: "{{ lookup('file', '/var/ssh/admin.pub') }}"
+        status_code: 201
+        HEADER_Content-Type: text/plain
+
+    - name: Create temp dir for Gerrit config update
+      shell: mktemp -d
+      register: gerrit_tmp
+    - name: Set All-Project repo location
+      set_fact:
+        all_projects_repo: "{{ gerrit_tmp.stdout }}/All-Projects"
+    - name: Checkout All-Projects config
+      git:
+        repo: ssh://gerrit:29418/All-Projects/
+        ssh_opts: "-o StrictHostKeyChecking=no -i /var/ssh/admin -l admin"
+        dest: "{{ all_projects_repo }}"
+        refspec: '+refs/meta/config:refs/meta/config'
+        version: refs/meta/config
+    - name: Copy new All-Projects config into place
+      copy:
+        src: "/var/playbooks/project.config"
+        dest: "{{ all_projects_repo }}/project.config"
+    - name: Update All-Projects config in Gerrit
+      shell: |
+        git config user.email 'admin@example.com'
+        git commit -a -m 'update config'
+        git push http://admin:secret@gerrit:8080/All-Projects +HEAD:refs/meta/config
+      args:
+        chdir: "{{ all_projects_repo }}"
+        warn: false

tools/projects.ini

@@ -0,0 +1,15 @@
+[projects]
+homepage=http://test.org
+acl-dir=tools/acls
+local-git-dir=/tmp/jeepyb-git
+jeepyb-cache-dir=/tmp/jeepyb-cache
+gerrit-host=localhost
+gerrit-user=admin
+gerrit-key=tools/sshkey/admin
+gerrit-system-user=zuul
+gerrit-system-group=zuul
+gerrit-committer=Zuul <admin@example.com>
+has-wiki=False
+has-issues=False
+has-downloads=False
+has-github=False

tools/projects.yaml

@@ -0,0 +1,17 @@
+- project: test/test-repo-1
+  description: One of several test repos
+  use-storyboard: true
+  acl-config: tools/acls/test/test.config
+  groups:
+    - testing
+- project: test/test-repo-2
+  description: Second of several test repos
+  use-storyboard: false
+  acl-config: tools/acls/test/test.config
+  groups:
+    - testing
+- project: test/test-repo-3
+  description: A retired repo
+  acl-config: tools/acls/test/retired.config
+  groups:
+    - testing

tools/sshkey/README

@@ -0,0 +1 @@
+This dir will contain Gerrit ssh keys.