From 9b5e5d3c57024dd23f28a20a053ee13665c45c37 Mon Sep 17 00:00:00 2001
From: Clark Boylan <clark.boylan@gmail.com>
Date: Wed, 24 Jun 2020 15:32:34 -0700
Subject: [PATCH] Deal with gitea pagination of repo lists

We list gitea repos to determine if we need to create a repo. If the
repo isn't listed by gitea we create it. New gitea paginates these
listings so we were only getting 30 repos listed when we had far more.
This resulted in us trying to create repos which already exist which is
a gitea http 409 error.

Fix this by paging through the listings until we've seen all the
repos. This should give us a complete listing.

To test this we run our manage-projects playbook twice in the
system-config-run-gitea job. The first pass creates all the new
projects. Then the second pass should noop cleanly.

Change-Id: I73b77b9ddaa0106d4dc0a49c4d4b7751a39a16f9
Co-Authored-By: Jeremy Stanley <fungi@yuggoth.org>
---
 .../library/gitea_create_repos.py             | 34 +++++++++++++++----
 zuul.d/system-config-run.yaml                 |  3 ++
 2 files changed, 31 insertions(+), 6 deletions(-)

diff --git a/playbooks/roles/gitea-git-repos/library/gitea_create_repos.py b/playbooks/roles/gitea-git-repos/library/gitea_create_repos.py
index 0674d19dbf..a0afec63fd 100755
--- a/playbooks/roles/gitea-git-repos/library/gitea_create_repos.py
+++ b/playbooks/roles/gitea-git-repos/library/gitea_create_repos.py
@@ -99,8 +99,22 @@ class Gitea(object):
             self.log("Added gerrit to team:", org)
 
     def get_org_repo_list(self, org):
-        return [x['full_name'] for x in
-                self.get('/api/v1/orgs/{org}/repos'.format(org=org)).json()]
+        params = { 'limit': 50, 'page': 1 }
+        repos = []
+        gitea_data = self.get(
+                '/api/v1/orgs/{org}/repos'.format(org=org),
+                params=params
+            ).json()
+        while gitea_data:
+            repos.extend([x['full_name'] for x in gitea_data])
+            # Gitea paginates and returns an empty list at the end of the
+            # listing. 50 items is the max limit.
+            params['page'] += 1
+            gitea_data = self.get(
+                    '/api/v1/orgs/{org}/repos'.format(org=org),
+                    params=params
+                ).json()
+        return repos
 
     def get_csrf_token(self):
         resp = self.get('/')
@@ -182,10 +196,18 @@ class Gitea(object):
     def make_projects(self, projects, gitea_repos, csrf_token,
                       settings_thread_pool, branches_thread_pool, futures):
         for project in projects:
-            if project['project'] in gitea_repos:
-                create = False
-            else:
-                create = True
+            create = False
+            if project['project'] not in gitea_repos:
+                try:
+                    self.get('/' + project['project'])
+                except requests.HTTPError:
+                    # If the project isn't in the listing we do an explicit
+                    # check for its existence. This is because gitea repo
+                    # listings require pagination and they don't use stable
+                    # sorting and that causes problems reliably producing a
+                    # complete repo list. If we cannot find the project
+                    # then create it.
+                    create = True
             if create:
                 # TODO: use threadpool when we're running with
                 # https://github.com/go-gitea/gitea/pull/7493
diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml
index 3c619c8ca9..b5acf18984 100644
--- a/zuul.d/system-config-run.yaml
+++ b/zuul.d/system-config-run.yaml
@@ -500,6 +500,9 @@
         - playbooks/service-gitea-lb.yaml
         - playbooks/service-gitea.yaml
         - playbooks/manage-projects.yaml
+        # Run twice to ensure that we noop properly when
+        # all projects are created in gitea.
+        - playbooks/manage-projects.yaml
       run_test_playbook: playbooks/test-gitea.yaml
     host-vars:
       gitea99.opendev.org: