Deal with gitea pagination of repo lists

We list gitea repos to determine if we need to create a repo. If the
repo isn't listed by gitea we create it. New gitea paginates these
listings so we were only getting 30 repos listed when we had far more.
This resulted in us trying to create repos which already exist which is
a gitea http 409 error.

Fix this by paging through the listings until we've seen all the
repos. This should give us a complete listing.

To test this we run our manage-projects playbook twice in the
system-config-run-gitea job. The first pass creates all the new
projects. Then the second pass should noop cleanly.

Change-Id: I73b77b9ddaa0106d4dc0a49c4d4b7751a39a16f9
Co-Authored-By: Jeremy Stanley <fungi@yuggoth.org>

playbooks/roles/gitea-git-repos/library/gitea_create_repos.py

@@ -99,8 +99,22 @@ class Gitea(object):
             self.log("Added gerrit to team:", org)
 
     def get_org_repo_list(self, org):
-        return [x['full_name'] for x in
-                self.get('/api/v1/orgs/{org}/repos'.format(org=org)).json()]
+        params = { 'limit': 50, 'page': 1 }
+        repos = []
+        gitea_data = self.get(
+                '/api/v1/orgs/{org}/repos'.format(org=org),
+                params=params
+            ).json()
+        while gitea_data:
+            repos.extend([x['full_name'] for x in gitea_data])
+            # Gitea paginates and returns an empty list at the end of the
+            # listing. 50 items is the max limit.
+            params['page'] += 1
+            gitea_data = self.get(
+                    '/api/v1/orgs/{org}/repos'.format(org=org),
+                    params=params
+                ).json()
+        return repos
 
     def get_csrf_token(self):
         resp = self.get('/')
@@ -182,9 +196,17 @@ class Gitea(object):
     def make_projects(self, projects, gitea_repos, csrf_token,
                       settings_thread_pool, branches_thread_pool, futures):
         for project in projects:
-            if project['project'] in gitea_repos:
             create = False
-            else:
+            if project['project'] not in gitea_repos:
+                try:
+                    self.get('/' + project['project'])
+                except requests.HTTPError:
+                    # If the project isn't in the listing we do an explicit
+                    # check for its existence. This is because gitea repo
+                    # listings require pagination and they don't use stable
+                    # sorting and that causes problems reliably producing a
+                    # complete repo list. If we cannot find the project
+                    # then create it.
                     create = True
             if create:
                 # TODO: use threadpool when we're running with

zuul.d/system-config-run.yaml

@@ -500,6 +500,9 @@
         - playbooks/service-gitea-lb.yaml
         - playbooks/service-gitea.yaml
         - playbooks/manage-projects.yaml
+        # Run twice to ensure that we noop properly when
+        # all projects are created in gitea.
+        - playbooks/manage-projects.yaml
       run_test_playbook: playbooks/test-gitea.yaml
     host-vars:
       gitea99.opendev.org: