From 1459ef107b50ac69587cd5c4f3bd2915ad48e232 Mon Sep 17 00:00:00 2001
From: Monty Taylor <mordred@inaugust.com>
Date: Thu, 3 Jan 2019 18:11:04 +0000
Subject: [PATCH] Update generate-gitmodules script for official deliverables

We should generate the list of deliverables from the projects.yaml file
in the governance repo. We should also make the submodule entries
relative rather than absolute so that browsing in other places works.

Also, when adding a git submodule, one needs to use the git submodule
add command. So do that, and trim out old no-longer-existing submodules
too.

Change-Id: I97c812f8497cc32a2ee1b418fce2e6bd20af3bc3
---
 generate-gitmodules    | 63 --------------------------------------
 generate-gitmodules.py | 68 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 68 insertions(+), 63 deletions(-)
 delete mode 100755 generate-gitmodules
 create mode 100755 generate-gitmodules.py

diff --git a/generate-gitmodules b/generate-gitmodules
deleted file mode 100755
index af6dd8d9e7..0000000000
--- a/generate-gitmodules
+++ /dev/null
@@ -1,63 +0,0 @@
-#!/usr/bin/python
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import json
-
-import requests
-
-TEMPLATE = """[submodule "%s"]
-\tpath = %s
-\turl = https://review.openstack.org/%s.git
-\tbranch = .
-"""
-
-# only return projects starting with openstack
-CONFIG = ("https://review.openstack.org:443/projects/?p=openstack")
-
-
-def find_integrated_gate_projects():
-    r = requests.get(CONFIG)
-    # strip off first few chars because 'the JSON response body starts with a
-    # magic prefix line that must be stripped before feeding the rest of the
-    # response body to a JSON parser'
-    # https://review.openstack.org/Documentation/rest-api.html
-    gerrit_projects = json.loads(r.text[4:])
-
-    # Ignore openstack-attic
-    orgs = ['openstack/', 'openstack-infra/', 'openstack-dev/']
-    blacklist = ['openstack/openstack']
-
-    projects = []
-    for project in gerrit_projects:
-        if any(project.startswith(org) for org in orgs):
-            if project not in blacklist:
-                projects.append(project)
-    return projects
-
-
-def gen_gitmodules(projects):
-    projects = sorted(projects)
-    with open(".gitmodules", 'w') as f:
-        for p in projects:
-            ns, name = p.split('/')
-            f.write(TEMPLATE % (name, name, p))
-
-
-def main():
-    projects = find_integrated_gate_projects()
-    gen_gitmodules(projects)
-
-
-if __name__ == '__main__':
-    main()
diff --git a/generate-gitmodules.py b/generate-gitmodules.py
new file mode 100755
index 0000000000..ec3de572d6
--- /dev/null
+++ b/generate-gitmodules.py
@@ -0,0 +1,68 @@
+#!/usr/bin/python
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+
+import requests
+import yaml
+
+TEMPLATE = """[submodule "%s"]
+\tpath = %s
+\turl = ../../%s.git
+\tbranch = .
+"""
+
+# only return projects starting with openstack
+CONFIG = ("https://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml")
+
+
+def find_integrated_gate_repos():
+    r = requests.get(CONFIG)
+    projects = yaml.safe_load(r.text)
+    repos = []
+    for project in projects.values():
+        for deliverable in project.get('deliverables', {}).values():
+            for repo in deliverable['repos']:
+                repos.append(repo)
+    return repos
+
+
+def gen_gitmodules(projects):
+    path_template = "https://git.openstack.org/{project}"
+    short_projects = []
+    for project in projects:
+        short = os.path.basename(project)
+        short_projects.append(short)
+        path = path_template.format(project=project)
+        if not os.path.isdir(short):
+            os.system('git submodule add {path}'.format(path=path))
+    for existing in os.listdir('.'):
+        if not os.path.isdir(existing) or existing.startswith('.'):
+            continue
+        if existing not in short_projects:
+            os.system('git rm {existing}'.format(existing=existing))
+    projects = sorted(projects)
+    with open(".gitmodules", 'w') as f:
+        for p in projects:
+            ns, name = p.split('/')
+            f.write(TEMPLATE % (name, name, p))
+
+
+def main():
+    repos = find_integrated_gate_repos()
+    gen_gitmodules(repos)
+
+
+if __name__ == '__main__':
+    main()