Update generate-gitmodules script for official deliverables

We should generate the list of deliverables from the projects.yaml file
in the governance repo. We should also make the submodule entries
relative rather than absolute so that browsing in other places works.

Also, when adding a git submodule, one needs to use the git submodule
add command. So do that, and trim out old no-longer-existing submodules
too.

Change-Id: I97c812f8497cc32a2ee1b418fce2e6bd20af3bc3

generate-gitmodules

@@ -1,63 +0,0 @@
-#!/usr/bin/python
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import json
-
-import requests
-
-TEMPLATE = """[submodule "%s"]
-\tpath = %s
-\turl = https://review.openstack.org/%s.git
-\tbranch = .
-"""
-
-# only return projects starting with openstack
-CONFIG = ("https://review.openstack.org:443/projects/?p=openstack")
-
-
-def find_integrated_gate_projects():
-    r = requests.get(CONFIG)
-    # strip off first few chars because 'the JSON response body starts with a
-    # magic prefix line that must be stripped before feeding the rest of the
-    # response body to a JSON parser'
-    # https://review.openstack.org/Documentation/rest-api.html
-    gerrit_projects = json.loads(r.text[4:])
-
-    # Ignore openstack-attic
-    orgs = ['openstack/', 'openstack-infra/', 'openstack-dev/']
-    blacklist = ['openstack/openstack']
-
-    projects = []
-    for project in gerrit_projects:
-        if any(project.startswith(org) for org in orgs):
-            if project not in blacklist:
-                projects.append(project)
-    return projects
-
-
-def gen_gitmodules(projects):
-    projects = sorted(projects)
-    with open(".gitmodules", 'w') as f:
-        for p in projects:
-            ns, name = p.split('/')
-            f.write(TEMPLATE % (name, name, p))
-
-
-def main():
-    projects = find_integrated_gate_projects()
-    gen_gitmodules(projects)
-
-
-if __name__ == '__main__':
-    main()

generate-gitmodules.py

@@ -0,0 +1,68 @@
+#!/usr/bin/python
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+
+import requests
+import yaml
+
+TEMPLATE = """[submodule "%s"]
+\tpath = %s
+\turl = ../../%s.git
+\tbranch = .
+"""
+
+# only return projects starting with openstack
+CONFIG = ("https://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml")
+
+
+def find_integrated_gate_repos():
+    r = requests.get(CONFIG)
+    projects = yaml.safe_load(r.text)
+    repos = []
+    for project in projects.values():
+        for deliverable in project.get('deliverables', {}).values():
+            for repo in deliverable['repos']:
+                repos.append(repo)
+    return repos
+
+
+def gen_gitmodules(projects):
+    path_template = "https://git.openstack.org/{project}"
+    short_projects = []
+    for project in projects:
+        short = os.path.basename(project)
+        short_projects.append(short)
+        path = path_template.format(project=project)
+        if not os.path.isdir(short):
+            os.system('git submodule add {path}'.format(path=path))
+    for existing in os.listdir('.'):
+        if not os.path.isdir(existing) or existing.startswith('.'):
+            continue
+        if existing not in short_projects:
+            os.system('git rm {existing}'.format(existing=existing))
+    projects = sorted(projects)
+    with open(".gitmodules", 'w') as f:
+        for p in projects:
+            ns, name = p.split('/')
+            f.write(TEMPLATE % (name, name, p))
+
+
+def main():
+    repos = find_integrated_gate_repos()
+    gen_gitmodules(repos)
+
+
+if __name__ == '__main__':
+    main()