show which repos use a library

Use the codesearch service to find repositories that use a library to
make it easier to anticipate the impact of releasing a new
version. Only official repositories are included.

Change-Id: I656d9bb09d1b4ed485a29061dede8ab86ef5bdca
Signed-off-by: Doug Hellmann <doug@doughellmann.com>

openstack_releases/cmds/list_changes.py

@@ -35,6 +35,8 @@ import requests
 from openstack_releases import defaults
 from openstack_releases import gitutils
 from openstack_releases import governance
+from openstack_releases import hound
+from openstack_releases import pythonutils
 from openstack_releases import release_notes
 from openstack_releases import yamlutils
 
@@ -180,6 +182,11 @@ def show_watched_queries(branch, repo):
         )
 
 
+def show_dependency_listings(package_name, official_repos):
+    header('Users of {}'.format(package_name))
+    hound.show_dependency_listings(package_name, official_repos)
+
+
 def main():
     if not sys.stdout.encoding:
         # Wrap sys.stdout with a writer that knows how to handle
@@ -238,6 +245,10 @@ def main():
     atexit.register(cleanup_workdir)
 
     team_data = governance.get_team_data()
+    official_repos = set(
+        r.name
+        for r in governance.get_repositories(team_data)
+    )
 
     # Remove any inherited PAGER environment variable to avoid
     # blocking the output waiting for input.
@@ -589,4 +600,10 @@ def main():
                 print('\n')
                 print(notes)
 
+            if 'library' in deliverable_info.get('type', 'other'):
+                show_dependency_listings(
+                    pythonutils.guess_sdist_name(project),
+                    official_repos,
+                )
+
     return 0

openstack_releases/hound.py

@@ -0,0 +1,71 @@
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Query codesearch.openstack.org.
+
+"""
+
+import requests
+
+
+_URL = 'http://codesearch.openstack.org/api/v1/search'
+
+
+def _query(q, **kwds):
+    params = {
+        'repos': '*',  # which repositories to search
+        'i': 'nope',  # ignore case
+        'ctx': 0,  # lines of extra context
+    }
+    params.update(kwds)
+    params['q'] = q
+
+    headers = {
+        'Content-Type': 'application/json',
+    }
+    response = requests.get(_URL, params=params, headers=headers)
+    return response.json()['Results']
+
+
+def get_dependency_listings(package_name):
+    return _query(
+        q=package_name,
+        # NOTE(dhellmann): Including setup.cfg shows *lots* of results
+        # for oslo.config because of the plugins for the config
+        # generator. It would be nice to figure out how to filter
+        # those.
+        files='(.*requirements.txt|.*constraint.*.txt)',
+    )
+
+
+def show_dependency_listings(package_name, official_repos):
+    to_show = set(
+        r.partition('/')[-1]
+        for r in official_repos
+    )
+    results = get_dependency_listings(package_name)
+    for repo, repo_matches in sorted(results.items()):
+        if repo not in to_show:
+            continue
+        for repo_match in repo_matches['Matches']:
+            for file_match in repo_match['Matches']:
+                if file_match['Line'].lstrip().startswith('#'):
+                    # ignore comments
+                    continue
+                print('{repo:30}:{filename:30}:{linenum:3}: {line}'.format(
+                    repo=repo,
+                    filename=repo_match['Filename'],
+                    linenum=file_match['LineNumber'],
+                    line=file_match['Line'],
+                    ))