Add tool to generate a health report for all reqs

This adds a tool script that loops through all of our requirements and
output basic information for each one along with any things we might be
concerned about with each one. For now it just checks a few things, but
we could extend this issue checking as we find other things to check on.

Also refactors the unused package tool to put some common things into a
shared function library used by both scripts.

Change-Id: I140c3a5fb71a8fb82ccb696bade5be13bb9b0ba0
Signed-off-by: Sean McGinnis <sean.mcginnis@gmail.com>

tools/functions

@@ -0,0 +1,45 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# Shared functions for shell scripts
+
+function enable_venv () {
+    BASE="${1}"
+    if [[ -z "${VIRTUAL_ENV}" ]]; then
+        if [[ ! -d ${BASE}/.tox/venv ]]; then
+            (cd ${BASE} && tox -e venv --notest > /dev/null)
+        fi
+        source ${BASE}/.tox/venv/bin/activate
+    fi
+}
+
+# Search for requirements used in openstack/ repos
+function search_reqs () {
+    beagle search --ignore-case --file '(.*requirement.*|setup.cfg)' "${1}" | \
+        grep "openstack/" | \
+        # Sometimes we get false positives from a package name being a
+        # substring within another package. This filter isn't working right
+        # though. This just means we might miss a package that isn't being
+        # used.
+        # grep "${1}[ |\!|>]" | \
+        grep -v "openstack.requirements"
+}
+
+# Get a list of all package names by filtering out comments, blank lines, and
+# any package modifiers like version constraints.
+function get_tracked_requirements () {
+    reqs=$(sed 's/[!|>|<|=|;].*//g' global-requirements.txt |
+            sed 's/  .*//g' |
+            sed '/^#/d' |
+            sed '/^$/d' |
+            sort | uniq)
+}

tools/get-health-report.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# Checks all of our tracked packages for any issues
+
+TOOLSDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+BASEDIR=$(dirname ${TOOLSDIR})
+
+source ${TOOLSDIR}/functions
+
+# Make sure we are using our venv
+enable_venv "${BASEDIR}"
+
+update=
+if [[ "$#" -eq 1 ]]; then
+    update="${1}"
+fi
+
+# Save off our current timestamp for use later
+current=$(date +%s)
+
+# Loop through each package to get details and check for issues
+get_tracked_requirements
+for req in $reqs; do
+    count=$(search_reqs ${req} |
+            grep -v " openstack/${req}  " |
+            wc -l)
+
+    metadata=$(curl -s -L "https://pypi.org/pypi/$req/json")
+    summary=$(echo "${metadata}" | jq -r '.info.summary')
+    last_release=$(echo "${metadata}" | jq -r '.info.version')
+    release_date=$(echo "${metadata}" | jq -r ".releases.\"${last_release}\" | .[0].upload_time")
+
+    # Print basic package information
+    echo "${req}"
+    if [[ "${summary}" != "" ]]; then
+        echo "    Summary:       ${summary}"
+    fi
+    echo "    Used by repos: ${count}"
+    echo "    Last release:  ${last_release}"
+    echo "    Release date:  ${release_date}"
+
+    # Check for various things to warn about
+    package_name=$(echo "${metadata}" | jq -r '.info.name')
+    if [[ "${req}" != "${package_name}" ]]; then
+        echo "    WARNING: In g-r as ${req} but actual name is ${package_name}"
+    fi
+
+    py3=$(echo "${metadata}" | \
+        jq -r '.info.classifiers | .[]' | \
+        grep "Programming Language :: Python :: 3")
+    if [[ -z ${py3} ]]; then
+        echo "    WARNING: No python 3 classifier in metadata"
+    fi
+
+    release=$(date -d $release_date +%s)
+    seconds_since_release=$((current-release))
+    years_since_release=$((seconds_since_release/60/60/24/365))
+    message=$(echo "It's been ${years_since_release} years since last release")
+    if [[ ${years_since_release} -gt 4 ]]; then
+        echo "    !!WARNING!! ${message}"
+    elif [[ ${years_since_release} -gt 2 ]]; then
+        echo "    WARNING ${message}"
+    fi
+done

tools/list-unused-packages.sh

@@ -17,41 +17,19 @@
 TOOLSDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 BASEDIR=$(dirname ${TOOLSDIR})
 
+source ${TOOLSDIR}/functions
+
 # Make sure we are using our venv
-if [[ -z "${VIRTUAL_ENV}" ]]; then
+enable_venv "${BASEDIR}"
-    if [[ ! -d ${BASEDIR}/.tox/venv ]]; then
-        (cd ${BASEDIR} && tox -e venv --notest > /dev/null)
-    fi
-    source ${BASEDIR}/.tox/venv/bin/activate
-fi
 
 update=
 if [[ "$#" -eq 1 ]]; then
     update="${1}"
 fi
 
-search_reqs ()
-{
-    beagle search --ignore-case --file '(.*requirement.*|setup.cfg)' "${1}" | \
-        grep "openstack/" | \
-        # Sometimes we get false positives from a package name being a
-        # substring within another package. This filter isn't working right
-        # though. This just means we might miss a package that isn't being
-        # used.
-        # grep "${1}[ |\!|>]" | \
-        grep -v "openstack.requirements"
-}
-
-# Get a list of all package names by filtering out comments, blank lines, and
-# any package modifiers like version constraints.
-reqs=$(sed 's/[!|>|<|=|;].*//g' global-requirements.txt |
-        sed 's/  .*//g' |
-        sed '/^#/d' |
-        sed '/^$/d' |
-        sort | uniq)
-
 # Loop through each package and check for its presence in any repo's
 # requirements files other than mentions in its own repo
+get_tracked_requirements
 for req in $reqs; do
     count=$(search_reqs ${req} |
             grep -v " openstack/${req}  " |