diff --git a/tools/functions b/tools/functions new file mode 100755 index 0000000000..f4e5b329f4 --- /dev/null +++ b/tools/functions @@ -0,0 +1,45 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Shared functions for shell scripts + +function enable_venv () { + BASE="${1}" + if [[ -z "${VIRTUAL_ENV}" ]]; then + if [[ ! -d ${BASE}/.tox/venv ]]; then + (cd ${BASE} && tox -e venv --notest > /dev/null) + fi + source ${BASE}/.tox/venv/bin/activate + fi +} + +# Search for requirements used in openstack/ repos +function search_reqs () { + beagle search --ignore-case --file '(.*requirement.*|setup.cfg)' "${1}" | \ + grep "openstack/" | \ + # Sometimes we get false positives from a package name being a + # substring within another package. This filter isn't working right + # though. This just means we might miss a package that isn't being + # used. + # grep "${1}[ |\!|>]" | \ + grep -v "openstack.requirements" +} + +# Get a list of all package names by filtering out comments, blank lines, and +# any package modifiers like version constraints. +function get_tracked_requirements () { + reqs=$(sed 's/[!|>|<|=|;].*//g' global-requirements.txt | + sed 's/ .*//g' | + sed '/^#/d' | + sed '/^$/d' | + sort | uniq) +} diff --git a/tools/get-health-report.sh b/tools/get-health-report.sh new file mode 100755 index 0000000000..66ccf21ea9 --- /dev/null +++ b/tools/get-health-report.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Checks all of our tracked packages for any issues + +TOOLSDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +BASEDIR=$(dirname ${TOOLSDIR}) + +source ${TOOLSDIR}/functions + +# Make sure we are using our venv +enable_venv "${BASEDIR}" + +update= +if [[ "$#" -eq 1 ]]; then + update="${1}" +fi + +# Save off our current timestamp for use later +current=$(date +%s) + +# Loop through each package to get details and check for issues +get_tracked_requirements +for req in $reqs; do + count=$(search_reqs ${req} | + grep -v " openstack/${req} " | + wc -l) + + metadata=$(curl -s -L "https://pypi.org/pypi/$req/json") + summary=$(echo "${metadata}" | jq -r '.info.summary') + last_release=$(echo "${metadata}" | jq -r '.info.version') + release_date=$(echo "${metadata}" | jq -r ".releases.\"${last_release}\" | .[0].upload_time") + + # Print basic package information + echo "${req}" + if [[ "${summary}" != "" ]]; then + echo " Summary: ${summary}" + fi + echo " Used by repos: ${count}" + echo " Last release: ${last_release}" + echo " Release date: ${release_date}" + + # Check for various things to warn about + package_name=$(echo "${metadata}" | jq -r '.info.name') + if [[ "${req}" != "${package_name}" ]]; then + echo " WARNING: In g-r as ${req} but actual name is ${package_name}" + fi + + py3=$(echo "${metadata}" | \ + jq -r '.info.classifiers | .[]' | \ + grep "Programming Language :: Python :: 3") + if [[ -z ${py3} ]]; then + echo " WARNING: No python 3 classifier in metadata" + fi + + release=$(date -d $release_date +%s) + seconds_since_release=$((current-release)) + years_since_release=$((seconds_since_release/60/60/24/365)) + message=$(echo "It's been ${years_since_release} years since last release") + if [[ ${years_since_release} -gt 4 ]]; then + echo " !!WARNING!! ${message}" + elif [[ ${years_since_release} -gt 2 ]]; then + echo " WARNING ${message}" + fi +done diff --git a/tools/list-unused-packages.sh b/tools/list-unused-packages.sh index 51681b0c73..1150525a3d 100755 --- a/tools/list-unused-packages.sh +++ b/tools/list-unused-packages.sh @@ -17,41 +17,19 @@ TOOLSDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" BASEDIR=$(dirname ${TOOLSDIR}) +source ${TOOLSDIR}/functions + # Make sure we are using our venv -if [[ -z "${VIRTUAL_ENV}" ]]; then - if [[ ! -d ${BASEDIR}/.tox/venv ]]; then - (cd ${BASEDIR} && tox -e venv --notest > /dev/null) - fi - source ${BASEDIR}/.tox/venv/bin/activate -fi +enable_venv "${BASEDIR}" update= if [[ "$#" -eq 1 ]]; then update="${1}" fi -search_reqs () -{ - beagle search --ignore-case --file '(.*requirement.*|setup.cfg)' "${1}" | \ - grep "openstack/" | \ - # Sometimes we get false positives from a package name being a - # substring within another package. This filter isn't working right - # though. This just means we might miss a package that isn't being - # used. - # grep "${1}[ |\!|>]" | \ - grep -v "openstack.requirements" -} - -# Get a list of all package names by filtering out comments, blank lines, and -# any package modifiers like version constraints. -reqs=$(sed 's/[!|>|<|=|;].*//g' global-requirements.txt | - sed 's/ .*//g' | - sed '/^#/d' | - sed '/^$/d' | - sort | uniq) - # Loop through each package and check for its presence in any repo's # requirements files other than mentions in its own repo +get_tracked_requirements for req in $reqs; do count=$(search_reqs ${req} | grep -v " openstack/${req} " |