Fix building anvils requirements.

It doesn't appear like the correct versions are being
built so create a helper tool that can search for the
correct versions and install those if they exist or
build the ones that are missing.

Change-Id: I03869556a8e321c06ad3ed0b81bb7e8f647e4c8e

smithy

@@ -7,6 +7,7 @@ cd "$(dirname "$0")"
 
 VERBOSE="${VERBOSE:-0}"
 PY2RPM_CMD="$PWD/tools/py2rpm"
+YUMFIND_CMD="$PWD/tools/yumfind"
 
 YUM_OPTS="--assumeyes --nogpgcheck"
 PIP_CMD=""
@@ -156,51 +157,54 @@ try:
 except KeyError:
     pass
 ")
-    # NOTE(aababilov): drop `<` and `<=` requirements because yum cannot
+    local python_names=$(cat requirements.txt test-requirements.txt | sed -r -e 's/#.*$//' | sort -u)
-    # handle them correctly
+    echo "Attemping to install python requirements: $(echo $python_names)"
-    local python_names=$(cat requirements.txt test-requirements.txt |
+    local missing_packages=""
-        sed -r -e 's/#.*$//' -e 's/,?<[ =.0-9]+//' | sort -u)
+    local found_packages=""
-    local rpm_names=$("$PY2RPM_CMD" --package-map $package_map --convert $python_names |
+    for name in $python_names; do
-        while read req pack; do echo $pack; done | sort -u)
+        local pkg_name=$("$PY2RPM_CMD" --package-map $package_map --convert "$name" | while read req pack; do echo $pack; done  | head -n1 | tr -s ' ' | cut -d' ' -f1)
-    # Install all available RPMs
+        local yum_name=$("$YUMFIND_CMD" "$pkg_name" "$name")
-    echo "Installing python requirement packages: $(echo $rpm_names)"
+        if [ -n "$yum_name" ]; then
-    for rpm in $rpm_names; do
+            found_packages="$found_packages $yum_name"
-        yum_install $rpm
+        else
-    done
+            missing_packages="$missing_packages $name"
-    # Build and install missing packages
-    local missing_python=""
-    for python_name in $python_names; do
-        rpm_name=$("$PY2RPM_CMD" --package-map $package_map --convert $python_name | awk '{print $2}' | sort -u)
-        if ! rpm_is_installed $rpm_name; then
-            missing_python="$missing_python $python_name"
         fi
     done
-    if [ -z "$missing_python" ]; then
+    if [ -n "$found_packages" ]; then
-        return
+        echo "Attemping to install python requirements found as packages: $(echo $found_packages)"
+        yum install $YUM_OPTS $found_packages
+        if [ "$?" != "0" ]; then
+            echo "Failed installing $(echo $found_packages)"
+            return 1
+        fi
     fi
-    # Some RPMs are not available, try to build them on fly.
+    if [ -z "$missing_packages" ]; then
-    # First download them...
+        return 0
+    fi
+    echo "Building missing python requirements: $(echo $missing_packages)"
     local pip_tmp_dir=$(mktemp -d)
     find_pip
     local pip_opts="$PIP_OPTS -U -I"
-    echo "Downloading missing python requirements:$missing_python"
+    echo "Downloading..."
-    $PIP_CMD install $pip_opts $missing_python --download "$pip_tmp_dir"
+    $PIP_CMD install $pip_opts $missing_packages --download "$pip_tmp_dir"
-    # Now build them
+    echo "Building RPMs..."
-    echo "Building RPMs for $missing_python"
     local rpm_names=$("$PY2RPM_CMD"  --package-map $package_map -- "$pip_tmp_dir/"* 2>/dev/null |
         awk '/^Wrote: /{ print $2 }' | grep -v '.src.rpm' | sort -u)
     if [ -z "$rpm_names" ]; then
-        echo "No binary RPMs were built for$missing_python"
+        echo "No binary RPMs were built!"
         return 1
     fi
-    echo "Installing missing python requirement packages: $(echo $rpm_names)"
+    local rpm_base_names=""
     for rpm in $rpm_names; do
-        yum_install "$rpm"
+        rpm_base_names="$rpm_base_names $(basename $rpm)"
-        if [ "$?" != "0" ]; then
-            echo "Failed installing $rpm"
-            return 1
-        fi
     done
+    echo "Installing missing python requirement packages: $(echo $rpm_base_names)"
+    yum install $YUM_OPTS $rpm_names
+    if [ "$?" != "0" ]; then
+        echo "Failed installing $(echo $rpm_base_names)"
+        return 1
+    fi
+    return 0
 }
 
 needs_bootstrap()

tools/yumfind

@@ -0,0 +1,42 @@
+#!/usr/bin/python
+
+import warnings
+warnings.filterwarnings("ignore")
+
+import logging
+import pkg_resources
+import sys
+
+from yum import YumBase
+
+
+if __name__ == '__main__':
+    pkg_name = sys.argv[1]
+    py_name = sys.argv[2]
+    req = pkg_resources.Requirement.parse(py_name)
+
+    base = YumBase()
+    base.doConfigSetup(debuglevel=-1, errorlevel=-1)
+    pkgs = base.doPackageLists(patterns=[pkg_name], ignore_case=True)
+    all_pkgs = list(pkgs.available)
+    all_pkgs.extend(pkgs.installed)
+
+    yum_map = {}
+    for pkg in all_pkgs:
+        for provides in pkg.provides:
+            pkg_info = (pkg.version, pkg.repo, pkg)
+            yum_map.setdefault(provides[0], set()).add(pkg_info)
+
+    matches = []
+    yum_versions = yum_map.get(pkg_name, [])
+    for (version, repo, pkg) in yum_versions:
+            if version in req:
+                matches.append(pkg)
+
+    if matches:
+        # Pick the newest match.
+        match = sorted(matches)[-1]
+        print(match)
+        sys.exit(0)
+    else:
+        sys.exit(1)