From ed868f3ab906c366317ce0228cea58c5af12c1a3 Mon Sep 17 00:00:00 2001
From: Joe Gordon <joe.gordon0@gmail.com>
Date: Thu, 15 Jan 2015 23:30:35 +1300
Subject: [PATCH] Import cap.py tool to cap  explicit dependencies

This is a forward port of the tool we used to cap stable/juno
requirements. Save a copy of it in master so its available when we are
ready to cut future stable branches.

pin all global-requirements with the version found in pip-freeze
along with a tool to generate the new requirements file

Issues:
* Some versions are only apt-get installable (suds 0.4.1)
* Not all packages are installed in our standard dsvm-tempest env
* Some versions are lower then the minimum requirement we previously had
  because python is awful (libvirt-python)
* Doesn't pin transitive dependencies
* library versions can vanish from pypi and still break us
* Grenade and tempest-full pip-freezes are different and do not work with
each other, so some tweaking is required.

Taking the installed library version from pip-freeze and use that as a
version cap.

Previously used pip-freeze from a tempest-dsvm-neutron-full job.

Change-Id: Iaf48bb069fdd7a19d614ce44b86abd9977c5f0c0
---
 tools/cap.py | 138 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 138 insertions(+)
 create mode 100755 tools/cap.py

diff --git a/tools/cap.py b/tools/cap.py
new file mode 100755
index 0000000000..11ecfc64e2
--- /dev/null
+++ b/tools/cap.py
@@ -0,0 +1,138 @@
+#! /usr/bin/env python
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+import argparse
+import re
+
+import pkg_resources
+
+overrides = dict()
+# List of overrides needed. Ignore version in pip-freeze and use the one here
+# instead. Example:
+# suds 0.4.1 isn't pip installable but is in distribution packages
+# overrides['suds'] = 'suds==0.4'
+# apt package of libvirt-python is lower then our minimum requirement
+# overrides['libvirt-python'] = None
+
+
+def cap(requirements, frozen):
+    """Cap requirements to version in freeze.
+
+    Go through every package in requirements and try to cap.
+
+    Input: two arrays of lines.
+    Output: Array of new lines.
+    """
+    output = []
+    for line in requirements:
+        try:
+            req = pkg_resources.Requirement.parse(line)
+            specifier = str(req.specifier)
+            if any(op in specifier for op in ['==', '~=', '<']):
+                # if already capped, continue
+                output.append(line)
+                continue
+        except ValueError:
+            # line was a comment, continue
+            output.append(line)
+            continue
+        if req.project_name in overrides:
+            new_line = overrides[req.project_name]
+            if new_line:
+                output.append(overrides[req.project_name])
+            else:
+                output.append(line)
+            continue
+        # add cap
+        new_cap = cap_requirement(req.project_name, frozen)
+        if new_cap:
+            output.append(pin(line, new_cap))
+        else:
+            output.append(line)
+    return output
+
+
+def pin(line, new_cap):
+    """Add new cap into existing line
+
+    Don't use pkg_resources so we can preserve the comments.
+    """
+    end = None
+    use_comma = False
+    parts = line.split(' #')
+    if len(split(parts[0].strip())) > 1:
+        use_comma = True
+    if "#" in line:
+        # if comment
+        end = parts[1]
+    # cap to new max version
+    if end:
+        new_end = "<=%s #%s" % (new_cap, end)
+    else:
+        new_end = "<=%s" % new_cap
+    if use_comma is True:
+        return "%s,%s" % (parts[0].strip(), new_end)
+    else:
+        return "%s%s" % (parts[0].strip(), new_end)
+
+
+def split(line):
+    return re.split('[><=]', line)
+
+
+def cap_requirement(requirement, frozen):
+    # Find current version of requirement in freeze
+    specifier = frozen.get(requirement, None)
+    if specifier:
+        return split(str(specifier))[-1]
+    return None
+
+
+def freeze(lines):
+    """Parse lines from freeze file into a dict.
+
+    Where k:v is project_name:specifier.
+    """
+    freeze = dict()
+
+    for line in lines:
+        try:
+            req = pkg_resources.Requirement.parse(line)
+            freeze[req.project_name] = req.specifier
+        except ValueError:
+            # not a valid requirement, can be a comment, blank line etc
+            continue
+    return freeze
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Take the output of "
+                    "'pip freeze' and use the installed versions to "
+                    "caps requirements.")
+    parser.add_argument('requirements', help='requirements file input')
+    parser.add_argument(
+        'freeze',
+        help='output of pip freeze, taken from a full tempest job')
+    args = parser.parse_args()
+    with open(args.requirements) as f:
+        requirements = [line.strip() for line in f.readlines()]
+    with open(args.freeze) as f:
+        frozen = freeze([line.strip() for line in f.readlines()])
+    for line in cap(requirements, frozen):
+        print(line)
+
+if __name__ == '__main__':
+    main()