Add the concept of a version map for constraints generation

We need to generate constraints for all python versions we use.  This is
python 2.7 (trusty and xenial), 3.4 (trusty) and 3.5(xenial).  There
isn't a supportable way to install all 3 versions on a single node.

Add a version-map that allows us to clone the output of a freeze between
python versions.  This assumes that the python version in question are
compatible during the transition window.

It's possible to run with:
    ... --version-map 3.4:3.5 --version-map 3.5:3.4 ...

This will mean that we generate constraints consistently between versions
regardless of which version /usr/bin/python3 is.

Change-Id: I55a3597ea2f2bf48e30dc564a4bf4b2ef7f178b3

openstack_requirements/cmds/generate.py

@@ -11,6 +11,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
+
+import copy
 import optparse
 import os.path
 import subprocess
@@ -131,6 +134,18 @@ def _combine_freezes(freezes, blacklist=None):
             yield '%s===%s\n' % (package, list(versions.keys())[0])
 
 
+def _clone_versions(freezes, options):
+    for freeze_data in freezes:
+        versions = [v for v, d in freezes]
+        version, freeze = freeze_data
+        if (version in options.version_map and
+                options.version_map[version] not in versions):
+            print("Duplicating %s freeze to %s" %
+                  (version, options.version_map[version]),
+                  file=sys.stderr)
+            freezes.append((options.version_map[version], copy.copy(freeze)))
+
+
 # -- untested UI glue from here down.
 
 def _validate_options(options):
@@ -154,6 +169,15 @@ def _validate_options(options):
         raise Exception(
             "Blacklist file %(path)s not found."
             % dict(path=options.blacklist))
+    version_map = {}
+    for map_entry in options.version_map:
+        if ':' not in map_entry:
+            raise Exception(
+                "Invalid version-map entry %(map_entry)s"
+                % dict(map_entry=map_entry))
+        src, dst = map_entry.split(':')
+        version_map[src] = dst
+    options.version_map = version_map
 
 
 def _parse_blacklist(path):
@@ -175,12 +199,20 @@ def main(argv=None, stdout=None):
     parser.add_option(
         "-b", dest="blacklist",
         help="Filename of a list of package names to exclude.")
+    parser.add_option(
+        "--version-map", dest='version_map', default=[], action='append',
+        help=('Add a : seperated list of versions to clone.  To \'clone\' '
+              'a freeze geberated by python3.4 to python3.5 specify 3.4:3.5.  '
+              'This is intented as as a way to transition between python '
+              'versions when it\'s not possible to have all versions '
+              'installed'))
     options, args = parser.parse_args(argv)
     if stdout is None:
         stdout = sys.stdout
     _validate_options(options)
     freezes = [
         _freeze(options.requirements, python) for python in options.pythons]
+    _clone_versions(freezes, options)
     blacklist = _parse_blacklist(options.blacklist)
     stdout.writelines(_combine_freezes(freezes, blacklist))
     stdout.flush()

openstack_requirements/tests/test_generate.py

@@ -98,3 +98,56 @@ class TestCombine(testtools.TestCase):
             ['enum===1.5.0\n'],
             list(generate._combine_freezes(
                 [freeze_27], blacklist=blacklist)))
+
+
+class Namespace(object):
+    def __init__(self, **kwargs):
+        self.__dict__.update(kwargs)
+
+
+class TestClone(testtools.TestCase):
+
+    def test_py34_clone_py35(self):
+        # Simulate an environment where we have python 3.4 data and need to
+        # clone that to python 3.5
+        options = Namespace(version_map={'3.4': '3.5', '3.5': '3.4'})
+        freeze_27 = ('2.7', [('dnspython', '1.15.0')])
+        freeze_34 = ('3.4', [('dnspython3', '1.12.0')])
+        freeze_35 = ('3.5', [('dnspython3', '1.12.0')])
+
+        freezes = [freeze_27, freeze_34]
+        expected_freezes = [freeze_27, freeze_34, freeze_35]
+
+        generate._clone_versions(freezes, options)
+
+        self.assertEqual(expected_freezes, freezes)
+
+    def test_py34_noclone_py35(self):
+        # Simulate an environment where we have python 3.4 and python 3.5 data
+        # so there is no need to clone.
+        options = Namespace(version_map={'3.4': '3.5', '3.5': '3.4'})
+        freeze_27 = ('2.7', [('dnspython', '1.15.0')])
+        freeze_34 = ('3.4', [('dnspython3', '1.12.0')])
+        freeze_35 = ('3.5', [('other-pkg', '1.0.0')])
+
+        freezes = [freeze_27, freeze_34, freeze_35]
+        expected_freezes = [freeze_27, freeze_34, freeze_35]
+
+        generate._clone_versions(freezes, options)
+
+        self.assertEqual(expected_freezes, freezes)
+
+    def test_py35_clone_py34(self):
+        # Simulate an environment where we have python 3.5 data and need to
+        # clone that to python 3.4
+        options = Namespace(version_map={'3.4': '3.5', '3.5': '3.4'})
+        freeze_27 = ('2.7', [('dnspython', '1.15.0')])
+        freeze_34 = ('3.4', [('dnspython3', '1.12.0')])
+        freeze_35 = ('3.5', [('dnspython3', '1.12.0')])
+
+        freezes = [freeze_27, freeze_35]
+        expected_freezes = [freeze_27, freeze_35, freeze_34]
+
+        generate._clone_versions(freezes, options)
+
+        self.assertEqual(expected_freezes, freezes)