From c8e25f50e4e5bf4ed6071745542de334a7dc80bd Mon Sep 17 00:00:00 2001
From: Kevin Carter <kevin.carter@rackspace.com>
Date: Thu, 21 Jul 2016 08:58:45 -0500
Subject: [PATCH] Updated py_pkgs to index requirement files

This change indexes all of the requirement files found within the
plugins search path. A new section has been added for
"role_requirement_files" which contains keys using the requirement
file name. In the event that a requirement file is found and it is
not within a known role it the vaules of the requirement file will
be added to the "default" section.

Change-Id: I5e981c818a1da50aa0ae15630655a8dfac4f3db7
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>
---
 lookup/py_pkgs.py | 52 +++++++++++++++++++++++++++++++++--------------
 1 file changed, 37 insertions(+), 15 deletions(-)

diff --git a/lookup/py_pkgs.py b/lookup/py_pkgs.py
index b1ac252..0a14429 100644
--- a/lookup/py_pkgs.py
+++ b/lookup/py_pkgs.py
@@ -37,6 +37,7 @@ GIT_PACKAGE_DEFAULT_PARTS = dict()
 
 
 ROLE_PACKAGES = dict()
+ROLE_REQUIREMENTS = dict()
 
 
 REQUIREMENTS_FILE_TYPES = [
@@ -283,15 +284,18 @@ class DependencyFileProcessor(object):
         # Process everything simply by calling the method
         self._process_files()
 
-    def _py_pkg_extend(self, packages):
+    def _py_pkg_extend(self, packages, py_package=None):
+        if py_package is None:
+            py_package = self.pip['py_package']
         for pkg in packages:
             pkg_name = _pip_requirement_split(pkg)[0]
-            for py_pkg in self.pip['py_package']:
+            for py_pkg in py_package:
                 py_pkg_name = _pip_requirement_split(py_pkg)[0]
                 if pkg_name == py_pkg_name:
-                    self.pip['py_package'].remove(py_pkg)
+                    py_package.remove(py_pkg)
         else:
-            self.pip['py_package'].extend([i.lower() for i in packages])
+            py_package.extend([i.lower() for i in packages])
+        return py_package
 
     @staticmethod
     def _filter_files(file_names, ext):
@@ -443,23 +447,24 @@ class DependencyFileProcessor(object):
                 git_data=git_data
             )
 
-    def _package_build_index(self, packages, role_name, var_name):
+    def _package_build_index(self, packages, role_name, var_name,
+                             pkg_index=ROLE_PACKAGES):
         self._py_pkg_extend(packages)
         if role_name:
-            if role_name in ROLE_PACKAGES:
-                role_pkgs = ROLE_PACKAGES[role_name]
+            if role_name in pkg_index:
+                role_pkgs = pkg_index[role_name]
             else:
-                role_pkgs = ROLE_PACKAGES[role_name] = dict()
+                role_pkgs = pkg_index[role_name] = dict()
 
             pkgs = role_pkgs.get(var_name, list())
             if 'optional' not in var_name:
                 pkgs.extend(packages)
-            ROLE_PACKAGES[role_name][var_name] = pkgs
+            pkg_index[role_name][var_name] = self._py_pkg_extend(packages, pkgs)
         else:
-            for k, v in ROLE_PACKAGES.items():
+            for k, v in pkg_index.items():
                 for item_name in v.keys():
                     if var_name == item_name:
-                        ROLE_PACKAGES[k][item_name].extend(packages)
+                        pkg_index[k][item_name] = self._py_pkg_extend(packages, pkg_index[k][item_name])
 
     def _process_files(self):
         """Process files."""
@@ -511,6 +516,8 @@ class DependencyFileProcessor(object):
                             role_name=role_name,
                             var_name=key
                         )
+        else:
+            role_name = None
 
         return_list = self._filter_files(self.file_names, 'txt')
         for file_name in return_list:
@@ -523,15 +530,28 @@ class DependencyFileProcessor(object):
             for file_name in return_list:
                 if file_name.endswith('other-requirements.txt'):
                     continue
+                if 'roles' in file_name:
+                    _role_name = file_name.split('roles%s' % os.sep)[-1]
+                    role_name = _role_name.split(os.sep)[0]
+                if not role_name:
+                    role_name = 'default'
                 with open(file_name, 'r') as f:
                     packages = [
-                        i.split()[0] for i in f.read().splitlines()
+                        i.split()[0].lower() for i in f.read().splitlines()
                         if i
                         if not i.startswith('#')
                     ]
-                    self._py_pkg_extend(packages)
-
-
+                    base_file_name = os.path.basename(file_name)
+                    if base_file_name.endswith('test-requirements.txt'):
+                        continue
+                    self._package_build_index(
+                        packages=packages,
+                        role_name=role_name,
+                        var_name='txt_file_packages',
+                        pkg_index=ROLE_REQUIREMENTS
+                    )
+            else:
+                role_name = None
 def _abs_path(path):
     return os.path.abspath(
         os.path.expanduser(
@@ -596,6 +616,7 @@ class LookupModule(BASECLASS):
             for key, value in return_data.items():
                 if isinstance(value, (list, set)):
                     return_data[key] = sorted(value)
+            return_data['role_requirement_files'] = ROLE_REQUIREMENTS
             return [return_data]
 
     def run_v1(self, terms, inject=None, **kwargs):
@@ -644,6 +665,7 @@ class LookupModule(BASECLASS):
             for key, value in return_data.items():
                 if isinstance(value, (list, set)):
                     return_data[key] = sorted(value)
+            return_data['role_requirement_files'] = ROLE_REQUIREMENTS
             return [return_data]
 
 # Used for testing and debuging usage: `python plugins/lookups/py_pkgs.py ../`