Fix neutron migrations with optional providers

The venv creation process would include all python packages into the venv
even if the optional provider was not used. This causes neutron migration
issues and creates unneeded table. To resolve this, the py_pkg lookup
plugin has been modified to ensure no "optional" pip packages are included
in the base venv.

The optional string was added to the "plumgrid_pip_packages" array to ensure
the lookup plugin is no longer indexing those packages for inclusion in the
base venv.

Note:
This patch includes a tox.ini configuration change to make it use
test-requirements.txt for the pep8 environment dependencies. This is
essential to allow the lint testing to pass and can be reverted later.

Change-Id: I999ebd52d99f984d7bfa19685fc0fa530c2b32a9
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>

playbooks/plugins/lookups/py_pkgs.py

@@ -356,6 +356,7 @@ class DependencyFileProcessor(object):
                             role_pkgs = ROLE_PACKAGES[role_name] = dict()
 
                         pkgs = role_pkgs.get(key, list())
+                        if 'optional' not in key:
                             pkgs.extend(values)
                         ROLE_PACKAGES[role_name][key] = pkgs
                     else:

playbooks/roles/os_neutron/defaults/main.yml

@@ -344,7 +344,7 @@ neutron_pip_packages:
   - python-novaclient
   - repoze.lru
 
-neutron_plumgrid_pip_packages:
+neutron_optional_plumgrid_pip_packages:
   - networking-plumgrid
 
 ## Tunable overrides

playbooks/roles/os_neutron/tasks/plumgrid_config.yml

@@ -23,19 +23,40 @@
   tags:
     - neutron_config
 
-- name: Install networking-plumgrid
+- name: Install plumgrid pip packages (venv)
+  pip:
+    name: "{{ item }}"
+    state: present
+    virtualenv: "{{ neutron_venv_bin | dirname }}"
+    virtualenv_site_packages: "no"
+    extra_args: "{{ pip_install_options|default('') }}"
+  register: install_packages
+  until: install_packages|success
+  retries: 5
+  delay: 2
+  with_items:
+    - neutron_optional_plumgrid_pip_packages
+  when:
+    - neutron_venv_enabled | bool
+    - inventory_hostname in groups['neutron_server']
+  tags:
+    - neutron-install
+    - neutron-pip-packages
+
+- name: Install plumgrid pip packages (no venv)
   pip:
     name: "{{ item }}"
     state: present
     extra_args: "{{ pip_install_options|default('') }}"
-  with_items: neutron_plumgrid_pip_packages
-  register: install_package
+  register: install_packages
   until: install_packages|success
   retries: 5
   delay: 2
+  with_items:
+    - neutron_optional_plumgrid_pip_packages
   when:
+    - not neutron_venv_enabled | bool
     - inventory_hostname in groups['neutron_server']
-    - neutron_get_venv | failed
   tags:
     - neutron-install
     - neutron-pip-packages

tox.ini

@@ -23,7 +23,7 @@ commands = {posargs}
 
 # Run hacking/flake8 check for all python files
 [testenv:pep8]
-deps = flake8
+deps = -r{toxinidir}/test-requirements.txt
 whitelist_externals = bash
 commands =
     bash -c "grep -Irl \