Switch to use a venv by default for testing

Also updates the method to use the python3 native
venv module instead of the virtualenv module.

Change-Id: Iae4248acec0b3bdf48be62134f64356d40375dde

bindep.txt

@@ -18,7 +18,7 @@ python3              [platform:rpm]
 python3-minimal      [platform:dpkg]
 python3-devel        [platform:rpm]
 python3-dev          [platform:dpkg]
-python3-virtualenv
+python3-venv         [platform:dpkg]
 wget
 sudo
 ebtables

playbooks/ci/run.yaml

@@ -1,7 +1,7 @@
 - hosts: all
   tasks:
     - shell:
-        cmd: scripts/test-bifrost.sh
+        cmd: bash scripts/test-bifrost.sh
         chdir: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/bifrost'].src_dir }}"
       environment:
         BUILD_IMAGE: "{{ build_image | default(false) | bool | lower }}"
@@ -10,5 +10,6 @@
         UPPER_CONSTRAINTS_FILE: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/requirements'].src_dir }}/upper-constraints.txt"
         WORKSPACE: "{{ ansible_user_dir }}/src/opendev.org"
         USE_DHCP: "{{ use_dhcp | default(false) | bool | lower }}"
-        USE_VENV: "{{ use_venv | default(false) | bool | lower }}"
+        USE_VENV: "{{ use_venv | default(true) | bool | lower }}"
         ZUUL_BRANCH: "{{ zuul.branch }}"
+        VENV: "/opt/stack/bifrost"

playbooks/roles/bifrost-create-vm-nodes/tasks/prepare_libvirt.yml

@@ -15,14 +15,30 @@
 # Setup libvirt - ensure network and storage pool are defined and active,
 # prepare dir for vm logs
 ---
-- name: install libvirt-python and lxml
+- name: set virtualenv_command
+  set_fact:
+    venv_command: "python3 -m venv"
+  when: enable_venv
+
+- name: install libvirt-python and lxml without virtualenv
   pip:
     name: "{{ item }}"
     extra_args: "-c {{ upper_constraints_file }}"
-    virtualenv: "{{ enable_venv | bool | ternary(bifrost_venv_dir, omit) }}"
   with_items:
     - libvirt-python
     - lxml
+  when: enable_venv | bool == false
+
+- name: install libvirt-python and lxml with virtualenv
+  pip:
+    name: "{{ item }}"
+    state: present
+    virtualenv: "{{ bifrost_venv_dir }}"
+    virtualenv_command: "{{ venv_command | default(omit) }}"
+  with_items:
+    - libvirt-python
+    - lxml
+  when: enable_venv | default(false) | bool
 
 - name: "Restart libvirt service"
   service: name="{{libvirt_service_name}}" state=restarted

playbooks/roles/bifrost-ironic-install/tasks/pip_install.yml

@@ -16,8 +16,8 @@
 
 - name: set virtualenv_command
   set_fact:
-    venv_command: "{{ hostvars[inventory_hostname].ansible_python.executable }} -m virtualenv"
-  when: enable_venv|bool
+    venv_command: "python3 -m venv"
+  when: enable_venv
 
 - name: "Install {{ package }} package from pip using virtualenv"
   pip:

playbooks/roles/bifrost-keystone-install/tasks/pip_install.yml

@@ -14,8 +14,8 @@
 
 - name: set virtualenv_command
   set_fact:
-    venv_command: "{{ hostvars[inventory_hostname].ansible_python.executable }} -m virtualenv"
-  when: enable_venv|bool
+    venv_command: "python3 -m venv"
+  when: enable_venv
 
 - name: "Install {{ package }} package from pip using virtualenv"
   pip:

scripts/env-setup.sh

@@ -10,7 +10,7 @@ ANSIBLE_PIP_VERSION=${ANSIBLE_PIP_VERSION:-${ANSIBLE_GIT_BRANCH:-stable-2.6}}
 ANSIBLE_PIP_STRING=$(${PYTHON} $(dirname $0)/ansible-pip-str.py ${ANSIBLE_PIP_VERSION})
 
 if [ -n "${VENV-}" ]; then
-    sudo -H -E ${PIP} install --upgrade "${ANSIBLE_PIP_STRING}"
+    sudo ${PIP} install --ignore-installed "${ANSIBLE_PIP_STRING}"
     ANSIBLE=${VENV}/bin/ansible
 else
     ${PIP} install --user --upgrade "${ANSIBLE_PIP_STRING}"

scripts/install-deps.sh

@@ -15,7 +15,6 @@ CHECK_CMD_PKGS=(
     net-tools
     python3-devel
     python3
-    venv
     wget
 )
 
@@ -34,7 +33,6 @@ case ${ID,,} in
         [net-tools]=net-tools
         [python]=python
         [python-devel]=python-devel
-        [venv]=python-virtualenv
         [wget]=wget
     )
     EXTRA_PKG_DEPS=( python-xml )
@@ -62,10 +60,9 @@ case ${ID,,} in
         [net-tools]=net-tools
         [python3]=python3-minimal
         [python3-devel]=libpython3-dev
-        [venv]=python3-virtualenv
         [wget]=wget
     )
-    EXTRA_PKG_DEPS=()
+    EXTRA_PKG_DEPS=( python3-apt )
     sudo apt-get update
     ;;
 
@@ -83,7 +80,6 @@ case ${ID,,} in
         [net-tools]=net-tools
         [python]=python3
         [python-devel]=python3-devel
-        [venv]=python3-virtualenv
         [wget]=wget
     )
     EXTRA_PKG_DEPS=()
@@ -115,7 +111,7 @@ if ! $(wget --version &>/dev/null); then
     ${INSTALLER_CMD} ${PKG_MAP[wget]}
 fi
 if [ -n "${VENV-}" ]; then
-    if ! $(python3 -m virtualenv --version &>/dev/null); then
+    if ! $(python3 -m venv --version &>/dev/null); then
         ${INSTALLER_CMD} ${PKG_MAP[venv]}
     fi
 fi
@@ -138,7 +134,7 @@ if [ -n "${VENV-}" ]; then
     echo "NOTICE: Using virtualenv for this installation."
     if [ ! -f ${VENV}/bin/activate ]; then
         # only create venv if one doesn't exist
-        sudo -H -E python3 -m virtualenv --no-site-packages ${VENV}
+        sudo -H -E python3 -m venv --system-site-packages ${VENV}
     fi
     # Note(cinerama): activate is not compatible with "set -u";
     # disable it just for this line.
@@ -160,19 +156,21 @@ PYTHON=$(which python3)
 # older versions of pip are incompatible with
 # requests, one of our indirect dependencies (bug 1459947).
 #
-
-if [ ! $($PYTHON -m pip install -U pip) ]; then
+ls $PYTHON
+sudo -H -E $PYTHON -m pip install -U pip --ignore-installed
+if [ "$?" != "0" ]; then
     wget -O /tmp/get-pip.py https://bootstrap.pypa.io/3.4/get-pip.py
     sudo -H -E ${PYTHON} /tmp/get-pip.py
 fi
 
-PIP=$(which pip3)
+ls -la /opt/stack/bifrost/bin
+
+PIP=$(echo $PYTHON | sed 's/python/pip/')
 
 if [ "$OS_FAMILY" == "RedHat" ]; then
     sudo -H -E ${PIP} freeze
     sudo -H -E ${PIP} install --ignore-installed pyparsing ipaddress
 fi
-
 sudo -H -E ${PIP} install -r "$(dirname $0)/../requirements.txt"
 
 # Install the rest of required packages using bindep

scripts/test-bifrost.sh

@@ -7,7 +7,7 @@ SCRIPT_HOME="$(cd "$(dirname "$0")" && pwd)"
 BIFROST_HOME=$SCRIPT_HOME/..
 ANSIBLE_INSTALL_ROOT=${ANSIBLE_INSTALL_ROOT:-/opt/stack}
 USE_DHCP="${USE_DHCP:-false}"
-USE_VENV="${USE_VENV:-false}"
+USE_VENV="${USE_VENV:-true}"
 BUILD_IMAGE="${BUILD_IMAGE:-false}"
 BAREMETAL_DATA_FILE=${BAREMETAL_DATA_FILE:-'/tmp/baremetal.json'}
 ENABLE_KEYSTONE="${ENABLE_KEYSTONE:-false}"
@@ -43,7 +43,7 @@ PROVISION_WAIT_TIMEOUT=${PROVISION_WAIT_TIMEOUT:-900}
 NOAUTH_MODE=true
 CLOUD_CONFIG=""
 WAIT_FOR_DEPLOY=true
-ENABLE_VENV=false
+ENABLE_VENV=true
 
 # This sets up the MySQL database like it's done for all OpenStack
 # projects for CI testing.
@@ -159,7 +159,10 @@ ${ANSIBLE} -vvvv \
        -e test_vm_memory_size=${VM_MEMORY_SIZE} \
        -e test_vm_domain_type=${VM_DOMAIN_TYPE} \
        -e baremetal_json_file=${BAREMETAL_DATA_FILE} \
-       -e enable_venv=${ENABLE_VENV}
+       -e enable_venv=${ENABLE_VENV} \
+       -e bifrost_venv_dir=${VENV}
+
+
 
 if [ ${USE_DHCP} = "true" ]; then
     # reduce the number of nodes in JSON file

tools/vagrant_dev_env/vagrant.yml

@@ -93,13 +93,15 @@
       environment:
           ANSIBLE_INSTALL_ROOT: "{{ansible_install_root}}"
           LANG: c
+          VENV: "/opt/stack/bifrost"
       args:
           chdir: /home/vagrant/bifrost
       become: yes
 
     - name: Install Bifrost
-      command: ansible-playbook -vvvv -i inventory/target install.yaml -e network_interface={{network_interface}} -e ansible_python_interpreter="/usr/bin/python3" -e download_ipa=true -e use_cirros=true
+      command: ansible-playbook -vvvv -i inventory/target install.yaml -e network_interface={{network_interface}} -e ansible_python_interpreter="/usr/bin/python3" -e download_ipa=true -e use_cirros=true -e enable_venv=true
       environment:
           PATH: "{{ansible_env.HOME}}/.local/bin:{{ansible_env.PATH}}"
+          VENV: "/opt/stack/bifrost"
       args:
           chdir: /home/vagrant/bifrost/playbooks