From 44196ea91bbb8dbd5f55bc01f1401b2c47f10f07 Mon Sep 17 00:00:00 2001 From: Julia Kreger Date: Wed, 8 Jan 2020 16:29:07 -0800 Subject: [PATCH] Switch to use a venv by default for testing Also updates the method to use the python3 native venv module instead of the virtualenv module. Change-Id: Iae4248acec0b3bdf48be62134f64356d40375dde --- bindep.txt | 2 +- playbooks/ci/run.yaml | 5 +++-- .../tasks/prepare_libvirt.yml | 20 +++++++++++++++++-- .../tasks/pip_install.yml | 4 ++-- .../tasks/pip_install.yml | 4 ++-- scripts/env-setup.sh | 2 +- scripts/install-deps.sh | 20 +++++++++---------- scripts/test-bifrost.sh | 9 ++++++--- tools/vagrant_dev_env/vagrant.yml | 4 +++- 9 files changed, 45 insertions(+), 25 deletions(-) diff --git a/bindep.txt b/bindep.txt index 41b1c1948..596002f2c 100644 --- a/bindep.txt +++ b/bindep.txt @@ -18,7 +18,7 @@ python3 [platform:rpm] python3-minimal [platform:dpkg] python3-devel [platform:rpm] python3-dev [platform:dpkg] -python3-virtualenv +python3-venv [platform:dpkg] wget sudo ebtables diff --git a/playbooks/ci/run.yaml b/playbooks/ci/run.yaml index 6b13755bf..53439ae34 100644 --- a/playbooks/ci/run.yaml +++ b/playbooks/ci/run.yaml @@ -1,7 +1,7 @@ - hosts: all tasks: - shell: - cmd: scripts/test-bifrost.sh + cmd: bash scripts/test-bifrost.sh chdir: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/bifrost'].src_dir }}" environment: BUILD_IMAGE: "{{ build_image | default(false) | bool | lower }}" @@ -10,5 +10,6 @@ UPPER_CONSTRAINTS_FILE: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/requirements'].src_dir }}/upper-constraints.txt" WORKSPACE: "{{ ansible_user_dir }}/src/opendev.org" USE_DHCP: "{{ use_dhcp | default(false) | bool | lower }}" - USE_VENV: "{{ use_venv | default(false) | bool | lower }}" + USE_VENV: "{{ use_venv | default(true) | bool | lower }}" ZUUL_BRANCH: "{{ zuul.branch }}" + VENV: "/opt/stack/bifrost" diff --git a/playbooks/roles/bifrost-create-vm-nodes/tasks/prepare_libvirt.yml b/playbooks/roles/bifrost-create-vm-nodes/tasks/prepare_libvirt.yml index bfbb54763..447ed829f 100644 --- a/playbooks/roles/bifrost-create-vm-nodes/tasks/prepare_libvirt.yml +++ b/playbooks/roles/bifrost-create-vm-nodes/tasks/prepare_libvirt.yml @@ -15,14 +15,30 @@ # Setup libvirt - ensure network and storage pool are defined and active, # prepare dir for vm logs --- -- name: install libvirt-python and lxml +- name: set virtualenv_command + set_fact: + venv_command: "python3 -m venv" + when: enable_venv + +- name: install libvirt-python and lxml without virtualenv pip: name: "{{ item }}" extra_args: "-c {{ upper_constraints_file }}" - virtualenv: "{{ enable_venv | bool | ternary(bifrost_venv_dir, omit) }}" with_items: - libvirt-python - lxml + when: enable_venv | bool == false + +- name: install libvirt-python and lxml with virtualenv + pip: + name: "{{ item }}" + state: present + virtualenv: "{{ bifrost_venv_dir }}" + virtualenv_command: "{{ venv_command | default(omit) }}" + with_items: + - libvirt-python + - lxml + when: enable_venv | default(false) | bool - name: "Restart libvirt service" service: name="{{libvirt_service_name}}" state=restarted diff --git a/playbooks/roles/bifrost-ironic-install/tasks/pip_install.yml b/playbooks/roles/bifrost-ironic-install/tasks/pip_install.yml index c42a9071b..61d6dec23 100644 --- a/playbooks/roles/bifrost-ironic-install/tasks/pip_install.yml +++ b/playbooks/roles/bifrost-ironic-install/tasks/pip_install.yml @@ -16,8 +16,8 @@ - name: set virtualenv_command set_fact: - venv_command: "{{ hostvars[inventory_hostname].ansible_python.executable }} -m virtualenv" - when: enable_venv|bool + venv_command: "python3 -m venv" + when: enable_venv - name: "Install {{ package }} package from pip using virtualenv" pip: diff --git a/playbooks/roles/bifrost-keystone-install/tasks/pip_install.yml b/playbooks/roles/bifrost-keystone-install/tasks/pip_install.yml index c4e184d82..38b4b1d6c 100644 --- a/playbooks/roles/bifrost-keystone-install/tasks/pip_install.yml +++ b/playbooks/roles/bifrost-keystone-install/tasks/pip_install.yml @@ -14,8 +14,8 @@ - name: set virtualenv_command set_fact: - venv_command: "{{ hostvars[inventory_hostname].ansible_python.executable }} -m virtualenv" - when: enable_venv|bool + venv_command: "python3 -m venv" + when: enable_venv - name: "Install {{ package }} package from pip using virtualenv" pip: diff --git a/scripts/env-setup.sh b/scripts/env-setup.sh index ceaa8cee2..65163b364 100755 --- a/scripts/env-setup.sh +++ b/scripts/env-setup.sh @@ -10,7 +10,7 @@ ANSIBLE_PIP_VERSION=${ANSIBLE_PIP_VERSION:-${ANSIBLE_GIT_BRANCH:-stable-2.6}} ANSIBLE_PIP_STRING=$(${PYTHON} $(dirname $0)/ansible-pip-str.py ${ANSIBLE_PIP_VERSION}) if [ -n "${VENV-}" ]; then - sudo -H -E ${PIP} install --upgrade "${ANSIBLE_PIP_STRING}" + sudo ${PIP} install --ignore-installed "${ANSIBLE_PIP_STRING}" ANSIBLE=${VENV}/bin/ansible else ${PIP} install --user --upgrade "${ANSIBLE_PIP_STRING}" diff --git a/scripts/install-deps.sh b/scripts/install-deps.sh index 15b10a74e..97ea7a9ff 100644 --- a/scripts/install-deps.sh +++ b/scripts/install-deps.sh @@ -15,7 +15,6 @@ CHECK_CMD_PKGS=( net-tools python3-devel python3 - venv wget ) @@ -34,7 +33,6 @@ case ${ID,,} in [net-tools]=net-tools [python]=python [python-devel]=python-devel - [venv]=python-virtualenv [wget]=wget ) EXTRA_PKG_DEPS=( python-xml ) @@ -62,10 +60,9 @@ case ${ID,,} in [net-tools]=net-tools [python3]=python3-minimal [python3-devel]=libpython3-dev - [venv]=python3-virtualenv [wget]=wget ) - EXTRA_PKG_DEPS=() + EXTRA_PKG_DEPS=( python3-apt ) sudo apt-get update ;; @@ -83,7 +80,6 @@ case ${ID,,} in [net-tools]=net-tools [python]=python3 [python-devel]=python3-devel - [venv]=python3-virtualenv [wget]=wget ) EXTRA_PKG_DEPS=() @@ -115,7 +111,7 @@ if ! $(wget --version &>/dev/null); then ${INSTALLER_CMD} ${PKG_MAP[wget]} fi if [ -n "${VENV-}" ]; then - if ! $(python3 -m virtualenv --version &>/dev/null); then + if ! $(python3 -m venv --version &>/dev/null); then ${INSTALLER_CMD} ${PKG_MAP[venv]} fi fi @@ -138,7 +134,7 @@ if [ -n "${VENV-}" ]; then echo "NOTICE: Using virtualenv for this installation." if [ ! -f ${VENV}/bin/activate ]; then # only create venv if one doesn't exist - sudo -H -E python3 -m virtualenv --no-site-packages ${VENV} + sudo -H -E python3 -m venv --system-site-packages ${VENV} fi # Note(cinerama): activate is not compatible with "set -u"; # disable it just for this line. @@ -160,19 +156,21 @@ PYTHON=$(which python3) # older versions of pip are incompatible with # requests, one of our indirect dependencies (bug 1459947). # - -if [ ! $($PYTHON -m pip install -U pip) ]; then +ls $PYTHON +sudo -H -E $PYTHON -m pip install -U pip --ignore-installed +if [ "$?" != "0" ]; then wget -O /tmp/get-pip.py https://bootstrap.pypa.io/3.4/get-pip.py sudo -H -E ${PYTHON} /tmp/get-pip.py fi -PIP=$(which pip3) +ls -la /opt/stack/bifrost/bin + +PIP=$(echo $PYTHON | sed 's/python/pip/') if [ "$OS_FAMILY" == "RedHat" ]; then sudo -H -E ${PIP} freeze sudo -H -E ${PIP} install --ignore-installed pyparsing ipaddress fi - sudo -H -E ${PIP} install -r "$(dirname $0)/../requirements.txt" # Install the rest of required packages using bindep diff --git a/scripts/test-bifrost.sh b/scripts/test-bifrost.sh index 64850d826..f9a449dbf 100755 --- a/scripts/test-bifrost.sh +++ b/scripts/test-bifrost.sh @@ -7,7 +7,7 @@ SCRIPT_HOME="$(cd "$(dirname "$0")" && pwd)" BIFROST_HOME=$SCRIPT_HOME/.. ANSIBLE_INSTALL_ROOT=${ANSIBLE_INSTALL_ROOT:-/opt/stack} USE_DHCP="${USE_DHCP:-false}" -USE_VENV="${USE_VENV:-false}" +USE_VENV="${USE_VENV:-true}" BUILD_IMAGE="${BUILD_IMAGE:-false}" BAREMETAL_DATA_FILE=${BAREMETAL_DATA_FILE:-'/tmp/baremetal.json'} ENABLE_KEYSTONE="${ENABLE_KEYSTONE:-false}" @@ -43,7 +43,7 @@ PROVISION_WAIT_TIMEOUT=${PROVISION_WAIT_TIMEOUT:-900} NOAUTH_MODE=true CLOUD_CONFIG="" WAIT_FOR_DEPLOY=true -ENABLE_VENV=false +ENABLE_VENV=true # This sets up the MySQL database like it's done for all OpenStack # projects for CI testing. @@ -159,7 +159,10 @@ ${ANSIBLE} -vvvv \ -e test_vm_memory_size=${VM_MEMORY_SIZE} \ -e test_vm_domain_type=${VM_DOMAIN_TYPE} \ -e baremetal_json_file=${BAREMETAL_DATA_FILE} \ - -e enable_venv=${ENABLE_VENV} + -e enable_venv=${ENABLE_VENV} \ + -e bifrost_venv_dir=${VENV} + + if [ ${USE_DHCP} = "true" ]; then # reduce the number of nodes in JSON file diff --git a/tools/vagrant_dev_env/vagrant.yml b/tools/vagrant_dev_env/vagrant.yml index a12ec8d30..e926635ba 100644 --- a/tools/vagrant_dev_env/vagrant.yml +++ b/tools/vagrant_dev_env/vagrant.yml @@ -93,13 +93,15 @@ environment: ANSIBLE_INSTALL_ROOT: "{{ansible_install_root}}" LANG: c + VENV: "/opt/stack/bifrost" args: chdir: /home/vagrant/bifrost become: yes - name: Install Bifrost - command: ansible-playbook -vvvv -i inventory/target install.yaml -e network_interface={{network_interface}} -e ansible_python_interpreter="/usr/bin/python3" -e download_ipa=true -e use_cirros=true + command: ansible-playbook -vvvv -i inventory/target install.yaml -e network_interface={{network_interface}} -e ansible_python_interpreter="/usr/bin/python3" -e download_ipa=true -e use_cirros=true -e enable_venv=true environment: PATH: "{{ansible_env.HOME}}/.local/bin:{{ansible_env.PATH}}" + VENV: "/opt/stack/bifrost" args: chdir: /home/vagrant/bifrost/playbooks