diff --git a/playbooks/tobiko-devstack-pre-run.yaml b/playbooks/tobiko-devstack-pre-run.yaml index ac9670017..715968eb1 100644 --- a/playbooks/tobiko-devstack-pre-run.yaml +++ b/playbooks/tobiko-devstack-pre-run.yaml @@ -36,14 +36,6 @@ - hosts: all roles: - - name: ensures has 'python3' and 'pip3' commands - role: python - vars: - python_command: python3 - python_version: 3.7 - python_release: 3.7.5 - pip_command: pip3 - - role: copy-build-sshkey copy_sshkey_target_user: stack diff --git a/playbooks/tobiko-tox-py38-pre-run.yaml b/playbooks/tobiko-tox-py38-pre-run.yaml index a84f8c306..c3989daad 100644 --- a/playbooks/tobiko-tox-py38-pre-run.yaml +++ b/playbooks/tobiko-tox-py38-pre-run.yaml @@ -4,8 +4,10 @@ roles: - role: ../roles/python vars: + python_command: python3.8 python_version: "3.8" python_release: "3.8.0" + pip_command: pip3.8 pip_install_packages: - virtualenv - tox diff --git a/roles/python/Vagrantfile b/roles/python/Vagrantfile index c5061670c..8d4cc3274 100644 --- a/roles/python/Vagrantfile +++ b/roles/python/Vagrantfile @@ -14,10 +14,10 @@ MEMORY = 512 # boxes at https://vagrantcloud.com/search. BOX = "generic/centos7" -HOSTNAME = "tobiko" - TOX_INI_DIR = '../..' +TEST_DIR = File.join(File.dirname(__FILE__), 'tests') + # All Vagrant configuration is done below. The "2" in Vagrant.configure # configures the configuration version (we support older styles for @@ -29,7 +29,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| # https://docs.vagrantup.com. config.vm.box = BOX - config.vm.hostname = HOSTNAME # Disable automatic box update checking. If you disable this, then # boxes will only be checked for updates when the user runs @@ -87,8 +86,15 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| ansible.playbook = "resolv_conf.yaml" end - config.vm.provision "ansible" do |ansible| - ansible.playbook = "tox-py38.yaml" + # Spawn a VM for each playbook in TEST_DIR + for playbook_file in Dir[File.join(TEST_DIR, 'test_*.yaml')] do + test_name = File.basename(playbook_file, '.yaml') + config.vm.define test_name do |node| + node.vm.hostname = test_name.gsub('_', '-') + node.vm.provision "ansible" do |ansible| + ansible.playbook = playbook_file + end + end end end diff --git a/roles/python/defaults/main.yaml b/roles/python/defaults/main.yaml index 628b046a8..60d0fa8ff 100644 --- a/roles/python/defaults/main.yaml +++ b/roles/python/defaults/main.yaml @@ -1,29 +1,45 @@ -python_release: "3.8.0" -python_version: "3.8" -python_command: "python{{ python_version }}" + +setup_python: true +setup_pip: true + +python_release: "3.7.5" +python_version: "3.7" +python_command: "python3" python_name: "Python-{{ python_release }}" python_prefix: "/usr/local" -python_executable: "{{ python_prefix }}/bin/{{ python_command }}" +python_executable: "{{ python_prefix }}/bin/python{{ python_version }}" +python_command_link: "{{ python_executable | dirname }}/{{ python_command }}" python_url: "https://www.python.org/ftp/python/{{ python_release }}/{{ python_name }}.tgz" python_tar: "{{ ansible_env.HOME }}/{{ python_url | basename }}" python_src_dir: "{{ ansible_env.HOME }}/{{ python_name }}" -python_configure_flags: "" # ""--enable-optimizations" -python_profile_file: "/etc/profile.d/{{ python_name }}.sh" -python_sudoers_file: "/etc/sudoers.d/60_{{python_name}}" +python_configure_flags: '--enable-shared LDFLAGS="-Wl,-rpath /usr/local/lib"' -pip_command: "pip{{ python_version }}" +pip_command: "pip3" pip_executable: "{{ python_prefix }}/bin/{{ pip_command }}" pip_url: "https://bootstrap.pypa.io/get-pip.py" pip_installer: "{{ ansible_env.HOME }}/{{ pip_url | basename }}" make_jobs: "{{ ansible_processor_vcpus }}" +make_install_goal: "install" + +bash_profile_file: "/etc/profile.d/{{ python_name }}.sh" +ldconfig_file: "/etc/ld.so.conf.d/{{ python_name }}.conf" + yum_install_packages: - "@Development tools" - - zlib-devel - - openssl-devel - bzip2-devel + - expat-devel + - gdbm-devel - libffi-devel + - libpcap-devel + - ncurses-devel + - openssl-devel + - readline-devel + - sqlite-devel + - tk-devel + - xz-devel + - zlib-devel pip_install_base_packages: - setuptools diff --git a/roles/python/tasks/check_pip.yaml b/roles/python/tasks/check_pip.yaml deleted file mode 100644 index 121702f74..000000000 --- a/roles/python/tasks/check_pip.yaml +++ /dev/null @@ -1,16 +0,0 @@ ---- - -- name: check '{{ pip_command }}' command - command: "'{{ pip_command }}' --version" - changed_when: false - - -- name: discover '{{ pip_command }}' executable path - command: "which '{{ pip_command }}'" - register: discover_pip_executable - changed_when: false - - -- name: register '{{ pip_command }}' executable as '{{ discover_pip_executable.stdout }}' - set_fact: - pip_executable: '{{ discover_pip_executable.stdout }}' diff --git a/roles/python/tasks/check_python.yaml b/roles/python/tasks/check_python.yaml deleted file mode 100644 index ca70f158f..000000000 --- a/roles/python/tasks/check_python.yaml +++ /dev/null @@ -1,14 +0,0 @@ ---- - -- name: check '{{ python_command }}' command - command: "'{{ python_command }}' --version" - changed_when: false - -- name: discover '{{ python_command }}' executable path - command: "which '{{ python_command }}'" - register: discover_python_executable - changed_when: false - -- name: register '{{ python_command }}' executable as '{{ discover_python_executable.stdout }}' - set_fact: - python_executable: '{{ discover_python_executable.stdout }}' diff --git a/roles/python/tasks/main.yaml b/roles/python/tasks/main.yaml index eeba6c1d9..d85cae08d 100644 --- a/roles/python/tasks/main.yaml +++ b/roles/python/tasks/main.yaml @@ -1,30 +1,7 @@ --- -- block: - - include: check_python.yaml +- include: setup_python.yaml + when: setup_python | bool - rescue: - - include: setup_python.yaml - - include: check_python.yaml - - -- block: - - include: check_pip.yaml - rescue: - - include: setup_pip.yaml - - include: check_pip.yaml - - -- become: yes - become_user: root - block: - - include: check_python.yaml - - include: check_pip.yaml - - rescue: - - include: setup_sudo.yaml - - include: check_python.yaml - - include: check_pip.yaml - - -- include: update_packages.yaml +- include: setup_pip.yaml + when: setup_pip | bool diff --git a/roles/python/tasks/setup_pip.yaml b/roles/python/tasks/setup_pip.yaml index 513d852c2..502e71db8 100644 --- a/roles/python/tasks/setup_pip.yaml +++ b/roles/python/tasks/setup_pip.yaml @@ -2,10 +2,19 @@ - block: - - name: check '{{ pip_executable }}' is installed - command: "'{{ pip_executable }}' --version" + - name: check '{{ pip_command }}' command + command: "'{{ pip_command }}' --version" changed_when: false + - name: discover '{{ pip_command }}' executable path + command: "which '{{ pip_command }}'" + register: discover_pip_executable + changed_when: false + + - name: register '{{ pip_command }}' executable as '{{ discover_pip_executable.stdout }}' + set_fact: + pip_executable: '{{ discover_pip_executable.stdout }}' + rescue: - name: download Pip installer from '{{ pip_url }}' @@ -21,3 +30,31 @@ - name: check Pip is installed for '{{ pip_executable }}' command: "'{{ pip_executable }}' --version" changed_when: false + + - name: check '{{ pip_command }}' command + command: "'{{ pip_command }}' --version" + changed_when: false + + +- name: "ensure required Python packages are installed and up-to-date" + become: true + become_user: root + pip: + name: "{{ item }}" + executable: '{{ pip_executable }}' + state: latest + vars: + ansible_python_interpreter: '{{ python_executable }}' + when: (item | length ) > 0 + loop: + - "{{ pip_install_base_packages }}" + - "{{ pip_install_packages }}" + + +- name: "remove '{{ pip_executable | dirname }}/pip'" + become: yes + become_user: root + file: + path: '{{ pip_executable | dirname }}/pip' + state: absent + when: pip_command != "pip" diff --git a/roles/python/tasks/setup_python.yaml b/roles/python/tasks/setup_python.yaml index 547a984d2..17ed0689c 100644 --- a/roles/python/tasks/setup_python.yaml +++ b/roles/python/tasks/setup_python.yaml @@ -1,15 +1,20 @@ --- - block: - - - name: check '{{ python_executable }}' is installed - shell: | - '{{ python_executable }}' --version 2>&1 | \ - grep 'Python {{ python_version}}' + - name: check '{{ python_command }}' command + command: "'{{ python_command }}' --version" changed_when: false - rescue: + - name: discover '{{ python_command }}' executable path + command: "which '{{ python_command }}'" + register: discover_python_executable + changed_when: false + - name: register '{{ python_command }}' executable as '{{ discover_python_executable.stdout }}' + set_fact: + python_executable: '{{ discover_python_executable.stdout }}' + + rescue: - name: install '{{ python_name }}' build requeirements become: yes become_user: root @@ -42,14 +47,14 @@ - name: compile '{{ python_name }}' command: - cmd: 'make -j {{ make_jobs }}' + cmd: "make -j '{{ make_jobs }}'" chdir: '{{ python_src_dir }}' - name: install '{{ python_name }}' become: yes become_user: root command: - cmd: 'make install' + cmd: "make '{{ make_install_goal }}'" chdir: '{{ python_src_dir }}' - name: check '{{ python_executable }}' is installed @@ -58,20 +63,45 @@ grep 'Python {{ python_version}}' changed_when: false + - name: add '{{ python_prefix }}/lib' to '{{ ldconfig_file }}' + become: yes + become_user: root + lineinfile: + path: '{{ ldconfig_file }}' + line: '{{ python_prefix }}/lib' + create: yes + owner: root + group: root + mode: '0644' + register: add_ldconfig_file -- name: add '{{ python_profile_file }}' - become: yes - become_user: root - template: - src: profile.sh.j2 - dest: '{{ python_profile_file }}' - owner: root - group: root - mode: '0644' + - name: run ldconfig after changing '{{ ldconfig_file }}' + become: yes + become_user: root + command: ldconfig + when: add_ldconfig_file.changed -- name: reset ssh connection to allow environment variable changes - meta: reset_connection + - name: add '{{ python_executable | dirname }}' to '{{ bash_profile_file }}' + become: yes + become_user: root + lineinfile: + path: '{{ bash_profile_file }}' + line: 'PATH={{ python_executable | dirname }}:$PATH' + create: yes + owner: root + group: root + mode: '0644' + when: (python_executable | dirname) not in ansible_env.PATH.split(':') + register: add_python_dir -- name: check '{{ python_command }}' command - command: "'{{ python_command }}' --version" - changed_when: false + - name: reset ssh connection after changing '{{ python_profile_file }}' + meta: reset_connection + + - name: check '{{ python_command }}' command + command: "'{{ python_command }}' --version" + changed_when: false + + - name: discover '{{ python_command }}' executable path + command: "which '{{ python_command }}'" + register: discover_python_executable + changed_when: false diff --git a/roles/python/tasks/setup_sudo.yaml b/roles/python/tasks/setup_sudo.yaml deleted file mode 100644 index 3dda1af92..000000000 --- a/roles/python/tasks/setup_sudo.yaml +++ /dev/null @@ -1,17 +0,0 @@ ---- - -- name: discover sudo secure_path - command: | - awk '($1 == "Defaults" && $2 == "secure_path"){print $NF}' /etc/sudoers - register: discover_sudo_secure_path - - -- name: add '{{ python_executable | dirname }}' to sudo 'secure_path' - lineinfile: - regex: '{{ discover_sudo_secure_path.stdout }}$' - path: /etc/sudoers - line: - 'Defaults secure_path = {{ python_executable | dirname }}:{{ discover_sudo_secure_path.stdout }}' - validate: '/usr/sbin/visudo -cf %s' - when: - - (python_executable | dirname) not in discover_sudo_secure_path.stdout diff --git a/roles/python/tasks/update_packages.yaml b/roles/python/tasks/update_packages.yaml deleted file mode 100644 index dfaa6cd57..000000000 --- a/roles/python/tasks/update_packages.yaml +++ /dev/null @@ -1,15 +0,0 @@ ---- - -- name: "ensure required Python packages are installed and up-to-date" - become: true - become_user: root - pip: - name: "{{ item }}" - executable: '{{ pip_executable }}' - state: latest - vars: - ansible_python_interpreter: '{{ python_executable }}' - when: (item | length ) > 0 - loop: - - "{{ pip_install_base_packages }}" - - "{{ pip_install_packages }}" diff --git a/roles/python/templates/profile.sh.j2 b/roles/python/templates/profile.sh.j2 deleted file mode 100644 index 519137559..000000000 --- a/roles/python/templates/profile.sh.j2 +++ /dev/null @@ -1 +0,0 @@ -PATH={{ python_executable | dirname }}:$PATH diff --git a/roles/python/templates/sudoers.j2 b/roles/python/templates/sudoers.j2 deleted file mode 100644 index f8795e243..000000000 --- a/roles/python/templates/sudoers.j2 +++ /dev/null @@ -1 +0,0 @@ -Defaults secure_path = {{ secure_path }} diff --git a/roles/python/tests/.gitignore b/roles/python/tests/.gitignore new file mode 100644 index 000000000..8731cb5c6 --- /dev/null +++ b/roles/python/tests/.gitignore @@ -0,0 +1 @@ +test_py38/ diff --git a/roles/python/tests/test_py38.yaml b/roles/python/tests/test_py38.yaml new file mode 100644 index 000000000..ee13bced9 --- /dev/null +++ b/roles/python/tests/test_py38.yaml @@ -0,0 +1,83 @@ +--- + +- hosts: all + tasks: + + - name: install default system python and pip packages + become: yes + yum: + state: present + name: + - python + - python-pip + + - name: check default command paths + command: "which '{{ item.command }}'" + register: which_command + changed_when: false + failed_when: 'which_command.stdout != item.path' + loop: + - command: python + path: /usr/bin/python + - command: pip + path: /usr/bin/pip + + +- hosts: all + roles: + - role: "{{ playbook_dir }}/../../python" + vars: + python_version: "3.8" + python_release: "3.8.0" + pip_install_packages: + - virtualenv + - tox + + +- hosts: all + tasks: + - name: check command paths + command: "which '{{ item.command }}'" + register: which_command + changed_when: false + failed_when: 'which_command.stdout != item.path' + loop: + - command: python + path: /usr/bin/python + - command: pip + path: /usr/bin/pip + - command: python3 + path: /usr/local/bin/python3 + - command: python3.8 + path: /usr/local/bin/python3.8 + - command: pip3 + path: /usr/local/bin/pip3 + - command: pip3.8 + path: /usr/local/bin/pip3.8 + + - name: run Tobiko test cases + shell: + cmd: tox -e py38 2>&1 + chdir: /vagrant + register: run_test_cases + ignore_errors: true + + - name: produce test reports + shell: + cmd: tox -e report 2>&1 + chdir: /vagrant + + - name: get test reports + fetch: + src: "/vagrant/tobiko_results.html" + dest: "{{ playbook_dir }}/test_py38/test_results.html" + flat: yes + + - name: check test cases result + assert: + that: run_test_cases.rc == 0 + fail_msg: | + Test cases failed: + {{ run_test_cases.stdout }} + success_msg: | + Test cases passed diff --git a/roles/python/tox-py38.yaml b/roles/python/tox-py38.yaml deleted file mode 100644 index 59d1daeb6..000000000 --- a/roles/python/tox-py38.yaml +++ /dev/null @@ -1,49 +0,0 @@ ---- - -- hosts: all - roles: - - - role: . - vars: - python_version: "3.8" - python_release: "3.8.0" - pip_install_packages: - - virtualenv - - tox - - tasks: - - - name: run test cases - shell: - cmd: tox -e py38 - chdir: /vagrant - register: run_test_cases - ignore_errors: true - - - name: Install Python devel package required to compile tox reports - yum: - name: python-devel - state: latest - - - name: produce test reports - shell: - cmd: tox -e report 2>&1 - chdir: /vagrant - - - name: get test reports - fetch: - src: "/vagrant/{{ item }}" - dest: "{{ playbook_dir }}/tox-py38/{{ inventory_hostname }}/{{ item }}" - flat: yes - loop: - - tobiko_results.html - - - name: check test cases result - assert: - that: run_test_cases.rc == 0 - fail_msg: | - Test cases failed - {{ run_test_cases.stdout }} - success_msg: | - Test cases passed - {{ run_test_cases.stdout }}