Compile Python with missing deps and shared libs

Change-Id: I99c5d518736004beed8686ff330a5c7150e3f7c6

playbooks/tobiko-devstack-pre-run.yaml

@@ -36,14 +36,6 @@
 - hosts: all
   roles:
 
-    - name: ensures has 'python3' and 'pip3' commands
-      role: python
-      vars:
-        python_command: python3
-        python_version: 3.7
-        python_release: 3.7.5
-        pip_command: pip3
-
     - role: copy-build-sshkey
       copy_sshkey_target_user: stack
 

playbooks/tobiko-tox-py38-pre-run.yaml

@@ -4,8 +4,10 @@
   roles:
     - role: ../roles/python
       vars:
+        python_command: python3.8
         python_version: "3.8"
         python_release: "3.8.0"
+        pip_command: pip3.8
         pip_install_packages:
           - virtualenv
           - tox

roles/python/Vagrantfile

@@ -14,10 +14,10 @@ MEMORY = 512
 # boxes at https://vagrantcloud.com/search.
 BOX = "generic/centos7"
 
-HOSTNAME = "tobiko"
-
 TOX_INI_DIR = '../..'
 
+TEST_DIR = File.join(File.dirname(__FILE__), 'tests')
+
 
 # All Vagrant configuration is done below. The "2" in Vagrant.configure
 # configures the configuration version (we support older styles for
@@ -29,7 +29,6 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   # https://docs.vagrantup.com.
 
   config.vm.box = BOX
-  config.vm.hostname = HOSTNAME
 
   # Disable automatic box update checking. If you disable this, then
   # boxes will only be checked for updates when the user runs
@@ -87,8 +86,15 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     ansible.playbook = "resolv_conf.yaml"
   end
 
-  config.vm.provision "ansible" do |ansible|
-    ansible.playbook = "tox-py38.yaml"
+  # Spawn a VM for each playbook in TEST_DIR
+  for playbook_file in Dir[File.join(TEST_DIR, 'test_*.yaml')] do
+    test_name = File.basename(playbook_file, '.yaml')
+    config.vm.define test_name do |node|
+      node.vm.hostname = test_name.gsub('_', '-')
+      node.vm.provision "ansible" do |ansible|
+        ansible.playbook = playbook_file
+      end
+    end
   end
 
 end

roles/python/defaults/main.yaml

@@ -1,29 +1,45 @@
-python_release: "3.8.0"
-python_version: "3.8"
-python_command: "python{{ python_version }}"
+
+setup_python: true
+setup_pip: true
+
+python_release: "3.7.5"
+python_version: "3.7"
+python_command: "python3"
 python_name: "Python-{{ python_release }}"
 python_prefix: "/usr/local"
-python_executable: "{{ python_prefix }}/bin/{{ python_command }}"
+python_executable: "{{ python_prefix }}/bin/python{{ python_version }}"
+python_command_link: "{{ python_executable | dirname }}/{{ python_command }}"
 python_url: "https://www.python.org/ftp/python/{{ python_release }}/{{ python_name }}.tgz"
 python_tar: "{{ ansible_env.HOME }}/{{ python_url | basename }}"
 python_src_dir: "{{ ansible_env.HOME }}/{{ python_name }}"
-python_configure_flags: "" # ""--enable-optimizations"
-python_profile_file: "/etc/profile.d/{{ python_name }}.sh"
-python_sudoers_file: "/etc/sudoers.d/60_{{python_name}}"
+python_configure_flags: '--enable-shared LDFLAGS="-Wl,-rpath /usr/local/lib"'
 
-pip_command: "pip{{ python_version }}"
+pip_command: "pip3"
 pip_executable: "{{ python_prefix }}/bin/{{ pip_command }}"
 pip_url: "https://bootstrap.pypa.io/get-pip.py"
 pip_installer: "{{ ansible_env.HOME }}/{{ pip_url | basename }}"
 
 make_jobs: "{{ ansible_processor_vcpus }}"
+make_install_goal: "install"
+
+bash_profile_file: "/etc/profile.d/{{ python_name }}.sh"
+ldconfig_file: "/etc/ld.so.conf.d/{{ python_name }}.conf"
+
 
 yum_install_packages:
   - "@Development tools"
-  - zlib-devel
-  - openssl-devel
   - bzip2-devel
+  - expat-devel
+  - gdbm-devel
   - libffi-devel
+  - libpcap-devel
+  - ncurses-devel
+  - openssl-devel
+  - readline-devel
+  - sqlite-devel
+  - tk-devel
+  - xz-devel
+  - zlib-devel
 
 pip_install_base_packages:
   - setuptools

roles/python/tasks/check_pip.yaml

@@ -1,16 +0,0 @@
----
-
-- name: check '{{ pip_command }}' command
-  command: "'{{ pip_command }}' --version"
-  changed_when: false
-
-
-- name: discover '{{ pip_command }}' executable path
-  command: "which '{{ pip_command }}'"
-  register: discover_pip_executable
-  changed_when: false
-
-
-- name: register '{{ pip_command }}' executable as '{{ discover_pip_executable.stdout }}'
-  set_fact:
-    pip_executable: '{{ discover_pip_executable.stdout }}'

roles/python/tasks/check_python.yaml

@@ -1,14 +0,0 @@
----
-
-- name: check '{{ python_command }}' command
-  command: "'{{ python_command }}' --version"
-  changed_when: false
-
-- name: discover '{{ python_command }}' executable path
-  command: "which '{{ python_command }}'"
-  register: discover_python_executable
-  changed_when: false
-
-- name: register '{{ python_command }}' executable as '{{ discover_python_executable.stdout }}'
-  set_fact:
-    python_executable: '{{ discover_python_executable.stdout }}'

roles/python/tasks/main.yaml

@@ -1,30 +1,7 @@
 ---
 
-- block:
-    - include: check_python.yaml
+- include: setup_python.yaml
+  when: setup_python | bool
 
-  rescue:
-    - include: setup_python.yaml
-    - include: check_python.yaml
-
-
-- block:
-    - include: check_pip.yaml
-  rescue:
-    - include: setup_pip.yaml
-    - include: check_pip.yaml
-
-
-- become: yes
-  become_user: root
-  block:
-    - include: check_python.yaml
-    - include: check_pip.yaml
-
-  rescue:
-    - include: setup_sudo.yaml
-    - include: check_python.yaml
-    - include: check_pip.yaml
-
-
-- include: update_packages.yaml
+- include: setup_pip.yaml
+  when: setup_pip | bool

roles/python/tasks/setup_pip.yaml

@@ -2,10 +2,19 @@
 
 - block:
 
-    - name: check '{{ pip_executable }}' is installed
-      command: "'{{ pip_executable }}' --version"
+    - name: check '{{ pip_command }}' command
+      command: "'{{ pip_command }}' --version"
       changed_when: false
 
+    - name: discover '{{ pip_command }}' executable path
+      command: "which '{{ pip_command }}'"
+      register: discover_pip_executable
+      changed_when: false
+
+    - name: register '{{ pip_command }}' executable as '{{ discover_pip_executable.stdout }}'
+      set_fact:
+        pip_executable: '{{ discover_pip_executable.stdout }}'
+
   rescue:
 
     - name: download Pip installer from '{{ pip_url }}'
@@ -21,3 +30,31 @@
     - name: check Pip is installed for '{{ pip_executable }}'
       command: "'{{ pip_executable }}' --version"
       changed_when: false
+
+    - name: check '{{ pip_command }}' command
+      command: "'{{ pip_command }}' --version"
+      changed_when: false
+
+
+- name: "ensure required Python packages are installed and up-to-date"
+  become: true
+  become_user: root
+  pip:
+    name: "{{ item }}"
+    executable: '{{ pip_executable }}'
+    state: latest
+  vars:
+    ansible_python_interpreter: '{{ python_executable }}'
+  when: (item | length ) > 0
+  loop:
+    - "{{ pip_install_base_packages }}"
+    - "{{ pip_install_packages }}"
+
+
+- name: "remove '{{ pip_executable | dirname }}/pip'"
+  become: yes
+  become_user: root
+  file:
+    path: '{{ pip_executable | dirname }}/pip'
+    state: absent
+  when: pip_command != "pip"

roles/python/tasks/setup_python.yaml

@@ -1,15 +1,20 @@
 ---
 
 - block:
-
-    - name: check '{{ python_executable }}' is installed
-      shell: |
-        '{{ python_executable }}' --version 2>&1 | \
-            grep 'Python {{ python_version}}'
+    - name: check '{{ python_command }}' command
+      command: "'{{ python_command }}' --version"
       changed_when: false
 
-  rescue:
+    - name: discover '{{ python_command }}' executable path
+      command: "which '{{ python_command }}'"
+      register: discover_python_executable
+      changed_when: false
 
+    - name: register '{{ python_command }}' executable as '{{ discover_python_executable.stdout }}'
+      set_fact:
+        python_executable: '{{ discover_python_executable.stdout }}'
+
+  rescue:
     - name: install '{{ python_name }}' build requeirements
       become: yes
       become_user: root
@@ -42,14 +47,14 @@
 
     - name: compile '{{ python_name }}'
       command:
-        cmd: 'make -j {{ make_jobs }}'
+        cmd: "make -j '{{ make_jobs }}'"
         chdir: '{{ python_src_dir }}'
 
     - name: install '{{ python_name }}'
       become: yes
       become_user: root
       command:
-        cmd: 'make install'
+        cmd: "make '{{ make_install_goal }}'"
         chdir: '{{ python_src_dir }}'
 
     - name: check '{{ python_executable }}' is installed
@@ -58,20 +63,45 @@
             grep 'Python {{ python_version}}'
       changed_when: false
 
+    - name: add '{{ python_prefix }}/lib' to '{{ ldconfig_file }}'
+      become: yes
+      become_user: root
+      lineinfile:
+        path: '{{ ldconfig_file }}'
+        line: '{{ python_prefix }}/lib'
+        create: yes
+        owner: root
+        group: root
+        mode: '0644'
+      register: add_ldconfig_file
 
-- name: add '{{ python_profile_file }}'
-  become: yes
-  become_user: root
-  template:
-    src: profile.sh.j2
-    dest: '{{ python_profile_file }}'
-    owner: root
-    group: root
-    mode: '0644'
+    - name: run ldconfig after changing '{{ ldconfig_file }}'
+      become: yes
+      become_user: root
+      command: ldconfig
+      when: add_ldconfig_file.changed
 
-- name: reset ssh connection to allow environment variable changes
-  meta: reset_connection
+    - name: add '{{ python_executable | dirname }}' to '{{ bash_profile_file }}'
+      become: yes
+      become_user: root
+      lineinfile:
+        path: '{{ bash_profile_file }}'
+        line: 'PATH={{ python_executable | dirname }}:$PATH'
+        create: yes
+        owner: root
+        group: root
+        mode: '0644'
+      when: (python_executable | dirname) not in ansible_env.PATH.split(':')
+      register: add_python_dir
 
-- name: check '{{ python_command }}' command
-  command: "'{{ python_command }}' --version"
-  changed_when: false
+    - name: reset ssh connection after changing '{{ python_profile_file }}'
+      meta: reset_connection
+
+    - name: check '{{ python_command }}' command
+      command: "'{{ python_command }}' --version"
+      changed_when: false
+
+    - name: discover '{{ python_command }}' executable path
+      command: "which '{{ python_command }}'"
+      register: discover_python_executable
+      changed_when: false

roles/python/tasks/setup_sudo.yaml

@@ -1,17 +0,0 @@
----
-
-- name: discover sudo secure_path
-  command: |
-    awk '($1 == "Defaults" && $2 == "secure_path"){print $NF}' /etc/sudoers
-  register: discover_sudo_secure_path
-
-
-- name: add '{{ python_executable | dirname }}' to sudo 'secure_path'
-  lineinfile:
-    regex: '{{ discover_sudo_secure_path.stdout }}$'
-    path: /etc/sudoers
-    line:
-      'Defaults    secure_path = {{ python_executable | dirname }}:{{ discover_sudo_secure_path.stdout }}'
-    validate: '/usr/sbin/visudo -cf %s'
-  when:
-    - (python_executable | dirname) not in discover_sudo_secure_path.stdout

roles/python/tasks/update_packages.yaml

@@ -1,15 +0,0 @@
----
-
-- name: "ensure required Python packages are installed and up-to-date"
-  become: true
-  become_user: root
-  pip:
-    name: "{{ item }}"
-    executable: '{{ pip_executable }}'
-    state: latest
-  vars:
-    ansible_python_interpreter: '{{ python_executable }}'
-  when: (item | length ) > 0
-  loop:
-    - "{{ pip_install_base_packages }}"
-    - "{{ pip_install_packages }}"

roles/python/templates/profile.sh.j2

@@ -1 +0,0 @@
-PATH={{ python_executable | dirname }}:$PATH

roles/python/templates/sudoers.j2

@@ -1 +0,0 @@
-Defaults    secure_path = {{ secure_path }}

roles/python/tests/.gitignore

@@ -0,0 +1 @@
+test_py38/

roles/python/tests/test_py38.yaml

@@ -0,0 +1,83 @@
+---
+
+- hosts: all
+  tasks:
+
+    - name: install default system python and pip packages
+      become: yes
+      yum:
+        state: present
+        name:
+          - python
+          - python-pip
+
+    - name: check default command paths
+      command: "which '{{ item.command }}'"
+      register: which_command
+      changed_when: false
+      failed_when: 'which_command.stdout != item.path'
+      loop:
+        - command: python
+          path: /usr/bin/python
+        - command: pip
+          path: /usr/bin/pip
+
+
+- hosts: all
+  roles:
+    - role: "{{ playbook_dir }}/../../python"
+      vars:
+        python_version: "3.8"
+        python_release: "3.8.0"
+        pip_install_packages:
+          - virtualenv
+          - tox
+
+
+- hosts: all
+  tasks:
+    - name: check command paths
+      command: "which '{{ item.command }}'"
+      register: which_command
+      changed_when: false
+      failed_when: 'which_command.stdout != item.path'
+      loop:
+        - command: python
+          path: /usr/bin/python
+        - command: pip
+          path: /usr/bin/pip
+        - command: python3
+          path: /usr/local/bin/python3
+        - command: python3.8
+          path: /usr/local/bin/python3.8
+        - command: pip3
+          path: /usr/local/bin/pip3
+        - command: pip3.8
+          path: /usr/local/bin/pip3.8
+
+    - name: run Tobiko test cases
+      shell:
+        cmd: tox -e py38 2>&1
+        chdir: /vagrant
+      register: run_test_cases
+      ignore_errors: true
+
+    - name: produce test reports
+      shell:
+        cmd: tox -e report 2>&1
+        chdir: /vagrant
+
+    - name: get test reports
+      fetch:
+        src: "/vagrant/tobiko_results.html"
+        dest: "{{ playbook_dir }}/test_py38/test_results.html"
+        flat: yes
+
+    - name: check test cases result
+      assert:
+        that: run_test_cases.rc == 0
+        fail_msg: |
+          Test cases failed:
+          {{ run_test_cases.stdout }}
+        success_msg: |
+          Test cases passed

roles/python/tox-py38.yaml

@@ -1,49 +0,0 @@
----
-
-- hosts: all
-  roles:
-
-    - role: .
-      vars:
-        python_version: "3.8"
-        python_release: "3.8.0"
-        pip_install_packages:
-          - virtualenv
-          - tox
-
-  tasks:
-
-    - name: run test cases
-      shell:
-        cmd: tox -e py38
-        chdir: /vagrant
-      register: run_test_cases
-      ignore_errors: true
-
-    - name: Install Python devel package required to compile tox reports
-      yum:
-        name: python-devel
-        state: latest
-
-    - name: produce test reports
-      shell:
-        cmd: tox -e report 2>&1
-        chdir: /vagrant
-
-    - name: get test reports
-      fetch:
-        src: "/vagrant/{{ item }}"
-        dest: "{{ playbook_dir }}/tox-py38/{{ inventory_hostname }}/{{ item }}"
-        flat: yes
-      loop:
-        - tobiko_results.html
-
-    - name: check test cases result
-      assert:
-        that: run_test_cases.rc == 0
-        fail_msg: |
-          Test cases failed
-          {{ run_test_cases.stdout }}
-        success_msg: |
-          Test cases passed
-          {{ run_test_cases.stdout }}