Merge "Eliminate installing pip on host/containers"

inventory/group_vars/utility_all.yml

@@ -23,9 +23,11 @@ galera_client_drop_config_file: true
 utility_package_state: "{{ package_state }}"
 utility_pip_package_state: "latest"
 
+# Path to the utility host openstack client venv binaries
+utility_venv_bin: "/openstack/venvs/utility-{{ openstack_release }}/bin"
+
 # Distribution packages to be installed into the utility container
 utility_distro_packages:
-  - curl
   - git
 
 utility_distro_openstack_clients_packages:

playbooks/ceph-rgw-keystone-setup.yml

@@ -14,120 +14,95 @@
 # limitations under the License.
 
 - name: Configure keystone for radosgw
-  hosts: utility_all
+  hosts: "{{ openstack_service_setup_host | default('localhost') }}"
   user: root
   vars_files:
     - "defaults/{{ install_method }}_install.yml"
+  vars:
+    ansible_python_interpreter: >-
+      {{ ((openstack_service_setup_host | default('localhost')) == 'localhost') | ternary(ansible_playbook_python, ansible_python['executable']) }}
+  tags:
+    - ceph-rgw
+    - ceph-rgw-setup
+    - rgw-service-add
   tasks:
-    - name: Ensure RGW service
+    - name: Add service to the keystone service catalog
-      keystone:
+      os_keystone_service:
-        command: "ensure_service"
+        cloud: default
-        endpoint: "{{ keystone_service_adminurl }}"
+        state: present
-        login_user: "{{ keystone_admin_user_name }}"
+        name: "{{ radosgw_service_name }}"
-        login_password: "{{ keystone_auth_admin_password }}"
-        login_project_name: "{{ keystone_admin_tenant_name }}"
-        service_name: "{{ radosgw_service_name }}"
         service_type: "{{ radosgw_service_type }}"
         description: "{{ radosgw_service_description }}"
-        insecure: "{{ keystone_service_adminuri_insecure }}"
+        interface: admin
-      run_once: true
+        verify: "{{ not (keystone_service_adminuri_insecure | bool) }}"
       register: add_service
       until: add_service is success
       retries: 5
-      delay: 2
+      delay: 10
-      no_log: True
       tags:
         - ceph-rgw-setup
         - rgw-service-add
 
-    - name: Ensure RGW user
+    - name: Add service user
-      keystone:
+      os_user:
-        command: "ensure_user"
+        cloud: default
-        endpoint: "{{ keystone_service_adminurl }}"
+        state: present
-        login_user: "{{ keystone_admin_user_name }}"
+        name: "{{ radosgw_admin_user }}"
-        login_password: "{{ keystone_auth_admin_password }}"
-        login_project_name: "{{ keystone_admin_tenant_name }}"
-        user_name: "{{ radosgw_admin_user }}"
-        tenant_name: "{{ radosgw_admin_tenant }}"
-        role_name: "{{ radosgw_role_name | default('service') }}"
         password: "{{ radosgw_admin_password }}"
-        insecure: "{{ keystone_service_adminuri_insecure }}"
+        domain: default
-      run_once: true
+        default_project: "{{ radosgw_admin_tenant }}"
+        interface: admin
+        verify: "{{ not (keystone_service_adminuri_insecure | bool) }}"
       register: add_user
       until: add_user is success
       retries: 5
       delay: 10
       no_log: True
-      tags:
-        - ceph-rgw-setup
-        - rgw-service-add
 
-    - name: Ensure RGW user to admin role
+    - name: Add service user to roles
-      keystone:
+      os_user_role:
-        command: "ensure_user_role"
+        cloud: default
-        endpoint: "{{ keystone_service_adminurl }}"
+        state: present
-        login_user: "{{ keystone_admin_user_name }}"
+        user: "{{ radosgw_admin_user }}"
-        login_password: "{{ keystone_auth_admin_password }}"
+        role: "{{ radosgw_role_name | default('admin') }}"
-        login_project_name: "{{ keystone_admin_tenant_name }}"
+        project: "{{ radosgw_admin_tenant }}"
-        user_name: "{{ radosgw_admin_user }}"
+        interface: admin
-        tenant_name: "{{ radosgw_admin_tenant }}"
+        verify: "{{ not (keystone_service_adminuri_insecure | bool) }}"
-        role_name: "{{ radosgw_role_name | default('admin') }}"
+      register: add_user_role
-        insecure: "{{ keystone_service_adminuri_insecure }}"
+      until: add_user_role is success
-      run_once: true
-      register: add_admin_role
-      until: add_admin_role is success
       retries: 5
       delay: 10
-      no_log: True
-      tags:
-        - ceph-rgw-setup
-        - rgw-service-add
 
-    - name: Ensure swiftoperator role
+    - name: Add service role
-      keystone:
+      os_keystone_role:
-        command: "ensure_role"
+        cloud: default
-        endpoint: "{{ keystone_service_adminurl }}"
+        state: present
-        login_user: "{{ keystone_admin_user_name }}"
+        name: "swiftoperator"
-        login_password: "{{ keystone_auth_admin_password }}"
+        interface: admin
-        login_project_name: "{{ keystone_admin_tenant_name }}"
+        verify: "{{ not (keystone_service_adminuri_insecure | bool) }}"
-        role_name: "swiftoperator"
+      register: add_role
-        insecure: "{{ keystone_service_adminuri_insecure }}"
+      until: add_role is success
-      run_once: true
-      register: add_swiftoperator_role
-      until: add_swiftoperator_role is success
       retries: 5
       delay: 10
-      no_log: True
-      tags:
-        - ceph-rgw-setup
-        - rgw-service-add
 
-    - name: Ensure RGW endpoint
+    - name: Add endpoints to keystone endpoint catalog
-      keystone:
+      os_keystone_endpoint:
-        command: "ensure_endpoint"
+        cloud: default
-        endpoint: "{{ keystone_service_adminurl }}"
+        state: present
-        login_user: "{{ keystone_admin_user_name }}"
+        service: "{{ radosgw_service_name }}"
-        login_password: "{{ keystone_auth_admin_password }}"
+        endpoint_interface: "{{ item.interface }}"
-        login_project_name: "{{ keystone_admin_tenant_name }}"
+        url: "{{ item.url }}"
-        region_name: "{{ radosgw_service_region }}"
+        region: "{{ radosgw_service_region }}"
-        service_name: "{{ radosgw_service_name }}"
+        interface: admin
-        service_type: "{{ radosgw_service_type }}"
+        verify: "{{ not (keystone_service_adminuri_insecure | bool) }}"
-        insecure: "{{ keystone_service_adminuri_insecure }}"
+      register: add_service
-        endpoint_list:
+      until: add_service is success
-          - url: "{{ radosgw_service_publicurl }}"
-            interface: "public"
-          - url: "{{ radosgw_service_adminurl }}"
-            interface: "admin"
-          - url: "{{ radosgw_service_internalurl }}"
-            interface: "internal"
-      run_once: true
-      register: add_endpoint
-      until: add_endpoint is success
       retries: 5
       delay: 10
-      no_log: True
+      with_items:
-      tags:
+        - interface: "public"
-        - ceph-rgw-setup
+          url: "{{ radosgw_service_publicurl }}"
-        - rgw-service-add
+        - interface: "internal"
-  tags:
+          url: "{{ radosgw_service_internalurl }}"
-    - ceph-rgw
+        - interface: "admin"
+          url: "{{ radosgw_service_adminurl }}"

playbooks/repo-use.yml

@@ -13,21 +13,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# TODO(evrardjp): Remove this playbook when the repo_build process
-# is done upfront. This would mean that the openstack_hosts role
-# can run once and for all directly after the
-# lxc-container-create playbook.
 - name: Configure all nodes to use the repo container for python/apt packages
   hosts: "{{ openstack_host_group | default('hosts') }}:all_containers"
+  vars:
+    pip_install: no
+    pip_configure: yes
   vars_files:
-    - defaults/repo_packages/openstack_services.yml
+    - "defaults/repo_packages/openstack_services.yml"
     - "defaults/{{ install_method }}_install.yml"
+  environment: "{{ deployment_environment_variables | default({}) }}"
+  tags:
+    - always
+    - repo-config
   pre_tasks:
     - include: common-tasks/package-cache-proxy.yml
     - include: common-tasks/set-pip-vars.yml
   roles:
     - pip_install
-  environment: "{{ deployment_environment_variables | default({}) }}"
-  tags:
-    - always
-    - repo-config

playbooks/utility-install.yml

@@ -16,6 +16,11 @@
 - name: Setup the utility location(s)
   hosts: utility_all
   user: root
+  environment: "{{ deployment_environment_variables | default({}) }}"
+  vars_files:
+    - "defaults/{{ install_method }}_install.yml"
+  tags:
+    - utility
   pre_tasks:
     - include: "common-tasks/os-{{ container_tech | default('lxc') }}-container-setup.yml"
       static: no
@@ -58,31 +63,6 @@
         update_cache: "{{ (ansible_pkg_mgr in ['apt', 'zypper']) | ternary('yes', omit) }}"
         cache_valid_time: "{{ (ansible_pkg_mgr == 'apt') | ternary(cache_timeout, omit) }}"
 
-    - name: Get list of python clients
-      shell: "curl -s {{ repo_release_path }}/requirements_absolute_requirements.txt  | grep client | cut -d'=' -f1"
-      args:
-        warn: no
-      register: client_list
-      run_once: true
-      changed_when: false
-      when: install_method == "source"
-      tags:
-        - always
-        - skip_ansible_lint
-
-    - name: Install pip packages
-      pip:
-        name: "{{ client_list.stdout_lines | union(utility_pip_packages) }}"
-        state: "{{ utility_pip_package_state }}"
-        extra_args: >-
-          {{ (pip_install_upper_constraints is defined) | ternary('--constraint ' + pip_install_upper_constraints | default(''),'') }}
-          {{ pip_install_options | default('') }}
-      register: install_packages
-      until: install_packages is success
-      retries: 5
-      delay: 2
-      when: install_method == "source"
-
     - name: Distribute private ssh key
       copy:
         content: "{{ utility_ssh_private_key }}"
@@ -91,8 +71,60 @@
         owner: root
         group: root
       when: utility_ssh_private_key is defined
-  environment: "{{ deployment_environment_variables | default({}) }}"
+
-  vars_files:
+    - name: Install openstack clients (source-based install)
-    - "defaults/{{ install_method }}_install.yml"
+      when:
-  tags:
+        - install_method == "source"
-    - utility
+      block:
+        - name: Get list of repo packages
+          uri:
+            url: "{{ repo_release_path }}/requirements_absolute_requirements.txt"
+            return_content: yes
+          register: _abs_reqs
+          run_once: true
+          tags:
+            - always
+
+        - name: Derive the list of openstack clients
+          set_fact:
+            _openstack_client_list: >-
+              {%- set package_list = [] %}
+              {%- for l in _abs_reqs.content.split('\n') %}
+              {%-   if (l is match('^python_.*client==.*$')) or (l is match('^(aodh|gnocchi)client==.*$')) %}
+              {%-     set _ = package_list.append(l | regex_replace('==.*$', '')) %}
+              {%-   endif %}
+              {%- endfor %}
+              {{- package_list }}
+          run_once: true
+          tags:
+            - always
+
+        - name: Create the virtualenv (if it does not exist)
+          command: "virtualenv --never-download --no-site-packages {{ utility_venv_bin | dirname }}"
+          args:
+            creates: "{{ utility_venv_bin }}/activate"
+
+        - name: Install pip packages
+          pip:
+            name: "{{ _openstack_client_list | union(utility_pip_packages) }}"
+            state: "{{ utility_pip_package_state }}"
+            virtualenv: "{{ utility_venv_bin | dirname }}"
+            virtualenv_site_packages: "no"
+            extra_args: >-
+              {{ (pip_install_upper_constraints is defined) | ternary('--constraint ' + pip_install_upper_constraints | default(''),'') }}
+              {{ pip_install_options | default('') }}
+          register: install_packages
+          until: install_packages is success
+          retries: 5
+          delay: 2
+
+        - name: Create symlinks for openstack clients
+          shell: |
+            {% set _bin_name = item | regex_replace('^(?:python_)?(\w*)(?:client)$', '\\1') %}
+            if [[ -e "{{ utility_venv_bin }}/{{ _bin_name }}" ]]; then
+              ln -sfn {{ utility_venv_bin }}/{{ _bin_name }} /usr/local/bin/{{ _bin_name }}
+            fi
+          args:
+            executable: /bin/bash
+          with_items: "{{ _openstack_client_list }}"
+

scripts/bootstrap-ansible.sh

@@ -83,19 +83,16 @@ case ${DISTRO_ID} in
           python3 python3-dev \
           libssl-dev libffi-dev \
           python-apt python3-apt \
-          python-pip \
           python-virtualenv
         ;;
     opensuse)
         zypper -n install -l git-core curl autoconf gcc gcc-c++ \
             netcat-openbsd python python-xml python-devel gcc \
-            libffi-devel libopenssl-devel python-pip \
+            libffi-devel libopenssl-devel python-virtualenv
-            python-virtualenv
         # Leap ships with python3.4 which is not supported by ansible and as
         # such we are using python2
         # See https://github.com/ansible/ansible/issues/24180
         PYTHON_EXEC_PATH="/usr/bin/python2"
-        alternatives --set pip /usr/bin/pip2.7 || true
         ;;
 esac
 

tests/bootstrap-aio.yml

@@ -19,7 +19,6 @@
   user: root
   roles:
     - role: "sshd"
-    - role: "pip_install"
     - role: "bootstrap-host"
   vars:
     ansible_python_interpreter: "/usr/bin/python"