Remove tripleo_ceph_fetch_dir role

The state information in the ceph-ansible fetch directory
is no longer necessary to replace a Ceph monitor as of
ceph-ansible 4.0.14-1. Stein, Train, Ussuri and newer
should be using this version of ceph-ansible or newer.
TripleO does not need to do extra work to maintain the
fetch directory. Remove tripleo_ceph_fetch_dir role and
modify code flow so that it's no longer used.

Also, remove unnecessary dependencies in tripleo-ceph-*
roles as described in LP #1844837.

Change-Id: Ic292dcc1bcf1e7575285ad56d56df2966e968b6d
Depends-On: I3f8c2571a6dd35029d71dd42df637a962d9b3235
Related-Bug: #1863809
Closes-Bug: #1844837

tripleo_ansible/playbooks/ceph.yaml

@@ -15,10 +15,28 @@
 # under the License.
 
 - hosts: Undercloud
-  roles:
-    - role: tripleo_ceph_run_ansible
-  post_tasks:
-    - name: backup and clean fetch directory after ceph-ansible run
+  tasks:
+    # tripleo heat templates call these on step1
+    - name: include common ceph defaults
       include_role:
-        name: tripleo_ceph_fetch_dir
-        tasks_from: backup_and_clean
+        name: tripleo_ceph_common
+    - name: create ceph-ansible working directory
+      include_role:
+        name: tripleo_ceph_work_dir
+        tasks_from: prepare
+    - name: prepare for ceph-ansible uuid gathering
+      include_role:
+        name: tripleo_ceph_uuid
+        tasks_from: prepare
+    # tripleo heat templates call these on step2
+    - name: get ssh private key
+      include_role:
+        name: tripleo_ceph_work_dir
+        tasks_from: get_ssh_private_key
+    - name: run nodes-uuid
+      include_role:
+        name: tripleo_ceph_uuid
+        tasks_from: gather
+    - name: run ceph-ansible
+      include_role:
+        name: tripleo_ceph_run_ansible

tripleo_ansible/roles/tripleo-ceph-fetch-dir

@@ -1 +0,0 @@
-tripleo_ceph_fetch_dir

tripleo_ansible/roles/tripleo_ceph_common/defaults/main.yml

@@ -128,9 +128,6 @@ ceph_ansible_extra_vars:
 uuid_content: {}
 ceph_ansible_playbook_verbosity: 3
 ceph_ansible_playbooks_param: ['default']
-local_ceph_ansible_fetch_directory_backup: "{{ playbook_dir }}/ceph-ansible/fetch_backup"
-swift_get_url: ''
-swift_put_url: ''
 ceph_ansible_skip_tags: 'package-install,with_pkg'
 ceph_ansible_environment_variables: []
 ceph_ansible_inherits_calling_ansible_environment: false

tripleo_ansible/roles/tripleo_ceph_common/tasks/main.yml

@@ -26,6 +26,12 @@
   when:
     - ansible_user_dir is undefined
 
+- name: set calling_ansible_environment_variables fact
+  set_fact:
+    calling_ansible_environment_variables: []
+  when:
+    - (not ceph_ansible_inherits_calling_ansible_environment | default(true)) | bool
+
 - name: Local environment inherit
   when:
     - (ceph_ansible_inherits_calling_ansible_environment | default(false)) | bool

tripleo_ansible/roles/tripleo_ceph_common/vars/main.yml

@@ -1,17 +0,0 @@
----
-# Copyright 2019 Red Hat, Inc.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-calling_ansible_environment_variables: []

tripleo_ansible/roles/tripleo_ceph_fetch_dir/meta/main.yml

@@ -1,47 +0,0 @@
----
-# Copyright 2019 Red Hat, Inc.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-
-galaxy_info:
-  author: OpenStack
-  description: TripleO OpenStack Role -- tripleo_ceph_fetch_dir
-  company: Red Hat
-  license: Apache-2.0
-  min_ansible_version: 2.7
-  #
-  # Provide a list of supported platforms, and for each platform a list of versions.
-  # If you don't wish to enumerate all versions for a particular platform, use 'all'.
-  # To view available platforms and versions (or releases), visit:
-  # https://galaxy.ansible.com/api/v1/platforms/
-  #
-  platforms:
-    - name: Fedora
-      versions:
-        - 28
-    - name: CentOS
-      versions:
-        - 7
-
-  galaxy_tags:
-    - tripleo
-
-dependencies:
-  - role: tripleo_ceph_common
-    when:
-      - ceph_common_done is undefined
-  - role: tripleo_ceph_work_dir
-    when:
-      - ceph_ansible_private_key_file is undefined

tripleo_ansible/roles/tripleo_ceph_fetch_dir/molecule/default/Dockerfile

@@ -1,37 +0,0 @@
-# Molecule managed
-# Copyright 2019 Red Hat, Inc.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-
-{% if item.registry is defined %}
-FROM {{ item.registry.url }}/{{ item.image }}
-{% else %}
-FROM {{ item.image }}
-{% endif %}
-
-RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \
-    elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install sudo python*-devel python*-dnf bash {{ item.pkg_extras | default('') }} && dnf clean all; \
-    elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl python-setuptools bash {{ item.pkg_extras | default('') }} && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
-    elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml {{ item.pkg_extras | default('') }} && zypper clean -a; \
-    elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates {{ item.pkg_extras | default('') }}; \
-    elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates {{ item.pkg_extras | default('') }} && xbps-remove -O; fi
-
-{% for pkg in item.easy_install | default([]) %}
-# install pip for centos where there is no python-pip rpm in default repos
-RUN easy_install {{ pkg }}
-{% endfor %}
-
-
-CMD ["sh", "-c", "while true; do sleep 10000; done"]

tripleo_ansible/roles/tripleo_ceph_fetch_dir/molecule/default/molecule.yml

@@ -1,52 +0,0 @@
----
-driver:
-  name: docker
-
-log: true
-
-platforms:
-  - name: centos7
-    hostname: centos7
-    image: centos:7
-    dockerfile: Dockerfile
-    pkg_extras: python-setuptools
-    volumes:
-      - /etc/ci/mirror_info.sh:/etc/ci/mirror_info.sh:ro
-    easy_install:
-      - pip
-    environment: &env
-      http_proxy: "{{ lookup('env', 'http_proxy') }}"
-      https_proxy: "{{ lookup('env', 'https_proxy') }}"
-
-  - name: centos8
-    hostname: centos8
-    image: centos:8
-    dockerfile: Dockerfile
-    pkg_extras: python*-setuptools
-    volumes:
-      - /etc/ci/mirror_info.sh:/etc/ci/mirror_info.sh:ro
-    environment:
-      <<: *env
-
-provisioner:
-  name: ansible
-  inventory:
-    hosts:
-      all:
-        vars:
-          ansible_user: root
-  log: true
-  env:
-    ANSIBLE_STDOUT_CALLBACK: yaml
-
-scenario:
-  test_sequence:
-    - destroy
-    - create
-    - prepare
-    - converge
-    - verify
-    - destroy
-
-verifier:
-  name: testinfra

tripleo_ansible/roles/tripleo_ceph_fetch_dir/molecule/default/playbook.yml

@@ -1,21 +0,0 @@
----
-# Copyright 2019 Red Hat, Inc.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-
-- name: Converge
-  hosts: all
-  roles:
-    - role: "tripleo_ceph_fetch_dir"

tripleo_ansible/roles/tripleo_ceph_fetch_dir/molecule/default/prepare.yml

@@ -1,21 +0,0 @@
----
-# Copyright 2019 Red Hat, Inc.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-
-- name: Prepare
-  hosts: all
-  roles:
-    - role: test_deps

tripleo_ansible/roles/tripleo_ceph_fetch_dir/molecule/default/verify.yml

@@ -1,15 +0,0 @@
----
-# Copyright 2019 Red Hat, Inc.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.

tripleo_ansible/roles/tripleo_ceph_fetch_dir/tasks/backup_and_clean.yml

@@ -1,68 +0,0 @@
----
-# Copyright 2019 Red Hat, Inc.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-- name: register contents of fetch_directory after ceph-ansible run
-  find:
-    paths: "{{ playbook_dir }}/ceph-ansible/fetch_dir/"
-    recurse: true
-  register: ceph_ansible_fetch_dir_contents
-
-# backup to local directory
-- name: create ceph-ansible fetch directory tarball in local backup
-  archive:
-    path: "{{ playbook_dir }}/ceph-ansible/fetch_dir/*"
-    dest: "{{ local_ceph_ansible_fetch_directory_backup }}/{{ ceph_ansible_tarball_name }}"
-    exclude_path:
-      - '{{ playbook_dir }}/ceph-ansible/fetch_dir/fetch_dir'
-  when:
-    - local_ceph_ansible_fetch_directory_backup | length > 0
-    - ceph_ansible_fetch_dir_contents.matched|int != 0
-
-# backup to swift
-- when:
-    - local_ceph_ansible_fetch_directory_backup | length == 0
-    - ceph_ansible_fetch_dir_contents.matched|int != 0
-  block:
-    - name: create temporary ceph-ansible fetch directory tarball for swift backup
-      archive:
-        path: "{{ playbook_dir }}/ceph-ansible/fetch_dir/*"
-        dest: "/tmp/{{ new_ceph_ansible_tarball_name }}"
-        exclude_path:
-          - '{{ playbook_dir }}/ceph-ansible/fetch_dir/fetch_dir'
-    - name: backup temporary ceph-ansible fetch directory tarball in swift
-      shell: "curl  -s -o /dev/null -w '%{http_code}' -X PUT -T /tmp/{{ new_ceph_ansible_tarball_name }} \"{{  swift_put_url  }}\""
-      register: curl_put_http_status
-    - fail:
-        msg: 'Received HTTP: {{ curl_put_http_status.stdout }} when attempting to PUT to {{ swift_put_url }}'
-      name: ensure we were able to backup temporary fetch directory to swift
-      when:
-        - curl_put_http_status is changed
-        - curl_put_http_status.stdout != "200"
-        - curl_put_http_status.stdout != "201"
-    - name: clean temporary fetch directory after swift backup
-      file:
-        path: "/tmp/{{ new_ceph_ansible_tarball_name }}"
-        state: absent
-      when: (curl_put_http_status is changed and
-             (curl_put_http_status.stdout == "200" or
-              curl_put_http_status.stdout == "201"))
-
-# remove copy of fetchdir in case another user unpacks it (LP #1823229)
-- name: Remove ceph-ansible fetch directory
-  become: true
-  file:
-    path: "{{ playbook_dir }}/ceph-ansible/fetch_dir/"
-    state: absent

tripleo_ansible/roles/tripleo_ceph_fetch_dir/tasks/create.yml

@@ -1,93 +0,0 @@
----
-# Copyright 2019 Red Hat, Inc.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-# local backup
-- when: local_ceph_ansible_fetch_directory_backup | length > 0
-  block:
-    - name: look for requested ceph-ansible fetch directory for local backup
-      stat: path="{{ local_ceph_ansible_fetch_directory_backup }}"
-      register: local_backup_directory
-      ignore_errors: true
-    - name: fail if ansible does not have necessary permissions on local_backup_directory
-      fail:
-        msg: >-
-          The user running Ansible needs to be able to read and write to
-          the '{{ local_ceph_ansible_fetch_directory_backup }}' directory.
-          Please set the LocalCephAnsibleFetchDirectoryBackup Heat parameter or
-          the local_ceph_ansible_fetch_directory_backup Ansible parameter to a
-          directory the user '{{ lookup('env','USER') }}' can read and write to.
-      when:
-        - (local_backup_directory.msg is defined and
-           local_backup_directory.msg == "Permission denied")
-          or
-          (local_backup_directory.stat is defined and
-           local_backup_directory.stat.exists and
-           not local_backup_directory.stat.writeable)
-    - name: autocreate new directory for ceph-ansible fetch directory backup
-      become: true
-      file:
-        path: "{{ local_ceph_ansible_fetch_directory_backup }}"
-        state: directory
-        owner: "{{ ansible_user }}"
-        mode: 0700
-      when:
-        - local_backup_directory.stat is defined
-        - not local_backup_directory.stat.exists
-    - name: look for tarball of ceph-ansible fetch directory in local backup
-      stat: path="{{ local_ceph_ansible_fetch_directory_backup }}/{{ ceph_ansible_tarball_name }}"
-      register: local_backup_file
-      ignore_errors: true
-    - name: untar local backup of ceph-ansible fetch directory
-      # unarchive module hit https://github.com/ansible/ansible/issues/35645
-      shell: >-
-        /usr/bin/gtar --gzip --extract --file \
-           {{ local_ceph_ansible_fetch_directory_backup }}/{{ ceph_ansible_tarball_name }} \
-           -C {{ playbook_dir }}/ceph-ansible/fetch_dir
-      when:
-        - local_backup_file.stat is defined
-        - local_backup_file.stat.exists
-
-# swift backup
-- when: local_ceph_ansible_fetch_directory_backup | length == 0
-  block:
-    - name: attempt download of fetch directory tarball from swift backup
-      shell: "curl -s -o /tmp/{{ old_ceph_ansible_tarball_name }} -w '%{http_code}' -X GET \"{{  swift_get_url  }}\""
-      register: curl_get_http_status
-      ignore_errors: true
-    - name: ensure we create a new fetch_directory or use the old fetch_directory
-      fail:
-        msg: "Received HTTP: {{ curl_get_http_status.stdout }} when attempting to GET from {{ swift_get_url }}"
-      when:
-        - curl_get_http_status is changed
-        - curl_get_http_status.stdout != "200"      # deployment update
-        - curl_get_http_status.stdout != "404"      # new deployment
-    - name: unpack downloaded ceph-ansible fetch tarball to fetch directory
-      # unarchive module hit https://github.com/ansible/ansible/issues/35645
-      shell: "/usr/bin/gtar --gzip --extract --file /tmp/{{ old_ceph_ansible_tarball_name }} -C {{ playbook_dir }}/ceph-ansible/fetch_dir"
-      when:
-        - curl_get_http_status is changed
-        - curl_get_http_status.stdout == "200"
-    - name: remove downloaded ceph-ansible fetch directory tarball from filesystem
-      file:
-        path: "/tmp/{{ old_ceph_ansible_tarball_name }}"
-        state: absent
-      when:
-        - curl_get_http_status is changed
-        - curl_get_http_status.stdout == "200"
-
-- name: Set cleaned fact
-  set_fact:
-    ceph_node_cleaned: true

tripleo_ansible/roles/tripleo_ceph_fetch_dir/tasks/main.yml

@@ -1,17 +0,0 @@
----
-# Copyright 2019 Red Hat, Inc.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-- import_tasks: create.yml

tripleo_ansible/roles/tripleo_ceph_fetch_dir/vars/main.yml

@@ -1,19 +0,0 @@
----
-# Copyright 2019 Red Hat, Inc.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-ceph_ansible_tarball_name: 'temporary_dir.tar.gz'
-old_ceph_ansible_tarball_name: 'temporary_dir_old.tar.gz'
-new_ceph_ansible_tarball_name: 'temporary_dir_new.tar.gz'

tripleo_ansible/roles/tripleo_ceph_run_ansible/meta/main.yml

@@ -37,17 +37,3 @@ galaxy_info:
 
   galaxy_tags:
     - tripleo
-
-dependencies:
-  - role: tripleo_ceph_common
-    when:
-      - ceph_common_done is undefined
-  - role: tripleo_ceph_work_dir
-    when:
-      - ceph_ansible_private_key_file is undefined
-  - role: tripleo_ceph_fetch_dir
-    when:
-      - ceph_node_cleaned is undefined
-  - role: tripleo_ceph_uuid
-    when:
-      - (nodes_uuid_list is undefined) or (nodes_data is undefined)

tripleo_ansible/roles/tripleo_ceph_run_ansible/molecule/default/playbook.yml

@@ -18,4 +18,7 @@
 - name: Converge
   hosts: all
   roles:
+    - role: "tripleo_ceph_common"
+    - role: "tripleo_ceph_work_dir"
+    - role: "tripleo_ceph_uuid"
     - role: "tripleo_ceph_run_ansible"

tripleo_ansible/roles/tripleo_ceph_run_ansible/tasks/main.yml

@@ -109,7 +109,11 @@
         ceph_scripts: "{{ ceph_scripts|default(ceph_default) + [ceph_prefix + item + ceph_suffix] }}"
       loop: "{{ ceph_external_multi_config | map(attribute='cluster') | list }}"
 
-- name: run {{ ceph_scripts|length|default(1) }} ceph-ansible playbook(s) (immediate log at {{ calling_ansible_log_path }})
+- name: "Notify user about upcoming ceph-ansible execution(s)"
+  debug:
+    msg: "Running {{ ceph_scripts|default(['ceph_ansible_command.sh'])|length }} ceph-ansible playbook(s) (immediate log at {{ calling_ansible_log_path }})"
+
+- name: run ceph-ansible
   # Needs become to be able to read the ssh private key
   become: true
   shell: "{{ item }}"

tripleo_ansible/roles/tripleo_ceph_uuid/meta/main.yml

@@ -37,11 +37,3 @@ galaxy_info:
 
   galaxy_tags:
     - tripleo
-
-dependencies:
-  - role: tripleo_ceph_common
-    when:
-      - ceph_common_done is undefined
-  - role: tripleo_ceph_work_dir
-    when:
-      - ceph_ansible_private_key_file is undefined