Add the ability to run the role on all hosts

We currently have spread out package/host management to multiple
roles, sometimes repeating ourselves in the process (see
pip_install and openstack_hosts overlap)

That is against Ansible principles, and we should have one role
that configures the minimum (to run openstack), applying it to
all the nodes, maybe behaving slightly differently depending on
some parameters. Here that parameter is if the host is a container
or not. If the host is a container, all the physical host
configuration (kernel and sysctl) is be skipped, the
rest of the configuration (packages/repos) still applies.

This needed a refactor to split the tasks into those two group
while remaining efficient and avoid multiple back and forth
of package installs/removal. For that last point, new defaults
variables were introduced, allowing overrides per host/group.
A node now member of a group x can now directly use this role
to setup all its necessary repos and keys.

Last, but not least, this override mechanism can now easily
trigger pip_install role, which can from now on, be removed
from every role. On top of that pip_install role can now
remove its repo management, and focus on installing pip on
hosts that don't have a proper version of pip installed.

Change-Id: Ibf145e561c80a12055bd4d5dca3914c4d495a748

defaults/main.yml

@@ -13,6 +13,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Package cache
+cache_timeout: 600
+
 # /etc/openstack-release settings
 openstack_distrib_id: "OSA"
 openstack_distrib_release: "{{ openstack_release | default('master') }}"
@@ -21,6 +24,8 @@ openstack_distrib_description: "OpenStack-Ansible"
 openstack_distrib_file: yes
 openstack_distrib_file_path: "/etc/openstack-release"
 
+is_container: "{{ ansible_virtualization_type == 'lxc' }}"
+
 openstack_host_sysstat_enabled: true
 openstack_host_sysstat_interval: 1
 openstack_host_sysstat_statistics_hour: 23
@@ -36,12 +41,28 @@ openstack_host_manage_hosts_file: true
 
 ## kernel modules for specific group hosts
 openstack_host_specific_kernel_modules: []
-# to include it in your play, an example is given below:
+# If you want to include some specific modules per group
+# of hosts, override this with a group/host var, like below:
 #openstack_host_specific_kernel_modules:
-#  - { name: "ebtables", pattern: "CONFIG_BRIDGE_NF_EBTABLES", group: "network_hosts" }
+#  - name: "ebtables"
+#    pattern: "CONFIG_BRIDGE_NF_EBTABLES"
+## Where:
 ## :param name: name of the kernel module
 ## :param pattern: pattern to grep for in /boot/config-$kernel_version to check how module is configured inside kernel
-## :param group: group of hosts where the module will be loaded
+## Our default overrides will be combined with your overrides.
+
+# Overridable package list is composed of the old override
+# named user_package_list and the standard defaults _package_list
+openstack_hosts_package_list: "{{ _package_list + (user_package_list | default([])) }}"
+
+# Overridable package repo is composed of the old override
+# named user_external_repo_lists and the standard defaults _package_repos
+openstack_hosts_package_repos: "{{ _package_repos + (user_external_repos_list | default([])) }}"
+
+# Overridable package repo gpg is composed of the old override
+# named user_external_repo_keys_list and the standard defaults _package_repos_keys
+openstack_hosts_package_repos_keys: "{{ _package_repos_keys + (user_external_repo_keys_list | default([])) }}"
+openstack_hosts_package_repos_priorities: "{{ _package_repos_priorities }}"
 
 # The following garbage collection values are set to better support lots of neutron networks/routers.
 #  Used for setting the net.ipv4/6.neigh.default.gc_thresh* values. This assumes that facts were
@@ -98,6 +119,9 @@ openstack_kernel_options:
 # above.
 openstack_user_kernel_options: []
 
+# Overridable set of packages to install on the host.
+openstack_host_metal_distro_packages: "{{ _openstack_host_metal_distro_packages }}"
+
 # Set the openstack domain name
 openstack_domain: openstack.local
 

releasenotes/notes/specific_kernel_modules_with_group_vars-8d169f564ffd450c.yaml

@@ -0,0 +1,25 @@
+---
+upgrade:
+  - |
+    If you have overriden your
+    ``openstack_host_specific_kernel_modules``, please
+    remove its group matching, and move that override
+    directly to the appropriate group.
+
+    Example, for an override like:
+
+    .. code-block:: yaml
+
+        - name: "ebtables"
+          pattern: "CONFIG_BRIDGE_NF_EBTABLES"
+          group: "network_hosts"
+
+    You can create a file for the network_host group,
+    inside its group vars folder
+    ``/etc/openstack_deploy/group_vars/network_hosts``,
+    with the content:
+
+    .. code-block:: yaml
+
+        - name: "ebtables"
+          pattern: "CONFIG_BRIDGE_NF_EBTABLES"

tasks/configure_metal_hosts.yml

@@ -0,0 +1,86 @@
+---
+# Copyright 2017, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Check Kernel Version
+  fail:
+    msg: >
+      Wrong kernel Version found
+      [ {{ ansible_kernel }} < {{ openstack_host_required_kernel }} ]
+      Resolve this issue before continuing.
+  when: ansible_kernel | version_compare(openstack_host_required_kernel, '<')
+
+- name: Disable cache for apt update for hosts
+  copy:
+    content: |
+      Acquire::http::No-Cache true;
+    dest: "/etc/apt/apt.conf.d/00apt-no-cache"
+  tags:
+    openstack_hosts-config
+  when:
+    - ansible_pkg_mgr == 'apt'
+    - >
+      global_environment_variables.http_proxy is defined or
+      global_environment_variables.HTTP_PROXY is defined or
+      global_environment_variables.https_proxy is defined or
+      global_environment_variables.HTTPS_PROXY is defined
+
+- name: Install distro packages for bare metal nodes
+  package:
+    name: "{{ openstack_host_metal_distro_packages }}"
+    state: "{{ openstack_hosts_package_state }}"
+  register: install_packages
+  until: install_packages | success
+  retries: 5
+  delay: 2
+
+- name: check how kernel modules are implemented (statically builtin, dynamic, not set)
+  slurp:
+    src: "/boot/config-{{ ansible_kernel }}"
+  register: modules
+  when: openstack_host_specific_kernel_modules | length > 0
+
+- name: Fail fast if we can't load a module
+  fail:
+    msg: "{{ item.pattern }} is not set"
+  with_items: "{{ openstack_host_specific_kernel_modules  }}"
+  when:
+    - (modules.content | b64decode).find(item.pattern + ' is not set') != -1
+
+- name: "Load kernel module(s)"
+  modprobe:
+    name: "{{ item.name }}"
+  with_items: "{{ openstack_host_kernel_modules + openstack_host_specific_kernel_modules }}"
+  when:
+    - item.name != ''
+    - item.pattern is undefined or (item.pattern is defined and (modules.content | b64decode).find(item.pattern + '=m') != -1)
+
+- name: Write list of modules to load at boot
+  template:
+    src: modprobe.conf.j2
+    dest: "{{ openstask_host_module_file }}"
+
+- name: Adding new system tuning
+  sysctl:
+    name: "{{ item.key }}"
+    value: "{{ item.value }}"
+    sysctl_set: "{{ item.set|default('yes') }}"
+    state: "{{ item.state|default('present') }}"
+    reload: no
+  with_items: "{{ openstack_kernel_options + openstack_user_kernel_options }}"
+  failed_when: false
+
+- name: Configure sysstat
+  include: openstack_sysstat.yml
+  when: openstack_host_sysstat_enabled | bool

tasks/main.yml

@@ -24,19 +24,6 @@
   tags:
     - always
 
-- include: openstack_kernel_check.yml
-  tags:
-    - openstack_hosts-install
-
-- include: openstack_proxy_settings.yml
-  tags:
-    - openstack_hosts-install
-    - openstack_hosts-config
-
-- include: openstack_host_install.yml
-  tags:
-    - openstack_hosts-install
-
 - name: Allow the usage of local facts
   file:
     path: /etc/ansible/facts.d/
@@ -44,30 +31,47 @@
   tags:
     - openstack_hosts-install
 
-- include: openstack_sysstat.yml
+# Drop the release file everywhere
+- include: openstack_release.yml
   tags:
     - openstack_hosts-install
+
+# Proxy configuration applies to all nodes
+- name: Add global_environment_variables to environment file
+  blockinfile:
+    dest: "/etc/environment"
+    state: present
+    marker: "# {mark} Managed by OpenStack-Ansible"
+    insertbefore: EOF
+    block: "{{ lookup('template', 'environment.j2') }}"
+  tags:
     - openstack_hosts-config
 
-- include: openstack_update_hosts_file.yml
+# Configure host files should apply to all nodes
-  static: no
+- name: Configure etc hosts files
+  include: openstack_update_hosts_file.yml
   when: openstack_host_manage_hosts_file | bool
+  tags:
+    - openstack_hosts-config
+
+# This allows to include this role to get all the distro
+# specific configuration for all the nodes.
+# It is also used for installing common packages to
+# all nodes
+- name: Apply package management distro specific configuration
+  include: "openstack_hosts_configure_{{ ansible_pkg_mgr | lower }}.yml"
+
+# Configure bare metal nodes: Kernel, sysctl, sysstat, hosts files, metal packages
+- include: configure_metal_hosts.yml
+  when:
+    - not is_container
   tags:
     - openstack_hosts-install
-    - openstack_hosts-config
-
-- include: openstack_kernel_modules.yml
-  tags:
-    - openstack_hosts-config
-
-- include: openstack_kernel_tuning.yml
-  tags:
-    - openstack_hosts-config
 
 - include: openstack_authorized_keys.yml
   tags:
     - openstack_hosts-config
 
-- include: openstack_release.yml
+# Now run the pip install role and your host should be ready!
-  tags:
+#- include_role: pip_install
-    - openstack_hosts-install
+#  when: host_need_pip | default(True) | bool

tasks/openstack_host_install.yml

@@ -1,16 +0,0 @@
----
-# Copyright 2014, Rackspace US, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- include: "openstack_host_install_{{ ansible_pkg_mgr }}.yml"

tasks/openstack_host_install_apt.yml

@@ -1,43 +0,0 @@
----
-# Copyright 2014, Rackspace US, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- name: Remove conflicting distro packages
-  apt:
-    name: "{{ openstack_host_distro_packages_remove | default([]) }}"
-    state: absent
-
-- name: Disable cache for apt update if behind proxy
-  copy:
-    content: |
-      Acquire::http::No-Cache true;
-    dest: "/etc/apt/apt.conf.d/00apt-no-cache"
-  when: >
-    global_environment_variables.http_proxy is defined or
-    global_environment_variables.HTTP_PROXY is defined or
-    global_environment_variables.https_proxy is defined or
-    global_environment_variables.HTTPS_PROXY is defined
-
-- name: Install distro packages
-  apt:
-    pkg: "{{ openstack_host_distro_packages }}"
-    state: "{{ openstack_hosts_package_state }}"
-    update_cache: yes
-    cache_valid_time: "{{ cache_timeout }}"
-  register: install_packages
-  until: install_packages | success
-  retries: 5
-  delay: 2
-  tags:
-    - openstack-apt-packages

tasks/openstack_host_install_dnf.yml

@@ -1 +0,0 @@
-openstack_host_install_yum.yml

tasks/openstack_host_install_zypper.yml

@@ -1,56 +0,0 @@
----
-# Copyright 2017, SUSE LINUX GmbH.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# NOTE(hwoarang) snapper make take significant amount of CPU time
-# when executing zypper over and over so it's best to disable the
-# background comparison process. Snapper may need further tuning
-# for example reducing the number of snapshots to keep, disable daily
-# cleanup job etc but these may come later in the future if needed.
-- name: Check if Snapper root configuration file exists
-  stat:
-    path: "/etc/snapper/configs/root"
-  register: snapper_root_config
-  tags:
-    - openstack_hosts-config
-
-- name: Disable background snapshot comparisons on Snapper
-  lineinfile:
-    path: "/etc/snapper/configs/root"
-    regexp: '^BACKGROUND_COMPARISON=.*'
-    line: 'BACKGROUND_COMPARISON="no"'
-    state: present
-  when: snapper_root_config.stat.exists
-  tags:
-    - openstack_hosts-config
-
-- name: Remove conflicting distro packages
-  zypper:
-    name: "{{ openstack_host_distro_packages_remove | default([]) }}"
-    state: absent
-  tags:
-    - openstack-zypper-packages
-    - openstack-packages
-
-- name: Install distro packages
-  zypper:
-    name: "{{ openstack_host_distro_packages }}"
-    state: "{{ openstack_hosts_package_state }}"
-  register: install_packages
-  until: install_packages|success
-  retries: 5
-  delay: 2
-  tags:
-    - openstack-zypper-packages
-    - openstack-packages

tasks/openstack_hosts_configure_apt.yml

@@ -0,0 +1,56 @@
+# Copyright 2017, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# APT configuration tasks that apply on all nodes.
+
+- name: Remove the blacklisted packages
+  package:
+    name: "{{ openstack_hosts_package_list | selectattr('state','equalto','absent') | map(attribute='name') | list }}"
+    state: absent
+
+- name: Add/Remove repositories gpg keys manually
+  apt_key:
+    id: "{{ key.id | default(omit) }}"
+    data: "{{ key.data | default(omit) }}"            # use lookup('file','armored_content.asc')
+    keyserver: "{{ key.keyserver | default(omit) }}"
+    url: "{{ key.url | default(omit) }}"
+    state: "{{ key.state | default('present') }}"
+  with_items: "{{ openstack_hosts_package_repos_keys }}"
+  loop_control:
+    loop_var: key
+  register: _add_apt_keys
+  until: _add_apt_keys | success
+  retries: 5
+  delay: 2
+
+- name: Add requirement packages (repositories gpg keys, toolkits...)
+  apt:
+    name: "{{ openstack_hosts_package_list | rejectattr('state','equalto','absent') | map(attribute='name') | list }}"
+    state: "{{ openstack_hosts_package_state }}"
+    update_cache: yes
+    cache_valid_time: "{{ cache_timeout }}"
+
+- name: Add/Remove/Update standard and user defined repositories
+  apt_repository:
+    repo: "{{ repo.repo }}"
+    state: "{{ repo.state | default('present') }}"
+    filename: "{{ repo.filename | default(omit) }}"
+    update_cache: "{{ repo == package_repos[-1] }}"
+  with_items: "{{ openstack_hosts_package_repos }}"
+  loop_control:
+    loop_var: repo
+  register: _adding_apt_repo
+  until: _adding_apt_repo | success
+  retries: 5
+  delay: 2

tasks/openstack_hosts_configure_dnf.yml

@@ -0,0 +1 @@
+openstack_hosts_configure_yum.yml

tasks/openstack_hosts_configure_yum.yml

@@ -13,35 +13,42 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- name: Install EPEL, and yum priorities plugin
+- name: Disable requiretty for root sudo on centos
+  template:
+    dest: /etc/sudoers.d/openstack-ansible
+    owner: root
+    group: root
+    mode: "0440"
+    src: sudoers.j2
+
+# yum configuration tasks that apply on all nodes.
+- name: Remove the blacklisted packages
   package:
-    name: "{{ openstack_host_required_distro_packages }}"
+    name: "{{ openstack_hosts_package_list | selectattr('state','equalto','absent') | map(attribute='name') | list }}"
-    state: "{{ openstack_hosts_package_state }}"
+    state: absent
-  tags:
-    - openstack-yum-packages
-    - openstack-packages
 
 # Copy all factored-in GPG keys.
 # KeyID 764429E6 from https://raw.githubusercontent.com/rdo-infra/centos-release-openstack/ocata-rdo/RPM-GPG-KEY-CentOS-SIG-Cloud
 # KeyID 61E8806C from keyserver for rdo-qemu-ev
-- name: Copy validated GPG keys
+- name: If a keyfile is provided, copy the gpg keyfile to the key location
   copy:
-    src: "gpg/{{ item | basename }}"
+    src: "{{ item.keyfile }}"
-    dest: /etc/pki/rpm-gpg/
+    dest: "{{ item.key }}"
     mode: '0644'
-  with_fileglob:
+  with_items: "{{ openstack_hosts_package_repos_keys | selectattr('keyfile','defined') | list }}"
-    - "gpg/*"
 
 - name: Ensure GPG keys have the correct SELinux contexts applied
   command: restorecon -Rv /etc/pki/rpm-gpg/
+  # TODO(evrardjp): Be more idempotent
+  changed_when: false
 
 # Handle gpg keys manually
 - name: Install gpg keys
   rpm_key:
-    key: "{{ key.keyfile | default(key.key) }}"
+    key: "{{ key.key }}"
     validate_certs: "{{ key.validate_certs | default(omit) }}"
     state: "{{ key.state | default('present') }}"
-  with_items: "{{ openstack_host_rdo_repos_keys }}"
+  with_items: "{{ openstack_hosts_package_repos_keys }}"
   loop_control:
     loop_var: key
   register: _add_yum_keys
@@ -49,28 +56,32 @@
   retries: 5
   delay: 2
 
+- name: Add requirement packages (repositories gpg keys packages, toolkits...)
+  package:
+    name: "{{ openstack_hosts_package_list | rejectattr('state','equalto','absent') | map(attribute='name') | list }}"
+    state: "{{ openstack_hosts_package_state }}"
+
 - name: Check for existing yum repositories
   shell: "yum-config-manager | grep 'repo:'"
   register: existing_yum_repos
-  tags:
-    - openstack-yum-packages
-    - openstack-packages
 
 - name: Add yum repositories if they do not exist
   yum_repository:
-    name: "{{ item.name }}"
+    name: "{{ repo.name }}"
-    description: "{{ item.description }}"
+    description: "{{ repo.description | default(omit) }}"
-    baseurl: "{{ item.baseurl }}"
+    baseurl: "{{ repo.baseurl | default(omit) }}"
-    file: "{{ item.file }}"
+    gpgkey: "{{ repo.gpgkey | default(omit) }}"
-    gpgcheck: "{{ item.gpgcheck }}"
+    gpgcheck: "{{ repo.gpgcheck | default(omit) }}"
-    enabled: "{{ item.enabled }}"
+    enabled: "{{ repo.enabled | default('yes') }}"
-  with_items:
+  with_items: "{{ openstack_hosts_package_repos  }}"
-    - "{{ openstack_host_rdo_repos }}"
+  loop_control:
+    loop_var: repo
   when:
-    - item.name not in existing_yum_repos.stdout
+    - repo.name not in existing_yum_repos.stdout
-  tags:
+  register: _adding_repo
-    - openstack-yum-packages
+  until: _adding_repo | success
-    - openstack-packages
+  retries: 5
+  delay: 2
 
 - name: Update yum repositories if they already exist
   command: >
@@ -79,47 +90,18 @@
     {% for key in item.keys() if key != 'file' %}
     --setopt="{{ item.name }}.{{ key }}={{ item[key] }}"
     {% endfor %}
-  with_items:
+  # TODO(evrardjp): Be more idempotent
-    - "{{ openstack_host_rdo_repos }}"
+  changed_when: false
+  with_items: "{{ openstack_hosts_package_repos }}"
   when:
     - item.name in existing_yum_repos.stdout
-  tags:
-    - openstack-yum-packages
-    - openstack-packages
 
-- name: Enable and set repo priorities
+- name: Update repo priorities
   command: >
     yum-config-manager
-    {% for repo_priority in openstack_host_repo_priorities %}
+    {% for repo_priority in openstack_hosts_package_repos_priorities %}
     --enable {{ repo_priority['name'] }} \
     --setopt="{{ repo_priority['name'] }}.priority={{ repo_priority['priority'] }}"
     {% endfor %}
+  # TODO(evrardjp): Be more idempotent
   changed_when: false
-  tags:
-    - openstack-yum-packages
-    - openstack-packages
-
-- name: Remove conflicting distro packages
-  package:
-    name: "{{ openstack_host_distro_packages_remove | default([]) }}"
-    state: absent
-
-- name: Install distro packages
-  package:
-    pkg: "{{ openstack_host_distro_packages }}"
-    state: "{{ openstack_hosts_package_state }}"
-  register: install_packages
-  until: install_packages | success
-  retries: 5
-  delay: 2
-  tags:
-    - openstack-yum-packages
-    - openstack-packages
-
-- name: Disable requiretty for root sudo on centos
-  template:
-    dest: /etc/sudoers.d/openstack-ansible
-    owner: root
-    group: root
-    mode: "0440"
-    src: sudoers.j2

tasks/openstack_hosts_configure_zypper.yml

@@ -0,0 +1,80 @@
+---
+# Copyright 2017, SUSE LINUX GmbH.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# NOTE(hwoarang) snapper make take significant amount of CPU time
+# when executing zypper over and over so it's best to disable the
+# background comparison process. Snapper may need further tuning
+# for example reducing the number of snapshots to keep, disable daily
+# cleanup job etc but these may come later in the future if needed.
+- name: Check if Snapper root configuration file exists
+  stat:
+    path: "/etc/snapper/configs/root"
+  register: snapper_root_config
+
+- name: Disable background snapshot comparisons on Snapper
+  lineinfile:
+    path: "/etc/snapper/configs/root"
+    regexp: '^BACKGROUND_COMPARISON=.*'
+    line: 'BACKGROUND_COMPARISON="no"'
+    state: present
+  when: snapper_root_config.stat.exists
+
+- name: Remove the blacklisted packages
+  package:
+    name: "{{ openstack_hosts_package_list | selectattr('state','equalto','absent') | map(attribute='name') | list }}"
+    state: absent
+
+- name: If a keyfile is provided, copy the gpg keyfile to the key location
+  copy:
+    src: "{{ item.keyfile }}"
+    dest: "{{ item.key }}"
+  with_items: "{{ openstack_hosts_package_repos_keys | selectattr('keyfile','defined') | list }}"
+
+- name: Add/Remove repositories gpg keys manually
+  rpm_key:
+    key: "{{ key.key }}"
+    state: "{{ key.state | default('present') }}"
+    validate_certs: "{{ key.validate_certs | default(omit) }}"
+  with_items: "{{ openstack_hosts_package_repos_keys }}"
+  loop_control:
+    loop_var: key
+  register: _add_rpm_keys
+  until: _add_rpm_keys | success
+  retries: 5
+  delay: 2
+
+- name: Add requirement packages (repositories gpg keys, toolkits...)
+  zypper:
+    name: "{{ openstack_hosts_package_list | rejectattr('state','equalto','absent') | map(attribute='name') | list }}"
+    state: "{{ openstack_hosts_package_state }}"
+  when: "{{ openstack_hosts_package_list | rejectattr('state','equalto','absent') | map(attribute='name') | list | length > 0}}"  
+
+- name: Add/Remove/Update standard and user defined repositories
+  zypper_repository:
+    repo: "{{ repo.repo }}"
+    state: "{{ repo.state | default('present') }}"
+    name: "{{ repo.name | default(omit) }}"
+    enabled: "{{ repo.enabled | default(omit) }}"
+    disable_gpg_check: "{{ repo.disable_gpg_check | default(omit) }}"
+    description: "{{ repo.description | default(omit) }}"
+    autorefresh: "{{ repo.autorefresh | default(omit) }}"
+    auto_import_keys: "{{ repo.auto_import_keys | default(omit) }}"
+  with_items: "{{ openstack_hosts_package_repos }}"
+  loop_control:
+    loop_var: repo
+  register: _adding_repo
+  until: _adding_repo | success
+  retries: 5
+  delay: 2

tasks/openstack_kernel_check.yml

@@ -1,22 +0,0 @@
----
-# Copyright 2014, Rackspace US, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- name: Check Kernel Version
-  fail:
-    msg: >
-      Wrong kernel Version found
-      [ {{ ansible_kernel }} < {{ openstack_host_required_kernel }} ]
-      Resolve this issue before continuing.
-  when: ansible_kernel | version_compare(openstack_host_required_kernel, '<')

tasks/openstack_kernel_modules.yml

@@ -1,61 +0,0 @@
----
-# Copyright 2014, Rackspace US, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- name: check how kernel modules are implemented (statically builtin, dynamic, not set)
-  slurp:
-    src: "/boot/config-{{ ansible_kernel }}"
-  register: modules
-  when: openstack_host_specific_kernel_modules | length > 0
-
-- name: fail if a specific kernel module is not set
-  fail:
-    msg: "{{ item.pattern }} is not set"
-  with_items: "{{ openstack_host_specific_kernel_modules }}"
-  when:
-    - groups[item.group] is defined
-    - inventory_hostname in groups[item.group]
-    - (modules.content | b64decode).find(item.pattern + ' is not set') != -1
-
-- name: fail if a specific pattern is not valid
-  fail:
-    msg: "{{ item.pattern }} is not a valid pattern"
-  with_items: "{{ openstack_host_specific_kernel_modules }}"
-  when:
-    - groups[item.group] is defined
-    - inventory_hostname in groups[item.group]
-    - (modules.content | b64decode).find(item.pattern + '=y') == -1
-    - (modules.content | b64decode).find(item.pattern + '=m') == -1
-
-- name: "Ensure kernel module(s)"
-  modprobe:
-    name: "{{ item }}"
-  with_items: "{{ openstack_host_kernel_modules }}"
-  when:
-    - openstack_host_kernel_modules | length > 0
-    - item != ''
-
-- name: "Ensure dynamic specific kernel module(s) are loaded"
-  modprobe:
-    name: "{{ item.name }}"
-  with_items: "{{ openstack_host_specific_kernel_modules }}"
-  when:
-    - groups[item.group] is defined
-    - inventory_hostname in groups[item.group]
-    - (modules.content | b64decode).find(item.pattern + '=m') != -1
-
-- name: Write list of modules to load at boot
-  template:
-    src: modprobe.conf.j2
-    dest: "{{ openstask_host_module_file }}"

tasks/openstack_kernel_tuning.yml

@@ -1,24 +0,0 @@
----
-# Copyright 2014, Rackspace US, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-- name: Adding new system tuning
-  sysctl:
-    name: "{{ item.key }}"
-    value: "{{ item.value }}"
-    sysctl_set: "{{ item.set|default('yes') }}"
-    state: "{{ item.state|default('present') }}"
-    reload: no
-  with_items: "{{ openstack_kernel_options + openstack_user_kernel_options }}"
-  failed_when: false

tasks/openstack_sysstat.yml

@@ -19,7 +19,6 @@
     dest: "{{ openstack_host_sysstat_file }}"
     mode: "0644"
   when:
-    - openstack_host_sysstat_enabled | bool
     - ansible_pkg_mgr == 'apt'
   notify: Restart sysstat
 
@@ -28,13 +27,10 @@
     src: "{{ openstack_host_cron_template }}"
     dest: "/etc/cron.d/sysstat"
     mode: "{{ openstack_host_sysstat_cron_mode }}"
-  when:
-    - openstack_host_sysstat_enabled | bool
 
 - name: Restore SELinux contexts on sysstat cron file
   command: restorecon -v /etc/cron.d/sysstat
   when:
-    - openstack_host_sysstat_enabled | bool
     - ansible_pkg_mgr in ['yum', 'dnf']
 
 - name: Start and enable the sysstat service
@@ -43,5 +39,4 @@
     state: started
     enabled: yes
   when:
-    - openstack_host_sysstat_enabled | bool
     - ansible_pkg_mgr in ['yum', 'dnf', 'zypper']

templates/modprobe.conf.j2

@@ -1,16 +1,5 @@
 # {{ ansible_managed }}
 # Modules from the openstack-ansible-openstack_hosts role
-{% for module in openstack_host_kernel_modules %}
+{% for module in openstack_host_kernel_modules + openstack_host_specific_kernel_modules %}
-{{ module }}
+{{ module.name }}
-{% endfor %}
-
-# Host-specific dynamic modules
-{%
-    for module in openstack_host_specific_kernel_modules if (
-        groups[module.group] is defined and
-        inventory_hostname in module.group and
-        item.pattern + '=m' in (modules.content | b64decode)
-    )
-%}
-{{ module }}
 {% endfor %}

tests/openstack_hosts-overrides.yml

@@ -1,3 +1,4 @@
 ---
 openstack_host_specific_kernel_modules:
-  - { name: "ebtables", pattern: "CONFIG_BRIDGE_NF_EBTABLES", group: "hosts" }
+  - name: "ebtables"
+    pattern: "CONFIG_BRIDGE_NF_EBTABLES"

vars/redhat-7.yml

@@ -23,34 +23,34 @@ openstack_host_sysstat_cron_mode: '0600'
 
 ## Kernel modules loaded on hosts
 openstack_host_kernel_modules:
-  - 8021q
+  - name: 8021q
-  - "{% if hostvars[inventory_hostname]['ansible_kernel'] | version_compare('3.10.0-514', '>=') %}br_netfilter{% endif %}"
+  - name: "{% if hostvars[inventory_hostname]['ansible_kernel'] | version_compare('3.10.0-514', '>=') %}br_netfilter{% endif %}"
-  - dm_multipath
+  - name: dm_multipath
-  - dm_snapshot
+  - name: dm_snapshot
-  - ebtables
+  - name: ebtables
-  - ip6table_filter
+  - name: ip6table_filter
-  - ip6_tables
+  - name: ip6_tables
-  - ip_tables
+  - name: ip_tables
-  - ipt_MASQUERADE
+  - name: ipt_MASQUERADE
-  - ipt_REJECT
+  - name: ipt_REJECT
-  - iptable_filter
+  - name: iptable_filter
-  - iptable_mangle
+  - name: iptable_mangle
-  - iptable_nat
+  - name: iptable_nat
-  - ip_vs
+  - name: ip_vs
-  - iscsi_tcp
+  - name: iscsi_tcp
-  - nf_conntrack
+  - name: nf_conntrack
-  - nf_conntrack_ipv4
+  - name: nf_conntrack_ipv4
-  - nf_defrag_ipv4
+  - name: nf_defrag_ipv4
-  - nf_nat
+  - name: nf_nat
-  - nf_nat_ipv4
+  - name: nf_nat_ipv4
   # TODO (odyssey4me): revise the minimum kernel version once this kernel version is commonplace
   #  If we end up with more requirements like this, then we should change the approach.
-  - "{% if hostvars[inventory_hostname]['ansible_kernel'] | version_compare('4.4.0-0', '<') %}scsi_dh{% endif %}"
+  - name: "{% if hostvars[inventory_hostname]['ansible_kernel'] | version_compare('4.4.0-0', '<') %}scsi_dh{% endif %}"
-  - vhost_net
+  - name: vhost_net
-  - x_tables
+  - name: x_tables
 
-## Base packages
+## Bare metal base packages
-openstack_host_distro_packages:
+_openstack_host_metal_distro_packages:
   - bridge-utils
   - curl
   - device-mapper-event
@@ -74,18 +74,23 @@ openstack_host_distro_packages:
   - time
   - wget
 
-openstack_host_required_distro_packages:
+_package_repos_keys:
-  - epel-release
+  - name: openstack-pike
-  - yum-plugin-priorities
+    key: /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud
-  - yum-utils
+    keyfile: "gpg/764429E6"
+  - name: rdo-qemu-ev
+    key: /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Virtualization-RDO
+    keyfile: "gpg/61E8806C"
 
-openstack_host_rdo_repos_keys:
+_package_list:
-  - repo: openstack-pike
+  - name: epel-release
-    keyfile: /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud
+    state: present
-  - repo: rdo-qemu-ev
+  - name: yum-plugin-priorities
-    keyfile: /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Virtualization-RDO
+    state: present
+  - name: yum-utils
+    state: present
 
-openstack_host_rdo_repos:
+_package_repos:
   - file: rdo-qemu-ev
     name: rdo-qemu-ev
     description: "RDO CentOS-7 - QEMU EV"
@@ -99,7 +104,7 @@ openstack_host_rdo_repos:
     gpgcheck: yes
     enabled: yes
 
-openstack_host_repo_priorities:
+_package_repos_priorities:
   - name: base
     priority: 50
   - name: epel

vars/suse-42.yml

@@ -22,34 +22,34 @@ openstask_host_module_file: /etc/modules-load.d/openstack-ansible.conf
 
 ## Kernel modules loaded on hosts
 openstack_host_kernel_modules:
-  - 8021q
+  - name: 8021q
-  - "{% if hostvars[inventory_hostname]['ansible_kernel'] | version_compare('4.4', '>=') %}br_netfilter{% endif %}"
+  - name: "{% if hostvars[inventory_hostname]['ansible_kernel'] | version_compare('4.4', '>=') %}br_netfilter{% endif %}"
-  - dm_multipath
+  - name: dm_multipath
-  - dm_snapshot
+  - name: dm_snapshot
-  - ebtables
+  - name: ebtables
-  - ip6table_filter
+  - name: ip6table_filter
-  - ip6_tables
+  - name: ip6_tables
-  - ip_tables
+  - name: ip_tables
-  - ipt_MASQUERADE
+  - name: ipt_MASQUERADE
-  - ipt_REJECT
+  - name: ipt_REJECT
-  - iptable_filter
+  - name: iptable_filter
-  - iptable_mangle
+  - name: iptable_mangle
-  - iptable_nat
+  - name: iptable_nat
-  - ip_vs
+  - name: ip_vs
-  - iscsi_tcp
+  - name: iscsi_tcp
-  - nf_conntrack
+  - name: nf_conntrack
-  - nf_conntrack_ipv4
+  - name: nf_conntrack_ipv4
-  - nf_defrag_ipv4
+  - name: nf_defrag_ipv4
-  - nf_nat
+  - name: nf_nat
-  - nf_nat_ipv4
+  - name: nf_nat_ipv4
   # TODO (odyssey4me): revise the minimum kernel version once this kernel version is commonplace
   #  If we end up with more requirements like this, then we should change the approach.
-  - "{% if hostvars[inventory_hostname]['ansible_kernel'] | version_compare('4.4.0-0', '<') %}scsi_dh{% endif %}"
+  - name: "{% if hostvars[inventory_hostname]['ansible_kernel'] | version_compare('4.4.0-0', '<') %}scsi_dh{% endif %}"
-  - vhost_net
+  - name: vhost_net
-  - x_tables
+  - name: x_tables
 
 ## Base packages
-openstack_host_distro_packages:
+_openstack_host_metal_distro_packages:
   - bridge-utils
   - patterns-openSUSE-devel_basis
   - curl
@@ -71,6 +71,27 @@ openstack_host_distro_packages:
   - time
   - wget
 
-## Packages to remove
+_package_repos_keys: []
-openstack_host_distro_packages_remove:
+## example:
-  - systemd-logger # conflicts with rsyslog
+# - key: "http://url_to_gpg_key"
+#   validate_certs:
+#   state:
+# - key: "/tmp/file1"
+#   keyfile: "gpg/file1"
+#   validate_certs:
+#   state:
+
+_package_list:
+  - name: systemd-logger
+    state: absent # conflicts with rsyslog
+
+_package_repos: []
+## example:
+# - repo:
+#   state:
+#   name:
+#   enabled:
+#   disable_gpg_check:
+#   description:
+#   autorefresh:
+#   auto_import_keys:

vars/ubuntu-16.04.yml

@@ -24,32 +24,32 @@ openstask_host_module_file: /etc/modules
 
 ## Kernel modules loaded on hosts
 openstack_host_kernel_modules:
-  - 8021q
+  - name: 8021q
-  - "{% if hostvars[inventory_hostname]['ansible_kernel'] | version_compare('4.4', '>=') %}br_netfilter{% endif %}"
+  - name: "{% if hostvars[inventory_hostname]['ansible_kernel'] | version_compare('4.4', '>=') %}br_netfilter{% endif %}"
-  - dm_multipath
+  - name: dm_multipath
-  - dm_snapshot
+  - name: dm_snapshot
-  - ebtables
+  - name: ebtables
-  - ip6table_filter
+  - name: ip6table_filter
-  - ip6_tables
+  - name: ip6_tables
-  - ip_tables
+  - name: ip_tables
-  - ipt_MASQUERADE
+  - name: ipt_MASQUERADE
-  - ipt_REJECT
+  - name: ipt_REJECT
-  - iptable_filter
+  - name: iptable_filter
-  - iptable_mangle
+  - name: iptable_mangle
-  - iptable_nat
+  - name: iptable_nat
-  - ip_vs
+  - name: ip_vs
-  - iscsi_tcp
+  - name: iscsi_tcp
-  - nbd
+  - name: nbd
-  - nf_conntrack
+  - name: nf_conntrack
-  - nf_conntrack_ipv4
+  - name: nf_conntrack_ipv4
-  - nf_defrag_ipv4
+  - name: nf_defrag_ipv4
-  - nf_nat
+  - name: nf_nat
-  - nf_nat_ipv4
+  - name: nf_nat_ipv4
-  - vhost_net
+  - name: vhost_net
-  - x_tables
+  - name: x_tables
 
 ## Base packages
-openstack_host_distro_packages:
+_openstack_host_metal_distro_packages:
   - apparmor-utils
   - apt-transport-https
   - bridge-utils
@@ -75,3 +75,14 @@ openstack_host_distro_packages:
   - time
   - vlan
   - wget
+
+_package_repos_keys: []
+## example:
+# - id:
+#   file:
+#   keyserver:
+#   url:
+#   state:
+
+_package_list: []
+_package_repos: []