Updated role for multi-distro support

Separate files have been created for vars and tasks related to a
specific package manager.

The 'repo_apt_packages' variable has been deprecated and renamed to
the more generalized 'repo_server_packages' to better describe its
purpose and to simplify reuse of existing install tasks between multiple
distros.

git daemon is configured to host git repositories from the repo servers
using the git protocol.

Currently, openstack-ansible uses git over http to access repositories
on servers created by this role.
fcgiwrap and its configuration within nginx should be removed in a
follow-up patch after openstack-ansible has been updated to use the git
protocol.

Change-Id: I62321a7b62dabca469eb072ddbf4e8f250ce0fb3

defaults/main.yml

@@ -1,5 +1,5 @@
 ---
-# Copyright 2014, Rackspace US, Inc.
+# Copyright 2016, Rackspace US, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -36,11 +36,5 @@ repo_auto_rebuild: false
 # Otherwise keys will be generated on the first run and not regenerated each run.
 repo_recreate_keys: False
 
-repo_apt_packages:
-  - fcgiwrap
-  - lsyncd
-  - nginx-extras
-  - rsync
-
 # Main web server port
 repo_server_port: 8181

handlers/main.yml

@@ -1,5 +1,5 @@
 ---
-# Copyright 2014, Rackspace US, Inc.
+# Copyright 2016, Rackspace US, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,16 +13,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+- name: Init reload
+  command: "initctl reload-configuration"
+
 - name: reload nginx
   service:
     name: "nginx"
     state: restarted
+    enabled: yes
     pattern: "nginx"
 
 - name: reload rsyncd
   service:
-    name: "rsync"
+    name: "{{ rsyncd_service_name }}"
     state: restarted
+    enabled: yes
     pattern: "rsync"
 
 - name: reload fcgiwrap
@@ -41,4 +46,18 @@
   service:
     name: "lsyncd"
     state: restarted
+    enabled: yes
     pattern: "lsyncd"
+
+- name: reload git-daemon
+  service:
+    name: "git-daemon"
+    state: restarted
+    enabled: yes
+    pattern: "git daemon"
+
+- name: reload git socket
+  service:
+    name: "git.socket"
+    state: restarted
+    enabled: yes

meta/main.yml

@@ -1,5 +1,5 @@
 ---
-# Copyright 2014, Rackspace US, Inc.
+# Copyright 2016, Rackspace US, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,11 +23,17 @@ galaxy_info:
     - name: Ubuntu
       versions:
         - trusty
+        - xenial
+    - name: EL
+      versions:
+        - 7
   categories:
     - cloud
     - python
     - development
     - openstack
 dependencies:
-  - apt_package_pinning
+  - role: apt_package_pinning
+    when:
+      - ansible_pkg_mgr == 'apt'
   - pip_install

other-requirements.txt

@@ -14,7 +14,13 @@
 # TODO(odyssey4me) remove this once https://review.openstack.org/288634 has merged
 # and the disk images are rebuilt and redeployed.
 curl
+wget
 
 # Requirements for Paramiko 2.0
-libssl-dev
-libffi-dev
+libssl-dev        [platform:dpkg]
+libffi-dev        [platform:dpkg]
+libffi-devel      [platform:rpm]
+openssl-devel     [platform:rpm]
+
+# For selinux
+libselinux-python [platform:rpm]

releasenotes/notes/deprecate-repo-apt-packages-f8c4a22fc60828bf.yaml

@@ -0,0 +1,5 @@
+---
+deprecations:
+  - The ``repo_apt_packages`` variable has been deprecated.
+    ``repo_server_packages`` should be used instead to override
+    packages required to install a repo server.

tasks/install_apt.yml

@@ -0,0 +1,43 @@
+---
+# Copyright 2016, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#TODO(evrardjp): Replace the next 2 tasks by a standard apt with cache
+#when https://github.com/ansible/ansible-modules-core/pull/1517 is merged
+#in 1.9.x or we move to 2.0 (if tested working)
+- name: Check apt last update file
+  stat:
+    path: /var/cache/apt
+  register: apt_cache_stat
+  tags:
+    - repo-packages
+
+- name: Update apt if needed
+  apt:
+    update_cache: yes
+  when: "ansible_date_time.epoch|float - apt_cache_stat.stat.mtime > {{cache_timeout}}"
+  tags:
+    - repo-packages
+
+- name: Install repo server packages
+  apt:
+    pkg: "{{ item }}"
+    state: latest
+  register: install_packages
+  until: install_packages|success
+  retries: 5
+  delay: 5
+  with_items: "{{ repo_server_packages | deprecated(repo_apt_packages, 'repo_apt_packages', 'repo_server_packages', 'Ocata') }}"
+  tags:
+    - repo-packages

tasks/install_yum.yml

@@ -0,0 +1,26 @@
+---
+# Copyright 2016, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Install repo server packages
+  yum:
+    pkg: "{{ item }}"
+    state: latest
+  register: install_packages
+  until: install_packages|success
+  retries: 5
+  delay: 5
+  with_items: "{{ repo_server_packages }}"
+  tags:
+    - repo-packages

tasks/main.yml

@@ -1,5 +1,5 @@
 ---
-# Copyright 2014, Rackspace US, Inc.
+# Copyright 2016, Rackspace US, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,6 +13,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+- name: Gather variables for each operating system
+  include_vars: "{{ item }}"
+  with_first_found:
+    - "{{ ansible_distribution | lower }}-{{ ansible_distribution_release | lower }}.yml"
+    - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml"
+    - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
+    - "{{ ansible_distribution | lower }}.yml"
+    - "{{ ansible_os_family | lower }}.yml"
+  tags:
+    - always
+
+- name: Check init system
+  command: cat /proc/1/comm
+  register: _pid1_name
+  tags:
+    - always
+
+- name: Set the name of pid1
+  set_fact:
+    pid1_name: "{{ _pid1_name.stdout }}"
+  tags:
+    - always
+
 - include: repo_pre_install.yml
 - include: repo_install.yml
 - include: repo_post_install.yml

tasks/repo_install.yml

@@ -1,5 +1,5 @@
 ---
-# Copyright 2014, Rackspace US, Inc.
+# Copyright 2016, Rackspace US, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,31 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-#TODO(evrardjp): Replace the next 2 tasks by a standard apt with cache
-#when https://github.com/ansible/ansible-modules-core/pull/1517 is merged
-#in 1.9.x or we move to 2.0 (if tested working)
-- name: Check apt last update file
-  stat:
-    path: /var/cache/apt
-  register: apt_cache_stat
+- include: install_apt.yml
+  when:
+    - ansible_pkg_mgr == 'apt'
   tags:
     - repo-apt-packages
 
-- name: Update apt if needed
-  apt:
-    update_cache: yes
-  when: "ansible_date_time.epoch|float - apt_cache_stat.stat.mtime > {{cache_timeout}}"
+- include: install_yum.yml
+  when:
+    - ansible_pkg_mgr == 'yum'
   tags:
-    - repo-apt-packages
-
-- name: Install apt packages
-  apt:
-    pkg: "{{ item }}"
-    state: latest
-  register: install_packages
-  until: install_packages|success
-  retries: 5
-  delay: 5
-  with_items: repo_apt_packages
-  tags:
-    - repo-apt-packages
+    - repo-yum-packages

tasks/repo_post_install.yml

@@ -1,5 +1,5 @@
 ---
-# Copyright 2014, Rackspace US, Inc.
+# Copyright 2016, Rackspace US, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -68,6 +68,8 @@
     - { path: "/etc/lsyncd", state: "directory", mode: "0755" }
     - { path: "/var/log/lsyncd", state: "directory", mode: "0755" }
     - { path: "/etc/nginx/sites-enabled/default", state: "absent", mode: "0644" }
+    - { path: "/etc/nginx/sites-available", state: "directory", mode: "0644" }
+    - { path: "/etc/nginx/sites-enabled", state: "directory", mode: "0644" }
   tags:
     - pkg-repo-dirs
 
@@ -85,7 +87,7 @@
     - pkg-repo-nginx
     - pkg-repo-config
 
-- name: Change fcgiwrap GID/UID
+- name: Change fcgiwrap GID/UID - Debian
   lineinfile:
     dest: "/etc/init.d/fcgiwrap"
     regexp: "{{ item.regexp }}"
@@ -96,6 +98,7 @@
   notify:
     - reload nginx
     - reload fcgiwrap
+  when: ansible_os_family == "Debian"
   tags:
     - pkg-repo-nginx
     - pkg-repo-config
@@ -110,3 +113,36 @@
   tags:
     - pkg-repo-nginx
     - pkg-repo-config
+
+- name: Place git daemon upstart init script
+  template:
+    src: "git-daemon-upstart-init.j2"
+    dest: "/etc/init/git-daemon.conf"
+    mode: "0644"
+    owner: "root"
+    group: "root"
+  when:
+    - pid1_name == "init"
+    - ansible_distribution == "Ubuntu"
+  notify:
+    - Init reload
+    - reload git-daemon
+  tags:
+    - git-daemon-init
+
+- name: Place git daemon upstart init script
+  template:
+    src: "{{ item.src }}"
+    dest: "{{ item.dest }}"
+    mode: "0644"
+    owner: "root"
+    group: "root"
+  with_items:
+    - { src: "git.service.j2", dest: "/lib/systemd/system/git@.service" }
+    - { src: "git.socket.j2", dest: "/lib/systemd/system/git.socket" }
+  when:
+    - pid1_name == "systemd"
+  notify:
+    - reload git socket
+  tags:
+    - git-daemon-init

templates/git-daemon-upstart-init.j2

@@ -0,0 +1,9 @@
+start on startup
+stop on shutdown
+exec /usr/bin/git daemon \
+    --user=nobody \
+    --export-all \
+    --reuseaddr \
+    --base-path={{ repo_service_home_folder }}/repo/openstackgit/ \
+    {{ repo_service_home_folder }}/repo/openstackgit/
+respawn

templates/git.service.j2

@@ -0,0 +1,8 @@
+[Unit]
+Description=Git Repositories Server Daemon
+Documentation=man:git-daemon(1)
+
+[Service]
+User=nobody
+ExecStart=-{{ git_daemon_path }} --base-path={{ repo_service_home_folder }}/repo/openstackgit/ --export-all --user-path=public_git --syslog --inetd --verbose
+StandardInput=socket

templates/git.socket.j2

@@ -0,0 +1,9 @@
+[Unit]
+Description=Git Activation Socket
+
+[Socket]
+ListenStream=9418
+Accept=true
+
+[Install]
+WantedBy=sockets.target

templates/nginx.conf.j2

@@ -1,5 +1,5 @@
 user {{ repo_service_user_name }} {{ repo_service_group_name }};
-pid /var/run/nginx.pid;
+pid {{ repo_nginx_pid }};
 
 {% set _api_threads = ansible_processor_vcpus|default(2) // 2 %}
 {% set api_threads = _api_threads if _api_threads > 0 else 1 %}

tests/test-prepare-containers.yml

@@ -15,7 +15,6 @@
 
 - name: Create test containers
   hosts: all_containers
-  connection: local
   gather_facts: false
   pre_tasks:
     - name: Destroy test containers

tests/test-prepare-host.yml

@@ -19,6 +19,7 @@
     - name: Ensure apt cache is always refreshed
       apt:
         update_cache: yes
+      when: ansible_pkg_mgr == 'apt'
     - name: Ensure root's new public ssh key is in authorized_keys
       authorized_key:
         user: root
@@ -30,20 +31,31 @@
       stat:
         path: /etc/nodepool/provider
       register: nodepool
-    - name: Set the files to copy into the container cache for OpenStack-CI instances
+    - name: Set the files to copy into the container cache for OpenStack-CI instances (deb)
       set_fact:
         lxc_container_cache_files:
           - { src: '/etc/pip.conf', dest: '/etc/pip.conf' }
           - { src: '/etc/apt/apt.conf.d/99unauthenticated', dest: '/etc/apt/apt.conf.d/99unauthenticated' }
-      when: nodepool.stat.exists | bool
+      when:
+        - nodepool.stat.exists | bool
+        - ansible_pkg_mgr == 'apt'
+    - name: Set the files to copy into the container cache for OpenStack-CI instances (rpm)
+      set_fact:
+        lxc_container_cache_files:
+          - { src: '/etc/pip.conf', dest: '/etc/pip.conf' }
+      when:
+        - nodepool.stat.exists | bool
+        - ansible_pkg_mgr == 'yum'
     - name: Determine the existing Ubuntu repo configuration
       shell: 'awk "/^deb .*ubuntu\/? {{ ansible_distribution_release }} main/ {print \$2; exit}" /etc/apt/sources.list'
       register: ubuntu_repo
       changed_when: false
+      when: ansible_pkg_mgr == 'apt'
     - name: Set apt repo facts based on discovered information
       set_fact:
         lxc_container_template_main_apt_repo: "{{ ubuntu_repo.stdout }}"
         lxc_container_template_security_apt_rep: "{{ ubuntu_repo.stdout }}"
+      when: ansible_pkg_mgr == 'apt'
   roles:
     - role: "lxc_hosts"
       lxc_net_address: 10.100.100.1

tests/test-repo-server-functional.yml

@@ -40,3 +40,14 @@
       assert:
         that:
           - "repo_dir.stat.exists"
+    - name: Clone repo for testing
+      git:
+        repo: "https://git.openstack.org/openstack/openstack-ansible-repo_server"
+        dest: /var/www/repo/openstackgit/repo_server
+        version: master
+    - name: Check git daemon is functioning
+      git:
+        repo: "git://localhost/repo_server"
+        dest: /tmp/repo_server
+        version: master
+        accept_hostkey: yes

vars/debian.yml

@@ -0,0 +1,26 @@
+---
+# Copyright 2016, Walmart Stores, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+repo_nginx_pid: /var/run/nginx.pid
+
+git_daemon_path: /usr/lib/git-core/git-daemon
+rsyncd_service_name: rsync
+
+repo_server_packages:
+  - fcgiwrap
+  - git
+  - lsyncd
+  - nginx-extras
+  - rsync

vars/redhat.yml

@@ -0,0 +1,25 @@
+---
+# Copyright 2016, Walmart Stores, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+repo_nginx_pid: /run/nginx.pid
+
+git_daemon_path: /usr/libexec/git-core/git-daemon
+rsyncd_service_name: rsyncd
+
+repo_server_packages:
+  - git-daemon
+  - lsyncd
+  - nginx
+  - rsync