Allow the use of a pre-staged git cache

This patch provides the ability for the repo server to take advantage
of a pre-staged git cache on the deployment host, such as that provided
on OpenStack-CI's nodes.

This reduces the execution time of the build process by ensuring that
the git clone process only has to complete an update, instead of a
fresh clone.

The patch includes a change to the AIO bootstrap which checks for the
existance of a git cache in the location it is typically found in
OpenStack-CI. If that path exists it sets the repo_build_git_cache
variable. In a case where the AIO is not being built in OpenStack-CI
then the deployer may either use the same path, or provide the path
as an extra option when bootstrapping.

Change-Id: I0c9d574a7f694cd3772a68fcdc71673616ea8e3b

playbooks/inventory/group_vars/repo_all.yml

@@ -16,3 +16,30 @@
 # Ensure that the package state matches the global setting
 repo_server_package_state: "{{ package_state }}"
 repo_build_package_state: "{{ package_state }}"
+
+# The default bind mount to hold the repo data
+repo_all_lxc_container_bind_mounts:
+  - mount_path: "/openstack/{{ inventory_hostname }}"
+    bind_dir_path: "/var/www"
+
+# Optionally set this variable to the location on the deployment
+# host where a set of git clones may be sourced to stage the repo
+# server.
+#repo_build_git_cache: /opt/git/openstack/
+
+# The folder in the repo container where the git clones should
+# be synchronised to.
+repo_build_git_dir: /var/www/repo/openstackgit
+
+# The folder in the repo container which is bind-mounted to
+# the host.
+repo_service_home_folder: /var/www
+
+# The folder on the repo container's host which will hold
+# the git clones via the container-host bind-mount
+repo_build_git_bind_mount: "/openstack/{{ inventory_hostname }}{{ repo_build_git_dir | replace(repo_service_home_folder, '') }}"
+
+# The appropriate user:group names for the repo_build_git_dir
+# folder/file attributes.
+repo_service_user_name: nginx
+repo_service_group_name: www-data

playbooks/repo-build.yml

@@ -39,6 +39,64 @@
       register: local_packages
       tags:
         - always
+
+    - name: Check if the git cache exists on deployment host
+      local_action:
+        module: stat
+        path: "{{ repo_build_git_cache }}"
+      register: _local_git_cache
+      when: repo_build_git_cache is defined
+
+    - name: Check if deployment host has the bind mount for the master repo container
+      local_action:
+        module: stat
+        path: "{{ repo_build_git_bind_mount }}"
+      register: _local_repo_bind_mount
+      when:
+        - inventory_hostname == groups['repo_all'][0]
+        - _local_git_cache.stat.exists
+
+    - name: Check if the repo container bind mount is empty
+      local_action: shell
+        ls -1A {{ repo_build_git_bind_mount }}
+      register: _local_repo_bind_mount_contents
+      when:
+        - _local_repo_bind_mount.stat is defined
+        - _local_repo_bind_mount.stat.exists
+        - not _local_repo_bind_mount.stat.islnk
+
+    - name: Remove the bind mounted git directory so we can symlink the cache folder to replace it
+      local_action:
+        module: file
+        path: "{{ repo_build_git_bind_mount }}"
+        state: absent
+      register: _local_repo_bind_mount_dir_remove
+      when:
+        - _local_repo_bind_mount.stat is defined
+        - _local_repo_bind_mount.stat.exists
+        - not _local_repo_bind_mount.stat.islnk
+        - _local_repo_bind_mount_contents.stdout_lines | length == 0
+
+    - name: Symlink the git cache into the repo container bind mount
+      local_action:
+        module: file
+        src: "{{ repo_build_git_cache }}"
+        dest: "{{ repo_build_git_bind_mount }}"
+        state: link
+      when:
+        - _local_git_cache.stat.exists
+        - _local_repo_bind_mount.stat is defined
+        - _local_repo_bind_mount.stat.exists
+
+    - name: Synchronise the contents of the git cache to the repo server
+      synchronize:
+        src: "{{ repo_build_git_cache }}"
+        dest: "{{ repo_build_git_dir }}"
+      when:
+        - _local_git_cache.stat.exists
+        - _local_repo_bind_mount.stat is defined
+        - not _local_repo_bind_mount.stat.exists
+
   roles:
     - role: "repo_build"
       repo_build_release_tag: "{{ openstack_release }}"
@@ -47,6 +105,7 @@
         - "inventory_hostname == groups['repo_servers_{{ ansible_architecture }}'][0]"
       tags:
         - "repo-build"
+
   vars:
     pkg_locations:
       - ../

playbooks/repo-server.yml

@@ -19,11 +19,29 @@
   max_fail_percentage: 20
   user: root
   pre_tasks:
+
+    - name: Check if the git cache exists on deployment host
+      local_action:
+        module: stat
+        path: "{{ repo_build_git_cache }}"
+      register: _local_git_cache
+      when: repo_build_git_cache is defined
+
     - include: common-tasks/os-lxc-container-setup.yml
       vars:
-        list_of_bind_mounts:
-          - bind_dir_path: "/var/www"
-            mount_path: "/openstack/{{ inventory_hostname }}"
+        list_of_bind_mounts: "{{ repo_all_lxc_container_bind_mounts }}"
+      when: repo_build_git_cache is not defined or not _local_git_cache.stat.exists
+
+    - include: common-tasks/os-lxc-container-setup.yml
+      vars:
+        repo_build_git_cache_bind_mount:
+          - mount_path: "{{ repo_build_git_cache }}"
+            bind_dir_path: "{{ repo_build_git_cache }}"
+        list_of_bind_mounts: "{{ repo_all_lxc_container_bind_mounts + repo_build_git_cache_bind_mount }}"
+      when:
+        - repo_build_git_cache is defined
+        - _local_git_cache.stat.exists
+
   roles:
     - { role: "repo_server", tags: [ "repo-server" ] }
     - role: "rsyslog_client"

releasenotes/notes/git-cache-staged-b9cb0e277478b19a.yaml

@@ -0,0 +1,9 @@
+---
+features:
+  - The repo build process is now able to synchronize a git cache from the
+    deployment node to the repo server. The git cache path on the deployment
+    node is set using the variable ``repo_build_git_cache``. If the
+    deployment node hosts the repo container, then the folder will be
+    symlinked into the bind mount for the repo container. If the deployment
+    node does not host the repo container, then the contents of the folder
+    will be synchronised into the repo container.

tests/roles/bootstrap-host/tasks/prepare_aio_config.yml

@@ -100,6 +100,16 @@
     repo_build_pip_extra_indexes: "[\"{{ fastest_wheel_mirror.stdout }}\"]"
   when: not pip_conf_file.stat.exists
 
+- name: Check whether the host has a git cache
+  stat:
+    path: /opt/git/openstack
+  register: _local_git_cache
+
+- name: Set repo_build_git_cache fact
+  set_fact:
+    repo_build_git_cache: /opt/git/openstack
+  when: _local_git_cache.stat.exists
+
 - name: Set the user_variables
   config_template:
     src: user_variables.aio.yml.j2

tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2

@@ -88,6 +88,11 @@ uca_apt_repo_url: {{ uca_apt_repo_url }}
 cache_timeout: {{ cache_timeout }}
 {% endif %}
 
+{% if repo_build_git_cache is defined %}
+## Git cache to use for the repo build process
+repo_build_git_cache: {{ repo_build_git_cache }}
+{% endif %}
+
 ## Instruct the gate to always build libvirt-python from pip source
 repo_build_pip_no_binary:
   - libvirt-python