From adebdb884ac962f4b76c1b99a981f6de51b08490 Mon Sep 17 00:00:00 2001 From: Jesse Pretorius Date: Sun, 24 Jul 2016 13:04:50 +0100 Subject: [PATCH] Allow the use of a pre-staged git cache This patch provides the ability for the repo server to take advantage of a pre-staged git cache on the deployment host, such as that provided on OpenStack-CI's nodes. This reduces the execution time of the build process by ensuring that the git clone process only has to complete an update, instead of a fresh clone. The patch includes a change to the AIO bootstrap which checks for the existance of a git cache in the location it is typically found in OpenStack-CI. If that path exists it sets the repo_build_git_cache variable. In a case where the AIO is not being built in OpenStack-CI then the deployer may either use the same path, or provide the path as an extra option when bootstrapping. Change-Id: I0c9d574a7f694cd3772a68fcdc71673616ea8e3b --- playbooks/inventory/group_vars/repo_all.yml | 27 +++++++++ playbooks/repo-build.yml | 59 +++++++++++++++++++ playbooks/repo-server.yml | 24 +++++++- .../git-cache-staged-b9cb0e277478b19a.yaml | 9 +++ .../tasks/prepare_aio_config.yml | 10 ++++ .../templates/user_variables.aio.yml.j2 | 5 ++ 6 files changed, 131 insertions(+), 3 deletions(-) create mode 100644 releasenotes/notes/git-cache-staged-b9cb0e277478b19a.yaml diff --git a/playbooks/inventory/group_vars/repo_all.yml b/playbooks/inventory/group_vars/repo_all.yml index de5a1fb5bf..b13fc265ff 100644 --- a/playbooks/inventory/group_vars/repo_all.yml +++ b/playbooks/inventory/group_vars/repo_all.yml @@ -16,3 +16,30 @@ # Ensure that the package state matches the global setting repo_server_package_state: "{{ package_state }}" repo_build_package_state: "{{ package_state }}" + +# The default bind mount to hold the repo data +repo_all_lxc_container_bind_mounts: + - mount_path: "/openstack/{{ inventory_hostname }}" + bind_dir_path: "/var/www" + +# Optionally set this variable to the location on the deployment +# host where a set of git clones may be sourced to stage the repo +# server. +#repo_build_git_cache: /opt/git/openstack/ + +# The folder in the repo container where the git clones should +# be synchronised to. +repo_build_git_dir: /var/www/repo/openstackgit + +# The folder in the repo container which is bind-mounted to +# the host. +repo_service_home_folder: /var/www + +# The folder on the repo container's host which will hold +# the git clones via the container-host bind-mount +repo_build_git_bind_mount: "/openstack/{{ inventory_hostname }}{{ repo_build_git_dir | replace(repo_service_home_folder, '') }}" + +# The appropriate user:group names for the repo_build_git_dir +# folder/file attributes. +repo_service_user_name: nginx +repo_service_group_name: www-data diff --git a/playbooks/repo-build.yml b/playbooks/repo-build.yml index d83fc16164..74a5769aa9 100644 --- a/playbooks/repo-build.yml +++ b/playbooks/repo-build.yml @@ -39,6 +39,64 @@ register: local_packages tags: - always + + - name: Check if the git cache exists on deployment host + local_action: + module: stat + path: "{{ repo_build_git_cache }}" + register: _local_git_cache + when: repo_build_git_cache is defined + + - name: Check if deployment host has the bind mount for the master repo container + local_action: + module: stat + path: "{{ repo_build_git_bind_mount }}" + register: _local_repo_bind_mount + when: + - inventory_hostname == groups['repo_all'][0] + - _local_git_cache.stat.exists + + - name: Check if the repo container bind mount is empty + local_action: shell + ls -1A {{ repo_build_git_bind_mount }} + register: _local_repo_bind_mount_contents + when: + - _local_repo_bind_mount.stat is defined + - _local_repo_bind_mount.stat.exists + - not _local_repo_bind_mount.stat.islnk + + - name: Remove the bind mounted git directory so we can symlink the cache folder to replace it + local_action: + module: file + path: "{{ repo_build_git_bind_mount }}" + state: absent + register: _local_repo_bind_mount_dir_remove + when: + - _local_repo_bind_mount.stat is defined + - _local_repo_bind_mount.stat.exists + - not _local_repo_bind_mount.stat.islnk + - _local_repo_bind_mount_contents.stdout_lines | length == 0 + + - name: Symlink the git cache into the repo container bind mount + local_action: + module: file + src: "{{ repo_build_git_cache }}" + dest: "{{ repo_build_git_bind_mount }}" + state: link + when: + - _local_git_cache.stat.exists + - _local_repo_bind_mount.stat is defined + - _local_repo_bind_mount.stat.exists + + - name: Synchronise the contents of the git cache to the repo server + synchronize: + src: "{{ repo_build_git_cache }}" + dest: "{{ repo_build_git_dir }}" + when: + - _local_git_cache.stat.exists + - _local_repo_bind_mount.stat is defined + - not _local_repo_bind_mount.stat.exists + roles: - role: "repo_build" repo_build_release_tag: "{{ openstack_release }}" @@ -47,6 +105,7 @@ - "inventory_hostname == groups['repo_servers_{{ ansible_architecture }}'][0]" tags: - "repo-build" + vars: pkg_locations: - ../ diff --git a/playbooks/repo-server.yml b/playbooks/repo-server.yml index 6d45587c0c..e65c7c6c91 100644 --- a/playbooks/repo-server.yml +++ b/playbooks/repo-server.yml @@ -19,11 +19,29 @@ max_fail_percentage: 20 user: root pre_tasks: + + - name: Check if the git cache exists on deployment host + local_action: + module: stat + path: "{{ repo_build_git_cache }}" + register: _local_git_cache + when: repo_build_git_cache is defined + - include: common-tasks/os-lxc-container-setup.yml vars: - list_of_bind_mounts: - - bind_dir_path: "/var/www" - mount_path: "/openstack/{{ inventory_hostname }}" + list_of_bind_mounts: "{{ repo_all_lxc_container_bind_mounts }}" + when: repo_build_git_cache is not defined or not _local_git_cache.stat.exists + + - include: common-tasks/os-lxc-container-setup.yml + vars: + repo_build_git_cache_bind_mount: + - mount_path: "{{ repo_build_git_cache }}" + bind_dir_path: "{{ repo_build_git_cache }}" + list_of_bind_mounts: "{{ repo_all_lxc_container_bind_mounts + repo_build_git_cache_bind_mount }}" + when: + - repo_build_git_cache is defined + - _local_git_cache.stat.exists + roles: - { role: "repo_server", tags: [ "repo-server" ] } - role: "rsyslog_client" diff --git a/releasenotes/notes/git-cache-staged-b9cb0e277478b19a.yaml b/releasenotes/notes/git-cache-staged-b9cb0e277478b19a.yaml new file mode 100644 index 0000000000..19bb3e9a54 --- /dev/null +++ b/releasenotes/notes/git-cache-staged-b9cb0e277478b19a.yaml @@ -0,0 +1,9 @@ +--- +features: + - The repo build process is now able to synchronize a git cache from the + deployment node to the repo server. The git cache path on the deployment + node is set using the variable ``repo_build_git_cache``. If the + deployment node hosts the repo container, then the folder will be + symlinked into the bind mount for the repo container. If the deployment + node does not host the repo container, then the contents of the folder + will be synchronised into the repo container. diff --git a/tests/roles/bootstrap-host/tasks/prepare_aio_config.yml b/tests/roles/bootstrap-host/tasks/prepare_aio_config.yml index 2d7f8d50ec..0177830198 100644 --- a/tests/roles/bootstrap-host/tasks/prepare_aio_config.yml +++ b/tests/roles/bootstrap-host/tasks/prepare_aio_config.yml @@ -100,6 +100,16 @@ repo_build_pip_extra_indexes: "[\"{{ fastest_wheel_mirror.stdout }}\"]" when: not pip_conf_file.stat.exists +- name: Check whether the host has a git cache + stat: + path: /opt/git/openstack + register: _local_git_cache + +- name: Set repo_build_git_cache fact + set_fact: + repo_build_git_cache: /opt/git/openstack + when: _local_git_cache.stat.exists + - name: Set the user_variables config_template: src: user_variables.aio.yml.j2 diff --git a/tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2 b/tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2 index 38067a202b..ee8a5a5733 100644 --- a/tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2 +++ b/tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2 @@ -88,6 +88,11 @@ uca_apt_repo_url: {{ uca_apt_repo_url }} cache_timeout: {{ cache_timeout }} {% endif %} +{% if repo_build_git_cache is defined %} +## Git cache to use for the repo build process +repo_build_git_cache: {{ repo_build_git_cache }} +{% endif %} + ## Instruct the gate to always build libvirt-python from pip source repo_build_pip_no_binary: - libvirt-python