Implement pypiserver and pypi proxy cache

This patch implements nginx as a reverse proxy for python
packages. The initial query will be to a local deployment
of pypiserver in order to serve any locally built packages,
but if the package is not available locally it will retry
the query against pypi and cache the response.

Depends-On: Id20a43fed833d53ca0f147f517deafba6587352d
Change-Id: Ic4fd64f4dc82121a65088f3d7f4ae53f373df608
Implements: blueprint python-build-install-simplification
Signed-off-by: Jesse Pretorius <jesse.pretorius@rackspace.co.uk>

defaults/main.yml

@@ -13,6 +13,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+## Verbosity Options
+debug: False
+
 ## Cap the maximum number of threads / workers when a user value is unspecified.
 repo_nginx_threads_max: 16
 repo_nginx_threads: "{{ [[ansible_processor_vcpus|default(2) // 2, 1] | max, repo_nginx_threads_max] | min }}"
@@ -20,9 +23,10 @@ repo_nginx_threads: "{{ [[ansible_processor_vcpus|default(2) // 2, 1] | max, rep
 ## APT Cache Options
 cache_timeout: 600
 
-# Set the package install state for distribution packages
+# Set the package install state for distribution and pip packages
 # Options are 'present' and 'latest'
 repo_server_package_state: "latest"
+repo_server_pip_package_state: "latest"
 
 repo_worker_connections: 1024
 repo_server_name: openstack-slushee
@@ -61,3 +65,42 @@ repo_pkg_cache_group: apt-cacher-ng
 
 # Set the log directory
 repo_service_log_dir: /var/log/apt-cacher-ng
+
+# Required packages to install on the host
+repo_requires_pip_packages:
+  - virtualenv
+  - virtualenv-tools
+
+# Set the list of packages for the pypiserver
+repo_pypiserver_pip_packages:
+ - "pypiserver==1.2.0"
+
+# Set the path to place all built python wheels
+# This is used by pypiserver to serve them
+repo_pypiserver_package_path: "{{ repo_service_home_folder }}/repo/python_packages"
+
+# Path to the pypiserver python virtualenv binaries
+repo_pypiserver_bin: "/openstack/venvs/pypiserver-1.2.0/bin"
+
+# Path to the pypiserver working directory
+repo_pypiserver_working_dir: "{{ repo_service_home_folder }}/pypiserver"
+
+# pypiserver service start options
+repo_pypiserver_start_options: >-
+  -i localhost
+  -p 8080
+  --log-file /var/log/pypiserver/pypiserver.log
+  --disable-fallback
+  {{ (debug | bool) | ternary('-vv', '-v') }}
+  {{ repo_pypiserver_package_path }}
+
+# config override var for systemd init file
+repo_pypiserver_init_overrides: {}
+
+# Set the options for the nginx proxy_cache_path directive.
+# The proxy cache is used for data downloaded from pypi.
+# The default is set to cache up to 1G worth of packages
+# for up to 1 month
+repo_nginx_proxy_cache_path: >-
+  /var/lib/nginx/pypi levels=1:2 keys_zone=pypi:16m inactive=1M max_size=1G
+

handlers/main.yml

@@ -79,3 +79,14 @@
   retries: 5
   delay: 2
 
+- name: reload pypiserver
+  service:
+    name: "pypiserver"
+    enabled: yes
+    state: restarted
+    daemon_reload: "{{ (ansible_service_mgr == 'systemd') | ternary('yes', omit) }}"
+  register: _restart
+  until: _restart | success
+  retries: 5
+  delay: 2
+

meta/main.yml

@@ -38,3 +38,4 @@ dependencies:
   - role: apt_package_pinning
     when:
       - ansible_pkg_mgr == 'apt'
+  - role: pip_install

releasenotes/notes/pypiserver-pypi-cache-216e9e087f6d3f24.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    The repo server now implements nginx as a reverse proxy for python
+    packages sourced from pypi. The initial query will be to a local
+    deployment of pypiserver in order to serve any locally built packages,
+    but if the package is not available locally it will retry
+    the query against pypi and cache the response.

tasks/repo_install.yml

@@ -13,15 +13,42 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- name: Install repo server packages
+- name: Install distro packages
   package:
     name: "{{ repo_server_distro_packages }}"
     state: "{{ repo_server_package_state }}"
     update_cache: "{{ (ansible_pkg_mgr == 'apt') | ternary('yes', omit) }}"
     cache_valid_time: "{{ (ansible_pkg_mgr == 'apt') | ternary(cache_timeout, omit) }}"
   register: install_packages
-  until: install_packages|success
+  until: install_packages | success
   retries: 5
   delay: 5
-  tags:
-    - repo-packages
+
+- name: Install required pip packages
+  pip:
+    name: "{{ repo_requires_pip_packages }}"
+    state: "{{ repo_server_pip_package_state }}"
+    extra_args: >-
+      {{ (pip_install_upper_constraints is defined) | ternary('--constraint ' + pip_install_upper_constraints | default(''),'') }}
+      {{ pip_install_options | default('') }}
+  register: install_packages
+  until: install_packages | success
+  retries: 5
+  delay: 2
+
+- name: Install pip packages
+  pip:
+    name: "{{ repo_pypiserver_pip_packages }}"
+    state: "{{ repo_server_pip_package_state }}"
+    virtualenv: "{{ repo_pypiserver_bin | dirname }}"
+    virtualenv_site_packages: "no"
+    extra_args: >-
+      {{ (pip_install_upper_constraints is defined) | ternary('--constraint ' + pip_install_upper_constraints | default(''),'') }}
+      {{ pip_install_options | default('') }}
+  register: install_packages
+  until: install_packages | success
+  retries: 5
+  delay: 2
+  notify:
+    - reload pypiserver
+

tasks/repo_post_install.yml

@@ -39,6 +39,8 @@
       dest: "/etc/rsyncd.conf"
     - src: "openstack-slushee.vhost.j2"
       dest: "/etc/nginx/sites-available/openstack-slushee.vhost"
+    - src: "nginx-pypi.conf.j2"
+      dest: "/etc/nginx/conf.d/pypi.conf"
   notify:
     - reload nginx
 
@@ -64,3 +66,16 @@
       dest: "{{ systemd_utils_prefix }}/system/git.socket"
   notify:
     - reload git socket
+
+- name: Place the pypiserver systemd init script
+  config_template:
+    src: "pypiserver-systemd-init.j2"
+    dest: "/etc/systemd/system/pypiserver.service"
+    mode: "0644"
+    owner: "root"
+    group: "root"
+    config_overrides: "{{ repo_pypiserver_init_overrides }}"
+    config_type: "ini"
+  notify:
+    - reload pypiserver
+

tasks/repo_pre_install.yml

@@ -62,6 +62,10 @@
     - path: "{{ repo_service_home_folder }}/repo/venvs"
     - path: "/var/log/nginx"
       mode: "0775"
+    - path: "/var/log/pypiserver"
+      mode: "0775"
+    - path: "{{ repo_pypiserver_working_dir }}"
+      mode: "0775"
 
 - name: Drop repo pre/post command script
   template:
@@ -96,6 +100,8 @@
     - path: "/var/log/lsyncd"
     - path: "/etc/nginx/sites-enabled/default"
       state: "absent"
+    - path: "/etc/nginx/conf.d"
     - path: "/etc/nginx/sites-available"
     - path: "/etc/nginx/sites-enabled"
+    - path: "{{ repo_pypiserver_package_path }}"
 

templates/nginx-pypi.conf.j2

@@ -0,0 +1,12 @@
+# {{ ansible_managed }}
+
+proxy_cache_path {{ repo_nginx_proxy_cache_path }};
+
+upstream pypiserver {
+    server localhost:8080;
+}
+
+upstream pypi {
+    server pypi.python.org:443;
+    keepalive 16;
+}

templates/openstack-slushee.vhost.j2

@@ -6,6 +6,40 @@ server {
     access_log /var/log/nginx/{{ repo_server_name }}.access.log gzip buffer=32k;
     error_log /var/log/nginx/{{ repo_server_name }}.error.log notice;
 
+    # Allow cached content to be used even when the upstream source is not available.
+    proxy_cache pypi;
+    proxy_cache_key $uri;
+    proxy_cache_lock on;
+    proxy_cache_revalidate on;
+    proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504;
+
+    proxy_http_version 1.1;
+    proxy_set_header Host $host:$server_port;
+    proxy_set_header Connection "";
+    proxy_set_header Accept-Encoding "";
+
+    # Rewrite any http redirects to use relative to proxy
+    proxy_redirect ~https?://pypi.python.org(.*) $1;
+
+    # Fallback mechanism from:
+    # http://linuxplayer.org/2013/06/nginx-try-files-on-multiple-named-location-or-server
+    location @pypi {
+        proxy_set_header Host pypi.python.org;
+        proxy_pass https://pypi;
+    }
+
+    location /simple {
+        proxy_intercept_errors on;
+        proxy_pass http://pypiserver;
+        error_page 404 = @pypi;
+    }
+
+    location /packages {
+        proxy_intercept_errors on;
+        proxy_pass http://pypiserver;
+        error_page 404 = @pypi;
+    }
+
     location / {
         root {{ repo_service_home_folder }}/repo/;
         autoindex on;

templates/pypiserver-systemd-init.j2

@@ -0,0 +1,32 @@
+# {{ ansible_managed }}
+
+[Unit]
+Description=pypiserver
+After=network.target
+
+[Service]
+Type=simple
+User={{ repo_service_user_name }}
+Group={{ repo_service_group_name }}
+
+ExecStart={{ repo_pypiserver_bin }}/pypi-server {{ repo_pypiserver_start_options }}
+ExecStop=/bin/kill -TERM $MAINPID
+WorkingDirectory={{ repo_pypiserver_working_dir }}
+
+# Give a reasonable amount of time for the server to start up/shut down
+TimeoutSec=120
+TimeoutStartSec=3
+Restart=on-failure
+RestartSec=2
+
+# This creates a specific slice which all services will operate from
+#  The accounting options give us the ability to see resource usage through
+#  the `systemd-cgtop` command.
+Slice=pypiserver.slice
+CPUAccounting=true
+BlockIOAccounting=true
+MemoryAccounting=false
+TasksAccounting=true
+
+[Install]
+WantedBy=multi-user.target