From 787b408d841b544fd532fea161f705a9fdafa6c0 Mon Sep 17 00:00:00 2001 From: Michael Krotscheck Date: Fri, 4 Dec 2015 06:25:30 -0800 Subject: [PATCH] Add wheel mirror to mirrors This patch adds a new "wheel" directory to the pypi mirrors, as an rsync target for our built wheel packages. A rewrite rule has been added in anticipation of serving the wheels from an AFS drive. Since AFS has a practical folder size limit, we are using /a/a /s/sp/split /s/st/style directory structure that should be AFS-tolerant. The rewrite rule creates the necessary mappings that make the packages available to pip. Example: HTTP GET /Babel/ -> /B/Ba/Babel/ Furthermore, a cron job has been added to periodically generate a human-readable index of these mappings, in accordance with PEP503. While frequently regenerated, this index should only change meaningfully if a new package is added to the wheel, as it only represents the package names themselves, rather than the available versions of said package. Change-Id: I743fc3ec629eea225c981d6e870751f33e77d7c6 --- .../files/mirror/rebuild_wheel_afs_index.sh | 30 +++++++++++++ modules/openstack_project/manifests/pypi.pp | 18 +++++++- .../manifests/wheel_mirror.pp | 43 +++++++++++++++++++ .../templates/pypi.vhost.erb | 37 ++++++++++++++++ 4 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 modules/openstack_project/files/mirror/rebuild_wheel_afs_index.sh create mode 100644 modules/openstack_project/manifests/wheel_mirror.pp create mode 100644 modules/openstack_project/templates/pypi.vhost.erb diff --git a/modules/openstack_project/files/mirror/rebuild_wheel_afs_index.sh b/modules/openstack_project/files/mirror/rebuild_wheel_afs_index.sh new file mode 100644 index 0000000000..9601232654 --- /dev/null +++ b/modules/openstack_project/files/mirror/rebuild_wheel_afs_index.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# This script generates an index file for every directory in the wheel mirror +# hierarchy. + +DATA_DIRECTORY=$1 + +function build_index { + index="" + index="$index" + echo $index > $2 +} + +for dir in $DATA_DIRECTORY/*; do + build_index $dir "$dir/index.html" +done diff --git a/modules/openstack_project/manifests/pypi.pp b/modules/openstack_project/manifests/pypi.pp index 0580021069..ec9f195b66 100644 --- a/modules/openstack_project/manifests/pypi.pp +++ b/modules/openstack_project/manifests/pypi.pp @@ -12,6 +12,8 @@ class openstack_project::pypi ( $mirror_root = '/srv/static' $pypi_root = "${mirror_root}/mirror" + $www_root = "${pypi_root}/web" + $wheel_root = "${www_root}/wheel" if ! defined(File[$mirror_root]) { file { $mirror_root: @@ -24,16 +26,28 @@ class openstack_project::pypi ( require => File[$mirror_root] } + class { 'openstack_project::wheel_mirror': + data_directory => "${wheel_root}", + require => Class['Openstack_project::Pypi_mirror'], + } + include ::httpd + if ! defined(Httpd::Mod['rewrite']) { + httpd::mod { 'rewrite': + ensure => present, + } + } + ::httpd::vhost { $vhost_name: port => 80, priority => '50', - docroot => "${pypi_root}/web", + docroot => $www_root, require => Class['Openstack_project::Pypi_mirror'], + template => 'openstack_project/pypi.vhost.erb', } - file { "${pypi_root}/web/robots.txt": + file { "${www_root}/robots.txt": ensure => present, owner => 'root', group => 'root', diff --git a/modules/openstack_project/manifests/wheel_mirror.pp b/modules/openstack_project/manifests/wheel_mirror.pp new file mode 100644 index 0000000000..9fcde74b0f --- /dev/null +++ b/modules/openstack_project/manifests/wheel_mirror.pp @@ -0,0 +1,43 @@ +# == Class: openstack_project::wheel_mirror +# +class openstack_project::wheel_mirror ( + $data_directory = '/srv/static/wheel', + $config_directory = '/etc/wheel_mirror' +) { + + # The wheel mirror is a directory of python wheels, which have been rsynced' + # from the wheel build slaves. + file { "${data_directory}": + ensure => directory, + owner => 'root', + group => 'root', + } + + file { "${config_directory}": + ensure => directory, + owner => 'root', + group => 'root', + } + + file { "${config_directory}/rebuild_wheel_afs_index.sh": + ensure => present, + owner => 'root', + group => 'root', + mode => '0755', + source => "puppet:///modules/openstack_project/mirror/rebuild_wheel_afs_index.sh", + require => [ + File["${config_directory}"], + ] + } + + # */15 * * * * + cron { 'rebuild wheel afs index': + name => 'rebuild-wheel-afs-index.cron', + command => "/bin/bash ${config_directory}/rebuild_wheel_afs_index.sh ${data_directory}", + user => root, + minute => '*/15', + require => [ + File["${config_directory}/rebuild_wheel_afs_index.sh"], + ] + } +} diff --git a/modules/openstack_project/templates/pypi.vhost.erb b/modules/openstack_project/templates/pypi.vhost.erb new file mode 100644 index 0000000000..aa60088670 --- /dev/null +++ b/modules/openstack_project/templates/pypi.vhost.erb @@ -0,0 +1,37 @@ +# ************************************ +# Managed by Puppet +# ************************************ + +NameVirtualHost <%= @vhost_name %>:<%= @port %> +:<%= @port %>> + ServerName <%= @srvname %> + <% if @serveraliases.is_a? Array -%> + <% @serveraliases.each do |name| -%><%= " ServerAlias #{name}\n" %><% end -%> + <% elsif @serveraliases != nil -%> + <%= " ServerAlias #{@serveraliases}" -%> + <% end -%> + DocumentRoot <%= @www_root %> + > + Options <%= @options %> + AllowOverride None + Order allow,deny + allow from all + Satisfy any + = 2.4> + Require all granted + + + + # URL's are: + # /wheel/{distro}-{distro-version}/a/a/a-etc.whl + # /wheel/{distro}-{distro-version}/a/ab/abcd/abcd-etc.whl + # /wheel/{distro}-{distro-version}/a/ab/abcde/abcde-etc.whl + RewriteEngine On + RewriteRule ^/wheel/([^/]+)/([^/])/(.*)$ /wheel/$1/$2/$2/$3 [L] + RewriteRule ^/wheel/([^/]+)/([^/])([^/])([^/]*)/(.*)$ /wheel/$1/$2/$2$3/$2$3$4/$5 [L] + + ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_error.log + LogLevel warn + CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_access.log combined + ServerSignature Off +