Add wheel mirror to mirrors
This patch adds a new "wheel" directory to the pypi mirrors, as an rsync target for our built wheel packages. A rewrite rule has been added in anticipation of serving the wheels from an AFS drive. Since AFS has a practical folder size limit, we are using /a/a /s/sp/split /s/st/style directory structure that should be AFS-tolerant. The rewrite rule creates the necessary mappings that make the packages available to pip. Example: HTTP GET /Babel/ -> /B/Ba/Babel/ Furthermore, a cron job has been added to periodically generate a human-readable index of these mappings, in accordance with PEP503. While frequently regenerated, this index should only change meaningfully if a new package is added to the wheel, as it only represents the package names themselves, rather than the available versions of said package. Change-Id: I743fc3ec629eea225c981d6e870751f33e77d7c6
This commit is contained in:
Michael Krotscheck
parent
5e63349100
commit
787b408d84
@ -0,0 +1,30 @@
|
||||
#!/bin/bash
|
||||
|
||||
# This script generates an index file for every directory in the wheel mirror
|
||||
# hierarchy.
|
||||
|
||||
DATA_DIRECTORY=$1
|
||||
|
||||
function build_index {
|
||||
index="<!DOCTYPE html>"
|
||||
index="$index<html><body><ul>"
|
||||
|
||||
# Match all strings that are /A/A(B/AB[C..])?$
|
||||
regex="\/([^/])\/\1(([^/])\/\1\3[^/]+)?$"
|
||||
for f in $(find $1 -type d -mindepth 2 -maxdepth 3); do
|
||||
# Pull only the AFS-Matching directories
|
||||
if [[ $f =~ $regex ]]; then
|
||||
# Get the last name in the folder path, the package name.
|
||||
dir=$(basename $BASH_REMATCH)
|
||||
# Echo it out.
|
||||
index="$index<li><a href=\"$dir/\">$dir</a></li>"
|
||||
fi
|
||||
done
|
||||
|
||||
index="$index</ul></body></html>"
|
||||
echo $index > $2
|
||||
}
|
||||
|
||||
for dir in $DATA_DIRECTORY/*; do
|
||||
build_index $dir "$dir/index.html"
|
||||
done
|
||||
@ -12,6 +12,8 @@ class openstack_project::pypi (
|
||||
|
||||
$mirror_root = '/srv/static'
|
||||
$pypi_root = "${mirror_root}/mirror"
|
||||
$www_root = "${pypi_root}/web"
|
||||
$wheel_root = "${www_root}/wheel"
|
||||
|
||||
if ! defined(File[$mirror_root]) {
|
||||
file { $mirror_root:
|
||||
@ -24,16 +26,28 @@ class openstack_project::pypi (
|
||||
require => File[$mirror_root]
|
||||
}
|
||||
|
||||
class { 'openstack_project::wheel_mirror':
|
||||
data_directory => "${wheel_root}",
|
||||
require => Class['Openstack_project::Pypi_mirror'],
|
||||
}
|
||||
|
||||
include ::httpd
|
||||
|
||||
if ! defined(Httpd::Mod['rewrite']) {
|
||||
httpd::mod { 'rewrite':
|
||||
ensure => present,
|
||||
}
|
||||
}
|
||||
|
||||
::httpd::vhost { $vhost_name:
|
||||
port => 80,
|
||||
priority => '50',
|
||||
docroot => "${pypi_root}/web",
|
||||
docroot => $www_root,
|
||||
require => Class['Openstack_project::Pypi_mirror'],
|
||||
template => 'openstack_project/pypi.vhost.erb',
|
||||
}
|
||||
|
||||
file { "${pypi_root}/web/robots.txt":
|
||||
file { "${www_root}/robots.txt":
|
||||
ensure => present,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
|
||||
43
modules/openstack_project/manifests/wheel_mirror.pp
Normal file
43
modules/openstack_project/manifests/wheel_mirror.pp
Normal file
@ -0,0 +1,43 @@
|
||||
# == Class: openstack_project::wheel_mirror
|
||||
#
|
||||
class openstack_project::wheel_mirror (
|
||||
$data_directory = '/srv/static/wheel',
|
||||
$config_directory = '/etc/wheel_mirror'
|
||||
) {
|
||||
|
||||
# The wheel mirror is a directory of python wheels, which have been rsynced'
|
||||
# from the wheel build slaves.
|
||||
file { "${data_directory}":
|
||||
ensure => directory,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
}
|
||||
|
||||
file { "${config_directory}":
|
||||
ensure => directory,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
}
|
||||
|
||||
file { "${config_directory}/rebuild_wheel_afs_index.sh":
|
||||
ensure => present,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
mode => '0755',
|
||||
source => "puppet:///modules/openstack_project/mirror/rebuild_wheel_afs_index.sh",
|
||||
require => [
|
||||
File["${config_directory}"],
|
||||
]
|
||||
}
|
||||
|
||||
# */15 * * * *
|
||||
cron { 'rebuild wheel afs index':
|
||||
name => 'rebuild-wheel-afs-index.cron',
|
||||
command => "/bin/bash ${config_directory}/rebuild_wheel_afs_index.sh ${data_directory}",
|
||||
user => root,
|
||||
minute => '*/15',
|
||||
require => [
|
||||
File["${config_directory}/rebuild_wheel_afs_index.sh"],
|
||||
]
|
||||
}
|
||||
}
|
||||
37
modules/openstack_project/templates/pypi.vhost.erb
Normal file
37
modules/openstack_project/templates/pypi.vhost.erb
Normal file
@ -0,0 +1,37 @@
|
||||
# ************************************
|
||||
# Managed by Puppet
|
||||
# ************************************
|
||||
|
||||
NameVirtualHost <%= @vhost_name %>:<%= @port %>
|
||||
<VirtualHost <%= @vhost_name %>:<%= @port %>>
|
||||
ServerName <%= @srvname %>
|
||||
<% if @serveraliases.is_a? Array -%>
|
||||
<% @serveraliases.each do |name| -%><%= " ServerAlias #{name}\n" %><% end -%>
|
||||
<% elsif @serveraliases != nil -%>
|
||||
<%= " ServerAlias #{@serveraliases}" -%>
|
||||
<% end -%>
|
||||
DocumentRoot <%= @www_root %>
|
||||
<Directory <%= @www_root %>>
|
||||
Options <%= @options %>
|
||||
AllowOverride None
|
||||
Order allow,deny
|
||||
allow from all
|
||||
Satisfy any
|
||||
<IfVersion >= 2.4>
|
||||
Require all granted
|
||||
</IfVersion>
|
||||
</Directory>
|
||||
|
||||
# URL's are:
|
||||
# /wheel/{distro}-{distro-version}/a/a/a-etc.whl
|
||||
# /wheel/{distro}-{distro-version}/a/ab/abcd/abcd-etc.whl
|
||||
# /wheel/{distro}-{distro-version}/a/ab/abcde/abcde-etc.whl
|
||||
RewriteEngine On
|
||||
RewriteRule ^/wheel/([^/]+)/([^/])/(.*)$ /wheel/$1/$2/$2/$3 [L]
|
||||
RewriteRule ^/wheel/([^/]+)/([^/])([^/])([^/]*)/(.*)$ /wheel/$1/$2/$2$3/$2$3$4/$5 [L]
|
||||
|
||||
ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_error.log
|
||||
LogLevel warn
|
||||
CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_access.log combined
|
||||
ServerSignature Off
|
||||
</VirtualHost>
|
||||
Loading…
Reference in New Issue
Block a user