Added NPM Mirror to mirror_update

This patch adds a cron job to our mirror_update servers which will
begin mirroring the npm registry into our own AFS mirror directories,
under the /npm path.

It uses two major components: First, the registry-static project,
available here: https://github.com/davglass/registry-static

Secondly, a custom implementation of the fs-blob-store named
afs-blob-store, which creates a virtual filesystem that automatically
translates paths to accomodate AFS' folder size limit. This project
is currently still hosted on github, however will be added to
OpenStack soon: https://github.com/krotscheck/js-afs-blob-store

This will require creating ACL's as well as the AFS principal before
this can be landed.

Change-Id: Ib78852fc5cde4fb04fe6439e01a591ca2781ce21

manifests/site.pp

@@ -543,6 +543,7 @@ node 'mirror-update.openstack.org' {
     bandersnatch_keytab => hiera('bandersnatch_keytab'),
     admin_keytab        => hiera('afsadmin_keytab'),
     reprepro_keytab     => hiera('reprepro_keytab'),
+    npm_keytab          => hiera('npm_keytab'),
     sysadmins           => hiera('sysadmins', []),
   }
 }

modules/openstack_project/manifests/mirror_update.pp

@@ -5,6 +5,7 @@ class openstack_project::mirror_update (
   $bandersnatch_keytab = '',
   $reprepro_keytab = '',
   $admin_keytab = '',
+  $npm_keytab = '',
 ) {
 
   class { 'openstack_project::server':
@@ -12,6 +13,11 @@ class openstack_project::mirror_update (
     afs       => true,
   }
 
+  class { 'openstack_project::npm_mirror':
+    data_directory => '/afs/.openstack.org/mirror/npm',
+    uri_rewrite    => 'localhost',
+  }
+
   class { 'bandersnatch':
     bandersnatch_source => 'https://bitbucket.org/jeblair/bandersnatch',
   }
@@ -30,6 +36,13 @@ class openstack_project::mirror_update (
     content => $bandersnatch_keytab,
   }
 
+  file { '/etc/npm.keytab':
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0400',
+    content => $npm_keytab,
+  }
+
   file { '/etc/afsadmin.keytab':
     owner   => 'root',
     group   => 'root',
@@ -45,6 +58,14 @@ class openstack_project::mirror_update (
     source  => 'puppet:///modules/openstack_project/bandersnatch-mirror-update.sh',
   }
 
+  file { '/usr/local/bin/npm-mirror-update':
+    ensure   => present,
+    owner    => 'root',
+    group    => 'root',
+    mode     => '0755',
+    content  => template('openstack_project/npm-mirror-update.sh'),
+  }
+
   cron { 'bandersnatch':
     user        => $user,
     minute      => '*/5',
@@ -58,6 +79,19 @@ class openstack_project::mirror_update (
     ]
   }
 
+  cron { 'npm-mirror-update':
+    user        => $user,
+    minute      => '*/5',
+    command     => 'flock -n /var/run/npm-mirror-update/mirror.lock npm-mirror-update >>/var/log/npm-mirror-update/mirror.log 2>&1',
+    environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
+    require     => [
+      File['/usr/local/bin/npm-mirror-update'],
+      File['/etc/afsadmin.keytab'],
+      File['/etc/npm.keytab'],
+      Class['openstack_project::npm_mirror'],
+    ]
+  }
+
   # TODO(clarkb) this setup needs to go in a class of its own. It is not
   # in the define because it is common to all reprepro mirrors.
   package { 'reprepro':

modules/openstack_project/manifests/npm_mirror.pp

@@ -0,0 +1,52 @@
+# == Class: openstack_project::npm_mirror
+#
+class openstack_project::npm_mirror (
+  $uri_rewrite,
+  $data_directory,
+) {
+
+  file { $data_directory:
+    ensure  => directory,
+    owner   => 'root',
+    group   => 'root',
+  }
+
+  class { '::nodejs':
+    repo_url_suffix => 'node_4.x',
+  }
+
+  # See: https://github.com/davglass/registry-static/pull/45
+  package { 'patch-package-json':
+    ensure   => '0.0.4',
+    provider => 'npm',
+    require  => Class['nodejs'],
+  }
+
+  package { 'follow-registry':
+    ensure   => '2.0.0',
+    provider => 'npm',
+    require  => [
+      Class['nodejs'],
+    ]
+  }
+
+  # The registry mirroring script.
+  package { 'registry-static':
+    ensure   => '2.2.0',
+    provider => 'npm',
+    require  => [
+      Class['nodejs'],
+      Package['follow-registry'],
+      Package['patch-package-json'],
+    ]
+  }
+
+  # The afs-blob-store file structure rewriter.
+  package { 'afs-blob-store':
+    ensure   => '1.0.1',
+    provider => 'npm',
+    require  => [
+      Class['nodejs'],
+    ]
+  }
+}

modules/openstack_project/templates/npm-mirror-update.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# Copyright 2016 Hewlett Packard Enterprise Development Corporation, LP
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+CMD="/usr/bin/registry-static"
+CMD_ARGS="-d <%= @uri_rewrite %> -o <%= @data_directory %> --blobstore afs-blob-store"
+
+echo "Obtaining npm tokens and running registry-static."
+k5start -t -f /etc/npm.keytab service/npm -- timeout -k 2m 30m $CMD $CMD_ARGS
+
+echo "registry-static completed successfully, running vos release."
+k5start -t -f /etc/afsadmin.keytab service/afsadmin -- vos release -v mirror.npm
+
+echo "Done."