Merge "Allow user specified registries for config_controller" into f/stein

puppet-manifests/src/modules/platform/manifests/docker.pp

@@ -3,6 +3,11 @@ class platform::docker::params (
   $http_proxy   = undef,
   $https_proxy  = undef,
   $no_proxy     = undef,
+  $k8s_registry    = undef,
+  $gcr_registry    = undef,
+  $quay_registry   = undef,
+  $docker_registry = undef,
+  $insecure_registry    = undef,
 ) { }
 
 class platform::docker::config

puppet-manifests/src/modules/platform/manifests/dockerdistribution.pp

@@ -7,9 +7,18 @@ class platform::dockerdistribution::config
 
   if $enabled {
     include ::platform::network::mgmt::params
+    include ::platform::docker::params
 
     $docker_registry_ip = $::platform::network::mgmt::params::controller_address
 
+    # check insecure registries
+    if $::platform::docker::params::insecure_registry {
+      # insecure registry is true means unified registry was set
+      $insecure_registries = "\"${::platform::docker::params::k8s_registry}\", \"${docker_registry_ip}:9001\""
+    } else {
+      $insecure_registries = "\"${docker_registry_ip}:9001\""
+    }
+
     # currently docker registry is running insecure mode
     # when proper authentication is implemented, this would go away
     file { '/etc/docker':
@@ -52,9 +61,18 @@ class platform::dockerdistribution::compute
   $enabled = $::platform::kubernetes::params::enabled
   if $enabled {
     include ::platform::network::mgmt::params
+    include ::platform::docker::params
 
     $docker_registry_ip = $::platform::network::mgmt::params::controller_address
 
+    # check insecure registries
+    if $::platform::docker::params::insecure_registry {
+      # insecure registry is true means unified registry was set
+      $insecure_registries = "\"${::platform::docker::params::k8s_registry}\", \"${docker_registry_ip}:9001\""
+    } else {
+      $insecure_registries = "\"${docker_registry_ip}:9001\""
+    }
+
     # currently docker registry is running insecure mode
     # when proper authentication is implemented, this would go away
     file { '/etc/docker':

puppet-manifests/src/modules/platform/manifests/helm.pp

@@ -7,6 +7,7 @@ class platform::helm
   inherits ::platform::helm::repository::params {
 
   include ::platform::kubernetes::params
+  include ::platform::docker::params
 
   if $::platform::kubernetes::params::enabled {
     file {$source_helm_repo_dir:
@@ -27,17 +28,29 @@ class platform::helm
 
       if str2bool($::is_initial_config_primary) {
 
+        if $::platform::docker::params::gcr_registry {
+          $gcr_registry = $::platform::docker::params::gcr_registry
+        } else {
+          $gcr_registry = 'gcr.io'
+        }
+
+        if $::platform::docker::params::quay_registry {
+          $quay_registry = $::platform::docker::params::quay_registry
+        } else {
+          $quay_registry = 'quay.io'
+        }
+
         Class['::platform::kubernetes::master']
 
         # TODO(jrichard): Upversion tiller image to v2.11.1 once released.
         -> exec { 'load tiller docker image':
-          command   => 'docker image pull gcr.io/kubernetes-helm/tiller:v2.12.1',
+          command   => "docker image pull ${gcr_registry}/kubernetes-helm/tiller:v2.12.1",
           logoutput => true,
         }
 
         # TODO(tngo): If and when tiller image is upversioned, please ensure armada compatibility as part of the test
         -> exec { 'load armada docker image':
-          command   => 'docker image pull quay.io/airshipit/armada:f807c3a1ec727c883c772ffc618f084d960ed5c9',
+          command   => "docker image pull ${quay_registry}/airshipit/armada:f807c3a1ec727c883c772ffc618f084d960ed5c9",
           logoutput => true,
         }
 
@@ -54,7 +67,7 @@ class platform::helm
         # TODO(jrichard): Upversion tiller image to v2.11.1 once released.
         -> exec { 'initialize helm':
           environment => [ 'KUBECONFIG=/etc/kubernetes/admin.conf', 'HOME=/home/wrsroot' ],
-          command     => 'helm init --skip-refresh --service-account tiller --node-selectors "node-role.kubernetes.io/master"="" --tiller-image=gcr.io/kubernetes-helm/tiller:v2.12.1', # lint:ignore:140chars
+          command     => "helm init --skip-refresh --service-account tiller --node-selectors \"node-role.kubernetes.io/master\"=\"\" --tiller-image=${gcr_registry}/kubernetes-helm/tiller:v2.12.1", # lint:ignore:140chars
           logoutput   => true,
           user        => 'wrsroot',
           group       => 'wrs',

puppet-manifests/src/modules/platform/manifests/kubernetes.pp

@@ -13,9 +13,27 @@ class platform::kubernetes::params (
 ) { }
 
 class platform::kubernetes::kubeadm {
+  include ::platform::docker::params
+
   $iptables_file = "net.bridge.bridge-nf-call-ip6tables = 1
     net.bridge.bridge-nf-call-iptables = 1"
 
+  if $::platform::docker::params::k8s_registry {
+    $k8s_registry = $::platform::docker::params::k8s_registry
+  } else {
+    $k8s_registry = undef
+  }
+
+  # kubelet use --pod-infra-container-image to indentify the specified image
+  # TODO: this is not needed after kubernetes upgraded to 1.13
+  #       because the imageRepository setting will be used
+  if $k8s_registry {
+    file { '/etc/sysconfig/kubelet':
+      ensure  => file,
+      content => template('platform/kubelet.conf.erb'),
+    }
+  }
+
   # Update iptables config. This is required based on:
   # https://kubernetes.io/docs/tasks/tools/install-kubeadm
   # This probably belongs somewhere else - initscripts package?
@@ -52,6 +70,21 @@ class platform::kubernetes::master::init
   inherits ::platform::kubernetes::params {
 
   include ::platform::params
+  include ::platform::docker::params
+
+  # This is used for imageRepository in template kubeadm.yaml.erb
+  if $::platform::docker::params::k8s_registry {
+    $k8s_registry = $::platform::docker::params::k8s_registry
+  } else {
+    $k8s_registry = undef
+  }
+
+  # This is used for calico image in template calico.yaml.erb
+  if $::platform::docker::params::quay_registry {
+    $quay_registry = $::platform::docker::params::quay_registry
+  } else {
+    $quay_registry = 'quay.io'
+  }
 
   if str2bool($::is_initial_config_primary) {
     # For initial controller install, configure kubernetes from scratch.
@@ -347,10 +380,11 @@ class platform::kubernetes::worker
 }
 
 # TODO: remove port 9001 once we have a public docker image registry using standard ports.
+# add 5000 as the default port for private registry
 class platform::kubernetes::firewall::params (
   $transport = 'tcp',
   $table = 'nat',
-  $dports = [80, 443, 9001],
+  $dports = [80, 443, 9001, 5000],
   $chain = 'POSTROUTING',
   $jump = 'SNAT',
 ) {}

puppet-manifests/src/modules/platform/templates/calico.yaml.erb

@@ -107,7 +107,7 @@ spec:
       # as a host-networked pod.
       serviceAccountName: calico-node
       containers:
-      - image: quay.io/calico/typha:v3.1.4
+      - image: <%= @quay_registry %>/calico/typha:v3.1.4
         name: calico-typha
         ports:
         - containerPort: 5473
@@ -198,7 +198,7 @@ spec:
         # container programs network policy and routes on each
         # host.
         - name: calico-node
-          image: quay.io/calico/node:v3.1.4
+          image: <%= @quay_registry %>/calico/node:v3.1.4
           env:
             # Use Kubernetes API as the backing datastore.
             - name: DATASTORE_TYPE
@@ -283,7 +283,7 @@ spec:
         # This container installs the Calico CNI binaries
         # and CNI network config file on each node.
         - name: install-cni
-          image: quay.io/calico/cni:v3.1.4
+          image: <%= @quay_registry %>/calico/cni:v3.1.4
           command: ["/install-cni.sh"]
           env:
             # Name of the CNI config file to create.

puppet-manifests/src/modules/platform/templates/insecuredockerregistry.conf.erb

@@ -1,3 +1,3 @@
 {
-    "insecure-registries" : [ "<%= @docker_registry_ip %>:9001" ]
+    "insecure-registries" : [ <%= @insecure_registries %> ]
 }

puppet-manifests/src/modules/platform/templates/kubeadm.yaml.erb

@@ -21,6 +21,9 @@ controllerManagerExtraArgs:
   node-monitor-period: "2s"
   node-monitor-grace-period: "20s"
   pod-eviction-timeout: "30s"
+<%- if @k8s_registry -%>
+imageRepository: "<%= @k8s_registry %>"
+<%- end -%>
 ---
 kind: KubeletConfiguration
 apiVersion: kubelet.config.k8s.io/v1beta1

puppet-manifests/src/modules/platform/templates/kubelet.conf.erb

@@ -0,0 +1,2 @@
+# Overrides config file for kubelet
+KUBELET_EXTRA_ARGS=--pod-infra-container-image=<%= @k8s_registry %>/pause:3.1