Merge "k8s_fedora: Add container_infra_prefix label"

doc/source/user/index.rst

@@ -344,6 +344,8 @@ the table are linked to more details elsewhere in the user guide.
 | `etcd_volume_size`_                   | etcd storage       | 0             |
 |                                       | volume size        |               |
 +---------------------------------------+--------------------+---------------+
+| `container_infra_prefix`_             | see below          | ""            |
++---------------------------------------+--------------------+---------------+
 
 =======
 Cluster
@@ -1083,6 +1085,28 @@ _`etcd_volume_size`
   This label sets the size of a volume holding the etcd storage data.
   The default value is 0, meaning the etcd data is not persisted (no volume).
 
+_`container_infra_prefix`
+  Prefix of all container images used in the cluster (kubernetes components,
+  coredns, kubernetes-dashboard, node-exporter). For example,
+  kubernetes-apiserver is pulled from
+  docker.io/openstackmagnum/kubernetes-apiserver, with this label it can be
+  changed to myregistry.example.com/mycloud/kubernetes-apiserver. Similarly,
+  all other components used in the cluster will be prefixed with this label,
+  which assumes an operator has cloned all expected images in
+  myregistry.example.com/mycloud.
+  Images that must be mirrored:
+  * docker.io/coredns/coredns:011
+  * docker.io/grafana/grafana:latest
+  * docker.io/openstackmagnum/kubernetes-apiserver
+  * docker.io/openstackmagnum/kubernetes-controller-manager
+  * docker.io/openstackmagnum/kubernetes-kubelet
+  * docker.io/openstackmagnum/kubernetes-proxy
+  * docker.io/openstackmagnum/kubernetes-scheduler
+  * docker.io/prom/node-exporter:latest
+  * docker.io/prom/prometheus:latest
+  * gcr.io/google_containers/kubernetes-dashboard-amd64:v1.5.1
+  * gcr.io/google_containers/pause:3.0
+
 External load balancer for services
 -----------------------------------
 

magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh

@@ -4,11 +4,12 @@
 
 echo "configuring kubernetes (master)"
 
-atomic install --storage ostree --system --system-package=no --name=kubelet docker.io/openstackmagnum/kubernetes-kubelet:${KUBE_TAG}
+_prefix=${CONTAINER_INFRA_PREFIX:-docker.io/openstackmagnum/}
-atomic install --storage ostree --system --system-package=no --name=kube-proxy docker.io/openstackmagnum/kubernetes-proxy:${KUBE_TAG}
+atomic install --storage ostree --system --system-package=no --name=kubelet ${_prefix}kubernetes-kubelet:${KUBE_TAG}
-atomic install --storage ostree --system --system-package=no --name=kube-apiserver docker.io/openstackmagnum/kubernetes-apiserver:${KUBE_TAG}
+atomic install --storage ostree --system --system-package=no --name=kube-proxy ${_prefix}kubernetes-proxy:${KUBE_TAG}
-atomic install --storage ostree --system --system-package=no --name=kube-controller-manager docker.io/openstackmagnum/kubernetes-controller-manager:${KUBE_TAG}
+atomic install --storage ostree --system --system-package=no --name=kube-apiserver ${_prefix}kubernetes-apiserver:${KUBE_TAG}
-atomic install --storage ostree --system --system-package=no --name=kube-scheduler docker.io/openstackmagnum/kubernetes-scheduler:${KUBE_TAG}
+atomic install --storage ostree --system --system-package=no --name=kube-controller-manager ${_prefix}kubernetes-controller-manager:${KUBE_TAG}
+atomic install --storage ostree --system --system-package=no --name=kube-scheduler ${_prefix}kubernetes-scheduler:${KUBE_TAG}
 
 sed -i '
     /^KUBE_ALLOW_PRIV=/ s/=.*/="--allow-privileged='"$KUBE_ALLOW_PRIV"'"/
@@ -72,8 +73,8 @@ KUBELET_ARGS="${KUBELET_ARGS} --cluster_dns=${DNS_SERVICE_IP} --cluster_domain=$
 # For using default log-driver, other options should be ignored
 sed -i 's/\-\-log\-driver\=journald//g' /etc/sysconfig/docker
 
+KUBELET_ARGS="${KUBELET_ARGS} --pod-infra-container-image=${CONTAINER_INFRA_PREFIX:-gcr.io/google_containers/}pause:3.0"
 if [ -n "${INSECURE_REGISTRY_URL}" ]; then
-    KUBELET_ARGS="${KUBELET_ARGS} --pod-infra-container-image=${INSECURE_REGISTRY_URL}/google_containers/pause\:0.8.0"
     echo "INSECURE_REGISTRY='--insecure-registry ${INSECURE_REGISTRY_URL}'" >> /etc/sysconfig/docker
 fi
 

magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh

@@ -4,8 +4,9 @@
 
 echo "configuring kubernetes (minion)"
 
-atomic install --storage ostree --system --system-package=no --name=kubelet docker.io/openstackmagnum/kubernetes-kubelet:${KUBE_TAG}
+_prefix=${CONTAINER_INFRA_PREFIX:-docker.io/openstackmagnum/}
-atomic install --storage ostree --system --system-package=no --name=kube-proxy docker.io/openstackmagnum/kubernetes-proxy:${KUBE_TAG}
+atomic install --storage ostree --system --system-package=no --name=kubelet ${_prefix}kubernetes-kubelet:${KUBE_TAG}
+atomic install --storage ostree --system --system-package=no --name=kube-proxy ${_prefix}kubernetes-proxy:${KUBE_TAG}
 
 CERT_DIR=/etc/kubernetes/certs
 PROTOCOL=https
@@ -66,8 +67,8 @@ fi
 # For using default log-driver, other options should be ignored
 sed -i 's/\-\-log\-driver\=journald//g' /etc/sysconfig/docker
 
+KUBELET_ARGS="${KUBELET_ARGS} --pod-infra-container-image=${CONTAINER_INFRA_PREFIX:-gcr.io/google_containers/}pause:3.0"
 if [ -n "${INSECURE_REGISTRY_URL}" ]; then
-    KUBELET_ARGS="${KUBELET_ARGS} --pod-infra-container-image=${INSECURE_REGISTRY_URL}/google_containers/pause\:0.8.0"
     echo "INSECURE_REGISTRY='--insecure-registry ${INSECURE_REGISTRY_URL}'" >> /etc/sysconfig/docker
 fi
 

magnum/drivers/common/templates/kubernetes/fragments/core-dns-service.sh

@@ -2,6 +2,7 @@
 
 . /etc/sysconfig/heat-params
 
+_prefix=${CONTAINER_INFRA_PREFIX:-docker.io/coredns/}
 CORE_DNS=/etc/kubernetes/manifests/kube-coredns.yaml
 [ -f ${CORE_DNS} ] || {
     echo "Writing File: $CORE_DNS"
@@ -47,7 +48,7 @@ spec:
     spec:
       containers:
       - name: coredns
-        image: coredns/coredns:011
+        image: ${_prefix}coredns:011
         imagePullPolicy: Always
         args: [ "-conf", "/etc/coredns/Corefile" ]
         volumeMounts:

magnum/drivers/common/templates/kubernetes/fragments/enable-node-exporter.sh

@@ -24,7 +24,7 @@ metadata:
 spec:
   containers:
   - name: node-exporter
-    image: prom/node-exporter
+    image: ${CONTAINER_INFRA_PREFIX:-docker.io/prom/}node-exporter
     ports:
     - containerPort: 9100
       hostPort: 9100

magnum/drivers/common/templates/kubernetes/fragments/kube-dashboard-service.sh

@@ -10,11 +10,7 @@ if [ "$(echo $KUBE_DASHBOARD_ENABLED | tr '[:upper:]' '[:lower:]')" == "false" ]
     exit 0
 fi
 
-if [ -n "${INSECURE_REGISTRY_URL}" ]; then
+KUBE_DASH_IMAGE="${CONTAINER_INFRA_PREFIX:-gcr.io/google_containers/}kubernetes-dashboard-amd64:${KUBE_DASHBOARD_VERSION}"
-    KUBE_DASH_IMAGE="${INSECURE_REGISTRY_URL}/google_containers/kubernetes-dashboard-amd64:${KUBE_DASHBOARD_VERSION}"
-else
-    KUBE_DASH_IMAGE="gcr.io/google_containers/kubernetes-dashboard-amd64:${KUBE_DASHBOARD_VERSION}"
-fi
 
 KUBE_DASH_DEPLOY=/srv/kubernetes/manifests/kube-dash-deploy.yaml
 

magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml

@@ -45,6 +45,7 @@ write_files:
       TRUST_ID="$TRUST_ID"
       AUTH_URL="$AUTH_URL"
       INSECURE_REGISTRY_URL="$INSECURE_REGISTRY_URL"
+      CONTAINER_INFRA_PREFIX="$CONTAINER_INFRA_PREFIX"
       SYSTEM_PODS_INITIAL_DELAY="$SYSTEM_PODS_INITIAL_DELAY"
       SYSTEM_PODS_TIMEOUT="$SYSTEM_PODS_TIMEOUT"
       ETCD_LB_VIP="$ETCD_LB_VIP"

magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.yaml

@@ -41,5 +41,6 @@ write_files:
       TRUSTEE_PASSWORD="$TRUSTEE_PASSWORD"
       TRUST_ID="$TRUST_ID"
       INSECURE_REGISTRY_URL="$INSECURE_REGISTRY_URL"
+      CONTAINER_INFRA_PREFIX="$CONTAINER_INFRA_PREFIX"
       DNS_SERVICE_IP="$DNS_SERVICE_IP"
       DNS_CLUSTER_DOMAIN="$DNS_CLUSTER_DOMAIN"

magnum/drivers/heat/k8s_fedora_template_def.py

@@ -82,6 +82,11 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition):
         if kube_tag:
             extra_params['kube_tag'] = kube_tag
 
+        container_infra_prefix = cluster_template.labels.get(
+            'container_infra_prefix')
+        if container_infra_prefix:
+            extra_params['container_infra_prefix'] = container_infra_prefix
+
         return super(K8sFedoraTemplateDefinition,
                      self).get_params(context, cluster_template, cluster,
                                       extra_params=extra_params,

magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml

@@ -328,6 +328,15 @@ parameters:
     description: insecure registry url
     default: ""
 
+  container_infra_prefix:
+    type: string
+    description: >
+      prefix of container images used in the cluster, kubernetes components,
+      kubernetes-dashboard, coredns etc
+    constraints:
+      - allowed_pattern: "^$|.*/"
+    default: ""
+
   dns_service_ip:
     type: string
     description: >
@@ -515,6 +524,7 @@ resources:
           trust_id: {get_param: trust_id}
           auth_url: {get_param: auth_url}
           insecure_registry_url: {get_param: insecure_registry_url}
+          container_infra_prefix: {get_param: container_infra_prefix}
           etcd_lb_vip: {get_attr: [etcd_lb, address]}
           dns_service_ip: {get_param: dns_service_ip}
           dns_cluster_domain: {get_param: dns_cluster_domain}
@@ -582,6 +592,7 @@ resources:
           trustee_domain_id: {get_param: trustee_domain_id}
           trust_id: {get_param: trust_id}
           insecure_registry_url: {get_param: insecure_registry_url}
+          container_infra_prefix: {get_param: container_infra_prefix}
           dns_service_ip: {get_param: dns_service_ip}
           dns_cluster_domain: {get_param: dns_cluster_domain}
 

magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml

@@ -235,6 +235,12 @@ parameters:
     type: string
     description: insecure registry url
 
+  container_infra_prefix:
+    type: string
+    description: >
+      prefix of container images used in the cluster, kubernetes components,
+      kubernetes-dashboard, coredns etc
+
   etcd_lb_vip:
     type: string
     description: >
@@ -332,6 +338,7 @@ resources:
             "$TRUSTEE_PASSWORD": {get_param: trustee_password}
             "$TRUST_ID": {get_param: trust_id}
             "$INSECURE_REGISTRY_URL": {get_param: insecure_registry_url}
+            "$CONTAINER_INFRA_PREFIX": {get_param: container_infra_prefix}
             "$ETCD_LB_VIP": {get_param: etcd_lb_vip}
             "$DNS_SERVICE_IP": {get_param: dns_service_ip}
             "$DNS_CLUSTER_DOMAIN": {get_param: dns_cluster_domain}

magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml

@@ -209,6 +209,12 @@ parameters:
     type: string
     description: insecure registry url
 
+  container_infra_prefix:
+    type: string
+    description: >
+      prefix of container images used in the cluster, kubernetes components,
+      kubernetes-dashboard, coredns etc
+
   dns_service_ip:
     type: string
     description: >
@@ -280,6 +286,7 @@ resources:
             $TRUST_ID: {get_param: trust_id}
             $AUTH_URL: {get_param: auth_url}
             $INSECURE_REGISTRY_URL: {get_param: insecure_registry_url}
+            $CONTAINER_INFRA_PREFIX: {get_param: container_infra_prefix}
             $DNS_SERVICE_IP: {get_param: dns_service_ip}
             $DNS_CLUSTER_DOMAIN: {get_param: dns_cluster_domain}
 

magnum/drivers/k8s_fedora_ironic_v1/templates/kubecluster.yaml

@@ -331,6 +331,15 @@ parameters:
     description: insecure registry url
     default: ""
 
+  container_infra_prefix:
+    type: string
+    description: >
+      prefix of container images used in the cluster, kubernetes components,
+      kubernetes-dashboard, coredns etc
+    constraints:
+      - allowed_pattern: "^$|.*/"
+    default: ""
+
 resources:
 
   api_lb:
@@ -488,6 +497,7 @@ resources:
           trust_id: {get_param: trust_id}
           auth_url: {get_param: auth_url}
           insecure_registry_url: {get_param: insecure_registry_url}
+          container_infra_prefix: {get_param: container_infra_prefix}
           wc_curl_cli: {get_attr: [master_wait_handle, curl_cli]}
           etcd_lb_vip: {get_attr: [etcd_lb, address]}
 
@@ -575,6 +585,7 @@ resources:
       trustee_domain_id: {get_param: trustee_domain_id}
       trust_id: {get_param: trust_id}
       insecure_registry_url: {get_param: insecure_registry_url}
+      container_infra_prefix: {get_param: container_infra_prefix}
       wc_curl_cli: {get_attr: [minion_wait_handle, curl_cli]}
 
   ######################################################################

magnum/drivers/k8s_fedora_ironic_v1/templates/kubemaster.yaml

@@ -219,6 +219,12 @@ parameters:
     type: string
     description: insecure registry url
 
+  container_infra_prefix:
+    type: string
+    description: >
+      prefix of container images used in the cluster, kubernetes components,
+      kubernetes-dashboard, coredns etc
+
   wc_curl_cli:
     type: string
     description : >
@@ -296,6 +302,7 @@ resources:
             "$TRUSTEE_PASSWORD": {get_param: trustee_password}
             "$TRUST_ID": {get_param: trust_id}
             "$INSECURE_REGISTRY_URL": {get_param: insecure_registry_url}
+            "$CONTAINER_INFRA_PREFIX": {get_param: container_infra_prefix}
             "$ENABLE_CINDER": "False"
             "$ETCD_LB_VIP": {get_param: etcd_lb_vip}
 

magnum/drivers/k8s_fedora_ironic_v1/templates/kubeminion_software_configs.yaml

@@ -160,6 +160,12 @@ parameters:
     type: string
     description: insecure registry url
 
+  container_infra_prefix:
+    type: string
+    description: >
+      prefix of container images used in the cluster, kubernetes components,
+      kubernetes-dashboard, coredns etc
+
   wc_curl_cli:
     type: string
     description : >
@@ -216,6 +222,7 @@ resources:
             $TRUST_ID: {get_param: trust_id}
             $AUTH_URL: {get_param: auth_url}
             $INSECURE_REGISTRY_URL: {get_param: insecure_registry_url}
+            $CONTAINER_INFRA_PREFIX: {get_param: container_infra_prefix}
             $ENABLE_CINDER: "False"
 
   write_kubeconfig:

magnum/tests/unit/drivers/test_template_definition.py

@@ -270,6 +270,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
         etcd_volume_size = mock_cluster.labels.get(
             'etcd_volume_size')
         kube_tag = mock_cluster_template.labels.get('kube_tag')
+        container_infra_prefix = mock_cluster_template.labels.get(
+            'container_infra_prefix')
 
         k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
 
@@ -293,7 +295,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
             'username': 'fake_user',
             'magnum_url': mock_osc.magnum_url.return_value,
             'region_name': mock_osc.cinder_region_name.return_value,
-            'kube_tag': kube_tag}}
+            'kube_tag': kube_tag,
+            'container_infra_prefix': container_infra_prefix}}
         mock_get_params.assert_called_once_with(mock_context,
                                                 mock_cluster_template,
                                                 mock_cluster,
@@ -350,6 +353,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
         etcd_volume_size = mock_cluster.labels.get(
             'etcd_volume_size')
         kube_tag = mock_cluster_template.labels.get('kube_tag')
+        container_infra_prefix = mock_cluster_template.labels.get(
+            'container_infra_prefix')
 
         k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
 
@@ -375,7 +380,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
             'region_name': mock_osc.cinder_region_name.return_value,
             'loadbalancing_protocol': 'HTTP',
             'kubernetes_port': 8080,
-            'kube_tag': kube_tag}}
+            'kube_tag': kube_tag,
+            'container_infra_prefix': container_infra_prefix}}
         mock_get_params.assert_called_once_with(mock_context,
                                                 mock_cluster_template,
                                                 mock_cluster,

releasenotes/notes/add-container_infra_prefix-516cc43fbc5a0617.yaml

@@ -0,0 +1,11 @@
+---
+features:
+  - |
+    Prefix of all container images used in the cluster (kubernetes components,
+    coredns, kubernetes-dashboard, node-exporter). For example,
+    kubernetes-apiserver is pulled from
+    docker.io/openstackmagnum/kubernetes-apiserver, with this label it can be
+    changed to myregistry.example.com/mycloud/kubernetes-apiserver. Similarly,
+    all other components used in the cluster will be prefixed with this label,
+    which assumes an operator has cloned all expected images in
+    myregistry.example.com/mycloud.