From 9641afd305e5f0aad4ec68770391668099006346 Mon Sep 17 00:00:00 2001 From: Eli Qiao Date: Mon, 28 Mar 2016 11:23:52 +0800 Subject: [PATCH] Support using insecure registry for k8s COE This path adds supporting to using insecure registry for k8s COE when deploy a k8s COE. Partially-Implements: blueprint support-insecure-registry Change-Id: I384358b1419085ed42d96239f97292fa32db9aed --- magnum/conductor/template_definition.py | 2 ++ .../fragments/configure-kubernetes-master.sh | 10 ++++++++-- .../fragments/configure-kubernetes-minion.sh | 6 ++++++ .../fragments/enable-kube-podmaster.sh | 17 +++++++++++++---- .../fragments/enable-kube-proxy-master.sh | 8 +++++++- .../fragments/enable-kube-proxy-minion.sh | 8 +++++++- .../kubernetes/fragments/kube-ui-service.sh | 9 ++++++++- .../fragments/write-heat-params-master.yaml | 1 + .../kubernetes/fragments/write-heat-params.yaml | 1 + magnum/templates/kubernetes/kubecluster.yaml | 7 +++++++ magnum/templates/kubernetes/kubemaster.yaml | 5 +++++ magnum/templates/kubernetes/kubeminion.yaml | 5 +++++ .../handlers/test_k8s_bay_conductor.py | 15 +++++++++++---- 13 files changed, 81 insertions(+), 13 deletions(-) diff --git a/magnum/conductor/template_definition.py b/magnum/conductor/template_definition.py index bc4280ed05..8b103d37e7 100644 --- a/magnum/conductor/template_definition.py +++ b/magnum/conductor/template_definition.py @@ -462,6 +462,8 @@ class K8sTemplateDefinition(BaseTemplateDefinition): self.add_parameter('bay_uuid', bay_attr='uuid', param_type=str) + self.add_parameter('insecure_registry_url', + baymodel_attr='insecure_registry') self.add_output('api_address', bay_attr='api_address', diff --git a/magnum/templates/kubernetes/fragments/configure-kubernetes-master.sh b/magnum/templates/kubernetes/fragments/configure-kubernetes-master.sh index 7f118ebf7a..3f1a52f4d6 100644 --- a/magnum/templates/kubernetes/fragments/configure-kubernetes-master.sh +++ b/magnum/templates/kubernetes/fragments/configure-kubernetes-master.sh @@ -42,9 +42,15 @@ sed -i ' /^KUBE_CONTROLLER_MANAGER_ARGS=/ s/KUBE_CONTROLLER_MANAGER_ARGS.*/#Uncomment the following line to enable Kubernetes Load Balancer feature \n#KUBE_CONTROLLER_MANAGER_ARGS="--cloud_config=\/etc\/sysconfig\/kube_openstack_config --cloud_provider=openstack"/ ' /etc/kubernetes/controller-manager -KUBE_CONFIG="--register-node=true --register-schedulable=false --config=/etc/kubernetes/manifests --hostname-override=$KUBE_NODE_IP" +KUBELET_ARGS="--register-node=true --register-schedulable=false --config=/etc/kubernetes/manifests --hostname-override=$KUBE_NODE_IP" + +if [ -n "${INSECURE_REGISTRY_URL}" ]; then + KUBELET_ARGS="${KUBELET_ARGS} --pod-infra-container-image=${INSECURE_REGISTRY_URL}/google_containers/pause\:0.8.0" + echo "INSECURE_REGISTRY='--insecure-registry ${INSECURE_REGISTRY_URL}'" >> /etc/sysconfig/docker +fi + sed -i ' /^KUBELET_ADDRESS=/ s/=.*/="--address=0.0.0.0"/ /^KUBELET_HOSTNAME=/ s/=.*/=""/ - /^KUBELET_ARGS=/ s|=.*|='"$KUBE_CONFIG"'| + /^KUBELET_ARGS=/ s|=.*|='"$KUBELET_ARGS"'| ' /etc/kubernetes/kubelet diff --git a/magnum/templates/kubernetes/fragments/configure-kubernetes-minion.sh b/magnum/templates/kubernetes/fragments/configure-kubernetes-minion.sh index 8df2c65ecf..942e54360b 100644 --- a/magnum/templates/kubernetes/fragments/configure-kubernetes-minion.sh +++ b/magnum/templates/kubernetes/fragments/configure-kubernetes-minion.sh @@ -21,6 +21,12 @@ sed -i ' ' /etc/kubernetes/config KUBELET_ARGS="--config=/etc/kubernetes/manifests --cadvisor-port=4194 --hostname-override=$KUBE_NODE_IP ${KUBE_CONFIG}" + +if [ -n "${INSECURE_REGISTRY_URL}" ]; then + KUBELET_ARGS="${KUBELET_ARGS} --pod-infra-container-image=${INSECURE_REGISTRY_URL}/google_containers/pause\:0.8.0" + echo "INSECURE_REGISTRY='--insecure-registry ${INSECURE_REGISTRY_URL}'" >> /etc/sysconfig/docker +fi + sed -i ' /^KUBELET_ADDRESS=/ s/=.*/="--address=0.0.0.0"/ /^KUBELET_HOSTNAME=/ s/=.*/=""/ diff --git a/magnum/templates/kubernetes/fragments/enable-kube-podmaster.sh b/magnum/templates/kubernetes/fragments/enable-kube-podmaster.sh index 3d9f0fcd47..2e95adc655 100644 --- a/magnum/templates/kubernetes/fragments/enable-kube-podmaster.sh +++ b/magnum/templates/kubernetes/fragments/enable-kube-podmaster.sh @@ -2,6 +2,15 @@ . /etc/sysconfig/heat-params +if [ -n "${INSECURE_REGISTRY_URL}" ]; then + PODMASTER_IMAGE="${INSECURE_REGISTRY_URL}/google_containers/podmaster:1.1" + HYPERKUBE_IMAGE="${INSECURE_REGISTRY_URL}/google_containers/hyperkube:${KUBE_VERSION}" +else + PODMASTER_IMAGE="gcr.io/google_containers/podmaster:1.1" + HYPERKUBE_IMAGE="gcr.io/google_containers/hyperkube:${KUBE_VERSION}" +fi + + init_templates () { local TEMPLATE=/etc/kubernetes/manifests/kube-podmaster.yaml [ -f ${TEMPLATE} ] || { @@ -17,7 +26,7 @@ spec: hostNetwork: true containers: - name: scheduler-elector - image: gcr.io/google_containers/podmaster:1.1 + image: ${PODMASTER_IMAGE} command: - /podmaster - --etcd-servers=http://127.0.0.1:2379 @@ -31,7 +40,7 @@ spec: - mountPath: /dst/manifests name: manifest-dst - name: controller-manager-elector - image: gcr.io/google_containers/podmaster:1.1 + image: ${PODMASTER_IMAGE} command: - /podmaster - --etcd-servers=http://127.0.0.1:2379 @@ -76,7 +85,7 @@ metadata: spec: containers: - name: kube-controller-manager - image: gcr.io/google_containers/hyperkube:${KUBE_VERSION} + image: ${HYPERKUBE_IMAGE} command: - /hyperkube - controller-manager @@ -122,7 +131,7 @@ spec: hostNetwork: true containers: - name: kube-scheduler - image: gcr.io/google_containers/hyperkube:${KUBE_VERSION} + image: ${HYPERKUBE_IMAGE} command: - /hyperkube - scheduler diff --git a/magnum/templates/kubernetes/fragments/enable-kube-proxy-master.sh b/magnum/templates/kubernetes/fragments/enable-kube-proxy-master.sh index d8a40a3dcb..9a6447bdbd 100644 --- a/magnum/templates/kubernetes/fragments/enable-kube-proxy-master.sh +++ b/magnum/templates/kubernetes/fragments/enable-kube-proxy-master.sh @@ -2,6 +2,12 @@ . /etc/sysconfig/heat-params +if [ -n "${INSECURE_REGISTRY_URL}" ]; then + HYPERKUBE_IMAGE="${INSECURE_REGISTRY_URL}/google_containers/hyperkube:${KUBE_VERSION}" +else + HYPERKUBE_IMAGE="gcr.io/google_containers/hyperkube:${KUBE_VERSION}" +fi + init_templates () { local TEMPLATE=/etc/kubernetes/manifests/kube-proxy.yaml [ -f ${TEMPLATE} ] || { @@ -17,7 +23,7 @@ spec: hostNetwork: true containers: - name: kube-proxy - image: gcr.io/google_containers/hyperkube:${KUBE_VERSION} + image: ${HYPERKUBE_IMAGE} command: - /hyperkube - proxy diff --git a/magnum/templates/kubernetes/fragments/enable-kube-proxy-minion.sh b/magnum/templates/kubernetes/fragments/enable-kube-proxy-minion.sh index 5910fbe3b3..a230040e4c 100644 --- a/magnum/templates/kubernetes/fragments/enable-kube-proxy-minion.sh +++ b/magnum/templates/kubernetes/fragments/enable-kube-proxy-minion.sh @@ -2,6 +2,12 @@ . /etc/sysconfig/heat-params +if [ -n "${INSECURE_REGISTRY_URL}" ]; then + HYPERKUBE_IMAGE="${INSECURE_REGISTRY_URL}/google_containers/hyperkube:${KUBE_VERSION}" +else + HYPERKUBE_IMAGE="gcr.io/google_containers/hyperkube:${KUBE_VERSION}" +fi + init_templates () { local KUBE_PROTOCOL="https" local KUBE_CONFIG="/srv/kubernetes/kubeconfig.yaml" @@ -25,7 +31,7 @@ spec: hostNetwork: true containers: - name: kube-proxy - image: gcr.io/google_containers/hyperkube:${KUBE_VERSION} + image: ${HYPERKUBE_IMAGE} command: - /hyperkube - proxy diff --git a/magnum/templates/kubernetes/fragments/kube-ui-service.sh b/magnum/templates/kubernetes/fragments/kube-ui-service.sh index e4bfd4eda1..734fc96e99 100644 --- a/magnum/templates/kubernetes/fragments/kube-ui-service.sh +++ b/magnum/templates/kubernetes/fragments/kube-ui-service.sh @@ -6,7 +6,14 @@ . /etc/sysconfig/heat-params +if [ -n "${INSECURE_REGISTRY_URL}" ]; then + KUBEUI_IMAGE="${INSECURE_REGISTRY_URL}/google_containers/kube-ui:v4" +else + KUBEUI_IMAGE="gcr.io/google_containers/kube-ui:v4" +fi + KUBE_UI_RC=/srv/kubernetes/manifests/kube-ui-rc.yaml + [ -f ${KUBE_UI_RC} ] || { echo "Writing File: $KUBE_UI_RC" mkdir -p $(dirname ${KUBE_UI_RC}) @@ -34,7 +41,7 @@ spec: spec: containers: - name: kube-ui - image: gcr.io/google_containers/kube-ui:v4 + image: ${KUBEUI_IMAGE} resources: limits: cpu: 100m diff --git a/magnum/templates/kubernetes/fragments/write-heat-params-master.yaml b/magnum/templates/kubernetes/fragments/write-heat-params-master.yaml index 5ab96f76e2..1d81ce1489 100644 --- a/magnum/templates/kubernetes/fragments/write-heat-params-master.yaml +++ b/magnum/templates/kubernetes/fragments/write-heat-params-master.yaml @@ -33,3 +33,4 @@ write_files: TRUSTEE_PASSWORD="$TRUSTEE_PASSWORD" TRUST_ID="$TRUST_ID" AUTH_URL="$AUTH_URL" + INSECURE_REGISTRY_URL="$INSECURE_REGISTRY_URL" diff --git a/magnum/templates/kubernetes/fragments/write-heat-params.yaml b/magnum/templates/kubernetes/fragments/write-heat-params.yaml index 9980c97222..c064fa076d 100644 --- a/magnum/templates/kubernetes/fragments/write-heat-params.yaml +++ b/magnum/templates/kubernetes/fragments/write-heat-params.yaml @@ -38,3 +38,4 @@ write_files: TRUSTEE_DOMAIN_ID="$TRUSTEE_DOMAIN_ID" TRUST_ID="$TRUST_ID" AUTH_URL="$AUTH_URL" + INSECURE_REGISTRY_URL="$INSECURE_REGISTRY_URL" diff --git a/magnum/templates/kubernetes/kubecluster.yaml b/magnum/templates/kubernetes/kubecluster.yaml index 63c0286f0d..18b47a98ed 100644 --- a/magnum/templates/kubernetes/kubecluster.yaml +++ b/magnum/templates/kubernetes/kubecluster.yaml @@ -251,6 +251,11 @@ parameters: description: version of kubernetes used for kubernetes cluster default: v1.2.0 + insecure_registry_url: + type: string + description: insecure registry url + default: "" + resources: ###################################################################### @@ -432,6 +437,7 @@ resources: trustee_password: {get_param: trustee_password} trust_id: {get_param: trust_id} auth_url: {get_param: auth_url} + insecure_registry_url: {get_param: insecure_registry_url} ###################################################################### # @@ -489,6 +495,7 @@ resources: trustee_domain_id: {get_param: trustee_domain_id} trust_id: {get_param: trust_id} auth_url: {get_param: auth_url} + insecure_registry_url: {get_param: insecure_registry_url} outputs: diff --git a/magnum/templates/kubernetes/kubemaster.yaml b/magnum/templates/kubernetes/kubemaster.yaml index 4a51714263..59d71f22f7 100644 --- a/magnum/templates/kubernetes/kubemaster.yaml +++ b/magnum/templates/kubernetes/kubemaster.yaml @@ -170,6 +170,10 @@ parameters: description: id of the trust which is used by the trustee hidden: true + insecure_registry_url: + type: string + description: insecure registry url + resources: master_wait_handle: @@ -224,6 +228,7 @@ resources: "$TRUSTEE_USER_ID": {get_param: trustee_user_id} "$TRUSTEE_PASSWORD": {get_param: trustee_password} "$TRUST_ID": {get_param: trust_id} + "$INSECURE_REGISTRY_URL": {get_param: insecure_registry_url} make_cert: type: OS::Heat::SoftwareConfig diff --git a/magnum/templates/kubernetes/kubeminion.yaml b/magnum/templates/kubernetes/kubeminion.yaml index 0819f610e7..adb495cb3c 100644 --- a/magnum/templates/kubernetes/kubeminion.yaml +++ b/magnum/templates/kubernetes/kubeminion.yaml @@ -187,6 +187,10 @@ parameters: type: string description: url for keystone + insecure_registry_url: + type: string + description: insecure registry url + resources: minion_wait_handle: @@ -246,6 +250,7 @@ resources: $TRUSTEE_PASSWORD: {get_param: trustee_password} $TRUST_ID: {get_param: trust_id} $AUTH_URL: {get_param: auth_url} + $INSECURE_REGISTRY_URL: {get_param: insecure_registry_url} write_kubeconfig: type: OS::Heat::SoftwareConfig diff --git a/magnum/tests/unit/conductor/handlers/test_k8s_bay_conductor.py b/magnum/tests/unit/conductor/handlers/test_k8s_bay_conductor.py index 5bed7172bc..0f483e33bd 100644 --- a/magnum/tests/unit/conductor/handlers/test_k8s_bay_conductor.py +++ b/magnum/tests/unit/conductor/handlers/test_k8s_bay_conductor.py @@ -45,7 +45,8 @@ class TestBayConductorWithK8s(base.TestCase): 'flannel_backend': 'vxlan'}, 'tls_disabled': False, 'server_type': 'vm', - 'registry_enabled': False + 'registry_enabled': False, + 'insecure_registry': '10.0.0.1:5000', } self.bay_dict = { 'uuid': '5d12f6fd-a196-4bf0-ae4c-1f639a523a52', @@ -123,6 +124,7 @@ class TestBayConductorWithK8s(base.TestCase): 'bay_uuid': self.bay_dict['uuid'], 'magnum_url': self.mock_osc.magnum_url.return_value, 'tls_disabled': False, + 'insecure_registry': '10.0.0.1:5000', } expected = { 'ssh_key_name': 'keypair_id', @@ -155,7 +157,8 @@ class TestBayConductorWithK8s(base.TestCase): 'trustee_password': 'fake_trustee_password', 'trustee_user_id': '7b489f04-b458-4541-8179-6a48a553e656', 'trust_id': 'bd11efc5-d4e2-4dac-bbce-25e348ddf7de', - 'auth_url': 'http://192.168.10.10:5000/v3' + 'auth_url': 'http://192.168.10.10:5000/v3', + 'insecure_registry_url': '10.0.0.1:5000', } if missing_attr is not None: expected.pop(mapping[missing_attr], None) @@ -212,7 +215,8 @@ class TestBayConductorWithK8s(base.TestCase): 'trustee_user_id': '7b489f04-b458-4541-8179-6a48a553e656', 'trustee_username': 'fake_trustee', 'username': 'fake_user', - 'volume_driver': 'volume_driver' + 'volume_driver': 'volume_driver', + 'insecure_registry_url': '10.0.0.1:5000', } self.assertEqual(expected, definition) @@ -258,6 +262,7 @@ class TestBayConductorWithK8s(base.TestCase): 'auth_url': 'http://192.168.10.10:5000/v3', 'bay_uuid': self.bay_dict['uuid'], 'magnum_url': self.mock_osc.magnum_url.return_value, + 'insecure_registry_url': '10.0.0.1:5000', } self.assertEqual(expected, definition) @@ -307,6 +312,7 @@ class TestBayConductorWithK8s(base.TestCase): 'auth_url': 'http://192.168.10.10:5000/v3', 'bay_uuid': self.bay_dict['uuid'], 'magnum_url': self.mock_osc.magnum_url.return_value, + 'insecure_registry_url': '10.0.0.1:5000', } self.assertEqual(expected, definition) @@ -427,7 +433,8 @@ class TestBayConductorWithK8s(base.TestCase): 'trustee_password': 'fake_trustee_password', 'trustee_user_id': '7b489f04-b458-4541-8179-6a48a553e656', 'trust_id': 'bd11efc5-d4e2-4dac-bbce-25e348ddf7de', - 'auth_url': 'http://192.168.10.10:5000/v3' + 'auth_url': 'http://192.168.10.10:5000/v3', + 'insecure_registry_url': '10.0.0.1:5000', } self.assertEqual(expected, definition) reqget.assert_called_once_with('http://etcd/test?size=1')