From d61dd1d5b5507644f618713d7a270bf36987acb3 Mon Sep 17 00:00:00 2001 From: Feilong Wang Date: Fri, 16 Aug 2019 09:50:01 +1200 Subject: [PATCH] [k8s] Support post install manifest URL A new config option `post_install_manifest_url` is added to support installing cloud provider/vendor specific manifest after booted the k8s cluster. It's an URL pointing to the manifest file. For example, cloud admin can set their specific storageclass into this file, then it will be automatically setup after created the cluster. Task: 35798 Story: 2006209 Change-Id: Ib5a2c5cd7970085db941f189613e175f622aea3f --- doc/source/user/index.rst | 13 +++++++++++++ magnum/conf/kubernetes.py | 6 ++++++ .../fragments/kube-apiserver-to-kubelet-role.sh | 5 +++++ .../fragments/write-heat-params-master.sh | 1 + magnum/drivers/heat/k8s_fedora_template_def.py | 2 ++ .../k8s_fedora_atomic_v1/templates/kubecluster.yaml | 8 ++++++++ .../k8s_fedora_atomic_v1/templates/kubemaster.yaml | 7 +++++++ .../k8s_fedora_coreos_v1/templates/kubecluster.yaml | 8 ++++++++ .../k8s_fedora_coreos_v1/templates/kubemaster.yaml | 7 +++++++ .../handlers/test_k8s_cluster_conductor.py | 4 ++++ .../tests/unit/drivers/test_template_definition.py | 2 ++ .../support-post-install-file-1fe7afe7698dd7b2.yaml | 8 ++++++++ 12 files changed, 71 insertions(+) create mode 100644 releasenotes/notes/support-post-install-file-1fe7afe7698dd7b2.yaml diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst index 60abf20613..5089eeab10 100644 --- a/doc/source/user/index.rst +++ b/doc/source/user/index.rst @@ -32,6 +32,7 @@ created and managed by Magnum to support the COE's. #. `Image Management`_ #. `Notification`_ #. `Container Monitoring`_ +#. `Kubernetes Post Install Manifest`_ #. `Kubernetes External Load Balancer`_ #. `Rolling Upgrade`_ #. `Keystone Authentication and Authorization for Kubernetes`_ @@ -3380,6 +3381,18 @@ the service is running, the Grafana dashboards can be accessed through port For both Prometheus and Grafana, there is an assigned *systemd* service called *kube-enable-monitoring*. +Kubernetes Post Install Manifest +================================ + +A new config option `post_install_manifest_url` under `[kubernetes]` section +has been added to support installing cloud provider/vendor specific manifest +after provisioning the k8s cluster. It's an URL pointing to the manifest file. +For example, cloud admin can set their specific `StorageClass` into this file, +then it will be automatically setup after the cluster is created by end user. + +**NOTE:** The URL must be reachable from the master nodes when creating the +cluster. + Kubernetes External Load Balancer ================================= diff --git a/magnum/conf/kubernetes.py b/magnum/conf/kubernetes.py index 2de9370e2b..35f40d0d39 100644 --- a/magnum/conf/kubernetes.py +++ b/magnum/conf/kubernetes.py @@ -22,6 +22,12 @@ kubernetes_opts = [ 'Keystone auth policy for Kubernetes cluster when ' 'the Keystone auth is enabled. Vendors can put their ' 'specific default policy here'), + cfg.StrOpt('post_install_manifest_url', + default="", + help='An URL of the manifest file will be installed after ' + 'the Kubernetes cluster created. For exmaple, this ' + 'could be a file including the vendor specific ' + 'storage class.'), ] diff --git a/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh b/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh index 9f67a45886..e647f293b1 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh @@ -443,4 +443,9 @@ kind: List metadata: {} EOF +# Post install file to setup some cloud provider/vendor specific configs +if [ "$POST_INSTALL_MANIFEST_URL" != "" ]; then + kubectl apply -f "$POST_INSTALL_MANIFEST_URL" +fi + printf "Finished running ${step}\n" diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh index f8ba6e4f7d..4768c081e1 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh @@ -137,6 +137,7 @@ CONTAINER_RUNTIME="$CONTAINER_RUNTIME" CONTAINERD_VERSION="$CONTAINERD_VERSION" CONTAINERD_TARBALL_URL="$CONTAINERD_TARBALL_URL" CONTAINERD_TARBALL_SHA256="$CONTAINERD_TARBALL_SHA256" +POST_INSTALL_MANIFEST_URL="$POST_INSTALL_MANIFEST_URL" EOF } diff --git a/magnum/drivers/heat/k8s_fedora_template_def.py b/magnum/drivers/heat/k8s_fedora_template_def.py index 59f69d76cc..48ac7f894e 100644 --- a/magnum/drivers/heat/k8s_fedora_template_def.py +++ b/magnum/drivers/heat/k8s_fedora_template_def.py @@ -127,6 +127,8 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition): csr_keys["private_key"].replace("\n", "\\n") extra_params['project_id'] = cluster.project_id + extra_params['post_install_manifest_url'] = \ + CONF.kubernetes.post_install_manifest_url if not extra_params.get('max_node_count'): extra_params['max_node_count'] = cluster.node_count + 1 diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml index 798af2cc67..990f60ef51 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml @@ -840,6 +840,13 @@ parameters: description: sha256 of the target containerd tarball. default: '1f2f0fb928179df90492a83c326a194b8e9d992538498efb44cbb6ef15465627' + post_install_manifest_url: + type: string + description: > + Post install manifest URL used to setup some cloud provider/vendor + specific configs + default: "" + resources: ###################################################################### @@ -1169,6 +1176,7 @@ resources: containerd_version: {get_param: containerd_version} containerd_tarball_url: {get_param: containerd_tarball_url} containerd_tarball_sha256: {get_param: containerd_tarball_sha256} + post_install_manifest_url: {get_param: post_install_manifest_url} kube_cluster_config: condition: create_cluster_resources diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml index 82137a122f..28d2ce33cb 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml @@ -591,6 +591,12 @@ parameters: type: string description: sha256 of the target containerd tarball. + post_install_manifest_url: + type: string + description: > + Post install manifest url to setup some cloud provider/vendor + specific configs + conditions: image_based: {equals: [{get_param: boot_volume_size}, 0]} @@ -769,6 +775,7 @@ resources: "$CONTAINERD_VERSION": {get_param: containerd_version} "$CONTAINERD_TARBALL_URL": {get_param: containerd_tarball_url} "$CONTAINERD_TARBALL_SHA256": {get_param: containerd_tarball_sha256} + "$POST_INSTALL_MANIFEST_URL": {get_param: post_install_manifest_url} - get_file: ../../common/templates/kubernetes/fragments/install-cri.sh - get_file: ../../common/templates/kubernetes/fragments/make-cert.sh - str_replace: diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml index 81f2829b75..2aaa797b69 100644 --- a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml @@ -842,6 +842,13 @@ parameters: description: sha256 of the target containerd tarball. default: '1f2f0fb928179df90492a83c326a194b8e9d992538498efb44cbb6ef15465627' + post_install_manifest_url: + type: string + description: > + Post install manifest URL used to setup some cloud provider/vendor + specific configs + default: "" + resources: ###################################################################### @@ -1173,6 +1180,7 @@ resources: containerd_version: {get_param: containerd_version} containerd_tarball_url: {get_param: containerd_tarball_url} containerd_tarball_sha256: {get_param: containerd_tarball_sha256} + post_install_manifest_url: {get_param: post_install_manifest_url} kube_cluster_config: condition: create_cluster_resources diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml index 16b6c71692..e37334ad51 100644 --- a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml @@ -601,6 +601,12 @@ parameters: type: string description: sha256 of the target containerd tarball. + post_install_manifest_url: + type: string + description: > + Post install manifest url to setup some cloud provider/vendor + specific configs + conditions: image_based: {equals: [{get_param: boot_volume_size}, 0]} @@ -782,6 +788,7 @@ resources: "$CONTAINERD_VERSION": {get_param: containerd_version} "$CONTAINERD_TARBALL_URL": {get_param: containerd_tarball_url} "$CONTAINERD_TARBALL_SHA256": {get_param: containerd_tarball_sha256} + "$POST_INSTALL_MANIFEST_URL": {get_param: post_install_manifest_url} - get_file: ../../common/templates/kubernetes/fragments/install-cri.sh - get_file: ../../common/templates/kubernetes/fragments/make-cert.sh - str_replace: diff --git a/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py b/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py index cfe35754d4..556759d5c6 100644 --- a/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py +++ b/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py @@ -373,6 +373,7 @@ class TestClusterConductorWithK8s(base.TestCase): 'worker_role': 'worker', 'master_nodegroup_name': 'master_ng', 'worker_nodegroup_name': 'worker_ng', + 'post_install_manifest_url': '', } if missing_attr is not None: @@ -524,6 +525,7 @@ class TestClusterConductorWithK8s(base.TestCase): 'worker_role': 'worker', 'master_nodegroup_name': 'master_ng', 'worker_nodegroup_name': 'worker_ng', + 'post_install_manifest_url': '', } self.assertEqual(expected, definition) @@ -657,6 +659,7 @@ class TestClusterConductorWithK8s(base.TestCase): 'worker_role': 'worker', 'master_nodegroup_name': 'master_ng', 'worker_nodegroup_name': 'worker_ng', + 'post_install_manifest_url': '', } self.assertEqual(expected, definition) self.assertEqual( @@ -1112,6 +1115,7 @@ class TestClusterConductorWithK8s(base.TestCase): 'worker_role': 'worker', 'master_nodegroup_name': 'master_ng', 'worker_nodegroup_name': 'worker_ng', + 'post_install_manifest_url': '', } self.assertEqual(expected, definition) self.assertEqual( diff --git a/magnum/tests/unit/drivers/test_template_definition.py b/magnum/tests/unit/drivers/test_template_definition.py index 2279da2c53..0e4cd78352 100644 --- a/magnum/tests/unit/drivers/test_template_definition.py +++ b/magnum/tests/unit/drivers/test_template_definition.py @@ -697,6 +697,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase): 'containerd_version': containerd_version, 'containerd_tarball_url': containerd_tarball_url, 'containerd_tarball_sha256': containerd_tarball_sha256, + 'post_install_manifest_url': '', }} mock_get_params.assert_called_once_with(mock_context, mock_cluster_template, @@ -1178,6 +1179,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase): 'containerd_version': containerd_version, 'containerd_tarball_url': containerd_tarball_url, 'containerd_tarball_sha256': containerd_tarball_sha256, + 'post_install_manifest_url': '', }} mock_get_params.assert_called_once_with(mock_context, mock_cluster_template, diff --git a/releasenotes/notes/support-post-install-file-1fe7afe7698dd7b2.yaml b/releasenotes/notes/support-post-install-file-1fe7afe7698dd7b2.yaml new file mode 100644 index 0000000000..4f5d802c4a --- /dev/null +++ b/releasenotes/notes/support-post-install-file-1fe7afe7698dd7b2.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + A new config option `post_install_manifest_url` is added to support installing + cloud provider/vendor specific manifest after booted the k8s cluster. + It's an URL pointing to the manifest file. For example, cloud admin + can set their specific storageclass into this file, then it will be + automatically setup after created the cluster.