[k8s] Support post install manifest URL

A new config option `post_install_manifest_url` is added to support
installing cloud provider/vendor specific manifest after booted
the k8s cluster. It's an URL pointing to the manifest file. For
example, cloud admin can set their specific storageclass into
this file, then it will be automatically setup after created
the cluster.

Task: 35798
Story: 2006209

Change-Id: Ib5a2c5cd7970085db941f189613e175f622aea3f

doc/source/user/index.rst

@@ -32,6 +32,7 @@ created and managed by Magnum to support the COE's.
 #. `Image Management`_
 #. `Notification`_
 #. `Container Monitoring`_
+#. `Kubernetes Post Install Manifest`_
 #. `Kubernetes External Load Balancer`_
 #. `Rolling Upgrade`_
 #. `Keystone Authentication and Authorization for Kubernetes`_
@@ -3380,6 +3381,18 @@ the service is running, the Grafana dashboards can be accessed through port
 For both Prometheus and Grafana, there is an assigned *systemd* service
 called *kube-enable-monitoring*.
 
+Kubernetes Post Install Manifest
+================================
+
+A new config option `post_install_manifest_url` under `[kubernetes]` section
+has been added to support installing cloud provider/vendor specific manifest
+after provisioning the k8s cluster. It's an URL pointing to the manifest file.
+For example, cloud admin can set their specific `StorageClass` into this file,
+then it will be automatically setup after the cluster is created by end user.
+
+**NOTE:** The URL must be reachable from the master nodes when creating the
+cluster.
+
 Kubernetes External Load Balancer
 =================================
 

magnum/conf/kubernetes.py

@@ -22,6 +22,12 @@ kubernetes_opts = [
                     'Keystone auth policy for Kubernetes cluster when '
                     'the Keystone auth is enabled. Vendors can put their '
                     'specific default policy here'),
+    cfg.StrOpt('post_install_manifest_url',
+               default="",
+               help='An URL of the manifest file will be installed after '
+                    'the Kubernetes cluster created. For exmaple, this '
+                    'could be a file including the vendor specific '
+                    'storage class.'),
 ]
 
 

magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh

@@ -443,4 +443,9 @@ kind: List
 metadata: {}
 EOF
 
+# Post install file to setup some cloud provider/vendor specific configs
+if [ "$POST_INSTALL_MANIFEST_URL" != "" ]; then
+    kubectl apply -f "$POST_INSTALL_MANIFEST_URL"
+fi
+
 printf "Finished running ${step}\n"

magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh

@@ -137,6 +137,7 @@ CONTAINER_RUNTIME="$CONTAINER_RUNTIME"
 CONTAINERD_VERSION="$CONTAINERD_VERSION"
 CONTAINERD_TARBALL_URL="$CONTAINERD_TARBALL_URL"
 CONTAINERD_TARBALL_SHA256="$CONTAINERD_TARBALL_SHA256"
+POST_INSTALL_MANIFEST_URL="$POST_INSTALL_MANIFEST_URL"
 EOF
 }
 

magnum/drivers/heat/k8s_fedora_template_def.py

@@ -127,6 +127,8 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition):
             csr_keys["private_key"].replace("\n", "\\n")
 
         extra_params['project_id'] = cluster.project_id
+        extra_params['post_install_manifest_url'] = \
+            CONF.kubernetes.post_install_manifest_url
 
         if not extra_params.get('max_node_count'):
             extra_params['max_node_count'] = cluster.node_count + 1

magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml

@@ -840,6 +840,13 @@ parameters:
     description: sha256 of the target containerd tarball.
     default: '1f2f0fb928179df90492a83c326a194b8e9d992538498efb44cbb6ef15465627'
 
+  post_install_manifest_url:
+    type: string
+    description: >
+      Post install manifest URL used to setup some cloud provider/vendor
+      specific configs
+    default: ""
+
 resources:
 
   ######################################################################
@@ -1169,6 +1176,7 @@ resources:
           containerd_version: {get_param: containerd_version}
           containerd_tarball_url: {get_param: containerd_tarball_url}
           containerd_tarball_sha256: {get_param: containerd_tarball_sha256}
+          post_install_manifest_url: {get_param: post_install_manifest_url}
 
   kube_cluster_config:
     condition: create_cluster_resources

magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml

@@ -591,6 +591,12 @@ parameters:
     type: string
     description: sha256 of the target containerd tarball.
 
+  post_install_manifest_url:
+    type: string
+    description: >
+      Post install manifest url to setup some cloud provider/vendor
+      specific configs
+
 conditions:
 
   image_based: {equals: [{get_param: boot_volume_size}, 0]}
@@ -769,6 +775,7 @@ resources:
                   "$CONTAINERD_VERSION": {get_param: containerd_version}
                   "$CONTAINERD_TARBALL_URL": {get_param: containerd_tarball_url}
                   "$CONTAINERD_TARBALL_SHA256": {get_param: containerd_tarball_sha256}
+                  "$POST_INSTALL_MANIFEST_URL": {get_param: post_install_manifest_url}
             - get_file: ../../common/templates/kubernetes/fragments/install-cri.sh
             - get_file: ../../common/templates/kubernetes/fragments/make-cert.sh
             - str_replace:

magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml

@@ -842,6 +842,13 @@ parameters:
     description: sha256 of the target containerd tarball.
     default: '1f2f0fb928179df90492a83c326a194b8e9d992538498efb44cbb6ef15465627'
 
+  post_install_manifest_url:
+    type: string
+    description: >
+      Post install manifest URL used to setup some cloud provider/vendor
+      specific configs
+    default: ""
+
 resources:
 
   ######################################################################
@@ -1173,6 +1180,7 @@ resources:
           containerd_version: {get_param: containerd_version}
           containerd_tarball_url: {get_param: containerd_tarball_url}
           containerd_tarball_sha256: {get_param: containerd_tarball_sha256}
+          post_install_manifest_url: {get_param: post_install_manifest_url}
 
   kube_cluster_config:
     condition: create_cluster_resources

magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml

@@ -601,6 +601,12 @@ parameters:
     type: string
     description: sha256 of the target containerd tarball.
 
+  post_install_manifest_url:
+    type: string
+    description: >
+      Post install manifest url to setup some cloud provider/vendor
+      specific configs
+
 conditions:
 
   image_based: {equals: [{get_param: boot_volume_size}, 0]}
@@ -782,6 +788,7 @@ resources:
                   "$CONTAINERD_VERSION": {get_param: containerd_version}
                   "$CONTAINERD_TARBALL_URL": {get_param: containerd_tarball_url}
                   "$CONTAINERD_TARBALL_SHA256": {get_param: containerd_tarball_sha256}
+                  "$POST_INSTALL_MANIFEST_URL": {get_param: post_install_manifest_url}
             - get_file: ../../common/templates/kubernetes/fragments/install-cri.sh
             - get_file: ../../common/templates/kubernetes/fragments/make-cert.sh
             - str_replace:

magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py

@@ -373,6 +373,7 @@ class TestClusterConductorWithK8s(base.TestCase):
             'worker_role': 'worker',
             'master_nodegroup_name': 'master_ng',
             'worker_nodegroup_name': 'worker_ng',
+            'post_install_manifest_url': '',
         }
 
         if missing_attr is not None:
@@ -524,6 +525,7 @@ class TestClusterConductorWithK8s(base.TestCase):
             'worker_role': 'worker',
             'master_nodegroup_name': 'master_ng',
             'worker_nodegroup_name': 'worker_ng',
+            'post_install_manifest_url': '',
         }
 
         self.assertEqual(expected, definition)
@@ -657,6 +659,7 @@ class TestClusterConductorWithK8s(base.TestCase):
             'worker_role': 'worker',
             'master_nodegroup_name': 'master_ng',
             'worker_nodegroup_name': 'worker_ng',
+            'post_install_manifest_url': '',
         }
         self.assertEqual(expected, definition)
         self.assertEqual(
@@ -1112,6 +1115,7 @@ class TestClusterConductorWithK8s(base.TestCase):
             'worker_role': 'worker',
             'master_nodegroup_name': 'master_ng',
             'worker_nodegroup_name': 'worker_ng',
+            'post_install_manifest_url': '',
         }
         self.assertEqual(expected, definition)
         self.assertEqual(

magnum/tests/unit/drivers/test_template_definition.py

@@ -697,6 +697,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
             'containerd_version': containerd_version,
             'containerd_tarball_url': containerd_tarball_url,
             'containerd_tarball_sha256': containerd_tarball_sha256,
+            'post_install_manifest_url': '',
         }}
         mock_get_params.assert_called_once_with(mock_context,
                                                 mock_cluster_template,
@@ -1178,6 +1179,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
             'containerd_version': containerd_version,
             'containerd_tarball_url': containerd_tarball_url,
             'containerd_tarball_sha256': containerd_tarball_sha256,
+            'post_install_manifest_url': '',
         }}
         mock_get_params.assert_called_once_with(mock_context,
                                                 mock_cluster_template,

releasenotes/notes/support-post-install-file-1fe7afe7698dd7b2.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    A new config option `post_install_manifest_url` is added to support installing
+    cloud provider/vendor specific manifest after booted the k8s cluster.
+    It's an URL pointing to the manifest file. For example, cloud admin
+    can set their specific storageclass into this file, then it will be
+    automatically setup after created the cluster.