Fix SSL certificate setting error

When initializing k8s client in InfraDriverV2, the SSL CA
certificate is set incorrectly.

To fix the issue, the following modifies are made in this patch:

* A temp file for ssl_ca_cert is created before initializing
  k8s client and the temp file path is set to k8s_config.ssl_ca_cert,
* The temp file is deleted until the lifetime of k8s client ends.
Note: This references the implementation in InfraDriverV1.

If set the ssl_ca_cert in instantiate request, the validation of
request is failed because of the length of ssl_ca_cert exceeds 1024.
For this issue, add a new type `keyvalue_pairs_no_length_limit`
which has no max length limitation to verify the request.
And the interfaceInfo, accessInfo, extra are all set to the new type
for unity.

In Zuul test environment, when registering default vim, ssl_ca_cert
is not set. So the case with ssl_ca_cert is not tested.
In this patch ssl_ca_cert is set into the default vim.

Closes-Bug: #1979413
Change-Id: I61dbd70690b737a72fc619e5a08b4bab51160a27

.zuul.yaml

@@ -522,6 +522,7 @@
             controller_worker:
               amp_active_retries: 9999
       kuryr_k8s_api_url: "https://{{ hostvars['controller-k8s']['nodepool']['private_ipv4'] }}:6443"
+      k8s_ssl_verify: true
       helm_version: "3.5.4"
       test_matrix_configs: [neutron]
       zuul_work_dir: src/opendev.org/openstack/tacker

roles/setup-default-vim/tasks/main.yaml

@@ -97,6 +97,14 @@
     become: yes
     become_user: stack
 
+  - name: Fetch k8s's CA Certificate
+    fetch:
+       src: "/etc/kubernetes/pki/ca.crt"
+       dest: "/tmp/"
+       flat: true
+    when:
+      - k8s_ssl_verify
+
   when:
     - inventory_hostname == 'controller-k8s'
     - kuryr_k8s_api_url is defined
@@ -171,6 +179,32 @@
     when:
       - p.stat.exists
 
+  - name: Copy k8s's CA Certificate to tacker
+    copy:
+       src: "/tmp/ca.crt"
+       dest: "/tmp/"
+    when:
+      - p.stat.exists
+      - k8s_ssl_verify
+
+  - name: Get k8s's CA Certificate
+    command: cat "/tmp/ca.crt"
+    register: ssl_ca_cert
+    when:
+      - p.stat.exists
+      - k8s_ssl_verify
+
+  - name: Replace k8s CA Certificate in local-k8s-vim.yaml
+    replace:
+       path: "{{ item }}"
+       regexp: "ssl_ca_cert: .*$"
+       replace: "ssl_ca_cert: '{{ ssl_ca_cert.stdout }}'"
+    with_items:
+       - "{{ zuul_work_dir }}/tacker/tests/etc/samples/local-k8s-vim.yaml"
+    when:
+      - p.stat.exists
+      - k8s_ssl_verify
+
   - name: Replace the config file path in the test-setup-k8s-vim.sh
     replace:
        path: "{{ zuul_work_dir }}/tools/test-setup-k8s-vim.sh"

tacker/api/validation/parameter_types.py

@@ -131,6 +131,23 @@ keyvalue_pairs = {
     'additionalProperties': False
 }
 
+keyvalue_pairs_no_length_limit = {
+    'type': 'object',
+    'patternProperties': {
+        '^[a-zA-Z0-9-_:. /]+$': {
+            'anyOf': [
+                {'type': 'array'},
+                {'type': 'string'},
+                {'type': 'object'},
+                {'type': 'null'},
+                {'type': 'boolean'},
+                {'type': 'number'}
+            ]
+        }
+    },
+    'additionalProperties': False
+}
+
 description = {
     'type': 'string', 'minLength': 0, 'maxLength': 1024,
     'pattern': valid_description_regex,

tacker/sol_refactored/api/schemas/common_types.py

@@ -61,9 +61,9 @@ VimConnectionInfo = {
     'properties': {
         'vimId': {'type': 'string', 'maxLength': 255},
         'vimType': {'type': 'string', 'minLength': 1, 'maxLength': 255},
-        'interfaceInfo': parameter_types.keyvalue_pairs,
+        'interfaceInfo': parameter_types.keyvalue_pairs_no_length_limit,
-        'accessInfo': parameter_types.keyvalue_pairs,
+        'accessInfo': parameter_types.keyvalue_pairs_no_length_limit,
-        'extra': parameter_types.keyvalue_pairs,
+        'extra': parameter_types.keyvalue_pairs_no_length_limit,
     },
     'required': ['vimType'],
     'additionalProperties': True,

tacker/sol_refactored/infra_drivers/kubernetes/kubernetes.py

@@ -63,17 +63,26 @@ class Kubernetes(object):
 
         # deploy k8s resources with sorted resources
         vim_info = inst_utils.select_vim_info(inst.vimConnectionInfo)
-        k8s_client = kubernetes_utils.KubernetesClient(vim_info)
+        # This is Context Manager for creation and deletion
-        created_k8s_reses = k8s_client.create_k8s_resource(
+        # of CA certificate temp file
-            sorted_k8s_reses, namespace)
+        with kubernetes_utils.CaCertFileContextManager(
+                vim_info.interfaceInfo.get('ssl_ca_cert')) as ca_cert_cm:
 
-        # wait k8s resource create complete
+            # add an item ca_cert_file:file_path into vim_info.interfaceInfo,
-        k8s_client.wait_k8s_res_create(created_k8s_reses)
+            # and will be deleted in KubernetesClient
+            vim_info.interfaceInfo['ca_cert_file'] = ca_cert_cm.file_path
 
-        # make instantiated info
+            k8s_client = kubernetes_utils.KubernetesClient(vim_info)
-        all_pods = k8s_client.list_namespaced_pods(namespace)
+            created_k8s_reses = k8s_client.create_k8s_resource(
-        self._make_cnf_instantiated_info(
+                sorted_k8s_reses, namespace)
-            req, inst, vnfd, namespace, created_k8s_reses, all_pods)
+
+            # wait k8s resource create complete
+            k8s_client.wait_k8s_res_create(created_k8s_reses)
+
+            # make instantiated info
+            all_pods = k8s_client.list_namespaced_pods(namespace)
+            self._make_cnf_instantiated_info(
+                req, inst, vnfd, namespace, created_k8s_reses, all_pods)
 
     def terminate(self, req, inst, grant_req, grant, vnfd):
         target_k8s_files = inst.metadata.get('lcm-kubernetes-def-files')
@@ -88,11 +97,20 @@ class Kubernetes(object):
 
         # delete k8s resources with sorted resources
         vim_info = inst_utils.select_vim_info(inst.vimConnectionInfo)
-        k8s_client = kubernetes_utils.KubernetesClient(vim_info)
+        # This is Context Manager for creation and deletion
-        k8s_client.delete_k8s_resource(req, sorted_k8s_reses, namespace)
+        # of CA certificate temp file
+        with kubernetes_utils.CaCertFileContextManager(
+                vim_info.interfaceInfo.get('ssl_ca_cert')) as ca_cert_cm:
 
-        # wait k8s resource delete complete
+            # add an item ca_cert_file:file_path into vim_info.interfaceInfo,
-        k8s_client.wait_k8s_res_delete(sorted_k8s_reses, namespace)
+            # and will be deleted in KubernetesClient
+            vim_info.interfaceInfo['ca_cert_file'] = ca_cert_cm.file_path
+
+            k8s_client = kubernetes_utils.KubernetesClient(vim_info)
+            k8s_client.delete_k8s_resource(req, sorted_k8s_reses, namespace)
+
+            # wait k8s resource delete complete
+            k8s_client.wait_k8s_res_delete(sorted_k8s_reses, namespace)
 
     def change_vnfpkg(self, req, inst, grant_req, grant, vnfd):
         if req.additionalParams.get('upgrade_type') == 'RollingUpdate':
@@ -103,43 +121,56 @@ class Kubernetes(object):
 
             # check deployment exists in kubernetes
             vim_info = inst_utils.select_vim_info(inst.vimConnectionInfo)
-            k8s_client = kubernetes_utils.KubernetesClient(vim_info)
+            # This is Context Manager for creation and deletion
-            k8s_client.check_deployment_exist(deployment_names, namespace)
+            # of CA certificate temp file
+            with kubernetes_utils.CaCertFileContextManager(
+                    vim_info.interfaceInfo.get('ssl_ca_cert')) as ca_cert_cm:
 
-            # get new deployment body
+                # add an item ca_cert_file:file_path
-            new_deploy_reses = kubernetes_utils.get_new_deployment_body(
+                # into vim_info.interfaceInfo,
-                req, inst, vnfd, deployment_names, operation='CHANGE_VNFPKG')
+                # and will be deleted in KubernetesClient
+                vim_info.interfaceInfo['ca_cert_file'] = ca_cert_cm.file_path
 
-            # apply new deployment
+                k8s_client = kubernetes_utils.KubernetesClient(vim_info)
-            k8s_client.update_k8s_resource(new_deploy_reses, namespace)
+                k8s_client.check_deployment_exist(deployment_names, namespace)
 
-            # wait k8s resource update complete
+                # get new deployment body
-            old_pods_names = [vnfc.computeResource.resourceId for vnfc in
+                new_deploy_reses = kubernetes_utils.get_new_deployment_body(
-                              inst.instantiatedVnfInfo.vnfcResourceInfo]
+                    req, inst, vnfd, deployment_names,
-            try:
+                    operation='CHANGE_VNFPKG')
-                k8s_client.wait_k8s_res_update(
+
-                    new_deploy_reses, namespace, old_pods_names)
+                # apply new deployment
-            except sol_ex.UpdateK8SResourceFailed as ex:
+                k8s_client.update_k8s_resource(new_deploy_reses, namespace)
+
+                # wait k8s resource update complete
+                old_pods_names = [vnfc.computeResource.resourceId for vnfc in
+                                  inst.instantiatedVnfInfo.vnfcResourceInfo]
+                try:
+                    k8s_client.wait_k8s_res_update(
+                        new_deploy_reses, namespace, old_pods_names)
+                except sol_ex.UpdateK8SResourceFailed as ex:
+                    self._update_cnf_instantiated_info(
+                        inst, deployment_names,
+                        k8s_client.list_namespaced_pods(
+                            namespace=namespace))
+                    raise ex
+
+                # execute coordinate vnf script
+                try:
+                    self._execute_coordinate_vnf_script(
+                        req, inst, grant_req, grant, vnfd, 'CHANGE_VNFPKG',
+                        namespace, new_deploy_reses)
+                except sol_ex.CoordinateVNFExecutionFailed as ex:
+                    self._update_cnf_instantiated_info(
+                        inst, deployment_names,
+                        k8s_client.list_namespaced_pods(
+                            namespace=namespace))
+                    raise ex
+
+                # update cnf instantiated info
+                all_pods = k8s_client.list_namespaced_pods(namespace)
                 self._update_cnf_instantiated_info(
-                    inst, deployment_names, k8s_client.list_namespaced_pods(
+                    inst, deployment_names, all_pods)
-                        namespace=namespace))
-                raise ex
-
-            # execute coordinate vnf script
-            try:
-                self._execute_coordinate_vnf_script(
-                    req, inst, grant_req, grant, vnfd, 'CHANGE_VNFPKG',
-                    namespace, new_deploy_reses)
-            except sol_ex.CoordinateVNFExecutionFailed as ex:
-                self._update_cnf_instantiated_info(
-                    inst, deployment_names, k8s_client.list_namespaced_pods(
-                        namespace=namespace))
-                raise ex
-
-            # update cnf instantiated info
-            all_pods = k8s_client.list_namespaced_pods(namespace)
-            self._update_cnf_instantiated_info(
-                inst, deployment_names, all_pods)
 
         else:
             # TODO(YiFeng): Blue-Green type will be supported in next version.
@@ -169,31 +200,41 @@ class Kubernetes(object):
 
             # apply old deployment
             vim_info = inst_utils.select_vim_info(inst.vimConnectionInfo)
-            k8s_client = kubernetes_utils.KubernetesClient(vim_info)
+            # This is Context Manager for creation and deletion
-            k8s_client.update_k8s_resource(old_deploy_reses, namespace)
+            # of CA certificate temp file
+            with kubernetes_utils.CaCertFileContextManager(
+                    vim_info.interfaceInfo.get('ssl_ca_cert')) as ca_cert_cm:
 
-            # wait k8s resource update complete
+                # add an item ca_cert_file:file_path
-            old_pods_names = [vnfc.computeResource.resourceId for vnfc in
+                # into vim_info.interfaceInfo,
-                              inst.instantiatedVnfInfo.vnfcResourceInfo]
+                # and will be deleted in KubernetesClient
-            try:
+                vim_info.interfaceInfo['ca_cert_file'] = ca_cert_cm.file_path
-                k8s_client.wait_k8s_res_update(
-                    old_deploy_reses, namespace, old_pods_names)
-            except sol_ex.UpdateK8SResourceFailed as ex:
-                raise ex
 
-            # execute coordinate vnf script
+                k8s_client = kubernetes_utils.KubernetesClient(vim_info)
-            try:
+                k8s_client.update_k8s_resource(old_deploy_reses, namespace)
-                self._execute_coordinate_vnf_script(
-                    req, inst, grant_req, grant, vnfd,
-                    'CHANGE_VNFPKG_ROLLBACK',
-                    namespace, old_deploy_reses)
-            except sol_ex.CoordinateVNFExecutionFailed as ex:
-                raise ex
 
-            # update cnf instantiated info
+                # wait k8s resource update complete
-            all_pods = k8s_client.list_namespaced_pods(namespace)
+                old_pods_names = [vnfc.computeResource.resourceId for vnfc in
-            self._update_cnf_instantiated_info(
+                                  inst.instantiatedVnfInfo.vnfcResourceInfo]
-                inst, deployment_names, all_pods)
+                try:
+                    k8s_client.wait_k8s_res_update(
+                        old_deploy_reses, namespace, old_pods_names)
+                except sol_ex.UpdateK8SResourceFailed as ex:
+                    raise ex
+
+                # execute coordinate vnf script
+                try:
+                    self._execute_coordinate_vnf_script(
+                        req, inst, grant_req, grant, vnfd,
+                        'CHANGE_VNFPKG_ROLLBACK',
+                        namespace, old_deploy_reses)
+                except sol_ex.CoordinateVNFExecutionFailed as ex:
+                    raise ex
+
+                # update cnf instantiated info
+                all_pods = k8s_client.list_namespaced_pods(namespace)
+                self._update_cnf_instantiated_info(
+                    inst, deployment_names, all_pods)
 
         else:
             # TODO(YiFeng): Blue-Green type will be supported in next version.

tacker/sol_refactored/infra_drivers/kubernetes/kubernetes_utils.py

@@ -17,6 +17,7 @@ import copy
 import ipaddress
 import os
 import re
+import tempfile
 import time
 from urllib.parse import urlparse
 import urllib.request as urllib2
@@ -583,6 +584,9 @@ def init_k8s_api_client(vim_info):
     k8s_config = client.Configuration()
     k8s_config.host = vim_info.interfaceInfo['endpoint']
 
+    ca_cert_file = (vim_info.interfaceInfo.pop('ca_cert_file')
+                    if 'ca_cert_file' in vim_info.interfaceInfo else None)
+
     if ('username' in vim_info.accessInfo and 'password'
             in vim_info.accessInfo and vim_info.accessInfo.get(
                 'password') is not None):
@@ -596,8 +600,8 @@ def init_k8s_api_client(vim_info):
         k8s_config.api_key['authorization'] = vim_info.accessInfo[
             'bearer_token']
 
-    if 'ssl_ca_cert' in vim_info.accessInfo:
+    if 'ssl_ca_cert' in vim_info.interfaceInfo and ca_cert_file:
-        k8s_config.ssl_ca_cert = vim_info.accessInfo['ssl_ca_cert']
+        k8s_config.ssl_ca_cert = ca_cert_file
         k8s_config.verify_ssl = True
     else:
         k8s_config.verify_ssl = False
@@ -724,3 +728,27 @@ def get_new_deployment_body(
             new_deploy_reses.append(k8s_res)
 
     return new_deploy_reses
+
+
+class CaCertFileContextManager:
+    def __init__(self, ca_cert_str):
+        self._file_descriptor = None
+        self.file_path = None
+        self.ca_cert_str = ca_cert_str
+
+    def __enter__(self):
+        if not self.ca_cert_str:
+            return self
+        self._file_descriptor, self.file_path = tempfile.mkstemp()
+        ca_cert = re.sub(r'\s', '\n', self.ca_cert_str)
+        ca_cert = re.sub(r'BEGIN\nCERT', r'BEGIN CERT', ca_cert)
+        ca_cert = re.sub(r'END\nCERT', r'END CERT', ca_cert)
+        # write ca cert file
+        os.write(self._file_descriptor, ca_cert.encode())
+        return self
+
+    def __exit__(self, exc_type, exc_value, exc_traceback):
+        if not self.ca_cert_str:
+            return
+        os.close(self._file_descriptor)
+        os.remove(self.file_path)

tacker/tests/functional/sol_kubernetes_v2/base_v2.py

@@ -49,6 +49,7 @@ class BaseVnfLcmKubernetesV2Test(base.BaseTestCase):
         k8s_vim_info = cls.get_k8s_vim_info()
         cls.auth_url = k8s_vim_info.interfaceInfo['endpoint']
         cls.bearer_token = k8s_vim_info.accessInfo['bearer_token']
+        cls.ssl_ca_cert = k8s_vim_info.interfaceInfo['ssl_ca_cert']
 
         vim_info = cls.get_vim_info()
         auth = http_client.KeystonePasswordAuthHandle(
@@ -85,7 +86,10 @@ class BaseVnfLcmKubernetesV2Test(base.BaseTestCase):
         vim_params = yaml.safe_load(base_utils.read_file('local-k8s-vim.yaml'))
 
         vim_info = objects.VimConnectionInfo(
-            interfaceInfo={'endpoint': vim_params['auth_url']},
+            interfaceInfo={
+                'endpoint': vim_params['auth_url'],
+                'ssl_ca_cert': vim_params.get('ssl_ca_cert')
+            },
             accessInfo={
                 'region': 'RegionOne',
                 'bearer_token': vim_params['bearer_token']

tacker/tests/functional/sol_kubernetes_v2/paramgen.py

@@ -47,7 +47,7 @@ def test_instantiate_cnf_resources_terminate():
     }
 
 
-def max_sample_instantiate(auth_url, bearer_token):
+def max_sample_instantiate(auth_url, bearer_token, ssl_ca_cert=None):
     # All attributes are set.
     # NOTE: All of the following cardinality attributes are set.
     # In addition, 0..N or 1..N attributes are set to 2 or more.
@@ -78,6 +78,9 @@ def max_sample_instantiate(auth_url, bearer_token):
         },
         "extra": {"dummy-key": "dummy-val"}
     }
+    if ssl_ca_cert:
+        vim_1["interfaceInfo"]["ssl_ca_cert"] = ssl_ca_cert
+        vim_2["interfaceInfo"]["ssl_ca_cert"] = ssl_ca_cert
     return {
         "flavourId": "simple",
         "vimConnectionInfo": {

tacker/tests/functional/sol_kubernetes_v2/test_vnflcm_basic.py

@@ -98,7 +98,7 @@ class VnfLcmKubernetesTest(base_v2.BaseVnfLcmKubernetesV2Test):
 
         # 2. Instantiate a VNF instance
         instantiate_req = paramgen.max_sample_instantiate(
-            self.auth_url, self.bearer_token)
+            self.auth_url, self.bearer_token, ssl_ca_cert=self.ssl_ca_cert)
         resp, body = self.instantiate_vnf_instance(inst_id, instantiate_req)
         self.assertEqual(202, resp.status_code)
         self.check_resp_headers_in_operation_task(resp)

tacker/tests/unit/sol_refactored/infra_drivers/kubernetes/test_kubernetes.py

@@ -146,7 +146,7 @@ class TestKubernetes(base.BaseTestCase):
             _instantiate_req_example)
         req.additionalParams['lcm-kubernetes-def-files'].append(
             'Files/kubernetes/namespace.yaml')
-        req.vimConnectionInfo['vim1']['interfaceInfo']['ssl_ca_cert '] = 'test'
+        req.vimConnectionInfo['vim1']['interfaceInfo']['ssl_ca_cert'] = 'test'
         req.additionalParams['namespace'] = 'curry'
         inst = objects.VnfInstanceV2(
             vimConnectionInfo=req.vimConnectionInfo
@@ -1009,6 +1009,7 @@ class TestKubernetes(base.BaseTestCase):
         # prepare instantiate
         req = objects.InstantiateVnfRequest.from_dict(
             _instantiate_req_example)
+        req.vimConnectionInfo['vim1']['interfaceInfo']['ssl_ca_cert'] = 'test'
         inst = objects.VnfInstanceV2(
             vimConnectionInfo=req.vimConnectionInfo
         )
@@ -1302,6 +1303,7 @@ class TestKubernetes(base.BaseTestCase):
         # prepare instantiate
         req = objects.InstantiateVnfRequest.from_dict(
             _instantiate_req_example)
+        req.vimConnectionInfo['vim1']['interfaceInfo']['ssl_ca_cert'] = 'test'
         inst = objects.VnfInstanceV2(
             id=uuidutils.generate_uuid(),
             vimConnectionInfo=req.vimConnectionInfo