From 98d3f4bf31644b8ab79e2ebad1f73ece2fc83ed5 Mon Sep 17 00:00:00 2001 From: Qibin Yao Date: Thu, 7 Jul 2022 11:33:46 +0900 Subject: [PATCH] Fix SSL certificate setting error When initializing k8s client in InfraDriverV2, the SSL CA certificate is set incorrectly. To fix the issue, the following modifies are made in this patch: * A temp file for ssl_ca_cert is created before initializing k8s client and the temp file path is set to k8s_config.ssl_ca_cert, * The temp file is deleted until the lifetime of k8s client ends. Note: This references the implementation in InfraDriverV1. If set the ssl_ca_cert in instantiate request, the validation of request is failed because of the length of ssl_ca_cert exceeds 1024. For this issue, add a new type `keyvalue_pairs_no_length_limit` which has no max length limitation to verify the request. And the interfaceInfo, accessInfo, extra are all set to the new type for unity. In Zuul test environment, when registering default vim, ssl_ca_cert is not set. So the case with ssl_ca_cert is not tested. In this patch ssl_ca_cert is set into the default vim. Closes-Bug: #1979413 Change-Id: I61dbd70690b737a72fc619e5a08b4bab51160a27 --- .zuul.yaml | 1 + roles/setup-default-vim/tasks/main.yaml | 34 ++++ tacker/api/validation/parameter_types.py | 17 ++ .../api/schemas/common_types.py | 6 +- .../infra_drivers/kubernetes/kubernetes.py | 177 +++++++++++------- .../kubernetes/kubernetes_utils.py | 32 +++- .../functional/sol_kubernetes_v2/base_v2.py | 6 +- .../functional/sol_kubernetes_v2/paramgen.py | 5 +- .../sol_kubernetes_v2/test_vnflcm_basic.py | 2 +- .../kubernetes/test_kubernetes.py | 4 +- 10 files changed, 207 insertions(+), 77 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index e0d0bc351..c3c073369 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -522,6 +522,7 @@ controller_worker: amp_active_retries: 9999 kuryr_k8s_api_url: "https://{{ hostvars['controller-k8s']['nodepool']['private_ipv4'] }}:6443" + k8s_ssl_verify: true helm_version: "3.5.4" test_matrix_configs: [neutron] zuul_work_dir: src/opendev.org/openstack/tacker diff --git a/roles/setup-default-vim/tasks/main.yaml b/roles/setup-default-vim/tasks/main.yaml index d03ce7f5b..afadc7fbe 100644 --- a/roles/setup-default-vim/tasks/main.yaml +++ b/roles/setup-default-vim/tasks/main.yaml @@ -97,6 +97,14 @@ become: yes become_user: stack + - name: Fetch k8s's CA Certificate + fetch: + src: "/etc/kubernetes/pki/ca.crt" + dest: "/tmp/" + flat: true + when: + - k8s_ssl_verify + when: - inventory_hostname == 'controller-k8s' - kuryr_k8s_api_url is defined @@ -171,6 +179,32 @@ when: - p.stat.exists + - name: Copy k8s's CA Certificate to tacker + copy: + src: "/tmp/ca.crt" + dest: "/tmp/" + when: + - p.stat.exists + - k8s_ssl_verify + + - name: Get k8s's CA Certificate + command: cat "/tmp/ca.crt" + register: ssl_ca_cert + when: + - p.stat.exists + - k8s_ssl_verify + + - name: Replace k8s CA Certificate in local-k8s-vim.yaml + replace: + path: "{{ item }}" + regexp: "ssl_ca_cert: .*$" + replace: "ssl_ca_cert: '{{ ssl_ca_cert.stdout }}'" + with_items: + - "{{ zuul_work_dir }}/tacker/tests/etc/samples/local-k8s-vim.yaml" + when: + - p.stat.exists + - k8s_ssl_verify + - name: Replace the config file path in the test-setup-k8s-vim.sh replace: path: "{{ zuul_work_dir }}/tools/test-setup-k8s-vim.sh" diff --git a/tacker/api/validation/parameter_types.py b/tacker/api/validation/parameter_types.py index 8a28bdab2..729d39d45 100644 --- a/tacker/api/validation/parameter_types.py +++ b/tacker/api/validation/parameter_types.py @@ -131,6 +131,23 @@ keyvalue_pairs = { 'additionalProperties': False } +keyvalue_pairs_no_length_limit = { + 'type': 'object', + 'patternProperties': { + '^[a-zA-Z0-9-_:. /]+$': { + 'anyOf': [ + {'type': 'array'}, + {'type': 'string'}, + {'type': 'object'}, + {'type': 'null'}, + {'type': 'boolean'}, + {'type': 'number'} + ] + } + }, + 'additionalProperties': False +} + description = { 'type': 'string', 'minLength': 0, 'maxLength': 1024, 'pattern': valid_description_regex, diff --git a/tacker/sol_refactored/api/schemas/common_types.py b/tacker/sol_refactored/api/schemas/common_types.py index a68121214..67523df92 100644 --- a/tacker/sol_refactored/api/schemas/common_types.py +++ b/tacker/sol_refactored/api/schemas/common_types.py @@ -61,9 +61,9 @@ VimConnectionInfo = { 'properties': { 'vimId': {'type': 'string', 'maxLength': 255}, 'vimType': {'type': 'string', 'minLength': 1, 'maxLength': 255}, - 'interfaceInfo': parameter_types.keyvalue_pairs, - 'accessInfo': parameter_types.keyvalue_pairs, - 'extra': parameter_types.keyvalue_pairs, + 'interfaceInfo': parameter_types.keyvalue_pairs_no_length_limit, + 'accessInfo': parameter_types.keyvalue_pairs_no_length_limit, + 'extra': parameter_types.keyvalue_pairs_no_length_limit, }, 'required': ['vimType'], 'additionalProperties': True, diff --git a/tacker/sol_refactored/infra_drivers/kubernetes/kubernetes.py b/tacker/sol_refactored/infra_drivers/kubernetes/kubernetes.py index e5d668a94..086d14529 100644 --- a/tacker/sol_refactored/infra_drivers/kubernetes/kubernetes.py +++ b/tacker/sol_refactored/infra_drivers/kubernetes/kubernetes.py @@ -63,17 +63,26 @@ class Kubernetes(object): # deploy k8s resources with sorted resources vim_info = inst_utils.select_vim_info(inst.vimConnectionInfo) - k8s_client = kubernetes_utils.KubernetesClient(vim_info) - created_k8s_reses = k8s_client.create_k8s_resource( - sorted_k8s_reses, namespace) + # This is Context Manager for creation and deletion + # of CA certificate temp file + with kubernetes_utils.CaCertFileContextManager( + vim_info.interfaceInfo.get('ssl_ca_cert')) as ca_cert_cm: - # wait k8s resource create complete - k8s_client.wait_k8s_res_create(created_k8s_reses) + # add an item ca_cert_file:file_path into vim_info.interfaceInfo, + # and will be deleted in KubernetesClient + vim_info.interfaceInfo['ca_cert_file'] = ca_cert_cm.file_path - # make instantiated info - all_pods = k8s_client.list_namespaced_pods(namespace) - self._make_cnf_instantiated_info( - req, inst, vnfd, namespace, created_k8s_reses, all_pods) + k8s_client = kubernetes_utils.KubernetesClient(vim_info) + created_k8s_reses = k8s_client.create_k8s_resource( + sorted_k8s_reses, namespace) + + # wait k8s resource create complete + k8s_client.wait_k8s_res_create(created_k8s_reses) + + # make instantiated info + all_pods = k8s_client.list_namespaced_pods(namespace) + self._make_cnf_instantiated_info( + req, inst, vnfd, namespace, created_k8s_reses, all_pods) def terminate(self, req, inst, grant_req, grant, vnfd): target_k8s_files = inst.metadata.get('lcm-kubernetes-def-files') @@ -88,11 +97,20 @@ class Kubernetes(object): # delete k8s resources with sorted resources vim_info = inst_utils.select_vim_info(inst.vimConnectionInfo) - k8s_client = kubernetes_utils.KubernetesClient(vim_info) - k8s_client.delete_k8s_resource(req, sorted_k8s_reses, namespace) + # This is Context Manager for creation and deletion + # of CA certificate temp file + with kubernetes_utils.CaCertFileContextManager( + vim_info.interfaceInfo.get('ssl_ca_cert')) as ca_cert_cm: - # wait k8s resource delete complete - k8s_client.wait_k8s_res_delete(sorted_k8s_reses, namespace) + # add an item ca_cert_file:file_path into vim_info.interfaceInfo, + # and will be deleted in KubernetesClient + vim_info.interfaceInfo['ca_cert_file'] = ca_cert_cm.file_path + + k8s_client = kubernetes_utils.KubernetesClient(vim_info) + k8s_client.delete_k8s_resource(req, sorted_k8s_reses, namespace) + + # wait k8s resource delete complete + k8s_client.wait_k8s_res_delete(sorted_k8s_reses, namespace) def change_vnfpkg(self, req, inst, grant_req, grant, vnfd): if req.additionalParams.get('upgrade_type') == 'RollingUpdate': @@ -103,43 +121,56 @@ class Kubernetes(object): # check deployment exists in kubernetes vim_info = inst_utils.select_vim_info(inst.vimConnectionInfo) - k8s_client = kubernetes_utils.KubernetesClient(vim_info) - k8s_client.check_deployment_exist(deployment_names, namespace) + # This is Context Manager for creation and deletion + # of CA certificate temp file + with kubernetes_utils.CaCertFileContextManager( + vim_info.interfaceInfo.get('ssl_ca_cert')) as ca_cert_cm: - # get new deployment body - new_deploy_reses = kubernetes_utils.get_new_deployment_body( - req, inst, vnfd, deployment_names, operation='CHANGE_VNFPKG') + # add an item ca_cert_file:file_path + # into vim_info.interfaceInfo, + # and will be deleted in KubernetesClient + vim_info.interfaceInfo['ca_cert_file'] = ca_cert_cm.file_path - # apply new deployment - k8s_client.update_k8s_resource(new_deploy_reses, namespace) + k8s_client = kubernetes_utils.KubernetesClient(vim_info) + k8s_client.check_deployment_exist(deployment_names, namespace) - # wait k8s resource update complete - old_pods_names = [vnfc.computeResource.resourceId for vnfc in - inst.instantiatedVnfInfo.vnfcResourceInfo] - try: - k8s_client.wait_k8s_res_update( - new_deploy_reses, namespace, old_pods_names) - except sol_ex.UpdateK8SResourceFailed as ex: + # get new deployment body + new_deploy_reses = kubernetes_utils.get_new_deployment_body( + req, inst, vnfd, deployment_names, + operation='CHANGE_VNFPKG') + + # apply new deployment + k8s_client.update_k8s_resource(new_deploy_reses, namespace) + + # wait k8s resource update complete + old_pods_names = [vnfc.computeResource.resourceId for vnfc in + inst.instantiatedVnfInfo.vnfcResourceInfo] + try: + k8s_client.wait_k8s_res_update( + new_deploy_reses, namespace, old_pods_names) + except sol_ex.UpdateK8SResourceFailed as ex: + self._update_cnf_instantiated_info( + inst, deployment_names, + k8s_client.list_namespaced_pods( + namespace=namespace)) + raise ex + + # execute coordinate vnf script + try: + self._execute_coordinate_vnf_script( + req, inst, grant_req, grant, vnfd, 'CHANGE_VNFPKG', + namespace, new_deploy_reses) + except sol_ex.CoordinateVNFExecutionFailed as ex: + self._update_cnf_instantiated_info( + inst, deployment_names, + k8s_client.list_namespaced_pods( + namespace=namespace)) + raise ex + + # update cnf instantiated info + all_pods = k8s_client.list_namespaced_pods(namespace) self._update_cnf_instantiated_info( - inst, deployment_names, k8s_client.list_namespaced_pods( - namespace=namespace)) - raise ex - - # execute coordinate vnf script - try: - self._execute_coordinate_vnf_script( - req, inst, grant_req, grant, vnfd, 'CHANGE_VNFPKG', - namespace, new_deploy_reses) - except sol_ex.CoordinateVNFExecutionFailed as ex: - self._update_cnf_instantiated_info( - inst, deployment_names, k8s_client.list_namespaced_pods( - namespace=namespace)) - raise ex - - # update cnf instantiated info - all_pods = k8s_client.list_namespaced_pods(namespace) - self._update_cnf_instantiated_info( - inst, deployment_names, all_pods) + inst, deployment_names, all_pods) else: # TODO(YiFeng): Blue-Green type will be supported in next version. @@ -169,31 +200,41 @@ class Kubernetes(object): # apply old deployment vim_info = inst_utils.select_vim_info(inst.vimConnectionInfo) - k8s_client = kubernetes_utils.KubernetesClient(vim_info) - k8s_client.update_k8s_resource(old_deploy_reses, namespace) + # This is Context Manager for creation and deletion + # of CA certificate temp file + with kubernetes_utils.CaCertFileContextManager( + vim_info.interfaceInfo.get('ssl_ca_cert')) as ca_cert_cm: - # wait k8s resource update complete - old_pods_names = [vnfc.computeResource.resourceId for vnfc in - inst.instantiatedVnfInfo.vnfcResourceInfo] - try: - k8s_client.wait_k8s_res_update( - old_deploy_reses, namespace, old_pods_names) - except sol_ex.UpdateK8SResourceFailed as ex: - raise ex + # add an item ca_cert_file:file_path + # into vim_info.interfaceInfo, + # and will be deleted in KubernetesClient + vim_info.interfaceInfo['ca_cert_file'] = ca_cert_cm.file_path - # execute coordinate vnf script - try: - self._execute_coordinate_vnf_script( - req, inst, grant_req, grant, vnfd, - 'CHANGE_VNFPKG_ROLLBACK', - namespace, old_deploy_reses) - except sol_ex.CoordinateVNFExecutionFailed as ex: - raise ex + k8s_client = kubernetes_utils.KubernetesClient(vim_info) + k8s_client.update_k8s_resource(old_deploy_reses, namespace) - # update cnf instantiated info - all_pods = k8s_client.list_namespaced_pods(namespace) - self._update_cnf_instantiated_info( - inst, deployment_names, all_pods) + # wait k8s resource update complete + old_pods_names = [vnfc.computeResource.resourceId for vnfc in + inst.instantiatedVnfInfo.vnfcResourceInfo] + try: + k8s_client.wait_k8s_res_update( + old_deploy_reses, namespace, old_pods_names) + except sol_ex.UpdateK8SResourceFailed as ex: + raise ex + + # execute coordinate vnf script + try: + self._execute_coordinate_vnf_script( + req, inst, grant_req, grant, vnfd, + 'CHANGE_VNFPKG_ROLLBACK', + namespace, old_deploy_reses) + except sol_ex.CoordinateVNFExecutionFailed as ex: + raise ex + + # update cnf instantiated info + all_pods = k8s_client.list_namespaced_pods(namespace) + self._update_cnf_instantiated_info( + inst, deployment_names, all_pods) else: # TODO(YiFeng): Blue-Green type will be supported in next version. diff --git a/tacker/sol_refactored/infra_drivers/kubernetes/kubernetes_utils.py b/tacker/sol_refactored/infra_drivers/kubernetes/kubernetes_utils.py index a5e0267f6..bab3442c2 100644 --- a/tacker/sol_refactored/infra_drivers/kubernetes/kubernetes_utils.py +++ b/tacker/sol_refactored/infra_drivers/kubernetes/kubernetes_utils.py @@ -17,6 +17,7 @@ import copy import ipaddress import os import re +import tempfile import time from urllib.parse import urlparse import urllib.request as urllib2 @@ -583,6 +584,9 @@ def init_k8s_api_client(vim_info): k8s_config = client.Configuration() k8s_config.host = vim_info.interfaceInfo['endpoint'] + ca_cert_file = (vim_info.interfaceInfo.pop('ca_cert_file') + if 'ca_cert_file' in vim_info.interfaceInfo else None) + if ('username' in vim_info.accessInfo and 'password' in vim_info.accessInfo and vim_info.accessInfo.get( 'password') is not None): @@ -596,8 +600,8 @@ def init_k8s_api_client(vim_info): k8s_config.api_key['authorization'] = vim_info.accessInfo[ 'bearer_token'] - if 'ssl_ca_cert' in vim_info.accessInfo: - k8s_config.ssl_ca_cert = vim_info.accessInfo['ssl_ca_cert'] + if 'ssl_ca_cert' in vim_info.interfaceInfo and ca_cert_file: + k8s_config.ssl_ca_cert = ca_cert_file k8s_config.verify_ssl = True else: k8s_config.verify_ssl = False @@ -724,3 +728,27 @@ def get_new_deployment_body( new_deploy_reses.append(k8s_res) return new_deploy_reses + + +class CaCertFileContextManager: + def __init__(self, ca_cert_str): + self._file_descriptor = None + self.file_path = None + self.ca_cert_str = ca_cert_str + + def __enter__(self): + if not self.ca_cert_str: + return self + self._file_descriptor, self.file_path = tempfile.mkstemp() + ca_cert = re.sub(r'\s', '\n', self.ca_cert_str) + ca_cert = re.sub(r'BEGIN\nCERT', r'BEGIN CERT', ca_cert) + ca_cert = re.sub(r'END\nCERT', r'END CERT', ca_cert) + # write ca cert file + os.write(self._file_descriptor, ca_cert.encode()) + return self + + def __exit__(self, exc_type, exc_value, exc_traceback): + if not self.ca_cert_str: + return + os.close(self._file_descriptor) + os.remove(self.file_path) diff --git a/tacker/tests/functional/sol_kubernetes_v2/base_v2.py b/tacker/tests/functional/sol_kubernetes_v2/base_v2.py index 6e9f550f1..5f8adaa49 100644 --- a/tacker/tests/functional/sol_kubernetes_v2/base_v2.py +++ b/tacker/tests/functional/sol_kubernetes_v2/base_v2.py @@ -49,6 +49,7 @@ class BaseVnfLcmKubernetesV2Test(base.BaseTestCase): k8s_vim_info = cls.get_k8s_vim_info() cls.auth_url = k8s_vim_info.interfaceInfo['endpoint'] cls.bearer_token = k8s_vim_info.accessInfo['bearer_token'] + cls.ssl_ca_cert = k8s_vim_info.interfaceInfo['ssl_ca_cert'] vim_info = cls.get_vim_info() auth = http_client.KeystonePasswordAuthHandle( @@ -85,7 +86,10 @@ class BaseVnfLcmKubernetesV2Test(base.BaseTestCase): vim_params = yaml.safe_load(base_utils.read_file('local-k8s-vim.yaml')) vim_info = objects.VimConnectionInfo( - interfaceInfo={'endpoint': vim_params['auth_url']}, + interfaceInfo={ + 'endpoint': vim_params['auth_url'], + 'ssl_ca_cert': vim_params.get('ssl_ca_cert') + }, accessInfo={ 'region': 'RegionOne', 'bearer_token': vim_params['bearer_token'] diff --git a/tacker/tests/functional/sol_kubernetes_v2/paramgen.py b/tacker/tests/functional/sol_kubernetes_v2/paramgen.py index 1a0d72c67..e33051744 100644 --- a/tacker/tests/functional/sol_kubernetes_v2/paramgen.py +++ b/tacker/tests/functional/sol_kubernetes_v2/paramgen.py @@ -47,7 +47,7 @@ def test_instantiate_cnf_resources_terminate(): } -def max_sample_instantiate(auth_url, bearer_token): +def max_sample_instantiate(auth_url, bearer_token, ssl_ca_cert=None): # All attributes are set. # NOTE: All of the following cardinality attributes are set. # In addition, 0..N or 1..N attributes are set to 2 or more. @@ -78,6 +78,9 @@ def max_sample_instantiate(auth_url, bearer_token): }, "extra": {"dummy-key": "dummy-val"} } + if ssl_ca_cert: + vim_1["interfaceInfo"]["ssl_ca_cert"] = ssl_ca_cert + vim_2["interfaceInfo"]["ssl_ca_cert"] = ssl_ca_cert return { "flavourId": "simple", "vimConnectionInfo": { diff --git a/tacker/tests/functional/sol_kubernetes_v2/test_vnflcm_basic.py b/tacker/tests/functional/sol_kubernetes_v2/test_vnflcm_basic.py index d17c54568..ed7bd6723 100644 --- a/tacker/tests/functional/sol_kubernetes_v2/test_vnflcm_basic.py +++ b/tacker/tests/functional/sol_kubernetes_v2/test_vnflcm_basic.py @@ -98,7 +98,7 @@ class VnfLcmKubernetesTest(base_v2.BaseVnfLcmKubernetesV2Test): # 2. Instantiate a VNF instance instantiate_req = paramgen.max_sample_instantiate( - self.auth_url, self.bearer_token) + self.auth_url, self.bearer_token, ssl_ca_cert=self.ssl_ca_cert) resp, body = self.instantiate_vnf_instance(inst_id, instantiate_req) self.assertEqual(202, resp.status_code) self.check_resp_headers_in_operation_task(resp) diff --git a/tacker/tests/unit/sol_refactored/infra_drivers/kubernetes/test_kubernetes.py b/tacker/tests/unit/sol_refactored/infra_drivers/kubernetes/test_kubernetes.py index 81d7e2d21..0b93c3f35 100644 --- a/tacker/tests/unit/sol_refactored/infra_drivers/kubernetes/test_kubernetes.py +++ b/tacker/tests/unit/sol_refactored/infra_drivers/kubernetes/test_kubernetes.py @@ -146,7 +146,7 @@ class TestKubernetes(base.BaseTestCase): _instantiate_req_example) req.additionalParams['lcm-kubernetes-def-files'].append( 'Files/kubernetes/namespace.yaml') - req.vimConnectionInfo['vim1']['interfaceInfo']['ssl_ca_cert '] = 'test' + req.vimConnectionInfo['vim1']['interfaceInfo']['ssl_ca_cert'] = 'test' req.additionalParams['namespace'] = 'curry' inst = objects.VnfInstanceV2( vimConnectionInfo=req.vimConnectionInfo @@ -1009,6 +1009,7 @@ class TestKubernetes(base.BaseTestCase): # prepare instantiate req = objects.InstantiateVnfRequest.from_dict( _instantiate_req_example) + req.vimConnectionInfo['vim1']['interfaceInfo']['ssl_ca_cert'] = 'test' inst = objects.VnfInstanceV2( vimConnectionInfo=req.vimConnectionInfo ) @@ -1302,6 +1303,7 @@ class TestKubernetes(base.BaseTestCase): # prepare instantiate req = objects.InstantiateVnfRequest.from_dict( _instantiate_req_example) + req.vimConnectionInfo['vim1']['interfaceInfo']['ssl_ca_cert'] = 'test' inst = objects.VnfInstanceV2( id=uuidutils.generate_uuid(), vimConnectionInfo=req.vimConnectionInfo