starlingx/distcloud
Code Issues Proposed changes

Pass root cert and intermediate cert to subcloud

Browse Source 
When a subcloud is created, create an intermediate CA certificate and
pass the certificate and key pair to subcloud for subcloud to create
its intermediate CA.
Also pass root CA certificate to subcloud so all certificates issued
directly or indirectly by DC root CA are trusted.
On the subcloud delete, delete the intermediate CA and certificate.

Test cases:

Add subclouds
  - intermediate cert and CA cert are passed to subcloud bootstrap
Delete subclouds
  - intermediate cert and secret are deleted from cert-manager

Story: 2007347
Task: 39432

Depends-on: https://review.opendev.org/#/c/720270

Change-Id: I899ddea6b6d2f4fae7a5209251d00bca315a2aa4
Signed-off-by: Bin Qian <bin.qian@windriver.com>
This commit is contained in:
Bin Qian 2020-04-07 23:23:58 -04:00
parent 1190428cd5
commit 1b7c111a48
5 changed files with 249 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
distributedcloud/centos/distributedcloud.spec
View File

@ -72,6 +72,7 @@ Distributed Cloud provides configuration and management of distributed clouds
# DC Common
%package dccommon
Summary: DC common module
Requires: python-kubernetes
%description dccommon
Distributed Cloud Common Module

147
distributedcloud/dccommon/kubeoperator.py Normal file
View File

@ -0,0 +1,147 @@
#
# Copyright (c) 2020 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
from kubernetes import client
from kubernetes.client import Configuration
from kubernetes.client.rest import ApiException
from kubernetes import config
from oslo_log import log as logging
from six.moves import http_client as httplib
LOG = logging.getLogger(__name__)
KUBE_CONFIG_PATH = '/etc/kubernetes/admin.conf'
CERT_MANAGER_GROUP = 'cert-manager.io'
CERT_MANAGER_VERSION = 'v1alpha2'
CERT_MANAGER_CERTIFICATE = 'certificates'
class KubeOperator(object):
def __init__(self):
self._kube_client_batch = None
self._kube_client_core = None
self._kube_client_custom_objects = None
def _load_kube_config(self):
config.load_kube_config(KUBE_CONFIG_PATH)
# Workaround: Turn off SSL/TLS verification
c = Configuration()
c.verify_ssl = False
Configuration.set_default(c)
def _get_kubernetesclient_batch(self):
if not self._kube_client_batch:
self._load_kube_config()
self._kube_client_batch = client.BatchV1Api()
return self._kube_client_batch
def _get_kubernetesclient_core(self):
if not self._kube_client_core:
self._load_kube_config()
self._kube_client_core = client.CoreV1Api()
return self._kube_client_core
def _get_kubernetesclient_custom_objects(self):
if not self._kube_client_custom_objects:
self._load_kube_config()
self._kube_client_custom_objects = client.CustomObjectsApi()
return self._kube_client_custom_objects
def kube_get_secret(self, name, namespace):
c = self._get_kubernetesclient_core()
try:
return c.read_namespaced_secret(name, namespace)
except ApiException as e:
if e.status == httplib.NOT_FOUND:
return None
else:
LOG.error("Failed to get Secret %s under "
"Namespace %s: %s" % (name, namespace, e.body))
raise
except Exception as e:
LOG.error("Kubernetes exception in kube_get_secret: %s" % e)
raise
def kube_delete_secret(self, name, namespace, **kwargs):
body = {}
if kwargs:
body.update(kwargs)
c = self._get_kubernetesclient_core()
try:
c.delete_namespaced_secret(name, namespace, body)
except ApiException as e:
if e.status == httplib.NOT_FOUND:
LOG.warn("Secret %s under Namespace %s "
"not found." % (name, namespace))
else:
LOG.error("Failed to clean up Secret %s under "
"Namespace %s: %s" % (name, namespace, e.body))
raise
except Exception as e:
LOG.error("Kubernetes exception in kube_delete_secret: %s" % e)
raise
def get_cert_manager_certificate(self, namespace, name):
custom_object_api = self._get_kubernetesclient_custom_objects()
try:
cert = custom_object_api.get_namespaced_custom_object(
CERT_MANAGER_GROUP,
CERT_MANAGER_VERSION,
namespace,
CERT_MANAGER_CERTIFICATE,
name)
except ApiException as e:
if e.status == httplib.NOT_FOUND:
return None
else:
LOG.error("Fail to access %s:%s. %s" % (namespace, name, e))
raise
else:
return cert
def apply_cert_manager_certificate(self, namespace, name, body):
custom_object_api = self._get_kubernetesclient_custom_objects()
cert = self.get_cert_manager_certificate(namespace, name)
if cert:
custom_object_api.patch_namespaced_custom_object(
CERT_MANAGER_GROUP,
CERT_MANAGER_VERSION,
namespace,
CERT_MANAGER_CERTIFICATE,
name,
body
)
else:
custom_object_api.create_namespaced_custom_object(
CERT_MANAGER_GROUP,
CERT_MANAGER_VERSION,
namespace,
CERT_MANAGER_CERTIFICATE,
body)
def delete_cert_manager_certificate(self, namespace, name):
custom_object_api = self._get_kubernetesclient_custom_objects()
try:
custom_object_api.delete_namespaced_custom_object(
CERT_MANAGER_GROUP,
CERT_MANAGER_VERSION,
namespace,
CERT_MANAGER_CERTIFICATE,
name,
{}
)
except ApiException as e:
if e.status != httplib.NOT_FOUND:
LOG.error("Fail to delete %s:%s. %s" % (namespace, name, e))
raise

89
distributedcloud/dcmanager/manager/subcloud_manager.py
View File

@ -28,6 +28,7 @@ import keyring
import netaddr
import os
import threading
import time
from oslo_log import log as logging
from oslo_messaging import RemoteError
@ -37,6 +38,7 @@ from tsconfig.tsconfig import SW_VERSION
from dccommon.drivers.openstack.keystone_v3 import KeystoneClient
from dccommon.drivers.openstack.sysinv_v1 import SysinvClient
from dccommon import kubeoperator
from dcorch.common import consts as dcorch_consts
from dcorch.rpc import client as dcorch_rpc_client
@ -47,6 +49,7 @@ from dcmanager.common import exceptions
from dcmanager.common.i18n import _
from dcmanager.common import manager
from dcmanager.common import utils
from dcmanager.db import api as db_api
from dcmanager.manager.subcloud_install import SubcloudInstall
@ -78,6 +81,10 @@ USERS_TO_REPLICATE = [
SERVICES_USER = 'services'
SC_INTERMEDIATE_CERT_DURATION = "87600h"
SC_INTERMEDIATE_CERT_RENEW_BEFORE = "720h"
CERT_NAMESPACE = "dc-cert"
def sync_update_subcloud_endpoint_status(func):
"""Synchronized lock decorator for _update_subcloud_endpoint_status. """
@ -108,6 +115,70 @@ class SubcloudManager(manager.Manager):
self.dcorch_rpc_client = dcorch_rpc_client.EngineClient()
self.fm_api = fm_api.FaultAPIs()
@staticmethod
def _get_subcloud_cert_name(subcloud_name):
cert_name = "%s-adminep-ca-certificate" % subcloud_name
return cert_name
@staticmethod
def _get_subcloud_cert_secret_name(subcloud_name):
secret_name = "%s-adminep-ca-certificate" % subcloud_name
return secret_name
@staticmethod
def _create_intermediate_ca_cert(payload):
subcloud_name = payload["name"]
cert_name = SubcloudManager._get_subcloud_cert_name(subcloud_name)
secret_name = SubcloudManager._get_subcloud_cert_secret_name(
subcloud_name)
cert = {
"apiVersion": "cert-manager.io/v1alpha2",
"kind": "Certificate",
"metadata": {
"namespace": CERT_NAMESPACE,
"name": cert_name
},
"spec": {
"secretName": secret_name,
"duration": SC_INTERMEDIATE_CERT_DURATION,
"renewBefore": SC_INTERMEDIATE_CERT_RENEW_BEFORE,
"issuerRef": {
"kind": "Issuer",
"name": "dc-adminep-root-ca-issuer"
},
"commonName": cert_name,
"isCA": True,
},
}
kube = kubeoperator.KubeOperator()
kube.apply_cert_manager_certificate(CERT_NAMESPACE, cert_name, cert)
for count in range(1, 20):
secret = kube.kube_get_secret(secret_name, CERT_NAMESPACE)
if not hasattr(secret, 'data'):
time.sleep(1)
LOG.debug('Wait for %s ... %s' % (secret_name, count))
continue
data = secret.data
if 'ca.crt' not in data or \
'tls.crt' not in data or 'tls.key' not in data:
# ca cert, certificate and key pair are needed and must exist
# for creating an intermediate ca. If not, certificate is not
# ready yet.
time.sleep(1)
LOG.debug('Wait for %s ... %s' % (secret_name, count))
continue
payload['dc_root_ca_cert'] = data['ca.crt']
payload['sc_ca_cert'] = data['tls.crt']
payload['sc_ca_key'] = data['tls.key']
return
raise Exception("Secret for certificate %s is not ready." % cert_name)
def add_subcloud(self, context, payload):
"""Add subcloud and notify orchestrators.
@ -295,6 +366,9 @@ class SubcloudManager(manager.Manager):
self._create_subcloud_inventory(payload,
ansible_subcloud_inventory_file)
# create subcloud intermediate certificate and pass in keys
self._create_intermediate_ca_cert(payload)
# Write this subclouds overrides to file
# NOTE: This file should not be deleted if subcloud add fails
# as it is used for debugging
@ -583,6 +657,18 @@ class SubcloudManager(manager.Manager):
subcloud.systemcontroller_gateway_ip)),
1)
@staticmethod
def _delete_subcloud_cert(subcloud_name):
cert_name = SubcloudManager._get_subcloud_cert_name(subcloud_name)
secret_name = SubcloudManager._get_subcloud_cert_secret_name(
subcloud_name)
kube = kubeoperator.KubeOperator()
kube.delete_cert_manager_certificate(CERT_NAMESPACE, cert_name)
kube.kube_delete_secret(secret_name, CERT_NAMESPACE)
LOG.info("cert %s and secret %s are deleted" % (cert_name, secret_name))
def _remove_subcloud_details(self, context,
subcloud,
ansible_subcloud_inventory_file,
@ -626,6 +712,9 @@ class SubcloudManager(manager.Manager):
# Delete the ansible inventory for the new subcloud
self._delete_subcloud_inventory(ansible_subcloud_inventory_file)
# Delete the subcloud intermediate certificate
SubcloudManager._delete_subcloud_cert(subcloud.name)
# Regenerate the addn_hosts_dc file
self._create_addn_hosts_dc(context)

14
distributedcloud/dcmanager/tests/unit/manager/test_subcloud_manager.py
View File

@ -10,7 +10,7 @@
# License for the specific language governing permissions and limitations
# under the License.
#
# Copyright (c) 2017 Wind River Systems, Inc.
# Copyright (c) 2017-2020 Wind River Systems, Inc.
#
# The right to copy, distribute, modify, or otherwise make use
# of this software may be licensed only pursuant to the terms
@ -160,6 +160,8 @@ class TestSubcloudManager(base.DCManagerTestCase):
self.assertEqual('localhost', sm.host)
self.assertEqual(self.ctx, sm.context)
@mock.patch.object(subcloud_manager.SubcloudManager,
'_create_intermediate_ca_cert')
@mock.patch.object(subcloud_manager.SubcloudManager,
'_delete_subcloud_inventory')
@mock.patch.object(subcloud_manager, 'KeystoneClient')
@ -180,7 +182,8 @@ class TestSubcloudManager(base.DCManagerTestCase):
mock_create_subcloud_inventory,
mock_create_addn_hosts, mock_sysinv_client,
mock_db_api, mock_keystone_client,
mock_delete_subcloud_inventory):
mock_delete_subcloud_inventory,
mock_create_intermediate_ca_cert):
values = utils.create_subcloud_dict(base.SUBCLOUD_SAMPLE_DATA_0)
controllers = FAKE_CONTROLLERS
services = FAKE_SERVICES
@ -202,7 +205,10 @@ class TestSubcloudManager(base.DCManagerTestCase):
mock_write_subcloud_ansible_config.assert_called_once()
mock_keyring.get_password.assert_called()
mock_thread_start.assert_called_once()
mock_create_intermediate_ca_cert.assert_called_once()
@mock.patch.object(subcloud_manager.SubcloudManager,
'_delete_subcloud_cert')
@mock.patch.object(subcloud_manager, 'db_api')
@mock.patch.object(subcloud_manager, 'SysinvClient')
@mock.patch.object(subcloud_manager, 'KeystoneClient')
@ -211,7 +217,8 @@ class TestSubcloudManager(base.DCManagerTestCase):
def test_delete_subcloud(self, mock_create_addn_hosts,
mock_keystone_client,
mock_sysinv_client,
mock_db_api):
mock_db_api,
mock_delete_subcloud_cert):
controllers = FAKE_CONTROLLERS
data = utils.create_subcloud_dict(base.SUBCLOUD_SAMPLE_DATA_0)
fake_subcloud = Subcloud(data, False)
@ -223,6 +230,7 @@ class TestSubcloudManager(base.DCManagerTestCase):
mock_keystone_client().delete_region.assert_called_once()
mock_db_api.subcloud_destroy.assert_called_once()
mock_create_addn_hosts.assert_called_once()
mock_delete_subcloud_cert.assert_called_once()
@mock.patch.object(subcloud_manager, 'db_api')
def test_update_subcloud(self, mock_db_api):

1
distributedcloud/requirements.txt
View File

@ -47,3 +47,4 @@ python-novaclient>=7.1.0 # Apache-2.0
python-keystoneclient>=3.8.0 # Apache-2.0
pycrypto>=2.6 # Public Domain
requests_toolbelt
kubernetes # Apache-2.0