Docker registry certificate management by cert-manager
Cert-mon changes to monitor 'system-docker-local-certificate' k8s secret and install StarlingX docker_registry certificate for registry.local:9001. Changes include: - New thread to watch registry certificate changes - Refactored the code to reduce code duplication - Call sysinv api for 'certificate_install' Design testing completed: - When k8s secret is added/modified (initiated by cert-manager), certificate installation is completed - sysinv api 'certificate_install' installs & confirmed via openssl s_client -connect registry.local:9001 - When certificate is renewed, keys get regenerated (no changes needed. Confirmed that existing infrastructure takes care of it) Story: 2007361 Task: 41717 Change-Id: Iffa68486764287a1b82a183ab9801a53c1e4885b Signed-off-by: Isac Souza <IsacSacchi.Souza@windriver.com>
This commit is contained in:
Isac Souza
parent
53e3797fca
commit
983add3417
|
|
@ -316,16 +316,16 @@ class CertificateController(rest.RestController):
|
|||
force = True
|
||||
else:
|
||||
force = False
|
||||
# if PLATFORM_CERT_SECRET_NAME secret is present in k8s, we
|
||||
# if RESTAPI_CERT_SECRET_NAME secret is present in k8s, we
|
||||
# assume that SSL cert is managed by cert-manager/cert-mon
|
||||
managed_by_cm = self._kube_op.kube_get_secret(
|
||||
constants.PLATFORM_CERT_SECRET_NAME,
|
||||
constants.RESTAPI_CERT_SECRET_NAME,
|
||||
constants.CERT_NAMESPACE_PLATFORM_CERTS)
|
||||
|
||||
if force is False and managed_by_cm is not None:
|
||||
msg = "Certificate is currently being managed by cert-manager. \n" \
|
||||
"To manage certificate with this command, first delete " \
|
||||
"the %s Certificate and Secret." % constants.PLATFORM_CERT_SECRET_NAME
|
||||
"the %s Certificate and Secret." % constants.RESTAPI_CERT_SECRET_NAME
|
||||
LOG.info(msg)
|
||||
return dict(success="", error=msg)
|
||||
|
||||
|
|
|
|||
|
|
@ -451,12 +451,12 @@ class SystemController(rest.RestController):
|
|||
# while 'ssl' cert is managed by cert-manager, return error
|
||||
# (Otherwise, cert-mon will turn https back on during cert-renewal process)
|
||||
managed_by_cm = self._kube_op.kube_get_secret(
|
||||
constants.PLATFORM_CERT_SECRET_NAME,
|
||||
constants.RESTAPI_CERT_SECRET_NAME,
|
||||
constants.CERT_NAMESPACE_PLATFORM_CERTS)
|
||||
if https_enabled == 'false' and managed_by_cm is not None:
|
||||
msg = "Certificate is currently being managed by cert-manager. " \
|
||||
"Remove %s Certificate and Secret before disabling https." % \
|
||||
constants.PLATFORM_CERT_SECRET_NAME
|
||||
constants.RESTAPI_CERT_SECRET_NAME
|
||||
raise wsme.exc.ClientSideError(_(msg))
|
||||
|
||||
if https_enabled != rpc_isystem['capabilities']['https_enabled']:
|
||||
|
|
|
|||
|
|
@ -54,11 +54,11 @@ CONF.register_opts(cert_mon_opts, 'certmon')
|
|||
class CertificateMonManager(periodic_task.PeriodicTasks):
|
||||
def __init__(self):
|
||||
super(CertificateMonManager, self).__init__(CONF)
|
||||
self.dc_mon_thread = None
|
||||
self.platcert_mon_thread = None
|
||||
self.mon_threads = []
|
||||
self.audit_thread = None
|
||||
self.dc_monitor = None
|
||||
self.platcert_monitor = None
|
||||
self.restapicert_monitor = None
|
||||
self.registrycert_monitor = None
|
||||
self.reattempt_tasks = []
|
||||
self.subclouds_to_audit = []
|
||||
|
||||
|
|
@ -190,17 +190,22 @@ class CertificateMonManager(periodic_task.PeriodicTasks):
|
|||
self.dc_monitor.initialize(
|
||||
audit_subcloud=lambda subcloud_name: self.requeue_audit(subcloud_name))
|
||||
|
||||
def init_platformcert_monitor(self):
|
||||
self.platcert_monitor = watcher.PlatCert_CertWatcher()
|
||||
self.platcert_monitor.initialize()
|
||||
def init_restapicert_monitor(self):
|
||||
self.restapicert_monitor = watcher.RestApiCert_CertWatcher()
|
||||
self.restapicert_monitor.initialize()
|
||||
|
||||
def init_registrycert_monitor(self):
|
||||
self.registrycert_monitor = watcher.RegistryCert_CertWatcher()
|
||||
self.registrycert_monitor.initialize()
|
||||
|
||||
def start_monitor(self):
|
||||
utils.init_keystone_auth_opts()
|
||||
dc_role = utils.get_dc_role()
|
||||
while True:
|
||||
try:
|
||||
# init platformcert monitor
|
||||
self.init_platformcert_monitor()
|
||||
# init platform cert monitors
|
||||
self.init_restapicert_monitor()
|
||||
self.init_registrycert_monitor()
|
||||
|
||||
# init dc monitor only if running in DC role
|
||||
if (dc_role == constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER or
|
||||
|
|
@ -213,20 +218,17 @@ class CertificateMonManager(periodic_task.PeriodicTasks):
|
|||
break
|
||||
|
||||
# spawn threads (DC thread spawned only if running in DC role)
|
||||
self.platcert_mon_thread = greenthread.spawn(self.platcert_monitor_cert)
|
||||
self.mon_threads.append(greenthread.spawn(self.monitor_cert, self.restapicert_monitor))
|
||||
self.mon_threads.append(greenthread.spawn(self.monitor_cert, self.registrycert_monitor))
|
||||
|
||||
if (dc_role == constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER or
|
||||
dc_role == constants.DISTRIBUTED_CLOUD_ROLE_SUBCLOUD):
|
||||
self.dc_mon_thread = greenthread.spawn(self.dc_monitor_cert)
|
||||
self.mon_threads.append(greenthread.spawn(self.monitor_cert, self.dc_monitor))
|
||||
|
||||
def stop_monitor(self):
|
||||
if self.dc_mon_thread:
|
||||
self.dc_mon_thread.kill()
|
||||
self.dc_mon_thread.wait()
|
||||
|
||||
if self.platcert_mon_thread:
|
||||
self.platcert_mon_thread.kill()
|
||||
self.platcert_mon_thread.wait()
|
||||
for mon_thread in self.mon_threads:
|
||||
mon_thread.kill()
|
||||
mon_thread.wait()
|
||||
|
||||
def stop_audit(self):
|
||||
if self.audit_thread:
|
||||
|
|
@ -243,26 +245,11 @@ class CertificateMonManager(periodic_task.PeriodicTasks):
|
|||
except Exception as e:
|
||||
LOG.exception(e)
|
||||
|
||||
def dc_monitor_cert(self):
|
||||
def monitor_cert(self, monitor):
|
||||
while True:
|
||||
# never exit until exit signal received
|
||||
try:
|
||||
self.dc_monitor.start_watch(
|
||||
on_success=lambda task_id: self._purge_reattempt_task(task_id),
|
||||
on_error=lambda task: self._add_reattempt_task(task),
|
||||
)
|
||||
except greenlet.GreenletExit:
|
||||
break
|
||||
except Exception as e:
|
||||
# A bug somewhere?
|
||||
# It shouldn't fall to here, but log and restart if it did
|
||||
LOG.exception(e)
|
||||
|
||||
def platcert_monitor_cert(self):
|
||||
while True:
|
||||
# never exit until exit signal received
|
||||
try:
|
||||
self.platcert_monitor.start_watch(
|
||||
monitor.start_watch(
|
||||
on_success=lambda task_id: self._purge_reattempt_task(task_id),
|
||||
on_error=lambda task: self._add_reattempt_task(task),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -576,8 +576,8 @@ def rest_api_upload(token, filepath, url, data=None):
|
|||
return upload_request_with_data(token, url, body=file_to_upload, data=data)
|
||||
|
||||
|
||||
def update_platformcert_pemfile(tls_crt, tls_key):
|
||||
LOG.info('Updating platformcert temporary pemfile')
|
||||
def update_pemfile(tls_crt, tls_key):
|
||||
LOG.info('Updating temporary pemfile')
|
||||
try:
|
||||
fd, tmppath = tempfile.mkstemp(suffix='.pem')
|
||||
with open(tmppath, 'w+') as f:
|
||||
|
|
@ -593,13 +593,20 @@ def update_platformcert_pemfile(tls_crt, tls_key):
|
|||
return tmppath
|
||||
|
||||
|
||||
def update_platform_cert(token, pem_file_path):
|
||||
LOG.info('Updating platform certificate. pem_file_path=%s' % pem_file_path)
|
||||
def update_platform_cert(token, cert_type, pem_file_path, force=False):
|
||||
"""Update a platform certificate using the sysinv API
|
||||
:param token: the token to access the sysinv API
|
||||
:param cert_type: the type of the certificate that is being updated
|
||||
:param pem_file_path: path to the certificate file in PEM format
|
||||
:param force: whether to bypass semantic checks and force the update,
|
||||
defaults to False
|
||||
"""
|
||||
LOG.info('Updating %s certificate. pem_file_path=%s' % (cert_type, pem_file_path))
|
||||
sysinv_url = token.get_service_internal_url(constants.SERVICE_TYPE_PLATFORM, constants.SYSINV_USERNAME)
|
||||
api_cmd = sysinv_url + '/certificate/certificate_install'
|
||||
|
||||
data = {'mode': 'ssl',
|
||||
'force': 'true'}
|
||||
data = {'mode': cert_type,
|
||||
'force': str(force).lower()}
|
||||
|
||||
response = rest_api_upload(token, pem_file_path, api_cmd, data)
|
||||
error = response.get('error')
|
||||
|
|
|
|||
|
|
@ -256,18 +256,32 @@ class DC_CertWatcher(CertWatcher):
|
|||
self.register_listener(RootCARenew(self.context))
|
||||
|
||||
|
||||
class PlatCert_CertWatcher(CertWatcher):
|
||||
class RestApiCert_CertWatcher(CertWatcher):
|
||||
def __init__(self):
|
||||
super(PlatCert_CertWatcher, self).__init__()
|
||||
super(RestApiCert_CertWatcher, self).__init__()
|
||||
|
||||
def initialize(self):
|
||||
self.context.initialize()
|
||||
|
||||
platcert_ns = constants.CERT_NAMESPACE_PLATFORM_CERTS
|
||||
LOG.info('setting ns : %s & registering listener' % platcert_ns)
|
||||
LOG.info('setting ns for restapi cert : %s & registering listener' % platcert_ns)
|
||||
self.namespace = platcert_ns
|
||||
self.context.kubernete_namespace = platcert_ns
|
||||
self.register_listener(PlatformCertRenew(self.context))
|
||||
self.register_listener(RestApiCertRenew(self.context))
|
||||
|
||||
|
||||
class RegistryCert_CertWatcher(CertWatcher):
|
||||
def __init__(self):
|
||||
super(RegistryCert_CertWatcher, self).__init__()
|
||||
|
||||
def initialize(self):
|
||||
self.context.initialize()
|
||||
|
||||
platcert_ns = constants.CERT_NAMESPACE_PLATFORM_CERTS
|
||||
LOG.info('setting ns for registry cert : %s & registering listener' % platcert_ns)
|
||||
self.namespace = platcert_ns
|
||||
self.context.kubernete_namespace = platcert_ns
|
||||
self.register_listener(RegistryCertRenew(self.context))
|
||||
|
||||
|
||||
class CertificateRenew(CertWatcherListener):
|
||||
|
|
@ -496,23 +510,58 @@ class RootCARenew(CertificateRenew):
|
|||
|
||||
|
||||
class PlatformCertRenew(CertificateRenew):
|
||||
def __init__(self, context):
|
||||
"""Handles a renew event for a certificate that must be installed as a platform cert.
|
||||
"""
|
||||
|
||||
def __init__(self, context, secret_name):
|
||||
super(PlatformCertRenew, self).__init__(context)
|
||||
self.secret_name = constants.PLATFORM_CERT_SECRET_NAME
|
||||
LOG.info('PlatformCertRenew init with secretname: %s' % self.secret_name)
|
||||
self.secret_name = secret_name
|
||||
LOG.info('%s init with secretname: %s' % (self.__class__.__name__, self.secret_name))
|
||||
|
||||
def check_filter(self, event_data):
|
||||
LOG.info('%s: Received event_data %s' % (self.secret_name, event_data))
|
||||
if self.secret_name == event_data.secret_name:
|
||||
return self.certificate_is_ready(event_data)
|
||||
else:
|
||||
return False
|
||||
|
||||
def update_platform_certificate(self, event_data, cert_type, force=False):
|
||||
"""Update a platform certificate
|
||||
|
||||
Save the certificate and key from the secret into a PEM file and send it to the
|
||||
platform to be installed. If force=True, the platform semantic checks will be
|
||||
skipped.
|
||||
|
||||
:param event_data: the event_data that triggered this renew
|
||||
:param cert_type: the type of the certificate that is being updated
|
||||
:param force: whether to bypass semantic checks and force the update,
|
||||
defaults to False
|
||||
"""
|
||||
pem_file_path = utils.update_pemfile(event_data.tls_crt, event_data.tls_key)
|
||||
|
||||
token = self.context.get_token()
|
||||
utils.update_platform_cert(token, cert_type, pem_file_path, force)
|
||||
|
||||
|
||||
class RestApiCertRenew(PlatformCertRenew):
|
||||
def __init__(self, context):
|
||||
super(RestApiCertRenew, self).__init__(context, constants.RESTAPI_CERT_SECRET_NAME)
|
||||
|
||||
def update_certificate(self, event_data):
|
||||
LOG.info('PlatformCertRenew: Secret changes detected. Initiating certificate update')
|
||||
LOG.info('RestApiCertRenew: Secret changes detected. Initiating certificate update')
|
||||
token = self.context.get_token()
|
||||
system_uuid = utils.get_isystems_uuid(token)
|
||||
ret = utils.enable_https(token, system_uuid)
|
||||
pem_file_path = utils.update_platformcert_pemfile(event_data.tls_crt,
|
||||
event_data.tls_key)
|
||||
|
||||
if ret is True:
|
||||
utils.update_platform_cert(token, pem_file_path)
|
||||
self.update_platform_certificate(event_data, constants.CERT_MODE_SSL, force=True)
|
||||
|
||||
|
||||
class RegistryCertRenew(PlatformCertRenew):
|
||||
def __init__(self, context):
|
||||
super(RegistryCertRenew, self).__init__(context, constants.REGISTRY_CERT_SECRET_NAME)
|
||||
|
||||
def update_certificate(self, event_data):
|
||||
LOG.info('RegistryCertRenew: Secret changes detected. Initiating certificate update')
|
||||
|
||||
self.update_platform_certificate(event_data, constants.CERT_MODE_DOCKER_REGISTRY)
|
||||
|
|
|
|||
|
|
@ -1735,5 +1735,6 @@ DC_ROOT_CA_CONFIG_PATH = \
|
|||
ADMIN_EP_CERT_FORMAT = '{tls_key}'
|
||||
|
||||
# Platform certificates
|
||||
PLATFORM_CERT_SECRET_NAME = "system-restapi-gui-certificate"
|
||||
RESTAPI_CERT_SECRET_NAME = "system-restapi-gui-certificate"
|
||||
REGISTRY_CERT_SECRET_NAME = "system-registry-local-certificate"
|
||||
CERT_NAMESPACE_PLATFORM_CERTS = 'deployment'
|
||||
|
|
|
|||
|
|
@ -42,19 +42,18 @@ class CertMonTestCase(base.DbTestCase):
|
|||
def tearDown(self):
|
||||
super(CertMonTestCase, self).tearDown()
|
||||
|
||||
def test_platformcert_secret_and_ns_check(self):
|
||||
def test_platform_certs_secret_and_ns_check(self):
|
||||
self.assertEqual("system-restapi-gui-certificate",
|
||||
constants.PLATFORM_CERT_SECRET_NAME)
|
||||
constants.RESTAPI_CERT_SECRET_NAME)
|
||||
self.assertEqual("system-registry-local-certificate",
|
||||
constants.REGISTRY_CERT_SECRET_NAME)
|
||||
self.assertEqual("deployment",
|
||||
constants.CERT_NAMESPACE_PLATFORM_CERTS)
|
||||
|
||||
def test_update_pemfile(self):
|
||||
reference_file = os.path.join(os.path.dirname(__file__),
|
||||
"data", "cert-with-key.pem")
|
||||
cert_filename = os.path.join(os.path.dirname(__file__),
|
||||
"data", "cert.pem")
|
||||
key_filename = os.path.join(os.path.dirname(__file__),
|
||||
"data", "key.pem")
|
||||
reference_file = self.get_data_file_path("cert-with-key.pem")
|
||||
cert_filename = self.get_data_file_path("cert.pem")
|
||||
key_filename = self.get_data_file_path("key.pem")
|
||||
|
||||
with open(cert_filename, 'r') as cfile:
|
||||
tls_cert = cfile.read()
|
||||
|
|
@ -62,14 +61,14 @@ class CertMonTestCase(base.DbTestCase):
|
|||
with open(key_filename, 'r') as kfile:
|
||||
tls_key = kfile.read()
|
||||
|
||||
generated_file = cert_mon_utils.update_platformcert_pemfile(tls_cert, tls_key)
|
||||
generated_file = cert_mon_utils.update_pemfile(tls_cert, tls_key)
|
||||
assert os.path.exists(generated_file)
|
||||
assert filecmp.cmp(generated_file, reference_file, shallow=False)
|
||||
|
||||
os.remove(generated_file)
|
||||
|
||||
def get_keystone_token(self):
|
||||
token_file = os.path.join(os.path.dirname(__file__), "data", "keystone-token")
|
||||
token_file = self.get_data_file_path("keystone-token")
|
||||
with open(token_file, 'r') as tfile:
|
||||
token_json = json.load(tfile)
|
||||
|
||||
|
|
@ -78,7 +77,7 @@ class CertMonTestCase(base.DbTestCase):
|
|||
return Token(token_json, token_id, region_name)
|
||||
|
||||
def test_get_isystems_uuid(self):
|
||||
isystems_file = os.path.join(os.path.dirname(__file__), "data", "isystems")
|
||||
isystems_file = self.get_data_file_path("isystems")
|
||||
with open(isystems_file, 'r') as ifile:
|
||||
self.rest_api_request_result = json.load(ifile)
|
||||
|
||||
|
|
@ -87,7 +86,7 @@ class CertMonTestCase(base.DbTestCase):
|
|||
assert ret == 'fdc60cf3-3330-4438-859d-b0da19e9663d'
|
||||
|
||||
def test_enable_https(self):
|
||||
isystems_file = os.path.join(os.path.dirname(__file__), "data", "isystems")
|
||||
isystems_file = self.get_data_file_path("isystems")
|
||||
with open(isystems_file, 'r') as ifile:
|
||||
isystems_json = json.load(ifile)
|
||||
|
||||
|
|
@ -96,3 +95,32 @@ class CertMonTestCase(base.DbTestCase):
|
|||
token = self.keystone_token
|
||||
ret = cert_mon_utils.enable_https(token, 'fdc60cf3-3330-4438-859d-b0da19e9663d')
|
||||
assert ret is True
|
||||
|
||||
def test_update_platform_cert_force_true(self):
|
||||
self.update_platform_cert(True)
|
||||
|
||||
def test_update_platform_cert_force_false(self):
|
||||
self.update_platform_cert(False)
|
||||
|
||||
def update_platform_cert(self, force):
|
||||
token = self.keystone_token
|
||||
cert_type = constants.CERT_MODE_DOCKER_REGISTRY
|
||||
pem_file_path = self.get_data_file_path('cert-with-key.pem')
|
||||
|
||||
patcher = mock.patch('sysinv.cert_mon.utils.rest_api_upload')
|
||||
mocked_rest_api_upload = patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
mocked_rest_api_upload.return_value = {'error': ''}
|
||||
|
||||
with mock.patch('sysinv.cert_mon.utils.os') as mocked_os:
|
||||
cert_mon_utils.update_platform_cert(token, cert_type, pem_file_path, force)
|
||||
|
||||
mocked_os.remove.assert_called_once_with(pem_file_path)
|
||||
|
||||
mocked_rest_api_upload.assert_called_once_with(token, pem_file_path, mock.ANY, mock.ANY)
|
||||
actual_data = mocked_rest_api_upload.call_args[0][3]
|
||||
|
||||
self.assertEqual(actual_data['force'], str(force).lower())
|
||||
|
||||
def get_data_file_path(self, file_name):
|
||||
return os.path.join(os.path.dirname(__file__), "data", file_name)
|
||||
|
|
|
|||
Loading…
Reference in New Issue