Kube rootca update orchestration integration

Updates to orchestration based on recent sysinv commits.

"Kube rootca update abort - API"
 -  changed the URL from /upload to /upload_cert
 - introduced a new state which requires the vim state
 machine to be updated to support resume from that state
 (it goes from abort to start)
 - required changes to how the the 'complete' step was invoked .

"CLIs for kube rootca update procedure"
-  changed the values for the states used by kube rootca
 update orchestration.

Improvements:
 - The subject and expiry_date validation is duplicated in the
VIM so the strategy does not need to be run to see if the inputs
are valid.
Note: if the strategy is created with a valid expiry date, and
not applied until after the date becomes invalid, that will not
be caught until that step is processed.

 - The Client was converting the error strings to lower case before
displaying them.  This made it very difficult to determine the
expected fields for things like certificate subject.

 - upload_cert option for the VIM now works.
Note: the path to the cert should be accessible from both controllers
otherwise it may fail to upload after a SWACT.

Story: 2008675
Task: 43131
Depends-On: https://review.opendev.org/c/starlingx/config/+/805375
Depends-On: https://review.opendev.org/c/starlingx/config/+/805878
Signed-off-by: albailey <Al.Bailey@windriver.com>
Change-Id: Ia27b65ba5142516d5a62c5225c8498997367fd6e
This commit is contained in:
albailey 2021-08-30 08:56:22 -05:00
parent 0a9e537c49
commit 04797d6c7f
13 changed files with 209 additions and 75 deletions

View File

@ -14,6 +14,8 @@ BuildRequires: python-setuptools
BuildRequires: python2-pip
BuildRequires: python2-wheel
Requires: python-requests
%description
StarlingX Network Function Virtualization

View File

@ -1,5 +1,5 @@
#
# Copyright (c) 2016 Wind River Systems, Inc.
# Copyright (c) 2016-2021 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -89,7 +89,7 @@ def request(token_id, method, api_cmd, api_cmd_headers=None,
response = json.loads(response_raw)
message = response.get('faultstring', None)
if message is not None:
reason = str(message.lower().rstrip('.'))
reason = str(message.rstrip('.'))
print("Operation failed: %s" % reason)
return

View File

@ -4,6 +4,8 @@
# SPDX-License-Identifier: Apache-2.0
#
import json
import os
from nfv_client.openstack import rest_api
@ -245,7 +247,10 @@ def create_strategy(token_id,
if 'subject' in kwargs and kwargs['subject']:
api_cmd_payload['subject'] = kwargs['subject']
if 'cert_file' in kwargs and kwargs['cert_file']:
api_cmd_payload['cert-file'] = kwargs['cert_file']
# The cert file needs to be converted to an absolute path.
# If the path is not accessible from both controllers, the strategy
# may not succeed if a SWACT occurs.
api_cmd_payload['cert-file'] = os.path.abspath(kwargs['cert_file'])
api_cmd_payload['default-instance-action'] = default_instance_action
elif 'kube-upgrade' == strategy_name:
# required: 'to_version' passed to strategy as 'to-version'

View File

@ -272,7 +272,7 @@ def setup_kube_rootca_update_parser(commands):
create_strategy_cmd.add_argument(
'--expiry-date',
required=False,
help='When the generated certificate should expire')
help='When the generated certificate should expire (yyyy-mm-dd)')
create_strategy_cmd.add_argument(
'--subject',
required=False,

View File

@ -3,6 +3,8 @@
#
# SPDX-License-Identifier: Apache-2.0
#
import datetime
import re
import uuid
@ -55,3 +57,69 @@ def valid_integer(integer_str):
return False
return True
def validate_certificate_subject(subject):
"""
Duplicate the get_subject validation logic defined in:
sysinv/api/controllers/v1/kube_rootca_update.py
Returns a tuple of True, "" if the input is None
Returns a tuple of True, "" if the input is valid
Returns a tuple of False, "<error details>" if the input is invalid
"""
if subject is None:
return True, ""
params_supported = ['C', 'OU', 'O', 'ST', 'CN', 'L']
subject_pairs = re.findall(r"([^=]+=[^=]+)(?:\s|$)", subject)
subject_dict = {}
for pair_value in subject_pairs:
key, value = pair_value.split("=")
subject_dict[key] = value
if not all([param in params_supported for param in subject_dict.keys()]):
return False, ("There are parameters not supported "
"for the certificate subject specification. "
"The subject parameter has to be in the "
"format of 'C=<Country> ST=<State/Province> "
"L=<Locality> O=<Organization> OU=<OrganizationUnit> "
"CN=<commonName>")
if 'CN' not in subject_dict.keys():
return False, ("The CN=<commonName> parameter is required to be "
"specified in subject argument")
return True, ""
def validate_expiry_date(expiry_date):
"""
Duplicate the expiry_date validation logic defined in:
sysinv/api/controllers/v1/kube_rootca_update.py
Returns a tuple of True, "" if the input is None
Returns a tuple of True, "" if the input is valid
Returns a tuple of False, "<error details>" if the input is invalid
"""
if expiry_date is None:
return True, ""
try:
date = datetime.datetime.strptime(expiry_date, "%Y-%m-%d")
except ValueError:
return False, ("expiry_date %s doesn't match format "
"YYYY-MM-DD" % expiry_date)
delta = date - datetime.datetime.now()
# we sum one day (24 hours) to accomplish the certificate expiry
# during the day specified by the user
duration = (delta.days * 24 + 24)
# Cert-manager manages certificates and renew them some time
# before it expires. Along this procedure we set renewBefore
# parameter for 24h, so we are checking if the duration sent
# has at least this amount of time. This is needed to avoid
# cert-manager to block the creation of the resources.
if duration <= 24:
return False, ("New k8s rootCA should have at least 24 hours of "
"validation before expiry.")
return True, ""

View File

@ -953,8 +953,8 @@ class NFVIInfrastructureAPI(nfvi.api.v1.NFVIInfrastructureAPI):
DLOG.error("%s did not complete." % action_type)
return
api_data = future.result.data
result_obj = nfvi.objects.v1.KubeRootcaUpdate(api_data['state'])
response['result-data'] = result_obj
new_cert_identifier = api_data['success']
response['result-data'] = new_cert_identifier
response['completed'] = True
except exceptions.OpenStackRestAPIException as e:
if httplib.UNAUTHORIZED == e.http_status_code:

View File

@ -5,6 +5,7 @@
#
import json
import re
import requests
from six.moves import BaseHTTPServer
from six.moves import http_client as httplib
from six.moves import socketserver as SocketServer
@ -287,8 +288,13 @@ def rest_api_get_server(host, port):
return RestAPIServer(host, port)
def _rest_api_request(token_id, method, api_cmd, api_cmd_headers,
api_cmd_payload, timeout_in_secs):
def _rest_api_request(token_id,
method,
api_cmd,
api_cmd_headers,
api_cmd_payload,
timeout_in_secs,
file_to_post):
"""
Internal: make a rest-api request
"""
@ -320,34 +326,44 @@ def _rest_api_request(token_id, method, api_cmd, api_cmd_headers,
# opener = urllib.request.build_opener(handler)
# urllib.request.install_opener(opener)
request = urllib.request.urlopen(request_info, timeout=timeout_in_secs)
if file_to_post is not None:
headers = {"X-Auth-Token": token_id}
files = {'file': ("for_upload", file_to_post)}
request = requests.post(api_cmd, headers=headers, files=files,
timeout=timeout_in_secs)
status_code = request.status_code
response_raw = request.text
request.close()
else:
request = urllib.request.urlopen(request_info,
timeout=timeout_in_secs)
headers = list() # list of tuples
for key, value in request.info().items():
if key not in headers_per_hop:
cap_key = '-'.join((ck.capitalize() for ck in key.split('-')))
headers.append((cap_key, value))
headers = list() # list of tuples
for key, value in request.info().items():
if key not in headers_per_hop:
cap_key = '-'.join((ck.capitalize() for ck in key.split('-')))
headers.append((cap_key, value))
response_raw = request.read()
status_code = request.code
request.close()
response_raw = request.read()
if response_raw == "":
response = dict()
else:
response = json.loads(response_raw)
request.close()
now_ms = timers.get_monotonic_timestamp_in_ms()
elapsed_ms = now_ms - start_ms
elapsed_secs = elapsed_ms // 1000
DLOG.verbose("Rest-API code=%s, headers=%s, response=%s"
% (request.code, headers, response))
% (status_code, headers, response))
log_info("Rest-API status=%s, %s, %s, hdrs=%s, payload=%s, elapsed_ms=%s"
% (request.code, method, api_cmd, api_cmd_headers,
% (status_code, method, api_cmd, api_cmd_headers,
api_cmd_payload, int(elapsed_ms)))
return Result(response, Object(status_code=request.code,
return Result(response, Object(status_code=status_code,
headers=headers,
response=response_raw,
execution_time=elapsed_secs))
@ -436,8 +452,13 @@ def _rest_api_request(token_id, method, api_cmd, api_cmd_headers,
api_cmd_payload, str(e), str(e))
def rest_api_request(token, method, api_cmd, api_cmd_headers=None,
api_cmd_payload=None, timeout_in_secs=20):
def rest_api_request(token,
method,
api_cmd,
api_cmd_headers=None,
api_cmd_payload=None,
timeout_in_secs=20,
file_to_post=None):
"""
Make a rest-api request using the given token
WARNING: Any change to the default timeout must be reflected in the timeout
@ -446,7 +467,7 @@ def rest_api_request(token, method, api_cmd, api_cmd_headers=None,
try:
return _rest_api_request(token.get_id(), method, api_cmd,
api_cmd_headers, api_cmd_payload,
timeout_in_secs)
timeout_in_secs, file_to_post)
except OpenStackRestAPIException as e:
if httplib.UNAUTHORIZED == e.http_status_code:
@ -454,13 +475,18 @@ def rest_api_request(token, method, api_cmd, api_cmd_headers=None,
raise
def rest_api_request_with_context(context, method, api_cmd,
api_cmd_headers=None, api_cmd_payload=None,
timeout_in_secs=20):
def rest_api_request_with_context(context,
method,
api_cmd,
api_cmd_headers=None,
api_cmd_payload=None,
timeout_in_secs=20,
file_to_post=None):
"""
Make a rest-api request using the given context
WARNING: Any change to the default timeout must be reflected in the timeout
calculations done in the TaskFuture class.
"""
return _rest_api_request(context.token_id, method, api_cmd, api_cmd_headers,
api_cmd_payload, timeout_in_secs)
return _rest_api_request(context.token_id, method, api_cmd,
api_cmd_headers, api_cmd_payload,
timeout_in_secs, file_to_post)

View File

@ -23,7 +23,7 @@ KUBE_ROOTCA_UPDATE_GENERATE_CERT_ENDPOINT = \
KUBE_ROOTCA_UPDATE_PODS_ENDPOINT = KUBE_ROOTCA_UPDATE_ENDPOINT + "/pods"
KUBE_ROOTCA_UPDATE_HOSTS_ENDPOINT = KUBE_ROOTCA_UPDATE_ENDPOINT + "/hosts"
KUBE_ROOTCA_UPDATE_UPLOAD_CERT_ENDPOINT = \
KUBE_ROOTCA_UPDATE_ENDPOINT + "/upload"
KUBE_ROOTCA_UPDATE_ENDPOINT + "/upload_cert"
# todo(abailey): refactor _api_get, etc.. into rest_api.py
@ -282,20 +282,38 @@ def kube_rootca_update_generate_cert(token, expiry_date=None, subject=None):
def kube_rootca_update_upload_cert(token, cert_file):
"""
Ask System Inventory to kube rootca update upload a cert file
This uses POST for a file, which urllib does not work well with.
"""
api_cmd = _api_cmd(token, KUBE_ROOTCA_UPDATE_UPLOAD_CERT_ENDPOINT)
api_cmd_headers = _api_cmd_headers()
api_cmd_payload = dict()
api_cmd_payload['cert_file'] = cert_file
return _api_post(token, KUBE_ROOTCA_UPDATE_UPLOAD_CERT_ENDPOINT,
api_cmd_payload)
# The API is expecting requests.post formatted data
with open(cert_file, "rb") as cert_file_handle:
# file handle automatically closed once this request is sent
response = rest_api_request(token,
"POST",
api_cmd,
api_cmd_headers,
json.dumps(api_cmd_payload),
timeout_in_secs=REST_API_REQUEST_TIMEOUT,
file_to_post=cert_file_handle)
return response
def kube_rootca_update_complete(token):
"""
Ask System Inventory to kube rootca update complete
"""
api_cmd_payload = list()
state_data = dict()
state_data['path'] = "/state"
state_data['value'] = 'update-completed'
state_data['op'] = "replace"
api_cmd_payload.append(state_data)
return _api_patch_dict(token,
KUBE_ROOTCA_UPDATE_ENDPOINT,
{'force': 'True'})
KUBE_ROOTCA_UPDATE_ENDPOINT + "?force=True",
api_cmd_payload)
def kube_rootca_update_host(token, host_uuid, phase):

View File

@ -236,8 +236,8 @@ class ApplyStageMixin(object):
{'name': 'kube-rootca-update-host-trustbothcas',
'entity_type': 'hosts',
'entity_names': [host, ],
'success_state': 'updated-host-trustBothCAs',
'fail_state': 'updating-host-trustBothCAs-failed',
'success_state': 'updated-host-trust-both-cas',
'fail_state': 'updating-host-trust-both-cas-failed',
})
return {
'name': 'kube-rootca-update-hosts-trustbothcas',
@ -248,8 +248,8 @@ class ApplyStageMixin(object):
def _kube_rootca_update_pods_trustbothcas_stage(self):
steps = [
{'name': 'kube-rootca-update-pods-trustbothcas',
'success_state': 'updated-pods-trustBothCAs',
'fail_state': 'updating-pods-trustBothCAs-failed',
'success_state': 'updated-pods-trust-both-cas',
'fail_state': 'updating-pods-trust-both-cas-failed',
},
]
return {
@ -265,8 +265,8 @@ class ApplyStageMixin(object):
{'name': 'kube-rootca-update-host-trustnewca',
'entity_type': 'hosts',
'entity_names': [host, ],
'success_state': 'updated-host-trustNewCA',
'fail_state': 'updating-host-trustNewCA-failed',
'success_state': 'updated-host-trust-new-ca',
'fail_state': 'updating-host-trust-new-ca-failed',
})
return {
'name': 'kube-rootca-update-hosts-trustnewca',
@ -277,8 +277,8 @@ class ApplyStageMixin(object):
def _kube_rootca_update_pods_trustnewca_stage(self):
steps = [
{'name': 'kube-rootca-update-pods-trustnewca',
'success_state': 'updated-pods-trustNewCA',
'fail_state': 'updating-pods-trustNewCA-failed',
'success_state': 'updated-pods-trust-new-ca',
'fail_state': 'updating-pods-trust-new-ca-failed',
},
]
return {
@ -294,8 +294,8 @@ class ApplyStageMixin(object):
{'name': 'kube-rootca-update-host-update-certs',
'entity_type': 'hosts',
'entity_names': [host, ],
'success_state': 'updated-host-updateCerts',
'fail_state': 'updating-host-updateCerts-failed',
'success_state': 'updated-host-update-certs',
'fail_state': 'updating-host-update-certs-failed',
})
return {
'name': 'kube-rootca-update-hosts-updatecerts',

View File

@ -705,6 +705,7 @@ class KubeRootcaUpdateStrategyAPI(SwUpdateStrategyAPI):
"""
Kubernetes Root CA Update Strategy Rest API
"""
@wsme_pecan.wsexpose(SwUpdateStrategyQueryData,
body=KubeRootcaUpdateStrategyCreateData,
status_code=httplib.OK)
@ -713,10 +714,21 @@ class KubeRootcaUpdateStrategyAPI(SwUpdateStrategyAPI):
rpc_request.sw_update_type = _get_sw_update_type_from_path(
pecan.request.path)
if wsme_types.Unset != request_data.expiry_date:
# Validate the expiry_date
is_valid, reason = validate.validate_expiry_date(
request_data.expiry_date)
if not is_valid:
return pecan.abort(httplib.BAD_REQUEST, reason)
rpc_request.expiry_date = request_data.expiry_date
if wsme_types.Unset != request_data.subject:
# Validate the subject
is_valid, reason = validate.validate_certificate_subject(
request_data.subject)
if not is_valid:
return pecan.abort(httplib.BAD_REQUEST, reason)
rpc_request.subject = request_data.subject
if wsme_types.Unset != request_data.cert_file:
# todo(abailey): Should investigate if cert_file can be validated
rpc_request.cert_file = request_data.cert_file
rpc_request.controller_apply_type = SW_UPDATE_APPLY_TYPE.SERIAL
rpc_request.storage_apply_type = request_data.storage_apply_type

View File

@ -21,22 +21,24 @@ class KubeRootcaUpdateState(Constants):
KUBE_ROOTCA_UPDATE_STARTED = Constant('update-started')
KUBE_ROOTCA_UPDATE_CERT_UPLOADED = Constant('update-new-rootca-cert-uploaded')
KUBE_ROOTCA_UPDATE_CERT_GENERATED = Constant('update-new-rootca-cert-generated')
KUBE_ROOTCA_UPDATING_PODS_TRUSTBOTHCAS = Constant('updating-pods-trustBothCAs')
KUBE_ROOTCA_UPDATED_PODS_TRUSTBOTHCAS = Constant('updated-pods-trustBothCAs')
KUBE_ROOTCA_UPDATING_PODS_TRUSTBOTHCAS_FAILED = Constant('updating-pods-trustBothCAs-failed')
KUBE_ROOTCA_UPDATING_PODS_TRUSTNEWCA = Constant('updating-pods-trustNewCA')
KUBE_ROOTCA_UPDATED_PODS_TRUSTNEWCA = Constant('updated-pods-trustNewCA')
KUBE_ROOTCA_UPDATING_PODS_TRUSTNEWCA_FAILED = Constant('updating-pods-trustNewCA-failed')
KUBE_ROOTCA_UPDATE_COMPLETED = Constant('update-completed')
KUBE_ROOTCA_UPDATING_HOST_TRUSTBOTHCAS = Constant('updating-host-trustBothCAs')
KUBE_ROOTCA_UPDATED_HOST_TRUSTBOTHCAS = Constant('updated-host-trustBothCAs')
KUBE_ROOTCA_UPDATING_HOST_TRUSTBOTHCAS_FAILED = Constant('updating-host-trustBothCAs-failed')
KUBE_ROOTCA_UPDATING_HOST_UPDATECERTS = Constant('updating-host-updateCerts')
KUBE_ROOTCA_UPDATED_HOST_UPDATECERTS = Constant('updated-host-updateCerts')
KUBE_ROOTCA_UPDATING_HOST_UPDATECERTS_FAILED = Constant('updating-host-updateCerts-failed')
KUBE_ROOTCA_UPDATING_HOST_TRUSTNEWCA = Constant('updating-host-trustNewCA')
KUBE_ROOTCA_UPDATED_HOST_TRUSTNEWCA = Constant('updated-host-trustNewCA')
KUBE_ROOTCA_UPDATING_HOST_TRUSTNEWCA_FAILED = Constant('updating-host-trustNewCA-failed')
KUBE_ROOTCA_UPDATING_PODS_TRUSTBOTHCAS = 'updating-pods-trust-both-cas'
KUBE_ROOTCA_UPDATED_PODS_TRUSTBOTHCAS = 'updated-pods-trust-both-cas'
KUBE_ROOTCA_UPDATING_PODS_TRUSTBOTHCAS_FAILED = 'updating-pods-trust-both-cas-failed'
KUBE_ROOTCA_UPDATING_PODS_TRUSTNEWCA = 'updating-pods-trust-new-ca'
KUBE_ROOTCA_UPDATED_PODS_TRUSTNEWCA = 'updated-pods-trust-new-ca'
KUBE_ROOTCA_UPDATING_PODS_TRUSTNEWCA_FAILED = 'updating-pods-trust-new-ca-failed'
KUBE_ROOTCA_UPDATE_COMPLETED = 'update-completed'
KUBE_ROOTCA_UPDATE_ABORTED = 'update-aborted'
KUBE_ROOTCA_UPDATING_HOST_TRUSTBOTHCAS = 'updating-host-trust-both-cas'
KUBE_ROOTCA_UPDATED_HOST_TRUSTBOTHCAS = 'updated-host-trust-both-cas'
KUBE_ROOTCA_UPDATING_HOST_TRUSTBOTHCAS_FAILED = 'updating-host-trust-both-cas-failed'
KUBE_ROOTCA_UPDATING_HOST_UPDATECERTS = 'updating-host-update-certs'
KUBE_ROOTCA_UPDATED_HOST_UPDATECERTS = 'updated-host-update-certs'
KUBE_ROOTCA_UPDATING_HOST_UPDATECERTS_FAILED = 'updating-host-update-certs-failed'
KUBE_ROOTCA_UPDATING_HOST_TRUSTNEWCA = 'updating-host-trust-new-ca'
KUBE_ROOTCA_UPDATED_HOST_TRUSTNEWCA = 'updated-host-trust-new-ca'
KUBE_ROOTCA_UPDATING_HOST_TRUSTNEWCA_FAILED = 'updating-host-trust-new-ca-failed'
# Kube Upgrade Constant Instantiation

View File

@ -2794,6 +2794,9 @@ class KubeRootcaUpdateStrategy(SwUpdateStrategy,
from nfv_vim import strategy
RESUME_STATE = {
# update was aborted, this means it needs to be recreated
nfvi.objects.v1.KUBE_ROOTCA_UPDATE_STATE.KUBE_ROOTCA_UPDATE_ABORTED:
self._add_kube_rootca_update_start_stage,
# after update-started -> generate or upload cert
nfvi.objects.v1.KUBE_ROOTCA_UPDATE_STATE.KUBE_ROOTCA_UPDATE_STARTED:
self._add_kube_rootca_update_cert_stage,

View File

@ -3352,9 +3352,8 @@ class KubeRootcaUpdateGenerateCertStep(AbstractKubeRootcaUpdateStep):
@coroutine
def _response_callback(self):
"""
Note: Generate Cert is a blocking call that returns an identifier
however polling the update is how we proceed to next state
"""Generate Cert is a blocking call that returns an identifier
Polling the update is how we proceed to next state
"""
response = (yield)
DLOG.debug("%s callback response=%s." % (self._name, response))
@ -3396,30 +3395,29 @@ class KubeRootcaUpdateGenerateCertStep(AbstractKubeRootcaUpdateStep):
return data
class KubeRootcaUpdateUploadCertStep(AbstractStrategyStep):
class KubeRootcaUpdateUploadCertStep(AbstractKubeRootcaUpdateStep):
"""Kube RootCA Update - Upload Cert - Strategy Step"""
def __init__(self, cert_file):
from nfv_vim import nfvi
super(KubeRootcaUpdateUploadCertStep, self).__init__(
STRATEGY_STEP_NAME.KUBE_ROOTCA_UPDATE_UPLOAD_CERT,
timeout_in_secs=120)
nfvi.objects.v1.KUBE_ROOTCA_UPDATE_STATE.KUBE_ROOTCA_UPDATE_CERT_UPLOADED,
None, # sysinv API does not have a FAILED state for this action
timeout_in_secs=300)
self._cert_file = cert_file
@coroutine
def _response_callback(self):
"""Upload Cert is a blocking call"""
"""Upload Cert is a blocking call that returns an identifier
Polling the update is how we proceed to next state
"""
response = (yield)
DLOG.debug("%s callback response=%s." % (self._name, response))
if response['completed']:
if self.strategy is not None:
self.strategy.nfvi_kube_rootca_update = response['result-data']
# todo(abailey): iMay want to check if the state is now:
# nfvi.objects.v1.KUBE_ROOTCA_UPDATE_STATE.KUBE_ROOTCA_UPDATE_CERT_UPLOADED
result = strategy.STRATEGY_STEP_RESULT.SUCCESS
self.stage.step_complete(result, "")
# We do not set 'success' here, let the handle_event do this
# The API returns a certificate identifier
pass
else:
result = strategy.STRATEGY_STEP_RESULT.FAILED
self.stage.step_complete(result, response['reason'])