openstack/magnum
Code Issues Proposed changes

Sync service account keys for multi masters

Browse Source 
Multi master deployments for k8s driver use different service account
keys for each api/controller manager server which leads to 401 errors
for service accounts. This patch will create a signed cert and private
key for k8s service account keys explicitly, dedicatedly for the k8s
cluster to avoid the inconsistent keys issue.

Task:  21653
Story: 1766546

Change-Id: I61547405f866d3c5a84da63de66724b55af1066a
This commit is contained in:
Feilong Wang 2018-06-18 14:37:49 +12:00
parent 4292b862a3
commit 043c57da74
11 changed files with 201 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
magnum/common/x509/operations.py
View File

@ -87,11 +87,14 @@ def _build_ca_extentions():
key_usage = x509.KeyUsage(False, False, False, False, False, True, False,
False, False)
key_usage = x509.Extension(key_usage.oid, True, key_usage)
extended_key_usage = x509.ExtendedKeyUsage([x509.OID_SERVER_AUTH])
extended_key_usage = x509.Extension(extended_key_usage.oid, False,
extended_key_usage)
basic_constraints = x509.BasicConstraints(ca=True, path_length=0)
basic_constraints = x509.Extension(basic_constraints.oid, True,
basic_constraints)
return [basic_constraints, key_usage]
return [basic_constraints, key_usage, extended_key_usage]
def _generate_self_signed_certificate(subject_name, extensions,
@ -231,6 +234,29 @@ def sign(csr, issuer_name, ca_key, ca_key_password=None,
return certificate
def generate_csr_and_key(common_name):
"""Return a dict with a new csr and key."""
key = rsa.generate_private_key(
public_exponent=65537,
key_size=2048,
backend=default_backend())
csr = x509.CertificateSigningRequestBuilder().subject_name(x509.Name([
x509.NameAttribute(x509.oid.NameOID.COMMON_NAME, common_name),
])).sign(key, hashes.SHA256(), default_backend())
result = {
'csr': csr.public_bytes(
encoding=serialization.Encoding.PEM).decode("utf-8"),
'key': key.private_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PrivateFormat.TraditionalOpenSSL,
encryption_algorithm=serialization.NoEncryption()).decode("utf-8"),
}
return result
def decrypt_key(encrypted_key, password):
private_key = _load_pem_private_key(encrypted_key, password)

4
magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh
View File

@ -42,7 +42,7 @@ else
KUBE_API_ARGS="$KUBE_API_ARGS --tls-private-key-file=$CERT_DIR/server.key"
KUBE_API_ARGS="$KUBE_API_ARGS --client-ca-file=$CERT_DIR/ca.crt"
KUBE_API_ARGS="$KUBE_API_ARGS --tls-ca-file=${CERT_DIR}/ca.crt"
KUBE_API_ARGS="$KUBE_API_ARGS --service-account-key-file=${CERT_DIR}/server.key"
KUBE_API_ARGS="$KUBE_API_ARGS --service-account-key-file=${CERT_DIR}/service_account.key"
KUBE_API_ARGS="$KUBE_API_ARGS --kubelet-certificate-authority=${CERT_DIR}/ca.crt --kubelet-client-certificate=${CERT_DIR}/server.crt --kubelet-client-key=${CERT_DIR}/server.key --kubelet-https=true"
fi
@ -68,7 +68,7 @@ sed -i '
KUBE_CONTROLLER_MANAGER_ARGS="--leader-elect=true"
KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS $KUBECONTROLLER_OPTIONS"
if [ -n "${ADMISSION_CONTROL_LIST}" ] && [ "${TLS_DISABLED}" == "False" ]; then
KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --service-account-private-key-file=$CERT_DIR/server.key --root-ca-file=$CERT_DIR/ca.crt"
KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --service-account-private-key-file=$CERT_DIR/service_account_private.key --root-ca-file=$CERT_DIR/ca.crt"
fi
if [ -n "$TRUST_ID" ]; then

4
magnum/drivers/common/templates/kubernetes/fragments/make-cert.sh
View File

@ -159,6 +159,10 @@ EOF
generate_certificates server ${cert_dir}/server.conf
generate_certificates kubelet ${cert_dir}/kubelet.conf
# Generate service account key and private key
echo -e "${KUBE_SERVICE_ACCOUNT_KEY}" > ${cert_dir}/service_account.key
echo -e "${KUBE_SERVICE_ACCOUNT_PRIVATE_KEY}" > ${cert_dir}/service_account_private.key
# Common certs and key are created for both etcd and kubernetes services.
# Both etcd and kube user should have permission to access the certs and key.
groupadd kube_etcd

2
magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml
View File

@ -71,3 +71,5 @@ write_files:
KUBEPROXY_OPTIONS="$KUBEPROXY_OPTIONS"
KUBESCHEDULER_OPTIONS="$KUBESCHEDULER_OPTIONS"
OCTAVIA_ENABLED="$OCTAVIA_ENABLED"
KUBE_SERVICE_ACCOUNT_KEY="$KUBE_SERVICE_ACCOUNT_KEY"
KUBE_SERVICE_ACCOUNT_PRIVATE_KEY="$KUBE_SERVICE_ACCOUNT_PRIVATE_KEY"

12
magnum/drivers/heat/k8s_fedora_template_def.py
View File

@ -101,6 +101,18 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition):
if label_value:
extra_params[label] = label_value
# NOTE(flwang): We're generating a signed cert and private key
# based on the cluster CA as the service account signing keys for
# k8s cluster, though a general public/private keypair works as well.
csr_key = x509.generate_csr_and_key(u"Kubernetes Service Account")
signed_cert = cert_manager.sign_node_certificate(cluster,
csr_key["csr"],
context=context)
extra_params['kube_service_account_key'] = \
signed_cert.replace("\n", "\\n")
extra_params['kube_service_account_private_key'] = \
csr_key["key"].replace("\n", "\\n")
cert_manager_api = cluster.labels.get('cert_manager_api')
if strutils.bool_from_string(cert_manager_api):
extra_params['cert_manager_api'] = cert_manager_api

15
magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml
View File

@ -468,6 +468,18 @@ parameters:
whether or not to use Octavia for LoadBalancer type service.
default: False
kube_service_account_key:
type: string
description: >
The signed cert will be used to verify the k8s service account tokens
during authentication.
kube_service_account_private_key:
type: string
description: >
The private key will be used to sign generated k8s service account
tokens.
resources:
######################################################################
@ -687,6 +699,9 @@ resources:
kubecontroller_options: {get_param: kubecontroller_options}
kubescheduler_options: {get_param: kubescheduler_options}
octavia_enabled: {get_param: octavia_enabled}
kube_service_account_key: {get_param: kube_service_account_key}
kube_service_account_private_key: {get_param: kube_service_account_private_key}
######################################################################
#

14
magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml
View File

@ -362,6 +362,18 @@ parameters:
the index of master node, index 0 means the master node is the primary,
bootstrapping node.
kube_service_account_key:
type: string
description: >
The signed cert will be used to verify the k8s service account tokens
during authentication.
kube_service_account_private_key:
type: string
description: >
The private key will be used to sign generated k8s service account
tokens.
resources:
master_wait_handle:
@ -467,6 +479,8 @@ resources:
"$KUBEPROXY_OPTIONS": {get_param: kubeproxy_options}
"$KUBESCHEDULER_OPTIONS": {get_param: kubescheduler_options}
"$OCTAVIA_ENABLED": {get_param: octavia_enabled}
"$KUBE_SERVICE_ACCOUNT_KEY": {get_param: kube_service_account_key}
"$KUBE_SERVICE_ACCOUNT_PRIVATE_KEY": {get_param: kube_service_account_private_key}
install_openstack_ca:
type: OS::Heat::SoftwareConfig

9
magnum/tests/unit/common/x509/test_operations.py
View File

@ -12,6 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.hazmat.primitives import serialization
import mock
@ -43,3 +44,11 @@ class TestX509Operations(base.BaseTestCase):
encryption_algorithm=mock_no_encryption_class.return_value
)
self.assertEqual(mock.sentinel.decrypted, actual_decrypted)
@mock.patch.object(operations, 'default_backend')
@mock.patch.object(rsa, 'generate_private_key')
def test_generate_csr_and_key(self, mock_generate_private_key,
mock_default_backend):
mock_generate_private_key.return_value = mock.MagicMock()
csr_key = operations.generate_csr_and_key(u"Test")
self.assertIsNotNone(csr_key)

85
magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py
View File

@ -135,16 +135,24 @@ class TestClusterConductorWithK8s(base.TestCase):
@patch('requests.get')
@patch('magnum.objects.ClusterTemplate.get_by_uuid')
@patch('magnum.drivers.common.driver.Driver.get_driver')
@patch('magnum.conductor.handlers.common.cert_manager'
'.sign_node_certificate')
@patch('magnum.common.x509.operations.generate_csr_and_key')
def test_extract_template_definition(
self,
mock_generate_csr_and_key,
mock_sign_node_certificate,
mock_driver,
mock_objects_cluster_template_get_by_uuid,
mock_get):
self._test_extract_template_definition(
mock_generate_csr_and_key, mock_sign_node_certificate,
mock_driver, mock_objects_cluster_template_get_by_uuid, mock_get)
def _test_extract_template_definition(
self,
mock_generate_csr_and_key,
mock_sign_node_certificate,
mock_driver,
mock_objects_cluster_template_get_by_uuid,
mock_get,
@ -155,6 +163,9 @@ class TestClusterConductorWithK8s(base.TestCase):
self.cluster_dict[missing_attr] = None
cluster_template = objects.ClusterTemplate(
self.context, **self.cluster_template_dict)
mock_generate_csr_and_key.return_value = {'csr': 'csr',
'key': 'private_key'}
mock_sign_node_certificate.return_value = 'signed_cert'
mock_objects_cluster_template_get_by_uuid.return_value = \
cluster_template
expected_result = str('{"action":"get","node":{"key":"test","value":'
@ -271,6 +282,8 @@ class TestClusterConductorWithK8s(base.TestCase):
'kubescheduler_options': '--kubescheduler',
'kubeproxy_options': '--kubeproxy',
'octavia_enabled': False,
'kube_service_account_key': 'signed_cert',
'kube_service_account_private_key': 'private_key',
}
if missing_attr is not None:
expected.pop(mapping[missing_attr], None)
@ -288,14 +301,22 @@ class TestClusterConductorWithK8s(base.TestCase):
@patch('requests.get')
@patch('magnum.objects.ClusterTemplate.get_by_uuid')
@patch('magnum.drivers.common.driver.Driver.get_driver')
@patch('magnum.conductor.handlers.common.cert_manager'
'.sign_node_certificate')
@patch('magnum.common.x509.operations.generate_csr_and_key')
def test_extract_template_definition_with_registry(
self,
mock_generate_csr_and_key,
mock_sign_node_certificate,
mock_driver,
mock_objects_cluster_template_get_by_uuid,
mock_get):
self.cluster_template_dict['registry_enabled'] = True
cluster_template = objects.ClusterTemplate(
self.context, **self.cluster_template_dict)
mock_generate_csr_and_key.return_value = {'csr': 'csr',
'key': 'private_key'}
mock_sign_node_certificate.return_value = 'signed_cert'
mock_objects_cluster_template_get_by_uuid.return_value = \
cluster_template
expected_result = str('{"action":"get","node":{"key":"test","value":'
@ -379,6 +400,8 @@ class TestClusterConductorWithK8s(base.TestCase):
'kubescheduler_options': '--kubescheduler',
'kubeproxy_options': '--kubeproxy',
'octavia_enabled': False,
'kube_service_account_key': 'signed_cert',
'kube_service_account_private_key': 'private_key',
}
self.assertEqual(expected, definition)
@ -394,8 +417,13 @@ class TestClusterConductorWithK8s(base.TestCase):
@patch('requests.get')
@patch('magnum.objects.ClusterTemplate.get_by_uuid')
@patch('magnum.drivers.common.driver.Driver.get_driver')
@patch('magnum.conductor.handlers.common.cert_manager'
'.sign_node_certificate')
@patch('magnum.common.x509.operations.generate_csr_and_key')
def test_extract_template_definition_only_required(
self,
mock_generate_csr_and_key,
mock_sign_node_certificate,
mock_driver,
mock_objects_cluster_template_get_by_uuid,
mock_get):
@ -411,6 +439,9 @@ class TestClusterConductorWithK8s(base.TestCase):
cluster_template = objects.ClusterTemplate(
self.context, **self.cluster_template_dict)
mock_generate_csr_and_key.return_value = {'csr': 'csr',
'key': 'private_key'}
mock_sign_node_certificate.return_value = 'signed_cert'
mock_objects_cluster_template_get_by_uuid.return_value = \
cluster_template
expected_result = str('{"action":"get","node":{"key":"test","value":'
@ -474,6 +505,8 @@ class TestClusterConductorWithK8s(base.TestCase):
'kubescheduler_options': '--kubescheduler',
'kubeproxy_options': '--kubeproxy',
'octavia_enabled': False,
'kube_service_account_key': 'signed_cert',
'kube_service_account_private_key': 'private_key',
}
self.assertEqual(expected, definition)
self.assertEqual(
@ -656,13 +689,20 @@ class TestClusterConductorWithK8s(base.TestCase):
@patch('requests.get')
@patch('magnum.objects.ClusterTemplate.get_by_uuid')
@patch('magnum.drivers.common.driver.Driver.get_driver')
@patch('magnum.conductor.handlers.common.cert_manager'
'.sign_node_certificate')
@patch('magnum.common.x509.operations.generate_csr_and_key')
def test_extract_template_definition_without_dns(
self,
mock_generate_csr_and_key,
mock_sign_node_certificate,
mock_driver,
mock_objects_cluster_template_get_by_uuid,
mock_get):
mock_driver.return_value = k8s_dr.Driver()
self._test_extract_template_definition(
mock_generate_csr_and_key,
mock_sign_node_certificate,
mock_driver,
mock_objects_cluster_template_get_by_uuid,
mock_get,
@ -671,13 +711,20 @@ class TestClusterConductorWithK8s(base.TestCase):
@patch('requests.get')
@patch('magnum.objects.ClusterTemplate.get_by_uuid')
@patch('magnum.drivers.common.driver.Driver.get_driver')
@patch('magnum.conductor.handlers.common.cert_manager'
'.sign_node_certificate')
@patch('magnum.common.x509.operations.generate_csr_and_key')
def test_extract_template_definition_without_server_image(
self,
mock_generate_csr_and_key,
mock_sign_node_certificate,
mock_driver,
mock_objects_cluster_template_get_by_uuid,
mock_get):
mock_driver.return_value = k8s_dr.Driver()
self._test_extract_template_definition(
mock_generate_csr_and_key,
mock_sign_node_certificate,
mock_driver,
mock_objects_cluster_template_get_by_uuid,
mock_get,
@ -686,13 +733,20 @@ class TestClusterConductorWithK8s(base.TestCase):
@patch('requests.get')
@patch('magnum.objects.ClusterTemplate.get_by_uuid')
@patch('magnum.drivers.common.driver.Driver.get_driver')
@patch('magnum.conductor.handlers.common.cert_manager'
'.sign_node_certificate')
@patch('magnum.common.x509.operations.generate_csr_and_key')
def test_extract_template_definition_without_docker_storage_driver(
self,
mock_generate_csr_and_key,
mock_sign_node_certificate,
mock_driver,
mock_objects_cluster_template_get_by_uuid,
mock_get):
mock_driver.return_value = k8s_dr.Driver()
self._test_extract_template_definition(
mock_generate_csr_and_key,
mock_sign_node_certificate,
mock_driver,
mock_objects_cluster_template_get_by_uuid,
mock_get,
@ -701,13 +755,20 @@ class TestClusterConductorWithK8s(base.TestCase):
@patch('requests.get')
@patch('magnum.objects.ClusterTemplate.get_by_uuid')
@patch('magnum.drivers.common.driver.Driver.get_driver')
@patch('magnum.conductor.handlers.common.cert_manager'
'.sign_node_certificate')
@patch('magnum.common.x509.operations.generate_csr_and_key')
def test_extract_template_definition_without_apiserver_port(
self,
mock_generate_csr_and_key,
mock_sign_node_certificate,
mock_driver,
mock_objects_cluster_template_get_by_uuid,
mock_get):
mock_driver.return_value = k8s_dr.Driver()
self._test_extract_template_definition(
mock_generate_csr_and_key,
mock_sign_node_certificate,
mock_driver,
mock_objects_cluster_template_get_by_uuid,
mock_get,
@ -716,13 +777,20 @@ class TestClusterConductorWithK8s(base.TestCase):
@patch('requests.get')
@patch('magnum.objects.ClusterTemplate.get_by_uuid')
@patch('magnum.drivers.common.driver.Driver.get_driver')
@patch('magnum.conductor.handlers.common.cert_manager'
'.sign_node_certificate')
@patch('magnum.common.x509.operations.generate_csr_and_key')
def test_extract_template_definition_without_node_count(
self,
mock_generate_csr_and_key,
mock_sign_node_certificate,
mock_driver,
mock_objects_cluster_template_get_by_uuid,
mock_get):
mock_driver.return_value = k8s_dr.Driver()
self._test_extract_template_definition(
mock_generate_csr_and_key,
mock_sign_node_certificate,
mock_driver,
mock_objects_cluster_template_get_by_uuid,
mock_get,
@ -731,13 +799,20 @@ class TestClusterConductorWithK8s(base.TestCase):
@patch('requests.get')
@patch('magnum.objects.ClusterTemplate.get_by_uuid')
@patch('magnum.drivers.common.driver.Driver.get_driver')
@patch('magnum.conductor.handlers.common.cert_manager'
'.sign_node_certificate')
@patch('magnum.common.x509.operations.generate_csr_and_key')
def test_extract_template_definition_without_master_count(
self,
mock_generate_csr_and_key,
mock_sign_node_certificate,
mock_driver,
mock_objects_cluster_template_get_by_uuid,
mock_get):
mock_driver.return_value = k8s_dr.Driver()
self._test_extract_template_definition(
mock_generate_csr_and_key,
mock_sign_node_certificate,
mock_driver,
mock_objects_cluster_template_get_by_uuid,
mock_get,
@ -746,13 +821,21 @@ class TestClusterConductorWithK8s(base.TestCase):
@patch('requests.get')
@patch('magnum.objects.ClusterTemplate.get_by_uuid')
@patch('magnum.drivers.common.driver.Driver.get_driver')
@patch('magnum.conductor.handlers.common.cert_manager'
'.sign_node_certificate')
@patch('magnum.common.x509.operations.generate_csr_and_key')
def test_extract_template_definition_without_discovery_url(
self,
mock_generate_csr_and_key,
mock_sign_node_certificate,
mock_driver,
mock_objects_cluster_template_get_by_uuid,
reqget):
cluster_template = objects.ClusterTemplate(
self.context, **self.cluster_template_dict)
mock_generate_csr_and_key.return_value = {'csr': 'csr',
'key': 'private_key'}
mock_sign_node_certificate.return_value = 'signed_cert'
mock_objects_cluster_template_get_by_uuid.return_value = \
cluster_template
cluster_dict = self.cluster_dict
@ -829,6 +912,8 @@ class TestClusterConductorWithK8s(base.TestCase):
'kubescheduler_options': '--kubescheduler',
'kubeproxy_options': '--kubeproxy',
'octavia_enabled': False,
'kube_service_account_key': 'signed_cert',
'kube_service_account_private_key': 'private_key',
}
self.assertEqual(expected, definition)
self.assertEqual(

24
magnum/tests/unit/drivers/test_template_definition.py
View File

@ -227,9 +227,17 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
'.get_params')
@mock.patch('magnum.drivers.heat.template_def.TemplateDefinition'
'.get_output')
def test_k8s_get_params(self, mock_get_output, mock_get_params,
@mock.patch('magnum.conductor.handlers.common.cert_manager'
'.sign_node_certificate')
@mock.patch('magnum.common.x509.operations.generate_csr_and_key')
def test_k8s_get_params(self, mock_generate_csr_and_key,
mock_sign_node_certificate,
mock_get_output, mock_get_params,
mock_get_discovery_url, mock_osc_class,
mock_enable_octavia):
mock_generate_csr_and_key.return_value = {'csr': 'csr',
'key': 'private_key'}
mock_sign_node_certificate.return_value = 'signed_cert'
mock_enable_octavia.return_value = False
mock_context = mock.MagicMock()
mock_context.auth_token = 'AUTH_TOKEN'
@ -354,6 +362,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
'ingress_controller': ingress_controller,
'ingress_controller_role': ingress_controller_role,
'octavia_enabled': False,
'kube_service_account_key': 'signed_cert',
'kube_service_account_private_key': 'private_key',
}}
mock_get_params.assert_called_once_with(mock_context,
mock_cluster_template,
@ -368,9 +378,17 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
'.get_params')
@mock.patch('magnum.drivers.heat.template_def.TemplateDefinition'
'.get_output')
def test_k8s_get_params_insecure(self, mock_get_output, mock_get_params,
@mock.patch('magnum.conductor.handlers.common.cert_manager'
'.sign_node_certificate')
@mock.patch('magnum.common.x509.operations.generate_csr_and_key')
def test_k8s_get_params_insecure(self, mock_generate_csr_and_key,
mock_sign_node_certificate,
mock_get_output, mock_get_params,
mock_get_discovery_url, mock_osc_class,
mock_enable_octavia):
mock_generate_csr_and_key.return_value = {'csr': 'csr',
'key': 'private_key'}
mock_sign_node_certificate.return_value = 'signed_cert'
mock_enable_octavia.return_value = False
mock_context = mock.MagicMock()
mock_context.auth_token = 'AUTH_TOKEN'
@ -497,6 +515,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
'ingress_controller': ingress_controller,
'ingress_controller_role': ingress_controller_role,
'octavia_enabled': False,
'kube_service_account_key': 'signed_cert',
'kube_service_account_private_key': 'private_key',
}}
mock_get_params.assert_called_once_with(mock_context,
mock_cluster_template,

9
releasenotes/notes/sync-service-account-keys-for-multi-masters-71217c4cf4dd472c.yaml Normal file
View File

@ -0,0 +1,9 @@
---
fixes:
- |
Multi master deployments for k8s driver use different service account
keys for each api/controller manager server which leads to 401 errors
for service accounts. This patch will create a signed cert and private key
for k8s service account keys explicitly, dedicatedly for the k8s
cluster to avoid the inconsistent keys issue.