Add separated CA cert for etcd and front-proxy

Support creating different for k8s, etcd and front-proxy for
security hardening. We're following some best practices[1][2] but
adjusted based on the current Magnum deployment approach.

[1] https://kubernetes.io/docs/setup/best-practices/certificates/
[2] https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-certs/

Task: 40687
Story: 2008031

Change-Id: I523a4a85867f82d234ba1f3e6fad8b8cd2291182
This commit is contained in:
Feilong Wang 2020-08-19 17:22:36 +12:00
parent 42f8c97bbf
commit 16344a5a95
20 changed files with 199 additions and 52 deletions

View File

@ -9,9 +9,10 @@ Generates and show CA certificates for bay/cluster.
Show details about the CA certificate for a bay/cluster
=======================================================
.. rest_method:: GET /v1/certificates/{bay_uuid/cluster_uuid}
.. rest_method:: GET /v1/certificates/{cluster_ident}?ca_cert_type={ca_cert_type}
Show CA certificate details that are associated with the created bay/cluster.
Show CA certificate details that are associated with the created bay/cluster based on the
given CA certificate type.
Response Codes
--------------
@ -30,7 +31,8 @@ Request
.. rest_parameters:: parameters.yaml
- bay_uuid: bay_id
- cluster_ident: cluster_ident
- ca_cert_type: ca_cert_type
.. note::

View File

@ -20,6 +20,12 @@ baymodel_ident:
in: path
required: true
type: string
ca_cert_type:
type: string
in: path
required: false
description: |
The CA certificate type. For Kubernetes, it could be kubelet, etcd or front-proxy.
cluster_ident:
type: string
in: path

View File

@ -276,7 +276,8 @@ class BayPatchType(types.JsonPatchType):
'/master_addresses', '/stack_id',
'/ca_cert_ref', '/magnum_cert_ref',
'/trust_id', '/trustee_user_name',
'/trustee_password', '/trustee_user_id']
'/trustee_password', '/trustee_user_id',
'/etcd_ca_cert_ref', '/front_proxy_ca_cert_ref']
return types.JsonPatchType.internal_attrs() + internal_attrs

View File

@ -88,6 +88,9 @@ class Certificate(base.APIBase):
pem = wtypes.StringType()
""""The Signed Certificate"""
ca_cert_type = wtypes.StringType()
""""The CA Certificate type the CSR will be signed by"""
def __init__(self, **kwargs):
super(Certificate, self).__init__()
@ -113,7 +116,7 @@ class Certificate(base.APIBase):
def _convert_with_links(certificate, url, expand=True):
if not expand:
certificate.unset_fields_except(['bay_uuid', 'cluster_uuid',
'csr', 'pem'])
'csr', 'pem', 'ca_cert_type'])
certificate.links = [link.Link.make_link('self', url,
'certificates',
@ -135,7 +138,8 @@ class Certificate(base.APIBase):
sample = cls(bay_uuid='7ae81bb3-dec3-4289-8d6c-da80bd8001ae',
cluster_uuid='7ae81bb3-dec3-4289-8d6c-da80bd8001ae',
created_at=timeutils.utcnow(),
csr='AAA....AAA')
csr='AAA....AAA',
ca_cert_type='kubernetes')
return cls._convert_with_links(sample, 'http://localhost:9511', expand)
@ -149,8 +153,8 @@ class CertificateController(base.Controller):
'detail': ['GET'],
}
@expose.expose(Certificate, types.uuid_or_name)
def get_one(self, cluster_ident):
@expose.expose(Certificate, types.uuid_or_name, wtypes.text)
def get_one(self, cluster_ident, ca_cert_type=None):
"""Retrieve CA information about the given cluster.
:param cluster_ident: UUID of a cluster or
@ -160,7 +164,8 @@ class CertificateController(base.Controller):
cluster = api_utils.get_resource('Cluster', cluster_ident)
policy.enforce(context, 'certificate:get', cluster.as_dict(),
action='certificate:get')
certificate = pecan.request.rpcapi.get_ca_certificate(cluster)
certificate = pecan.request.rpcapi.get_ca_certificate(cluster,
ca_cert_type)
return Certificate.convert_with_links(certificate)
@expose.expose(Certificate, body=Certificate, status_code=201)

View File

@ -294,7 +294,8 @@ class ClusterPatchType(types.JsonPatchType):
'/master_addresses', '/stack_id',
'/ca_cert_ref', '/magnum_cert_ref',
'/trust_id', '/trustee_user_name',
'/trustee_password', '/trustee_user_id']
'/trustee_password', '/trustee_user_id',
'/etcd_ca_cert_ref', '/front_proxy_ca_cert_ref']
return types.JsonPatchType.internal_attrs() + internal_attrs

View File

@ -126,8 +126,9 @@ class API(rpc_service.API):
return self._call('sign_certificate', cluster=cluster,
certificate=certificate)
def get_ca_certificate(self, cluster):
return self._call('get_ca_certificate', cluster=cluster)
def get_ca_certificate(self, cluster, ca_cert_type=None):
return self._call('get_ca_certificate', cluster=cluster,
ca_cert_type=ca_cert_type)
def rotate_ca_certificate(self, cluster):
return self._call('rotate_ca_certificate', cluster=cluster)

View File

@ -43,8 +43,15 @@ class Handler(object):
def sign_certificate(self, context, cluster, certificate):
LOG.debug("Creating self signed x509 certificate")
try:
ca_cert_type = certificate.ca_cert_type
except Exception as e:
LOG.debug("There is no CA cert type specified for the CSR")
ca_cert_type = "kubernetes"
signed_cert = cert_manager.sign_node_certificate(cluster,
certificate.csr,
ca_cert_type,
context=context)
if six.PY3 and isinstance(signed_cert, six.binary_type):
certificate.pem = signed_cert.decode()
@ -52,9 +59,9 @@ class Handler(object):
certificate.pem = signed_cert
return certificate
def get_ca_certificate(self, context, cluster):
ca_cert = cert_manager.get_cluster_ca_certificate(cluster,
context=context)
def get_ca_certificate(self, context, cluster, ca_cert_type=None):
ca_cert = cert_manager.get_cluster_ca_certificate(
cluster, context=context, ca_cert_type=ca_cert_type)
certificate = objects.Certificate.from_object_cluster(cluster)
if six.PY3 and isinstance(ca_cert.get_certificate(), six.binary_type):
certificate.pem = ca_cert.get_certificate().decode()

View File

@ -110,6 +110,10 @@ def generate_certificates_to_cluster(cluster, context=None):
ca_cert_ref, ca_cert, ca_password = _generate_ca_cert(issuer_name,
context=context)
etcd_ca_cert_ref, _, _ = _generate_ca_cert(issuer_name,
context=context)
fp_ca_cert_ref, _, _ = _generate_ca_cert(issuer_name,
context=context)
magnum_cert_ref = _generate_client_cert(issuer_name,
ca_cert,
ca_password,
@ -117,15 +121,23 @@ def generate_certificates_to_cluster(cluster, context=None):
cluster.ca_cert_ref = ca_cert_ref
cluster.magnum_cert_ref = magnum_cert_ref
cluster.etcd_ca_cert_ref = etcd_ca_cert_ref
cluster.front_proxy_ca_cert_ref = fp_ca_cert_ref
except Exception:
LOG.exception('Failed to generate certificates for Cluster: %s',
cluster.uuid)
raise exception.CertificatesToClusterFailed(cluster_uuid=cluster.uuid)
def get_cluster_ca_certificate(cluster, context=None):
def get_cluster_ca_certificate(cluster, context=None, ca_cert_type=None):
ref = cluster.ca_cert_ref
if ca_cert_type == "etcd":
ref = cluster.etcd_ca_cert_ref
elif ca_cert_type in ["front_proxy", "front-proxy"]:
ref = cluster.front_proxy_ca_cert_ref
ca_cert = cert_manager.get_backend().CertManager.get_cert(
cluster.ca_cert_ref,
ref,
resource_ref=cluster.uuid,
context=context
)
@ -202,9 +214,15 @@ def create_client_files(cluster, context=None):
return ca_file, key_file, cert_file
def sign_node_certificate(cluster, csr, context=None):
def sign_node_certificate(cluster, csr, ca_cert_type=None, context=None):
ref = cluster.ca_cert_ref
if ca_cert_type == "etcd":
ref = cluster.etcd_ca_cert_ref
elif ca_cert_type in ["front_proxy", "front-proxy"]:
ref = cluster.front_proxy_ca_cert_ref
ca_cert = cert_manager.get_backend().CertManager.get_cert(
cluster.ca_cert_ref,
ref,
resource_ref=cluster.uuid,
context=context
)

View File

@ -0,0 +1,42 @@
# Copyright 2020 Catalyst IT LTD. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""separated CA cert for etcd and front-proxy
Revision ID: 7da8489d6a68
Revises: f1d8b0ab8b8d
Create Date: 2020-08-19 17:18:27.634467
"""
# revision identifiers, used by Alembic.
revision = '7da8489d6a68'
down_revision = 'f1d8b0ab8b8d'
from alembic import op # noqa: E402 # noqa: E402
from oslo_db.sqlalchemy.types import String # noqa: E402
import sqlalchemy as sa # noqa: E402
from sqlalchemy.dialects.mysql import TEXT # noqa: E402
def upgrade():
op.add_column('cluster', sa.Column('etcd_ca_cert_ref',
String(512, mysql_ndb_type=TEXT),
nullable=True))
op.add_column('cluster', sa.Column('front_proxy_ca_cert_ref',
String(512, mysql_ndb_type=TEXT),
nullable=True))

View File

@ -145,6 +145,8 @@ class Cluster(Base):
# so, we use 512 chars to get some buffer.
ca_cert_ref = Column(String(512, mysql_ndb_type=mysql_TEXT))
magnum_cert_ref = Column(String(512, mysql_ndb_type=mysql_TEXT))
etcd_ca_cert_ref = Column(String(512, mysql_ndb_type=mysql_TEXT))
front_proxy_ca_cert_ref = Column(String(512, mysql_ndb_type=mysql_TEXT))
fixed_network = Column(String(255, mysql_ndb_type=TINYTEXT))
fixed_subnet = Column(String(255, mysql_ndb_type=TINYTEXT))
floating_ip_enabled = Column(Boolean, default=True)

View File

@ -317,10 +317,10 @@ KUBE_API_ARGS="$KUBE_API_ARGS --service-account-issuer=https://kubernetes.defaul
KUBE_API_ARGS="$KUBE_API_ARGS --kubelet-certificate-authority=${CERT_DIR}/ca.crt --kubelet-client-certificate=${CERT_DIR}/server.crt --kubelet-client-key=${CERT_DIR}/server.key --kubelet-https=true"
# Allow for metrics-server/aggregator communication
KUBE_API_ARGS="${KUBE_API_ARGS} \
--proxy-client-cert-file=${CERT_DIR}/server.crt \
--proxy-client-key-file=${CERT_DIR}/server.key \
--requestheader-allowed-names=front-proxy-client,kube,kubernetes \
--requestheader-client-ca-file=${CERT_DIR}/ca.crt \
--proxy-client-cert-file=${CERT_DIR}/front-proxy/server.crt \
--proxy-client-key-file=${CERT_DIR}/front-proxy/server.key \
--requestheader-allowed-names=front-proxy,kube,kubernetes \
--requestheader-client-ca-file=${CERT_DIR}/front-proxy/ca.crt \
--requestheader-extra-headers-prefix=X-Remote-Extra- \
--requestheader-group-headers=X-Remote-Group \
--requestheader-username-headers=X-Remote-User"

View File

@ -82,6 +82,7 @@ function generate_certificates {
_CSR=$cert_dir/${1}.csr
_KEY=$cert_dir/${1}.key
_CONF=$2
_CA_CERT_TYPE=$3
#Get a token by user credentials and trust
auth_json=$(cat << EOF
@ -108,11 +109,11 @@ EOF
USER_TOKEN=`curl $VERIFY_CA -s -i -X POST -H "$content_type" -d "$auth_json" $url \
| grep -i X-Subject-Token | awk '{print $2}' | tr -d '[[:space:]]'`
# Get CA certificate for this cluster
# Get CA certificate for this cluster. If the CA_CERT_TYPE is not etcd or front-proxy, it will return the default CA cert for kubelet
curl $VERIFY_CA -X GET \
-H "X-Auth-Token: $USER_TOKEN" \
-H "OpenStack-API-Version: container-infra latest" \
$MAGNUM_URL/certificates/$CLUSTER_UUID | python -c 'import sys, json; print(json.load(sys.stdin)["pem"])' >> ${CA_CERT}
$MAGNUM_URL/certificates/$CLUSTER_UUID"?ca_cert_type="${_CA_CERT_TYPE} | python -c 'import sys, json; print(json.load(sys.stdin)["pem"])' > ${CA_CERT}
# Generate server's private key and csr
$ssh_cmd openssl genrsa -out "${_KEY}" 4096
@ -124,7 +125,7 @@ EOF
-config "${_CONF}"
# Send csr to Magnum to have it signed
csr_req=$(python -c "import json; fp = open('${_CSR}'); print(json.dumps({'cluster_uuid': '$CLUSTER_UUID', 'csr': fp.read()})); fp.close()")
csr_req=$(python -c "import json; fp = open('${_CSR}'); print(json.dumps({'ca_cert_type': '$_CA_CERT_TYPE', 'cluster_uuid': '$CLUSTER_UUID', 'csr': fp.read()})); fp.close()")
curl $VERIFY_CA -X POST \
-H "X-Auth-Token: $USER_TOKEN" \
-H "OpenStack-API-Version: container-infra latest" \
@ -182,9 +183,9 @@ L=Austin
extendedKeyUsage= clientAuth
EOF
generate_certificates server ${cert_dir}/server.conf
generate_certificates kubelet ${cert_dir}/kubelet.conf
generate_certificates admin ${cert_dir}/admin.conf
generate_certificates server ${cert_dir}/server.conf kubelet
generate_certificates kubelet ${cert_dir}/kubelet.conf kubelet
generate_certificates admin ${cert_dir}/admin.conf kubelet
# Generate service account key and private key
echo -e "${KUBE_SERVICE_ACCOUNT_KEY}" > ${cert_dir}/service_account.key
@ -199,6 +200,52 @@ if [ -z "`cat /etc/group | grep kube_etcd`" ]; then
$ssh_cmd chmod 550 "${cert_dir}"
$ssh_cmd chown -R kube:kube_etcd "${cert_dir}"
$ssh_cmd chmod 440 "$cert_dir/server.key"
$ssh_cmd mkdir -p /etc/etcd/certs
$ssh_cmd cp ${cert_dir}/* /etc/etcd/certs
fi
# Create certs for etcd
cert_dir=/etc/etcd/certs
$ssh_cmd mkdir -p "$cert_dir"
CA_CERT=${cert_dir}/ca.crt
cat > ${cert_dir}/server.conf <<EOF
[req]
distinguished_name = req_distinguished_name
req_extensions = req_ext
prompt = no
[req_distinguished_name]
CN = etcd
[req_ext]
subjectAltName = ${sans}
extendedKeyUsage = clientAuth,serverAuth
EOF
generate_certificates server ${cert_dir}/server.conf etcd
generate_certificates admin ${cert_dir}/server.conf etcd
if [ -z "`cat /etc/group | grep kube_etcd`" ]; then
$ssh_cmd chown -R etcd:kube_etcd "${cert_dir}"
fi
# Create certs for front-proxy
cert_dir=/etc/kubernetes/certs/front-proxy
$ssh_cmd mkdir -p "$cert_dir"
CA_CERT=${cert_dir}/ca.crt
cat > ${cert_dir}/server.conf <<EOF
[req]
distinguished_name = req_distinguished_name
req_extensions = req_ext
prompt = no
[req_distinguished_name]
CN = front-proxy
[req_ext]
subjectAltName = ${sans}
extendedKeyUsage = clientAuth,serverAuth
EOF
generate_certificates server ${cert_dir}/server.conf front-proxy
generate_certificates admin ${cert_dir}/server.conf front-proxy
if [ -z "`cat /etc/group | grep kube_etcd`" ]; then
$ssh_cmd chown -R kube:kube_etcd "${cert_dir}"
fi

View File

@ -19,7 +19,10 @@ from magnum.objects import base
class Certificate(base.MagnumPersistentObject, base.MagnumObject):
# Version 1.0: Initial version
# Version 1.1: Rename bay_uuid to cluster_uuid
VERSION = '1.1'
# Version 1.2: Add ca_cert_type to indicate what's the CA cert type the
# CSR being signed
VERSION = '1.2'
fields = {
'project_id': fields.StringField(nullable=True),
@ -27,6 +30,7 @@ class Certificate(base.MagnumPersistentObject, base.MagnumObject):
'cluster_uuid': fields.StringField(nullable=True),
'csr': fields.StringField(nullable=True),
'pem': fields.StringField(nullable=True),
'ca_cert_type': fields.StringField(nullable=True),
}
@classmethod
@ -40,3 +44,4 @@ class Certificate(base.MagnumPersistentObject, base.MagnumObject):
return cls(project_id=cluster['project_id'],
user_id=cluster['user_id'],
cluster_uuid=cluster['uuid'])

View File

@ -53,8 +53,9 @@ class Cluster(base.MagnumPersistentObject, base.MagnumObject,
# master_addresses are now properties.
# Version 1.21 Added fixed_network, fixed_subnet, floating_ip_enabled
# Version 1.22 Added master_lb_enabled
# Version 1.23 Added etcd_ca_cert_ref and front_proxy_ca_cert_ref
VERSION = '1.22'
VERSION = '1.23'
dbapi = dbapi.get_instance()
@ -80,6 +81,8 @@ class Cluster(base.MagnumPersistentObject, base.MagnumObject,
'discovery_url': fields.StringField(nullable=True),
'ca_cert_ref': fields.StringField(nullable=True),
'magnum_cert_ref': fields.StringField(nullable=True),
'etcd_ca_cert_ref': fields.StringField(nullable=True),
'front_proxy_ca_cert_ref': fields.StringField(nullable=True),
'cluster_template': fields.ObjectField('ClusterTemplate'),
'trust_id': fields.StringField(nullable=True),
'trustee_username': fields.StringField(nullable=True),

View File

@ -0,0 +1,4 @@
---
features:
- |
Support creating different CA for kubernetes, etcd and front-proxy.

View File

@ -131,8 +131,8 @@ class CertManagerTestCase(base.BaseTestCase):
self.assertEqual(expected_ca_cert_ref, mock_cluster.ca_cert_ref)
self.assertEqual(expected_cert_ref, mock_cluster.magnum_cert_ref)
mock_generate_ca_cert.assert_called_once_with(expected_ca_name,
context=None)
mock_generate_ca_cert.assert_called_with(expected_ca_name,
context=None)
mock_generate_client_cert.assert_called_once_with(
expected_ca_name, expected_ca_cert, expected_ca_password,
context=None)
@ -189,7 +189,6 @@ class CertManagerTestCase(base.BaseTestCase):
self.CertManager.get_cert.return_value = mock_ca_cert
mock_csr = mock.MagicMock()
mock_x509_sign.return_value = mock.sentinel.signed_cert
cluster_ca_cert = cert_manager.sign_node_certificate(mock_cluster,
mock_csr)
@ -238,6 +237,20 @@ class CertManagerTestCase(base.BaseTestCase):
context=None)
self.assertEqual(mock_ca_cert, cluster_ca_cert)
def test_get_cluster_ca_certificate_ca_cert_type(self):
mock_cluster = mock.MagicMock()
mock_cluster.uuid = "mock_cluster_uuid"
mock_ca_cert = mock.MagicMock()
self.CertManager.get_cert.return_value = mock_ca_cert
cluster_ca_cert = cert_manager.get_cluster_ca_certificate(
mock_cluster, ca_cert_type="front-proxy")
self.CertManager.get_cert.assert_called_once_with(
mock_cluster.front_proxy_ca_cert_ref, resource_ref=mock_cluster.uuid,
context=None)
self.assertEqual(mock_ca_cert, cluster_ca_cert)
def test_get_cluster_magnum_cert(self):
mock_cluster = mock.MagicMock()
mock_cluster.uuid = "mock_cluster_uuid"

View File

@ -28,6 +28,7 @@ class TestSignConductor(base.TestCase):
mock_cluster = mock.MagicMock()
mock_certificate = mock.MagicMock()
mock_certificate.csr = 'fake-csr'
mock_certificate.ca_cert_type = 'kubernetes'
mock_cert_manager.sign_node_certificate.return_value = 'fake-pem'
actual_cert = self.ca_handler.sign_certificate(self.context,
@ -35,7 +36,7 @@ class TestSignConductor(base.TestCase):
mock_certificate)
mock_cert_manager.sign_node_certificate.assert_called_once_with(
mock_cluster, 'fake-csr', context=self.context
mock_cluster, 'fake-csr', 'kubernetes', context=self.context
)
self.assertEqual('fake-pem', actual_cert.pem)

View File

@ -103,6 +103,8 @@ def get_test_cluster(**kw):
'fixed_subnet': kw.get('fixed_subnet', None),
'floating_ip_enabled': kw.get('floating_ip_enabled', True),
'master_lb_enabled': kw.get('master_lb_enabled', True),
'etcd_ca_cert_ref': kw.get('etcd_ca_cert_ref', None),
'front_proxy_ca_cert_ref': kw.get('front_proxy_ca_cert_ref', None)
}
if kw.pop('for_api_use', False):

View File

@ -355,9 +355,9 @@ class TestObject(test_base.TestCase, _TestObject):
# For more information on object version testing, read
# https://docs.openstack.org/magnum/latest/contributor/objects.html
object_data = {
'Cluster': '1.22-39ae1aa9ed1e90ee05f67f64b5fce4bb',
'Cluster': '1.23-dfaf9ecb65a5fcab4f6c36497a8bc866',
'ClusterTemplate': '1.20-85469623f678e916f26e3cb5924ae664',
'Certificate': '1.1-1924dc077daa844f0f9076332ef96815',
'Certificate': '1.2-64f24db0e10ad4cbd72aea21d2075a80',
'MyObj': '1.0-34c4b1aadefd177b13f9a2f894cc23cd',
'X509KeyPair': '1.2-d81950af36c59a71365e33ce539d24f9',
'MagnumService': '1.0-2d397ec59b0046bd5ec35cd3e06efeca',

13
tox.ini
View File

@ -92,19 +92,6 @@ commands =
find . -type f -name "*.py[c|o]" -delete
stestr run {posargs}
[testenv:pep8]
commands =
doc8 -e .rst specs/ doc/source/ contrib/ CONTRIBUTING.rst HACKING.rst README.rst
bash tools/flake8wrap.sh {posargs}
bandit -r magnum -x tests -n5 -ll
bash -c "find {toxinidir} \
-not \( -type d -name .?\* -prune \) \
-not \( -type d -name doc -prune \) \
-not \( -type d -name contrib -prune \) \
-type f \
-name \*.sh \
-print0 | xargs -0 bashate -v -iE006,E010,E042 -eE005"
[testenv:venv]
commands = {posargs}