diff --git a/api-ref/source/certificates.inc b/api-ref/source/certificates.inc index aff0e244c1..0e91a975f9 100644 --- a/api-ref/source/certificates.inc +++ b/api-ref/source/certificates.inc @@ -9,9 +9,10 @@ Generates and show CA certificates for bay/cluster. Show details about the CA certificate for a bay/cluster ======================================================= -.. rest_method:: GET /v1/certificates/{bay_uuid/cluster_uuid} +.. rest_method:: GET /v1/certificates/{cluster_ident}?ca_cert_type={ca_cert_type} -Show CA certificate details that are associated with the created bay/cluster. +Show CA certificate details that are associated with the created bay/cluster based on the +given CA certificate type. Response Codes -------------- @@ -30,7 +31,8 @@ Request .. rest_parameters:: parameters.yaml - - bay_uuid: bay_id + - cluster_ident: cluster_ident + - ca_cert_type: ca_cert_type .. note:: diff --git a/api-ref/source/parameters.yaml b/api-ref/source/parameters.yaml index c921a5c3c1..3163fbdbf4 100644 --- a/api-ref/source/parameters.yaml +++ b/api-ref/source/parameters.yaml @@ -20,6 +20,12 @@ baymodel_ident: in: path required: true type: string +ca_cert_type: + type: string + in: path + required: false + description: | + The CA certificate type. For Kubernetes, it could be kubelet, etcd or front-proxy. cluster_ident: type: string in: path diff --git a/magnum/api/controllers/v1/bay.py b/magnum/api/controllers/v1/bay.py index 79c1de6372..56729cc1e1 100755 --- a/magnum/api/controllers/v1/bay.py +++ b/magnum/api/controllers/v1/bay.py @@ -276,7 +276,8 @@ class BayPatchType(types.JsonPatchType): '/master_addresses', '/stack_id', '/ca_cert_ref', '/magnum_cert_ref', '/trust_id', '/trustee_user_name', - '/trustee_password', '/trustee_user_id'] + '/trustee_password', '/trustee_user_id', + '/etcd_ca_cert_ref', '/front_proxy_ca_cert_ref'] return types.JsonPatchType.internal_attrs() + internal_attrs diff --git a/magnum/api/controllers/v1/certificate.py b/magnum/api/controllers/v1/certificate.py index f7680a8bd1..353827fa77 100644 --- a/magnum/api/controllers/v1/certificate.py +++ b/magnum/api/controllers/v1/certificate.py @@ -88,6 +88,9 @@ class Certificate(base.APIBase): pem = wtypes.StringType() """"The Signed Certificate""" + ca_cert_type = wtypes.StringType() + """"The CA Certificate type the CSR will be signed by""" + def __init__(self, **kwargs): super(Certificate, self).__init__() @@ -113,7 +116,7 @@ class Certificate(base.APIBase): def _convert_with_links(certificate, url, expand=True): if not expand: certificate.unset_fields_except(['bay_uuid', 'cluster_uuid', - 'csr', 'pem']) + 'csr', 'pem', 'ca_cert_type']) certificate.links = [link.Link.make_link('self', url, 'certificates', @@ -135,7 +138,8 @@ class Certificate(base.APIBase): sample = cls(bay_uuid='7ae81bb3-dec3-4289-8d6c-da80bd8001ae', cluster_uuid='7ae81bb3-dec3-4289-8d6c-da80bd8001ae', created_at=timeutils.utcnow(), - csr='AAA....AAA') + csr='AAA....AAA', + ca_cert_type='kubernetes') return cls._convert_with_links(sample, 'http://localhost:9511', expand) @@ -149,8 +153,8 @@ class CertificateController(base.Controller): 'detail': ['GET'], } - @expose.expose(Certificate, types.uuid_or_name) - def get_one(self, cluster_ident): + @expose.expose(Certificate, types.uuid_or_name, wtypes.text) + def get_one(self, cluster_ident, ca_cert_type=None): """Retrieve CA information about the given cluster. :param cluster_ident: UUID of a cluster or @@ -160,7 +164,8 @@ class CertificateController(base.Controller): cluster = api_utils.get_resource('Cluster', cluster_ident) policy.enforce(context, 'certificate:get', cluster.as_dict(), action='certificate:get') - certificate = pecan.request.rpcapi.get_ca_certificate(cluster) + certificate = pecan.request.rpcapi.get_ca_certificate(cluster, + ca_cert_type) return Certificate.convert_with_links(certificate) @expose.expose(Certificate, body=Certificate, status_code=201) diff --git a/magnum/api/controllers/v1/cluster.py b/magnum/api/controllers/v1/cluster.py index 6b3edf42e9..dccf10f874 100755 --- a/magnum/api/controllers/v1/cluster.py +++ b/magnum/api/controllers/v1/cluster.py @@ -294,7 +294,8 @@ class ClusterPatchType(types.JsonPatchType): '/master_addresses', '/stack_id', '/ca_cert_ref', '/magnum_cert_ref', '/trust_id', '/trustee_user_name', - '/trustee_password', '/trustee_user_id'] + '/trustee_password', '/trustee_user_id', + '/etcd_ca_cert_ref', '/front_proxy_ca_cert_ref'] return types.JsonPatchType.internal_attrs() + internal_attrs diff --git a/magnum/conductor/api.py b/magnum/conductor/api.py index 7c4986ae45..8f638bcd98 100644 --- a/magnum/conductor/api.py +++ b/magnum/conductor/api.py @@ -126,8 +126,9 @@ class API(rpc_service.API): return self._call('sign_certificate', cluster=cluster, certificate=certificate) - def get_ca_certificate(self, cluster): - return self._call('get_ca_certificate', cluster=cluster) + def get_ca_certificate(self, cluster, ca_cert_type=None): + return self._call('get_ca_certificate', cluster=cluster, + ca_cert_type=ca_cert_type) def rotate_ca_certificate(self, cluster): return self._call('rotate_ca_certificate', cluster=cluster) diff --git a/magnum/conductor/handlers/ca_conductor.py b/magnum/conductor/handlers/ca_conductor.py index 5696aa28f7..f7c4fc6abc 100644 --- a/magnum/conductor/handlers/ca_conductor.py +++ b/magnum/conductor/handlers/ca_conductor.py @@ -43,8 +43,15 @@ class Handler(object): def sign_certificate(self, context, cluster, certificate): LOG.debug("Creating self signed x509 certificate") + try: + ca_cert_type = certificate.ca_cert_type + except Exception as e: + LOG.debug("There is no CA cert type specified for the CSR") + ca_cert_type = "kubernetes" + signed_cert = cert_manager.sign_node_certificate(cluster, certificate.csr, + ca_cert_type, context=context) if six.PY3 and isinstance(signed_cert, six.binary_type): certificate.pem = signed_cert.decode() @@ -52,9 +59,9 @@ class Handler(object): certificate.pem = signed_cert return certificate - def get_ca_certificate(self, context, cluster): - ca_cert = cert_manager.get_cluster_ca_certificate(cluster, - context=context) + def get_ca_certificate(self, context, cluster, ca_cert_type=None): + ca_cert = cert_manager.get_cluster_ca_certificate( + cluster, context=context, ca_cert_type=ca_cert_type) certificate = objects.Certificate.from_object_cluster(cluster) if six.PY3 and isinstance(ca_cert.get_certificate(), six.binary_type): certificate.pem = ca_cert.get_certificate().decode() diff --git a/magnum/conductor/handlers/common/cert_manager.py b/magnum/conductor/handlers/common/cert_manager.py index 4acbd1b014..180c0b9025 100755 --- a/magnum/conductor/handlers/common/cert_manager.py +++ b/magnum/conductor/handlers/common/cert_manager.py @@ -110,6 +110,10 @@ def generate_certificates_to_cluster(cluster, context=None): ca_cert_ref, ca_cert, ca_password = _generate_ca_cert(issuer_name, context=context) + etcd_ca_cert_ref, _, _ = _generate_ca_cert(issuer_name, + context=context) + fp_ca_cert_ref, _, _ = _generate_ca_cert(issuer_name, + context=context) magnum_cert_ref = _generate_client_cert(issuer_name, ca_cert, ca_password, @@ -117,15 +121,23 @@ def generate_certificates_to_cluster(cluster, context=None): cluster.ca_cert_ref = ca_cert_ref cluster.magnum_cert_ref = magnum_cert_ref + cluster.etcd_ca_cert_ref = etcd_ca_cert_ref + cluster.front_proxy_ca_cert_ref = fp_ca_cert_ref except Exception: LOG.exception('Failed to generate certificates for Cluster: %s', cluster.uuid) raise exception.CertificatesToClusterFailed(cluster_uuid=cluster.uuid) -def get_cluster_ca_certificate(cluster, context=None): +def get_cluster_ca_certificate(cluster, context=None, ca_cert_type=None): + ref = cluster.ca_cert_ref + if ca_cert_type == "etcd": + ref = cluster.etcd_ca_cert_ref + elif ca_cert_type in ["front_proxy", "front-proxy"]: + ref = cluster.front_proxy_ca_cert_ref + ca_cert = cert_manager.get_backend().CertManager.get_cert( - cluster.ca_cert_ref, + ref, resource_ref=cluster.uuid, context=context ) @@ -202,9 +214,15 @@ def create_client_files(cluster, context=None): return ca_file, key_file, cert_file -def sign_node_certificate(cluster, csr, context=None): +def sign_node_certificate(cluster, csr, ca_cert_type=None, context=None): + ref = cluster.ca_cert_ref + if ca_cert_type == "etcd": + ref = cluster.etcd_ca_cert_ref + elif ca_cert_type in ["front_proxy", "front-proxy"]: + ref = cluster.front_proxy_ca_cert_ref + ca_cert = cert_manager.get_backend().CertManager.get_cert( - cluster.ca_cert_ref, + ref, resource_ref=cluster.uuid, context=context ) diff --git a/magnum/db/sqlalchemy/alembic/versions/7da8489d6a68_separated_ca_cert_for_etcd_and_front_.py b/magnum/db/sqlalchemy/alembic/versions/7da8489d6a68_separated_ca_cert_for_etcd_and_front_.py new file mode 100644 index 0000000000..3c3ccc4828 --- /dev/null +++ b/magnum/db/sqlalchemy/alembic/versions/7da8489d6a68_separated_ca_cert_for_etcd_and_front_.py @@ -0,0 +1,42 @@ +# Copyright 2020 Catalyst IT LTD. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""separated CA cert for etcd and front-proxy + +Revision ID: 7da8489d6a68 +Revises: f1d8b0ab8b8d +Create Date: 2020-08-19 17:18:27.634467 + +""" + +# revision identifiers, used by Alembic. +revision = '7da8489d6a68' +down_revision = 'f1d8b0ab8b8d' + +from alembic import op # noqa: E402 # noqa: E402 + +from oslo_db.sqlalchemy.types import String # noqa: E402 + +import sqlalchemy as sa # noqa: E402 + +from sqlalchemy.dialects.mysql import TEXT # noqa: E402 + + +def upgrade(): + op.add_column('cluster', sa.Column('etcd_ca_cert_ref', + String(512, mysql_ndb_type=TEXT), + nullable=True)) + op.add_column('cluster', sa.Column('front_proxy_ca_cert_ref', + String(512, mysql_ndb_type=TEXT), + nullable=True)) diff --git a/magnum/db/sqlalchemy/models.py b/magnum/db/sqlalchemy/models.py index b1436223b4..74bc8ca6cc 100644 --- a/magnum/db/sqlalchemy/models.py +++ b/magnum/db/sqlalchemy/models.py @@ -145,6 +145,8 @@ class Cluster(Base): # so, we use 512 chars to get some buffer. ca_cert_ref = Column(String(512, mysql_ndb_type=mysql_TEXT)) magnum_cert_ref = Column(String(512, mysql_ndb_type=mysql_TEXT)) + etcd_ca_cert_ref = Column(String(512, mysql_ndb_type=mysql_TEXT)) + front_proxy_ca_cert_ref = Column(String(512, mysql_ndb_type=mysql_TEXT)) fixed_network = Column(String(255, mysql_ndb_type=TINYTEXT)) fixed_subnet = Column(String(255, mysql_ndb_type=TINYTEXT)) floating_ip_enabled = Column(Boolean, default=True) diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh index 95ce1c3961..67c680ca1c 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh @@ -317,10 +317,10 @@ KUBE_API_ARGS="$KUBE_API_ARGS --service-account-issuer=https://kubernetes.defaul KUBE_API_ARGS="$KUBE_API_ARGS --kubelet-certificate-authority=${CERT_DIR}/ca.crt --kubelet-client-certificate=${CERT_DIR}/server.crt --kubelet-client-key=${CERT_DIR}/server.key --kubelet-https=true" # Allow for metrics-server/aggregator communication KUBE_API_ARGS="${KUBE_API_ARGS} \ - --proxy-client-cert-file=${CERT_DIR}/server.crt \ - --proxy-client-key-file=${CERT_DIR}/server.key \ - --requestheader-allowed-names=front-proxy-client,kube,kubernetes \ - --requestheader-client-ca-file=${CERT_DIR}/ca.crt \ + --proxy-client-cert-file=${CERT_DIR}/front-proxy/server.crt \ + --proxy-client-key-file=${CERT_DIR}/front-proxy/server.key \ + --requestheader-allowed-names=front-proxy,kube,kubernetes \ + --requestheader-client-ca-file=${CERT_DIR}/front-proxy/ca.crt \ --requestheader-extra-headers-prefix=X-Remote-Extra- \ --requestheader-group-headers=X-Remote-Group \ --requestheader-username-headers=X-Remote-User" diff --git a/magnum/drivers/common/templates/kubernetes/fragments/make-cert.sh b/magnum/drivers/common/templates/kubernetes/fragments/make-cert.sh index 58227b1d79..84bf83945d 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/make-cert.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/make-cert.sh @@ -82,6 +82,7 @@ function generate_certificates { _CSR=$cert_dir/${1}.csr _KEY=$cert_dir/${1}.key _CONF=$2 + _CA_CERT_TYPE=$3 #Get a token by user credentials and trust auth_json=$(cat << EOF @@ -108,11 +109,11 @@ EOF USER_TOKEN=`curl $VERIFY_CA -s -i -X POST -H "$content_type" -d "$auth_json" $url \ | grep -i X-Subject-Token | awk '{print $2}' | tr -d '[[:space:]]'` - # Get CA certificate for this cluster + # Get CA certificate for this cluster. If the CA_CERT_TYPE is not etcd or front-proxy, it will return the default CA cert for kubelet curl $VERIFY_CA -X GET \ -H "X-Auth-Token: $USER_TOKEN" \ -H "OpenStack-API-Version: container-infra latest" \ - $MAGNUM_URL/certificates/$CLUSTER_UUID | python -c 'import sys, json; print(json.load(sys.stdin)["pem"])' >> ${CA_CERT} + $MAGNUM_URL/certificates/$CLUSTER_UUID"?ca_cert_type="${_CA_CERT_TYPE} | python -c 'import sys, json; print(json.load(sys.stdin)["pem"])' > ${CA_CERT} # Generate server's private key and csr $ssh_cmd openssl genrsa -out "${_KEY}" 4096 @@ -124,7 +125,7 @@ EOF -config "${_CONF}" # Send csr to Magnum to have it signed - csr_req=$(python -c "import json; fp = open('${_CSR}'); print(json.dumps({'cluster_uuid': '$CLUSTER_UUID', 'csr': fp.read()})); fp.close()") + csr_req=$(python -c "import json; fp = open('${_CSR}'); print(json.dumps({'ca_cert_type': '$_CA_CERT_TYPE', 'cluster_uuid': '$CLUSTER_UUID', 'csr': fp.read()})); fp.close()") curl $VERIFY_CA -X POST \ -H "X-Auth-Token: $USER_TOKEN" \ -H "OpenStack-API-Version: container-infra latest" \ @@ -182,9 +183,9 @@ L=Austin extendedKeyUsage= clientAuth EOF -generate_certificates server ${cert_dir}/server.conf -generate_certificates kubelet ${cert_dir}/kubelet.conf -generate_certificates admin ${cert_dir}/admin.conf +generate_certificates server ${cert_dir}/server.conf kubelet +generate_certificates kubelet ${cert_dir}/kubelet.conf kubelet +generate_certificates admin ${cert_dir}/admin.conf kubelet # Generate service account key and private key echo -e "${KUBE_SERVICE_ACCOUNT_KEY}" > ${cert_dir}/service_account.key @@ -199,6 +200,52 @@ if [ -z "`cat /etc/group | grep kube_etcd`" ]; then $ssh_cmd chmod 550 "${cert_dir}" $ssh_cmd chown -R kube:kube_etcd "${cert_dir}" $ssh_cmd chmod 440 "$cert_dir/server.key" - $ssh_cmd mkdir -p /etc/etcd/certs - $ssh_cmd cp ${cert_dir}/* /etc/etcd/certs +fi + +# Create certs for etcd +cert_dir=/etc/etcd/certs +$ssh_cmd mkdir -p "$cert_dir" +CA_CERT=${cert_dir}/ca.crt + +cat > ${cert_dir}/server.conf < ${cert_dir}/server.conf <