From 11be0f16157d0daea286b20caeca97af00848eab Mon Sep 17 00:00:00 2001 From: yuchengde Date: Thu, 15 Jul 2021 11:24:57 +0800 Subject: [PATCH] Separate CA for k8s and etcd Sharing cert between kubernetes-ca and etcd-ca will dramatically increase the complexity of updating etcd-ca cert. Therefore do the separating CA work. Test Plan: 0) General -- PASS check the communication status between apiserver and etcd check the etcd status and configuration on controllers verify applications are upgraded check certs are split check certs are persisted 1) Deployment test on simplex -- PASS (changeset 31) Lock/unlock simplex Simplex spontaneous reboot 2) Deployment test on duplex -- PASS (changeset 31) Switch active controller (changeset 33) Reinstall controller and Swact (changeset 33) 3) Backup and restore starlingx platform on simplex -- PASS (changeset 31) 4) Backup and restore starlingx platform on duplex -- PASS with config-out-of-date on c0 (changeset 31) retest in progress 5) Upgrade starlingx platform on simplex -- PASS (changeset 31) 6) Upgrade starlingx platform on duplex -- PASS Story: 2008833 Task: 42320 Depends-on: https://review.opendev.org/#/c/800412/ Change-Id: Ic11a2cba27db03c07e0d748e093b5451482db380 Signed-off-by: yuchengde Signed-off-by: Michel Thebeau --- .../controllerconfig/upgrades/controller.py | 18 ---- .../71-enable-separate-etcd-ca.sh | 89 +++++++++++++++++++ .../sysinv/sysinv/sysinv/conductor/manager.py | 33 +++++++ tsconfig/tsconfig/tsconfig/tsconfig.py | 1 + 4 files changed, 123 insertions(+), 18 deletions(-) create mode 100644 controllerconfig/controllerconfig/upgrade-scripts/71-enable-separate-etcd-ca.sh diff --git a/controllerconfig/controllerconfig/controllerconfig/upgrades/controller.py b/controllerconfig/controllerconfig/controllerconfig/upgrades/controller.py index d7613fe21e..63a6f45642 100644 --- a/controllerconfig/controllerconfig/controllerconfig/upgrades/controller.py +++ b/controllerconfig/controllerconfig/controllerconfig/upgrades/controller.py @@ -714,24 +714,6 @@ def migrate_hiera_data(from_release, to_release, role=None): "hieradata") to_hiera_path = constants.HIERADATA_PERMDIR - # For simplex upgrade, we already set etcd security config during - # apply-bootstrap-manifest. Need to get it and update to target - # static.yaml. - static_file = os.path.join(to_hiera_path, "static.yaml") - etcd_security_config = {} - - if os.path.exists(static_file): - with open(static_file, 'r') as yaml_file: - static_config = yaml.load(yaml_file) - - if 'platform::etcd::params::security_enabled' in static_config.keys(): - etcd_security_config['platform::etcd::params::security_enabled'] = \ - static_config['platform::etcd::params::security_enabled'] - etcd_security_config['platform::etcd::params::bind_address'] = \ - static_config['platform::etcd::params::bind_address'] - etcd_security_config['platform::etcd::params::bind_address_version'] = \ - static_config['platform::etcd::params::bind_address_version'] - shutil.rmtree(to_hiera_path, ignore_errors=True) os.makedirs(to_hiera_path) diff --git a/controllerconfig/controllerconfig/upgrade-scripts/71-enable-separate-etcd-ca.sh b/controllerconfig/controllerconfig/upgrade-scripts/71-enable-separate-etcd-ca.sh new file mode 100644 index 0000000000..23f91a8dd6 --- /dev/null +++ b/controllerconfig/controllerconfig/upgrade-scripts/71-enable-separate-etcd-ca.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +# +# Copyright (c) 2021 Intel Corporation. +# +# SPDX-License-Identifier: Apache-2.0 +# + +# Enable separate etcd ca during upgrade. +# +# Note: this can be removed in the release after STX6.0 + +. /etc/platform/platform.conf + +# This will log to /var/log/platform.log +function log { + logger -p local1.info $1 +} + + +FROM_REL=$1 +TO_REL=$2 +ACTION=$3 + +# below function is cloned from ../scripts/controller_config +get_ip() +{ + HOST_NAME=$1 + + # Check /etc/hosts for the hostname + HOST_IP=$(cat /etc/hosts | grep "${HOST_NAME}" | awk '{print $1}') + if [ -n "${HOST_IP}" ]; then + echo "${HOST_IP}" + return + fi + + # Try the DNS query + # Because dnsmasq can resolve both a hostname to both an IPv4 and an IPv6 + # address in certain situations, and the last address is the IPv6, which + # would be the management, this is preferred over the IPv4 pxeboot address, + # so take the last address only. + HOST_IP=$(dig +short ANY $host|tail -1) + if [[ "${HOST_IP}" =~ ^[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*$ ]]; then + echo "${HOST_IP}" + return + fi + if [[ "${HOST_IP}" =~ ^[0-9a-z]*\:[0-9a-z\:]*$ ]]; then + echo "${HOST_IP}" + return + fi +} + +enable_separate_etcd_ca() +{ + STATIC_YAML="/opt/platform/puppet/${sw_version}/hieradata/static.yaml" + SYSTEM_YAML="/opt/platform/puppet/${sw_version}/hieradata/system.yaml" + + if [[ ! -f ${STATIC_YAML} ]] || [[ ! -f ${SYSTEM_YAML} ]]; then + log "Could not find specific static/system yaml files in /opt/platform/puppet/${sw_version}/hieradata!" + exit 1 + fi + + CLUSTER_FLOATING_ADDRESS=$(grep "platform::network::cluster_host::params::controller_address" ${SYSTEM_YAML} | awk '{print $2}') + CLUSTER_FLOATING_ADDRESS_VERSION=$(grep "platform::network::cluster_host::params::subnet_version" ${SYSTEM_YAML} | awk '{print $2}') + HOST_ADDR=$(get_ip $(hostname)) + + ansible-playbook /usr/share/ansible/stx-ansible/playbooks/separate_etcd_ca.yml \ + -e "cluster_floating_address=${CLUSTER_FLOATING_ADDRESS}" \ + -e "etcd_listen_address_version=${CLUSTER_FLOATING_ADDRESS_VERSION}" \ + -e "puppet_permdir=/opt/platform/puppet/${sw_version}" \ + -e "config_permdir=/opt/platform/config/${sw_version}" \ + -e "ipaddress=${HOST_ADDR}" \ + -e "etcd_root_ca_cert=''" \ + -e "etcd_root_ca_key=''" + if [ $? -ne 0 ]; then + log "Failed to run ansible playbook!" + exit 1 + fi +} + +log "${0} invoked with from_release = ${FROM_REL} to_release = ${TO_REL} action = ${ACTION}" + +if [ ${FROM_REL} == "21.05" -a ${ACTION} == "activate" ]; then + enable_separate_etcd_ca +else + log "Only execute this upgrade code when the activate action is being done and the from release is 21.05!" +fi + +exit 0 diff --git a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py index be7efa0c2f..501fba9c7b 100644 --- a/sysinv/sysinv/sysinv/sysinv/conductor/manager.py +++ b/sysinv/sysinv/sysinv/sysinv/conductor/manager.py @@ -1611,6 +1611,36 @@ class ConductorManager(service.PeriodicService): "Skipping deleting ceph monitor." % str(host.hostname)) + def _split_etcd_security_config(self, context): + """Update the manifests for separating etcd ca + + Note: this can be removed in the release after STX6.0 + returns True if runtime manifests were applied + """ + controllers = self.dbapi.ihost_get_by_personality(constants.CONTROLLER) + for host in controllers: + if not utils.is_host_active_controller(host): + # Just update etcd certs on the standby controller. + # Etcd certs were updated on the active controller with + # migration script 71-enable-separate-etcd-ca.sh + personalities = [constants.CONTROLLER] + host_uuids = [host.uuid] + config_uuid = self._config_update_hosts( + context, personalities, host_uuids) + config_dict = { + "personalities": personalities, + "host_uuids": host_uuids, + "classes": ['platform::etcd::upgrade::runtime'], + puppet_common.REPORT_STATUS_CFG: + puppet_common.REPORT_UPGRADE_ACTIONS + } + self._config_apply_runtime_manifest(context, + config_uuid=config_uuid, + config_dict=config_dict) + return True + + return False + def update_remotelogging_config(self, context): """Update the remotelogging configuration""" @@ -11208,6 +11238,9 @@ class ConductorManager(service.PeriodicService): {'state': constants.UPGRADE_ACTIVATION_FAILED}) manifests_applied = False + if from_version == tsc.SW_VERSION_21_05: + # Apply etcd split ca puppet manifest for standby controller. + manifests_applied = self._split_etcd_security_config(context) if manifests_applied: LOG.info("Running upgrade activation manifests") diff --git a/tsconfig/tsconfig/tsconfig/tsconfig.py b/tsconfig/tsconfig/tsconfig/tsconfig.py index c3a8fcef72..c42345d065 100644 --- a/tsconfig/tsconfig/tsconfig/tsconfig.py +++ b/tsconfig/tsconfig/tsconfig/tsconfig.py @@ -13,6 +13,7 @@ import six from six.moves import configparser SW_VERSION = "" +SW_VERSION_21_05 = "21.05" nodetype = None subfunctions = []