From 50c17f8e570d69fe4b770e42d027b5106a3c5d7e Mon Sep 17 00:00:00 2001
From: Michel Thebeau <Michel.Thebeau@windriver.com>
Date: Thu, 27 Jul 2023 20:29:29 +0000
Subject: [PATCH] collector: add collect of certificates
Add explicit collect of certificates files for the platform.
Delete all crt, pem and key files in collect before including those
files explicitly listed in certs.include file. Use openssl command to
omit all but certificate information from the files.
Test Plan:
PASS AIO-SX, AIO-DX+, DC with AIO-SX subcloud
PASS options --skip-mask, --omit-certs, --subcloud
PASS bashate
PASS unit test for collect_certificates
PASS collect output contains no crt, pem and key files except those
listed in certs.include
PASS If a file listed in certs.include does not exist on the filesystem
then the absence is ignored - it is ok for a specified file not to
exist.
PASS file with key omits key - only certs are copied
PASS manual verify of file paths (including those on DC subcloud)
Closes-Bug: 2029302
Change-Id: I9fafe5fde39a1a7de9a887424f274986b13e053a
Signed-off-by: Michel Thebeau <Michel.Thebeau@windriver.com>
---
tools/collector/debian-scripts/certs.include | 42 +++++++++++
tools/collector/debian-scripts/collect | 12 +++-
.../debian-scripts/collect_certificates | 70 +++++++++++++++++++
tools/collector/debian-scripts/collect_host | 8 +++
.../debian-scripts/collect_mask_passwords | 13 ++--
tools/collector/debian/deb_folder/rules | 2 +
6 files changed, 137 insertions(+), 10 deletions(-)
create mode 100644 tools/collector/debian-scripts/certs.include
create mode 100644 tools/collector/debian-scripts/collect_certificates
diff --git a/tools/collector/debian-scripts/certs.include b/tools/collector/debian-scripts/certs.include
new file mode 100644
index 00000000..9107f5aa
--- /dev/null
+++ b/tools/collector/debian-scripts/certs.include
@@ -0,0 +1,42 @@
+# certs.include format:
+#
+# Ignore anything that does not start with slash
+# Replace "%%RELEASE%%" with the cluster's current release
+# If the line ends with slash treat it like a directory
+# Otherwise, treat it like a file
+
+# 1. k8s certificates:
+/etc/kubernetes/pki/
+/etc/etcd/
+/var/lib/kubelet/pki/kubelet-client-current.pem
+/var/lib/kubelet/pki/kubelet.crt
+
+# 2. DC admin endpoint certificates
+/etc/ssl/private/admin-ep-cert.pem
+/opt/platform/config/%%RELEASE%%/dc-adminep-root-ca.crt
+
+# 3. docker registry certificates
+/etc/ssl/private/registry-cert.crt
+/etc/docker/certs.d/registry.local:9001/registry-cert.crt
+/etc/docker/certs.d/registry.central:9001/registry-cert.crt
+
+# 4. openldap certificates
+/etc/ldap/certs/openldap-cert.crt
+
+# 5. GUI/REST API certificates
+/etc/ssl/private/server-cert.pem
+
+# 6. Installed ssl CA certificates
+/etc/pki/ca-trust/source/anchors/
+# The following path is hardcoded with regex in collect_certificates:
+# /opt/platform/config/%%RELEASE%%/ssl_ca/ssl_ca_[0-9]{20}
+
+# 7. ceph
+/run/ceph/mgr/restful.crt
+
+# 8. platform config
+/opt/platform/config/%%RELEASE%%/
+/opt/platform/config/%%RELEASE%%/etcd/
+/opt/platform/config/%%RELEASE%%/kubernetes/pki/
+/opt/platform/config/%%RELEASE%%/registry.central/registry-cert.crt
+/opt/platform/config/%%RELEASE%%/ca-cert.pem
diff --git a/tools/collector/debian-scripts/collect b/tools/collector/debian-scripts/collect
index 78021867..1d7fd704 100644
--- a/tools/collector/debian-scripts/collect
+++ b/tools/collector/debian-scripts/collect
@@ -382,6 +382,8 @@ function print_help()
echo ""
echo "collect [--skip-mask] ... skip masking of collect data"
echo ""
+ echo "collect [--omit-certs] ... do not include certificates in the collect data"
+ echo ""
echo "Create a collect report"
echo ""
echo "collect [--report | -r ] ... run the collect report tool on the collected bundle"
@@ -408,6 +410,7 @@ CLEAN=false
REPORT=false
VERBOSE=false
SKIP_MASK=false
+OMIT_CERTS=false
INVENTORY=false
SUBCLOUD_COLLECT=false
SUBCLOUD_LOGIN_PROMPT="controller-"
@@ -745,6 +748,10 @@ while [[ ${#} -gt 0 ]] ; do
SKIP_MASK=true
;;
+ --omit-certs)
+ OMIT_CERTS=true
+ ;;
+
-in|--inline)
# switch to inline ; one-after-the-other (legacy) mode
PARALLEL_COLLECT_MODE=false
@@ -815,6 +822,7 @@ dlog "INVENTORY = ${INVENTORY}"
dlog "STARTDATE = ${STARTDATE}"
dlog "ENDDATE = ${ENDDATE}"
dlog "SKIPMASK = ${SKIP_MASK}"
+dlog "OMITCERTS = ${OMIT_CERTS}"
dlog "ALLHOSTS = ${ALLHOSTS}"
dlog "LISTING = ${LISTING}"
dlog "CLEAN = ${CLEAN}"
@@ -1813,7 +1821,7 @@ function collect_host_run()
spawn bash -i
set timeout ${TIMEOUT}
- send "sudo SKIP_MASK=${SKIP_MASK} ${collect_host} ${TARNAME} ${STARTDATE_OPTION} ${STARTDATE} ${STARTTIME} ${ENDDATE_OPTION} ${ENDDATE} ${ENDTIME} ${VERBOSE} ${INVENTORY}\n"
+ send "sudo OMIT_CERTS=${OMIT_CERTS} SKIP_MASK=${SKIP_MASK} ${collect_host} ${TARNAME} ${STARTDATE_OPTION} ${STARTDATE} ${STARTTIME} ${ENDDATE_OPTION} ${ENDDATE} ${ENDTIME} ${VERBOSE} ${INVENTORY}\n"
expect {
"assword:" {
send "${pw}\r"
@@ -1847,7 +1855,7 @@ EOF
expect {
"${host}:" {
set timeout ${COLLECT_HOST_TIMEOUT}
- send "sudo SKIP_MASK=${SKIP_MASK} ${collect_host} ${TARNAME} ${STARTDATE_OPTION} ${STARTDATE} ${STARTTIME} ${ENDDATE_OPTION} ${ENDDATE} ${ENDTIME} ${VERBOSE} ${INVENTORY}\n"
+ send "sudo OMIT_CERTS=${OMIT_CERTS} SKIP_MASK=${SKIP_MASK} ${collect_host} ${TARNAME} ${STARTDATE_OPTION} ${STARTDATE} ${STARTTIME} ${ENDDATE_OPTION} ${ENDDATE} ${ENDTIME} ${VERBOSE} ${INVENTORY}\n"
expect {
"assword:" {
send "${pw}\r"
diff --git a/tools/collector/debian-scripts/collect_certificates b/tools/collector/debian-scripts/collect_certificates
new file mode 100644
index 00000000..ee9d07f1
--- /dev/null
+++ b/tools/collector/debian-scripts/collect_certificates
@@ -0,0 +1,70 @@
+#! /bin/bash
+#
+# Copyright (c) 2023 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+EXTRA_DIR="$1"
+CERT_DIR="${EXTRA_DIR}/certs"
+CERTS_INCLUDE="/etc/collect/certs.include"
+
+# sw_version is exported by collect_host
+RELEASE="$sw_version"
+
+# Log file is exported from collect_host
+LOGF="$COLLECT_ERROR_LOG"
+
+# Read only lines beginning with slash,
+# and replace %%RELEASE%% with the sw_version
+# Include a hardcoded search for ssl_ca in /opt/platform/config
+INCLUDE_LIST="$(
+ grep "^/" "$CERTS_INCLUDE" \
+ | sed "s;%%RELEASE%%;${RELEASE};";
+ ls "/opt/platform/config/$RELEASE/ssl_ca/ssl_ca_"* 2>/dev/null \
+ | grep "/ssl_ca_[0-9]\{20\}$" )"
+
+function read_cert {
+ local certf="$1"
+ local outf
+ local based
+
+ # Put a copy of the cert file in EXTRA_DIR, using the file's full
+ # path within that space.
+ # All listed files start with slash, per global INCLUDE_LIST.
+ outf="${CERT_DIR}${certf}"
+ based="$( dirname "$outf" )"
+
+ if [ -f "$certf" ]; then
+ # Use openssl to retrieve only certificates from the file.
+ # The output includes some extra lines like this, but which
+ # doesn't affect inspection using openssl command
+ # 0: Certificate
+ # <snip>certificate data</snip>
+ # 1: Certificate
+ # <snip>certificate data</snip>
+ # Total found: 2
+ mkdir -p "$based"
+ openssl storeutl -certs "$certf" > "${outf}" 2>>$LOGF
+ fi
+}
+
+function read_certs_path {
+ local certd="$1"
+ local crtf
+
+ # copy certificates in certd, from files ending in .crt
+ while read crtf; do
+ read_cert "$crtf"
+ done <<<"$( ls -1 "${certd}"*.crt 2>/dev/null )"
+}
+
+while read fpath; do
+ if [[ "$fpath" =~ /$ ]]; then
+ # the path is a directory
+ read_certs_path "$fpath"
+ else
+ read_cert "$fpath"
+ fi
+done <<<"$INCLUDE_LIST"
+
diff --git a/tools/collector/debian-scripts/collect_host b/tools/collector/debian-scripts/collect_host
index 881f4588..97990c05 100755
--- a/tools/collector/debian-scripts/collect_host
+++ b/tools/collector/debian-scripts/collect_host
@@ -450,6 +450,14 @@ if [ "${SKIP_MASK}" != "true" ]; then
log_space "after passwd masking :"
fi
+if [ "${OMIT_CERTS}" != "true" ]; then
+ # Collect certificates from the host
+ dlog "running /usr/local/sbin/collect_certificates ${EXTRA_DIR}"
+ COLLECT_ERROR_LOG="$COLLECT_ERROR_LOG" \
+ /usr/local/sbin/collect_certificates ${EXTRA_DIR}
+ log_space "after collecting certificates :"
+fi
+
(cd ${COLLECT_BASE_DIR} ; ${IONICE_CMD} ${NICE_CMD} ${TAR_ZIP_CMD} ${COLLECT_NAME_DIR}.tgz ${COLLECT_NAME} 2>/dev/null 1>/dev/null )
log_space "after first tarball .:"
diff --git a/tools/collector/debian-scripts/collect_mask_passwords b/tools/collector/debian-scripts/collect_mask_passwords
index 357a4fd4..3f1233c8 100644
--- a/tools/collector/debian-scripts/collect_mask_passwords
+++ b/tools/collector/debian-scripts/collect_mask_passwords
@@ -60,18 +60,15 @@ do
s/\{default_pass, <<\".*\">>\}/\{default_pass, <<\"xxxxxx\">>\}/' $conffile
done
-find ${COLLECT_NAME_DIR} -name server-cert.pem | xargs --no-run-if-empty rm -f
+# Remove all certificate and key files. Certificates without secrets
+# will be added to collect by collect_certificates.
+find ${COLLECT_NAME_DIR} -name "*.pem" -o -name "*.crt" -o -name "*.key" \
+ | xargs --no-run-if-empty rm -f
+# Remove ssh config and platform secrets files
rm -rf ${COLLECT_NAME_DIR}/var/extra/platform/config/*/ssh_config
rm -f ${COLLECT_NAME_DIR}/var/extra/platform/puppet/*/hieradata/secure*.yaml
rm -f ${COLLECT_NAME_DIR}/etc/puppet/cache/hieradata/secure*.yaml
-# dir /etc/kubernetes/pki was etc.excluded
-if [ -d "/etc/kubernetes/pki" ] ; then
- # grab the public certificates if /etc/kubernetes/pki exists
- mkdir -p ${COLLECT_NAME_DIR}/etc/kubernetes/pki
- cp -a /etc/kubernetes/pki/*.crt ${COLLECT_NAME_DIR}/etc/kubernetes/pki 2>/dev/null 1>/dev/null
-fi
-
# Mask user passwords in sysinv db dump
if [ -f ${COLLECT_NAME_DIR}/var/extra/database/sysinv.db.sql.txt ]; then
sed -i -r '/COPY i_user/, /^--/ s/^(([^\t]*\t){10})[^\t]*(\t.*)/\1xxxxxx\3/;
diff --git a/tools/collector/debian/deb_folder/rules b/tools/collector/debian/deb_folder/rules
index c760c2b9..7a31076f 100755
--- a/tools/collector/debian/deb_folder/rules
+++ b/tools/collector/debian/deb_folder/rules
@@ -28,6 +28,7 @@ override_dh_auto_install:
install -m 755 -p collect_parms $(ROOT)/usr/local/sbin/collect_parms
install -m 755 -p collect_timeouts $(SYSCONFDIR)/collect/collect_timeouts
install -m 755 -p collect_mask_passwords $(ROOT)/usr/local/sbin/collect_mask_passwords
+ install -m 755 -p collect_certificates $(ROOT)/usr/local/sbin/collect_certificates
install -m 755 -p expect_done $(ROOT)/usr/local/sbin/expect_done
install -m 755 -p mariadb-cli.sh $(ROOT)/usr/local/sbin/mariadb-cli
@@ -93,6 +94,7 @@ override_dh_auto_install:
install -m 755 -p etc.exclude $(SYSCONFDIR)/collect/etc.exclude
install -m 755 -p run.exclude $(SYSCONFDIR)/collect/run.exclude
install -m 755 -p varlog.exclude $(SYSCONFDIR)/collect/varlog.exclude
+ install -m 755 -p certs.include $(SYSCONFDIR)/collect/certs.include
ln -sf /usr/local/sbin/collect $(SBINDIR)/collect
ln -sf /usr/local/sbin/collect $(ROOT)/usr/local/bin/collect