starlingx/config
Code Issues Proposed changes

Add upgrade script to enable IPsec for multi-node upgrades

Browse Source 
This commit adds an upgrade-script to enable and configure IPsec on
multi-node systems. It is required that IPsec is enabled on systems
after all upgrade-scripts are executed to prevent any occurrence of
network instability.

This script should prepare active controller environment and execute
initial-auth operation on each node pending to be IPsec configured.
An ansible-playbook is executed to contact and trigger initial-auth
operation request from other nodes to IPsec server. As a result of
the execution of the playbook, IPsec is configured on nodes. If any
node is missing to be configured, the script exits w/ an exception.
Notice that mtce_heartbeat_failure is updated to its default value
only after IPsec is successfully enabled per the execution of this
ansible-playbook.

The IPsec server port is set to 64764 as 54724 may be used for k8s
services.

Test Plan:
PASS: Deploy AIO-DX system and upgrade software version from stx 8 to
      stx 9. Observe that 100-enable-ipsec-on-hosts.py script is
      executed successfully and IPsec is enabled/configured on all
      nodes. The nodes remain online on unlocked enabled available
      state.
PASS: Deploy AIO-DX system on stx 9 version and manually execute
      100-enable-ipsec-on-hosts.py script. Observe that IPsec is
      already enabled/configured on all nodes, script is successfully
      executed with no additional changes applied on system and nodes
      remain online on unlocked enabled available state.

Depends-on: https://review.opendev.org/c/starlingx/ansible-playbooks/+/923294

Story: 2010940
Task: 50720

Change-Id: I3b3fde8f18d6c3f6d9f3ad548ff633aaabf40362
Signed-off-by: Manoel Benedito Neto <Manoel.BeneditoNeto@windriver.com>
This commit is contained in:
Manoel Benedito Neto 2024-06-11 08:50:59 -03:00
parent b69a4b432a
commit 8dfed5b6bb
6 changed files with 208 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
controllerconfig/controllerconfig/scripts/controller_config
View File

@ -43,7 +43,7 @@ IPSEC_DELAY=5
SWANCTL_CONF_FILE=/etc/swanctl/swanctl.conf
SWANCTL_ACTIVE_CONF_FILE=/etc/swanctl/swanctl_active.conf
SWANCTL_STANDBY_CONF_FILE=/etc/swanctl/swanctl_standby.conf
IPSEC_SERVER_PORT=54724
IPSEC_SERVER_PORT=64764
OS_ID=$(grep '^ID=' /etc/os-release | cut -f2- -d= | sed -e 's/\"//g')
if [ "$OS_ID" == "debian" ]

202
controllerconfig/controllerconfig/upgrade-scripts/100-enable-ipsec-on-hosts.py Normal file
View File

@ -0,0 +1,202 @@
#!/usr/bin/python
# Copyright (c) 2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
# This script enables IPsec on all hosts and should be executed
# at the end of upgrade-activate stage.
import json
import os
import subprocess
import sys
from controllerconfig.common import log
from sysinv.common import constants
from sysinv.common import service_parameter as sp_constants
from sysinv.ipsec_auth.common import constants as ips_constants
LOG = log.get_logger(__name__)
DEFAULT_POSTGRES_PORT = 5432
def main():
action = None
from_release = None
to_release = None
port = DEFAULT_POSTGRES_PORT
arg = 1
while arg < len(sys.argv):
if arg == 1:
from_release = sys.argv[arg]
elif arg == 2:
to_release = sys.argv[arg]
elif arg == 3:
action = sys.argv[arg]
elif arg == 4:
# optional port parameter for USM upgrade
port = sys.argv[arg]
pass
else:
print(f"Invalid option {sys.argv[arg]}.")
return 1
arg += 1
log.configure()
if from_release == "22.12" and action == "activate":
try:
LOG.info(f"Enable IPsec on system from the release "
f"{from_release} to {to_release}")
LOG.info("Update mtce_heartbeat_failure_action to alarm, "
"before IPsec is enabled.")
update_heartbeat_failure(
sp_constants.SERVICE_PARAM_PLAT_MTCE_HBS_FAILURE_ACTION_ALARM)
LOG.info("Remove mgmt_ipsec in capabilities of "
"sysinv i_host table")
remove_mgmt_ipsec(port)
LOG.info("Start ipsec-server service")
start_ipsec_server()
LOG.info("Configure IPsec on each node of the environment")
configure_ipsec_on_nodes(to_release)
LOG.info("Update heartbeat_failure_action to default value "
"(fail). IPsec is enabled.")
update_heartbeat_failure(
constants.SERVICE_PARAM_PLAT_MTCE_HBS_FAILURE_ACTION_DEFAULT)
except Exception as ex:
LOG.exception(ex)
print(ex)
return 1
else:
LOG.info(f"Nothing to do for action {action}.")
def start_ipsec_server():
cmd = "systemctl enable ipsec-server.service --now"
sub = subprocess.Popen(
cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = sub.communicate()
if sub.returncode == 0:
return stdout.decode('utf-8').rstrip('\n')
else:
LOG.error('Command failed:\n %s\n. %s\n%s\n'
% (cmd, stdout.decode('utf-8'), stderr.decode('utf-8')))
raise Exception("Failed to start ipsec-server.")
def remove_mgmt_ipsec(postgres_port):
"""This function removes mgmt_ipsec in capabilities of sysinv
i_host table.
"""
env = os.environ.copy()
sub_sel = subprocess.Popen(
['sudo', '-u', 'postgres',
'psql', '-p', f'{postgres_port}',
'-d', 'sysinv', '-c',
'select uuid, capabilities from i_host'],
env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
universal_newlines=True)
stdout, stderr = sub_sel.communicate()
if sub_sel.returncode == 0 and stdout:
rows = [item for item in stdout.split('\n') if '|' in item]
# Remove header from sql stdout
rows.pop(0)
for records in rows:
record = records.split('|')
host_uuid = record[0].strip()
capabilities = json.loads(record[1].strip())
if 'mgmt_ipsec' in capabilities and \
capabilities['mgmt_ipsec'] != ips_constants.MGMT_IPSEC_ENABLED:
del capabilities['mgmt_ipsec']
capabilities = json.dumps(capabilities)
sqlcom = (f"update i_host set capabilities='{capabilities}'"
f"where uuid='{host_uuid}'")
sub_update = subprocess.Popen(
['sudo', '-u', 'postgres', 'psql',
'-p', f'{postgres_port}',
'-d', 'sysinv', '-c', sqlcom],
env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
universal_newlines=True)
stdout, stderr = sub_update.communicate()
if sub_update.returncode != 0:
LOG.error('Failed to remove mgmt_ipsec flag:'
'\n%s. \n%s' % (stdout, stderr))
raise Exception(stderr)
else:
LOG.error('Failed to connect to sysinv database:'
'\n%s. \n%s.' % (stdout, stderr))
raise Exception(stderr)
def execute_system_cmd(api_cmd, exc_msg):
cmd = f'source /etc/platform/openrc && {api_cmd}'
sub = subprocess.Popen(
cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = sub.communicate()
if sub.returncode == 0:
return stdout.decode('utf-8').rstrip('\n')
else:
LOG.error('Command failed:\n %s\n. %s\n%s\n'
% (cmd, stdout.decode('utf-8'), stderr.decode('utf-8')))
raise Exception(exc_msg)
def update_heartbeat_failure(action):
cmd = f'system service-parameter-modify platform ' \
f'maintenance heartbeat_failure_action={action} && ' \
'system service-parameter-apply platform'
exc_msg = f'Cannot modify heartbeat_failure_action to {action}.'
return execute_system_cmd(cmd, exc_msg)
def get_admin_credentials():
cmd = 'echo $OS_PASSWORD'
exc_msg = 'Cannot retrieve user credential.'
passwd = execute_system_cmd(cmd, exc_msg)
if passwd == '':
raise Exception('Failed to retrieve sysadmin credentials.')
credentials = []
credentials.append('sysadmin')
credentials.append(passwd)
return credentials
def configure_ipsec_on_nodes(to_release):
"""Run ansible playbook to enable and configure IPsec on nodes
"""
playbooks_root = '/usr/share/ansible/stx-ansible/playbooks'
upgrade_script = 'enable-ipsec-on-nodes-in-upgrade.yml'
ssh_credentials = get_admin_credentials()
cmd = 'ansible-playbook {}/{} -e "software_version={} \
ansible_ssh_user={} ansible_ssh_pass={} \
ansible_become_pass={}"'.format(
playbooks_root, upgrade_script, to_release, ssh_credentials[0],
ssh_credentials[1], ssh_credentials[1])
sub = subprocess.Popen(
cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = sub.communicate()
if sub.returncode != 0:
LOG.error('Command failed:\n %s\n. %s\n%s\n'
% (cmd, stdout.decode('utf-8'), stderr.decode('utf-8')))
raise Exception('Failed to enable IPsec on all hosts.')
LOG.info('Successfully enabled IPsec on all hosts. Output:\n%s\n'
% stdout.decode('utf-8'))
if __name__ == "__main__":
sys.exit(main())

2
storageconfig/storageconfig/storage_config
View File

@ -30,7 +30,7 @@ IMA_POLICY=/etc/ima.policy
FIRST_BOOT="/etc/platform/.first_boot"
IPSEC_RETRIES=3
IPSEC_DELAY=5
IPSEC_SERVER_PORT=54724
IPSEC_SERVER_PORT=64764
fatal_error()
{

2
sysinv/sysinv/sysinv/sysinv/ipsec_auth/common/constants.py
View File

@ -6,7 +6,7 @@
PROCESS_ID = '/var/run/ipsec-server.pid'
DEFAULT_BIND_ADDR = "0.0.0.0"
DEFAULT_LISTEN_PORT = 54724
DEFAULT_LISTEN_PORT = 64764
TCP_SERVER = (DEFAULT_BIND_ADDR, DEFAULT_LISTEN_PORT)
PLATAFORM_CONF_FILE = '/etc/platform/platform.conf'

3
sysinv/sysinv/sysinv/sysinv/ipsec_auth/server/server.py
View File

@ -30,6 +30,7 @@ LOG = logging.getLogger(__name__)
class IPsecServer(object):
sel = selectors.DefaultSelector()
host = constants.DEFAULT_BIND_ADDR
def __init__(self, port=constants.DEFAULT_LISTEN_PORT):
self.port = port
@ -40,7 +41,7 @@ class IPsecServer(object):
ssocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
ssocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
ssocket.setblocking(False)
ssocket.bind(constants.TCP_SERVER)
ssocket.bind((self.host, self.port))
ssocket.listen()
self._create_pid_file()

2
workerconfig/workerconfig/worker_config
View File

@ -30,7 +30,7 @@ IMA_POLICY=/etc/ima.policy
FIRST_BOOT="/etc/platform/.first_boot"
IPSEC_RETRIES=3
IPSEC_DELAY=5
IPSEC_SERVER_PORT=54724
IPSEC_SERVER_PORT=64764
# Copy of /opt/platform required for worker_services
VOLATILE_PLATFORM_PATH=$VOLATILE_PATH/cpe_upgrade_opt_platform