Configure SQL as helm storage backend
Configmap is the default helmv2 storage backend to store release information but its 1MB resource limit prevents scaling up stx openstack worker nodes, so we want to use SQL as helm storage backend. To configure SQL backend, generate helm database hieradata that will be used in puppet to create helm database. The helm database password is stored in keyring which can be retrieved in ansible playbook to configure database connection address. System upgrade support: The helm DB is new in the release stx5.0, so a password is generated for helm user. Helm user and password are written into the hieradata of release stx5.0. For AIO-SX upgrade, helm DB is created when applying bootstrap puppet manifest during ansible upgrade playbook. For two controllers upgrade, helm DB is created when applying upgrade puppet manifest during controller-1 upgrade. A migration script is created to migrate helm releases from configmap to postgresql. Partial-Bug: 1887677 Depends-On: https://review.opendev.org/#/c/761642/ Change-Id: I2f4f414068af297b5f4a3792c061443b7d3bdb32 Signed-off-by: Angie Wang <angie.wang@windriver.com>
This commit is contained in:
Angie Wang
parent
c3d82de2d3
commit
efa5f521c3
|
|
@ -778,6 +778,30 @@ def migrate_hiera_data(from_release, to_release, role=None):
|
|||
if (from_release == SW_VERSION_20_06 and etcd_security_config):
|
||||
static_config.update(etcd_security_config)
|
||||
|
||||
if from_release == SW_VERSION_20_06:
|
||||
# The helm db is new in the release stx5.0 and requires
|
||||
# a password to be generated and a new user to access the DB.
|
||||
# This is required for all types of system upgrade. Should
|
||||
# removed in the release that follows stx5.0
|
||||
static_config.update({
|
||||
'platform::helm::v2::db::postgresql::user': 'admin-helmv2'
|
||||
})
|
||||
|
||||
helmv2_db_pw = utils.get_password_from_keyring('helmv2', 'database')
|
||||
if not helmv2_db_pw:
|
||||
helmv2_db_pw = utils.set_password_in_keyring('helmv2', 'database')
|
||||
|
||||
secure_static_file = os.path.join(
|
||||
constants.HIERADATA_PERMDIR, "secure_static.yaml")
|
||||
with open(secure_static_file, 'r') as yaml_file:
|
||||
secure_static_config = yaml.load(yaml_file)
|
||||
secure_static_config.update({
|
||||
'platform::helm::v2::db::postgresql::password': helmv2_db_pw
|
||||
})
|
||||
with open(secure_static_file, 'w') as yaml_file:
|
||||
yaml.dump(secure_static_config, yaml_file,
|
||||
default_flow_style=False)
|
||||
|
||||
with open(static_file, 'w') as yaml_file:
|
||||
yaml.dump(static_config, yaml_file, default_flow_style=False)
|
||||
|
||||
|
|
|
|||
|
|
@ -27,12 +27,13 @@ LOG = log.getLogger(__name__)
|
|||
def get_upgrade_databases(system_role, shared_services):
|
||||
|
||||
UPGRADE_DATABASES = ('postgres', 'template1', 'sysinv',
|
||||
'barbican', 'fm')
|
||||
'barbican', 'fm', 'helmv2')
|
||||
|
||||
UPGRADE_DATABASE_SKIP_TABLES = {'postgres': (), 'template1': (),
|
||||
'sysinv': (),
|
||||
'barbican': (),
|
||||
'fm': ('alarm',)}
|
||||
'fm': ('alarm',),
|
||||
'helmv2': ()}
|
||||
|
||||
if system_role == sysinv_constants.DISTRIBUTED_CLOUD_ROLE_SYSTEMCONTROLLER:
|
||||
UPGRADE_DATABASES += ('dcmanager', 'dcorch',)
|
||||
|
|
|
|||
|
|
@ -27,9 +27,7 @@ from tsconfig.tsconfig import PLATFORM_PATH
|
|||
from controllerconfig import utils as cutils
|
||||
from controllerconfig.common import constants
|
||||
from sysinv.common import constants as sysinv_constants
|
||||
# sysinv common utils is needed for adding new service account and endpoints
|
||||
# during upgrade.
|
||||
# from sysinv.common import utils as sysinv_utils
|
||||
from sysinv.common import utils as sysinv_utils
|
||||
|
||||
from oslo_log import log
|
||||
|
||||
|
|
@ -114,6 +112,22 @@ def get_password_from_keyring(service, username):
|
|||
return password
|
||||
|
||||
|
||||
def set_password_in_keyring(service, username):
|
||||
"""Generate random password and store in keyring"""
|
||||
os.environ["XDG_DATA_HOME"] = constants.KEYRING_PERMDIR
|
||||
try:
|
||||
password = sysinv_utils.generate_random_password(length=16)
|
||||
keyring.set_password(service, username, password)
|
||||
except Exception as e:
|
||||
LOG.exception("Received exception when attempting to generate "
|
||||
"password for service %s, username %s: %s" %
|
||||
(service, username, e))
|
||||
raise
|
||||
finally:
|
||||
del os.environ["XDG_DATA_HOME"]
|
||||
return password
|
||||
|
||||
|
||||
def get_upgrade_token(from_release,
|
||||
config,
|
||||
secure_config):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,170 @@
|
|||
#!/usr/bin/python
|
||||
#
|
||||
# Copyright (c) 2020 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# This migration script is used for migrating all helm releases
|
||||
# from configmaps to postgresql during the activate stage of
|
||||
# a platform upgrade.
|
||||
#
|
||||
# This script can be removed in the release that follows stx5.0
|
||||
#
|
||||
|
||||
import collections
|
||||
from datetime import datetime
|
||||
import psycopg2
|
||||
import subprocess
|
||||
import sys
|
||||
import json
|
||||
import keyring
|
||||
|
||||
from controllerconfig.common import log
|
||||
|
||||
LOG = log.get_logger(__name__)
|
||||
|
||||
Release = collections.namedtuple(
|
||||
'release', 'key body name version status owner created_at modified_at')
|
||||
|
||||
|
||||
def main():
|
||||
action = None
|
||||
from_release = None
|
||||
to_release = None
|
||||
arg = 1
|
||||
while arg < len(sys.argv):
|
||||
if arg == 1:
|
||||
from_release = sys.argv[arg]
|
||||
elif arg == 2:
|
||||
to_release = sys.argv[arg]
|
||||
elif arg == 3:
|
||||
action = sys.argv[arg]
|
||||
else:
|
||||
print ("Invalid option %s." % sys.argv[arg])
|
||||
return 1
|
||||
arg += 1
|
||||
|
||||
log.configure()
|
||||
|
||||
if from_release == '20.06' and action == 'activate':
|
||||
LOG.info("%s invoked with from_release = %s to_release = %s "
|
||||
"action = %s"
|
||||
% (sys.argv[0], from_release, to_release, action))
|
||||
migrate_helm_releases()
|
||||
LOG.info("Complete helm releases migration for release %s "
|
||||
"to %s with action %s."
|
||||
% (from_release, to_release, action))
|
||||
|
||||
|
||||
def execute_command(cmd):
|
||||
sub = subprocess.Popen(cmd, shell=True,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
|
||||
stdout, stderr = sub.communicate()
|
||||
if sub.returncode != 0:
|
||||
LOG.error("Command failed:\n %s\n%s\n%s" % (cmd, stdout, stderr))
|
||||
raise Exception("Failed to execute command: %s" % cmd)
|
||||
return stdout
|
||||
|
||||
|
||||
def get_helm_releases():
|
||||
# Get all configmaps that store helm releases
|
||||
cmd = "kubectl --kubeconfig=/etc/kubernetes/admin.conf get configmaps " \
|
||||
"-n kube-system -l OWNER=TILLER --sort-by '{.metadata.name}' " \
|
||||
"--template '{{range .items}}{{.metadata.name}}{{\"\\n\"}}{{end}}'"
|
||||
|
||||
releases = execute_command(cmd)
|
||||
releases_list = [r for r in releases.split('\n') if r]
|
||||
return releases_list
|
||||
|
||||
|
||||
def delete_helm_releases():
|
||||
# Delete all configmaps that store helm releases
|
||||
cmd = "kubectl --kubeconfig=/etc/kubernetes/admin.conf delete configmaps " \
|
||||
"-n kube-system -l OWNER=TILLER"
|
||||
execute_command(cmd)
|
||||
|
||||
|
||||
def get_helm_release_from_configmap(release_name):
|
||||
# Get the content of a specific helm release from configmap
|
||||
cmd = "kubectl --kubeconfig=/etc/kubernetes/admin.conf get configmaps " \
|
||||
"-n kube-system {} -o json".format(release_name)
|
||||
release_data = execute_command(cmd)
|
||||
|
||||
return json.loads(release_data)
|
||||
|
||||
|
||||
def map_helm_release(release):
|
||||
# Map the format of a helm release from configmap to postgresql
|
||||
try:
|
||||
key = str(release['metadata']['name'])
|
||||
body = str(release['data']['release'])
|
||||
name = str(release['metadata']['labels']['NAME'])
|
||||
version = int(release['metadata']['labels']['VERSION'])
|
||||
status = str(release['metadata']['labels']['STATUS'])
|
||||
owner = str(release['metadata']['labels']['OWNER'])
|
||||
created_at = int(datetime.strftime(datetime.strptime(
|
||||
release['metadata']['creationTimestamp'],
|
||||
"%Y-%m-%dT%H:%M:%SZ"), "%s"))
|
||||
modified_at = int(release['metadata']['labels']['MODIFIED_AT'])
|
||||
|
||||
mapped_release = Release(
|
||||
key=key, body=body, name=name, version=version, status=status,
|
||||
owner=owner, created_at=created_at, modified_at=modified_at)
|
||||
except Exception as e:
|
||||
LOG.exception("Failed to convert helm release: %s" % e)
|
||||
raise
|
||||
|
||||
return mapped_release
|
||||
|
||||
|
||||
def create_helm_release_in_db(conn, release):
|
||||
with conn:
|
||||
with conn.cursor() as cur:
|
||||
try:
|
||||
cur.execute(
|
||||
"insert into releases(key, body, name, version,"
|
||||
"status, owner, created_at, modified_at) "
|
||||
"values(%s, %s, %s, %s, %s, %s, %s, %s)",
|
||||
release)
|
||||
except psycopg2.IntegrityError:
|
||||
# release already exists
|
||||
pass
|
||||
except Exception as e:
|
||||
LOG.exception("Failed to create release in db:\n%s" % e)
|
||||
raise
|
||||
|
||||
|
||||
def migrate_helm_releases():
|
||||
releases = get_helm_releases()
|
||||
|
||||
if not releases:
|
||||
LOG.info("No helm releases need to be migrated.")
|
||||
return
|
||||
|
||||
LOG.info("Start migrating helm releases:\n%s" % releases)
|
||||
|
||||
helmv2_db_pw = keyring.get_password("helmv2", "database")
|
||||
if not helmv2_db_pw:
|
||||
raise Exception("Unable to get password to access helmv2 database.")
|
||||
|
||||
try:
|
||||
conn = psycopg2.connect(user="admin-helmv2",
|
||||
password=helmv2_db_pw,
|
||||
host="localhost",
|
||||
database="helmv2")
|
||||
except Exception as e:
|
||||
LOG.exception("Failed to connect helmv2 database: %s" % e)
|
||||
raise
|
||||
|
||||
for release in releases:
|
||||
release_data = get_helm_release_from_configmap(release)
|
||||
mapped_release = map_helm_release(release_data)
|
||||
create_helm_release_in_db(conn, mapped_release)
|
||||
LOG.info("Migrated release: %s" % release)
|
||||
|
||||
delete_helm_releases()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
|
@ -65,6 +65,7 @@ systemconfig.puppet_plugins =
|
|||
035_dockerdistribution = sysinv.puppet.dockerdistribution:DockerDistributionPuppet
|
||||
036_pciirqaffinity = sysinv.puppet.pci_irq_affinity:PciIrqAffinityPuppet
|
||||
038_certmon = sysinv.puppet.certmon:CertMonPuppet
|
||||
039_helm = sysinv.puppet.helm:HelmPuppet
|
||||
099_service_parameter = sysinv.puppet.service_parameter:ServiceParamPuppet
|
||||
|
||||
systemconfig.armada.manifest_ops =
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
#
|
||||
|
||||
import abc
|
||||
import keyring
|
||||
import six
|
||||
|
||||
from sqlalchemy.orm.exc import NoResultFound
|
||||
|
|
@ -75,6 +76,23 @@ class BasePuppet(object):
|
|||
def _generate_random_password(length=16):
|
||||
return utils.generate_random_password(length=length)
|
||||
|
||||
def _get_database_password(self, service):
|
||||
passwords = self.context.setdefault('_database_passwords', {})
|
||||
if service not in passwords:
|
||||
passwords[service] = self._get_keyring_password(service,
|
||||
'database')
|
||||
return passwords[service]
|
||||
|
||||
def _get_database_username(self, service):
|
||||
return 'admin-%s' % service
|
||||
|
||||
def _get_keyring_password(self, service, user):
|
||||
password = keyring.get_password(service, user)
|
||||
if not password:
|
||||
password = self._generate_random_password()
|
||||
keyring.set_password(service, user, password)
|
||||
return password
|
||||
|
||||
def _get_system(self):
|
||||
system = self.context.get('_system', None)
|
||||
if system is None:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,25 @@
|
|||
#
|
||||
# Copyright (c) 2020 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
from sysinv.puppet import base
|
||||
|
||||
|
||||
class HelmPuppet(base.BasePuppet):
|
||||
"""Class to encapsulate puppet operations for helm configuration"""
|
||||
|
||||
SERVICE_NAME = 'helmv2'
|
||||
|
||||
def get_static_config(self):
|
||||
dbuser = self._get_database_username(self.SERVICE_NAME)
|
||||
return {
|
||||
'platform::helm::v2::db::postgresql::user': dbuser,
|
||||
}
|
||||
|
||||
def get_secure_static_config(self):
|
||||
dbpass = self._get_database_password(self.SERVICE_NAME)
|
||||
|
||||
return {
|
||||
'platform::helm::v2::db::postgresql::password': dbpass,
|
||||
}
|
||||
|
|
@ -5,7 +5,6 @@
|
|||
#
|
||||
|
||||
import abc
|
||||
import keyring
|
||||
import os
|
||||
|
||||
from sysinv.common import constants
|
||||
|
|
@ -90,23 +89,6 @@ class OpenstackBasePuppet(base.BasePuppet):
|
|||
url = service_config.capabilities.get('internal_uri')
|
||||
return url
|
||||
|
||||
def _get_database_password(self, service):
|
||||
passwords = self.context.setdefault('_database_passwords', {})
|
||||
if service not in passwords:
|
||||
passwords[service] = self._get_keyring_password(service,
|
||||
'database')
|
||||
return passwords[service]
|
||||
|
||||
def _get_database_username(self, service):
|
||||
return 'admin-%s' % service
|
||||
|
||||
def _get_keyring_password(self, service, user):
|
||||
password = keyring.get_password(service, user)
|
||||
if not password:
|
||||
password = self._generate_random_password()
|
||||
keyring.set_password(service, user, password)
|
||||
return password
|
||||
|
||||
def _get_public_protocol(self):
|
||||
return 'https' if self._https_enabled() else 'http'
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue