Get swift working on containerized openstack

In order to get swift working on containerized openstack,
changes were needed both on platform and application side.

From platform side, settings from ceph.conf file were replaced.
A runtime manifest was added to update ceph.conf after a successful
application apply:
    1. Keystone auth url was updated with keystone openstack url
    2. 'rgw_keystone_admin_domain' and 'rgw_keystone_project' settings
    were updated with 'service'.

From application side the following changes have been implemented:
    1. Ceph-rgw chart from openstack-helm-infra repo was included
    in stx-openstack
    2. A chart schema for ceph-rgw was added
    3. An override file was generated

Signed-off-by: Elena Taivan <elena.taivan@windriver.com>
Story: 2003909
Task: 30606
Change-Id: I01f7cf412264394f4f9bfb31f3c5a5ebd73f49dc

kubernetes/applications/stx-openstack/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml

@@ -470,6 +470,81 @@ data:
     - helm-toolkit
 ---
 schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: openstack-ceph-rgw
+data:
+  chart_name: ceph-rgw
+  release: openstack-ceph-rgw
+  namespace: openstack
+  wait:
+    timeout: 1800
+    labels:
+      release_group: osh-openstack-ceph-rgw
+  test:
+    enabled: false
+  install:
+    no_hooks: false
+  upgrade:
+    no_hooks: false
+    pre:
+      delete:
+        - type: job
+          labels:
+            release_group: osh-openstack-ceph-rgw
+        - type: pod
+          labels:
+            release_group: osh-openstack-ceph-rgw
+            component: test
+  values:
+    conf:
+      ceph:
+        global:
+          cephx: false
+      rgw_ks:
+        enabled: true
+    endpoints:
+      object_store:
+        path:
+          default: '/swift/v1'
+        port:
+          api:
+            default: null
+            admin: 7480
+            internal: 7480
+            public: 7480
+    images:
+      tags:
+        ks_endpoints: docker.io/starlingx/stx-heat:master-centos-stable-latest
+        ks_service: docker.io/starlingx/stx-heat:master-centos-stable-latest
+        ks_user: docker.io/starlingx/stx-heat:master-centos-stable-latest
+    labels:
+      api:
+        node_selector_key: openstack-control-plane
+        node_selector_value: enabled
+      job:
+        node_selector_key: openstack-control-plane
+        node_selector_value: enabled
+      registry:
+        node_selector_key: openstack-control-plane
+        node_selector_value: enabled
+    pod:
+      replicas:
+        api: 2
+        registry: 2
+      affinity:
+        anti:
+          type:
+            default: requiredDuringSchedulingIgnoredDuringExecution
+  source:
+    type: tar
+    location: http://172.17.0.1/helm_charts/starlingx/ceph-rgw-0.1.0.tgz
+    subpath: ceph-rgw
+    reference: master
+  dependencies:
+    - helm-toolkit
+---
+schema: armada/Chart/v1
 metadata:
   schema: metadata/Document/v1
   name: openstack-glance
@@ -2968,6 +3043,16 @@ data:
     - openstack-glance
 ---
 schema: armada/ChartGroup/v1
+metadata:
+  schema: metadata/Document/v1
+  name: openstack-ceph-rgw
+data:
+  description: "Deploy swift"
+  sequenced: true
+  chart_group:
+    - openstack-ceph-rgw
+---
+schema: armada/ChartGroup/v1
 metadata:
   schema: metadata/Document/v1
   name: openstack-cinder
@@ -3039,6 +3124,7 @@ data:
     - openstack-keystone
     - openstack-barbican
     - openstack-glance
+    - openstack-ceph-rgw
     - openstack-compute-kit
     - openstack-heat
     - openstack-horizon

puppet-manifests/src/modules/openstack/manifests/keystone.pp

@@ -3,6 +3,7 @@ class openstack::keystone::params(
   $identity_uri,
   $auth_uri,
   $host_url,
+  $openstack_auth_uri = undef,
   $api_port = 5000,
   $admin_port = 5000,
   $region_name = undef,

puppet-manifests/src/modules/platform/manifests/ceph.pp

@@ -26,6 +26,9 @@ class platform::ceph::params(
   $rgw_frontend_type = 'civetweb',
   $rgw_port = 7480,
   $rgw_log_file = '/var/log/radosgw/radosgw.log',
+  $rgw_service_domain = undef,
+  $rgw_service_project = undef,
+  $rgw_service_password = undef,
   $rgw_admin_domain = undef,
   $rgw_admin_project = undef,
   $rgw_admin_user = 'swift',
@@ -406,6 +409,56 @@ class platform::ceph::haproxy
   }
 }
 
+class platform::ceph::rgw::runtime
+  inherits ::platform::ceph::params {
+    if $service_enabled {
+      include ::platform::params
+
+      include ::openstack::keystone::params
+
+      ceph::rgw::keystone { $rgw_client_name:
+        rgw_keystone_admin_token    => '',
+        rgw_keystone_url            => $::openstack::keystone::params::openstack_auth_uri,
+        rgw_keystone_version        => $::openstack::keystone::params::api_version,
+        rgw_keystone_accepted_roles => 'admin,_member_',
+        user                        => $rgw_user_name,
+        use_pki                     => false,
+        rgw_keystone_admin_domain   => $rgw_service_domain,
+        rgw_keystone_admin_project  => $rgw_service_project,
+        rgw_keystone_admin_user     => $rgw_admin_user,
+        rgw_keystone_admin_password => $rgw_service_password,
+      }
+      exec { 'sm-restart-safe service ceph-radosgw':
+        command => 'sm-restart-safe service ceph-radosgw'
+      }
+    }
+}
+
+class platform::ceph::rgw::runtime_revert
+  inherits ::platform::ceph::params {
+    if $service_enabled {
+      include ::platform::params
+
+      include ::openstack::keystone::params
+
+      ceph::rgw::keystone { $rgw_client_name:
+        rgw_keystone_admin_token    => '',
+        rgw_keystone_url            => $::openstack::keystone::params::auth_uri,
+        rgw_keystone_version        => $::openstack::keystone::params::api_version,
+        rgw_keystone_accepted_roles => 'admin,_member_',
+        user                        => $rgw_user_name,
+        use_pki                     => false,
+        rgw_keystone_admin_domain   => $rgw_admin_domain,
+        rgw_keystone_admin_project  => $rgw_admin_project,
+        rgw_keystone_admin_user     => $rgw_admin_user,
+        rgw_keystone_admin_password => $rgw_admin_password,
+      }
+      exec { 'sm-restart-safe service ceph-radosgw':
+        command => 'sm-restart-safe service ceph-radosgw'
+      }
+    }
+}
+
 class platform::ceph::rgw
   inherits ::platform::ceph::params {
 

sysinv/sysinv/sysinv/setup.cfg

@@ -104,6 +104,7 @@ systemconfig.helm_plugins.stx_openstack =
     020_helm-toolkit = sysinv.helm.helm_toolkit:HelmToolkitHelm
     021_barbican = sysinv.helm.barbican:BarbicanHelm
     022_keystone-api-proxy = sysinv.helm.keystone_api_proxy:KeystoneApiProxyHelm
+    023_ceph-rgw = sysinv.helm.swift:SwiftHelm
 
 sysinv.agent.lldp.drivers =
     lldpd = sysinv.agent.lldp.drivers.lldpd.driver:SysinvLldpdAgentDriver

sysinv/sysinv/sysinv/sysinv/common/constants.py

@@ -1450,6 +1450,7 @@ HELM_CHART_RBD_PROVISIONER = 'rbd-provisioner'
 HELM_CHART_CEPH_POOLS_AUDIT = 'ceph-pools-audit'
 HELM_CHART_HELM_TOOLKIT = 'helm-toolkit'
 HELM_CHART_KEYSTONE_API_PROXY = 'keystone-api-proxy'
+HELM_CHART_SWIFT = 'ceph-rgw'
 
 # Helm: Supported application (aka chart bundles)
 HELM_APP_OPENSTACK = 'stx-openstack'

sysinv/sysinv/sysinv/sysinv/conductor/manager.py

@@ -6483,6 +6483,21 @@ class ConductorManager(service.PeriodicService):
                                             config_uuid,
                                             config_dict)
 
+    def _revert_cephrgw_config(self, context):
+        """ Revert ceph rgw configuration. """
+        personalities = [constants.CONTROLLER]
+
+        config_uuid = self._config_update_hosts(context, personalities)
+
+        config_dict = {
+            "personalities": personalities,
+            "classes": ['platform::ceph::rgw::runtime_revert']
+        }
+
+        self._config_apply_runtime_manifest(context,
+                                            config_uuid,
+                                            config_dict)
+
     def _update_config_for_stx_openstack(self, context):
         """ Update the runtime configurations that are required
             for stx-openstack application
@@ -6501,6 +6516,21 @@ class ConductorManager(service.PeriodicService):
                                             config_uuid,
                                             config_dict)
 
+    def _update_cephrgw_config(self, context):
+        """ Update ceph rgw configuration. """
+        personalities = [constants.CONTROLLER]
+
+        config_uuid = self._config_update_hosts(context, personalities)
+
+        config_dict = {
+            "personalities": personalities,
+            "classes": ['platform::ceph::rgw::runtime']
+        }
+
+        self._config_apply_runtime_manifest(context,
+                                            config_uuid,
+                                            config_dict)
+
     def report_lvm_cinder_config_success(self, context, host_uuid):
         """ Callback for Sysinv Agent
 
@@ -10795,6 +10825,7 @@ class ConductorManager(service.PeriodicService):
         appname = self._app.get_appname(rpc_app)
         if constants.HELM_APP_OPENSTACK == appname and app_applied \
                 and not was_applied:
+            self._update_cephrgw_config(context)
             # apply any runtime configurations that are needed for
             # stx_openstack application
             self._update_config_for_stx_openstack(context)
@@ -10820,6 +10851,7 @@ class ConductorManager(service.PeriodicService):
 
         app_removed = self._app.perform_app_remove(rpc_app)
         if constants.HELM_APP_OPENSTACK == appname and app_removed:
+            self._revert_cephrgw_config(context)
             # Update the VIM and PciIrqAffinity configuration.
             self._update_vim_config(context)
             self._update_pciirqaffinity_config(context)

sysinv/sysinv/sysinv/sysinv/helm/swift.py

@@ -0,0 +1,59 @@
+#
+# Copyright (c) 2018 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+from sysinv.common import constants
+from sysinv.common import exception
+
+from sysinv.openstack.common import log as logging
+
+from sysinv.helm import common
+from sysinv.helm import openstack
+
+LOG = logging.getLogger(__name__)
+
+
+class SwiftHelm(openstack.OpenstackBaseHelm):
+    """Class to encapsulate helm operations for the swift chart"""
+
+    CHART = constants.HELM_CHART_SWIFT
+
+    SERVICE_NAME = 'swift'
+    SERVICE_TYPE = 'object-store'
+    AUTH_USERS = ['swift']
+
+    def get_overrides(self, namespace=None):
+        overrides = {
+            common.HELM_NS_OPENSTACK: {
+                'endpoints': self._get_endpoints_overrides(),
+            }
+        }
+
+        if namespace in self.SUPPORTED_NAMESPACES:
+            return overrides[namespace]
+        elif namespace:
+            raise exception.InvalidHelmNamespace(chart=self.CHART,
+                                                 namespace=namespace)
+        else:
+            return overrides
+
+    def _get_object_store_overrides(self):
+        return {
+            'hosts': {
+                'default': 'null',
+                'admin': self._get_management_address(),
+                'internal': self._get_management_address(),
+                'public': self._get_oam_address()
+            },
+        }
+
+    def _get_endpoints_overrides(self):
+        return {
+            'identity': {
+                'auth': self._get_endpoints_identity_overrides(
+                    self.SERVICE_NAME, self.AUTH_USERS),
+            },
+            'object_store': self._get_object_store_overrides(),
+        }

sysinv/sysinv/sysinv/sysinv/puppet/base.py

@@ -24,6 +24,7 @@ class BasePuppet(object):
     CONFIG_WORKDIR = '/tmp/config'
     DEFAULT_REGION_NAME = 'RegionOne'
     DEFAULT_SERVICE_PROJECT_NAME = 'services'
+    SWIFT_SERVICE_PROJECT_NAME = 'service'
     DEFAULT_KERNEL_OPTIONS = constants.SYSTEM_SECURITY_FEATURE_SPECTRE_MELTDOWN_DEFAULT_OPTS
 
     SYSTEM_CONTROLLER_SERVICES = [
@@ -132,6 +133,9 @@ class BasePuppet(object):
         system = self._get_system()
         return system.region_name
 
+    def _get_swift_service_project_name(self):
+        return self.SWIFT_SERVICE_PROJECT_NAME
+
     def _get_service_project_name(self):
         if self.dbapi is None:
             return self.DEFAULT_SERVICE_PROJECT_NAME

sysinv/sysinv/sysinv/sysinv/puppet/ceph.py

@@ -11,6 +11,7 @@ from sysinv.common import constants
 from sysinv.common import exception
 from sysinv.common import utils
 from sysinv.common.storage_backend_conf import StorageBackendConfig
+from sysinv.helm import common
 
 from sysinv.puppet import openstack
 
@@ -82,7 +83,7 @@ class CephPuppet(openstack.OpenstackBasePuppet):
 
         ksuser = self._get_service_user_name(self.SERVICE_NAME_RGW)
 
-        return {
+        config = {
             'ceph::ms_bind_ipv6': ms_bind_ipv6,
 
             'platform::ceph::params::service_enabled': True,
@@ -113,6 +114,23 @@ class CephPuppet(openstack.OpenstackBasePuppet):
                 self._get_service_tenant_name(),
         }
 
+        if utils.is_openstack_installed(self.dbapi):
+            override = self.dbapi.helm_override_get(
+                        self.SERVICE_NAME_RGW, common.HELM_NS_OPENSTACK)
+            password = override.system_overrides.get(
+                        self.SERVICE_NAME_RGW, None)
+            if password:
+                swift_auth_password = password.encode('utf8', 'strict')
+                config['platform::ceph::params::rgw_service_password'] = \
+                    swift_auth_password
+
+            config['platform::ceph::params::rgw_service_domain'] = \
+                self._get_swift_service_user_domain_name()
+            config['platform::ceph::params::rgw_service_project'] = \
+                self._get_swift_service_tenant_name()
+
+        return config
+
     def _is_ceph_mon_required(self, host, operator):
         # Two conditions that we need to check for:
         # 1) If cinder is a shared service and it has a ceph backend

sysinv/sysinv/sysinv/sysinv/puppet/keystone.py

@@ -7,6 +7,7 @@
 from six.moves import configparser
 import os
 
+from sysinv.common import utils
 from sysinv.common import constants
 
 from tsconfig import tsconfig
@@ -30,6 +31,7 @@ class KeystonePuppet(openstack.OpenstackBasePuppet):
     ADMIN_USER = 'admin'
 
     DEFAULT_DOMAIN_NAME = 'Default'
+    SWIFT_DOMAIN_NAME = 'service'
 
     def _region_config(self):
         # A wrapper over the Base region_config check.
@@ -125,6 +127,10 @@ class KeystonePuppet(openstack.OpenstackBasePuppet):
             'CONFIG_KEYSTONE_ADMIN_USERNAME': self.get_admin_user_name(),
         }
 
+        if utils.is_openstack_installed(self.dbapi):
+            config['openstack::keystone::params::openstack_auth_uri'] = \
+                self.get_openstack_auth_uri()
+
         config.update(self._get_service_parameter_config())
         config.update(self._get_password_rule())
         return config
@@ -298,6 +304,14 @@ class KeystonePuppet(openstack.OpenstackBasePuppet):
             return "http://%s:5000" % self._format_url_address(
                 self._get_management_address())
 
+    def get_openstack_auth_uri(self):
+        location = self._get_service_default_dns_name(
+            self.SERVICE_NAME)
+
+        url = "%s://%s:80" % (self._get_public_protocol(),
+                              location)
+        return url
+
     def get_identity_uri(self):
         if self._region_config():
             service_config = self._get_service_config(self.SERVICE_NAME)
@@ -349,6 +363,9 @@ class KeystonePuppet(openstack.OpenstackBasePuppet):
                 return service_config.capabilities.get('admin_project_domain')
         return self.DEFAULT_DOMAIN_NAME
 
+    def get_swift_service_user_domain(self):
+        return self.SWIFT_DOMAIN_NAME
+
     def get_service_user_domain(self):
         if self._region_config():
             service_config = self._get_service_config(self.SERVICE_NAME)

sysinv/sysinv/sysinv/sysinv/puppet/openstack.py

@@ -10,6 +10,7 @@ import keyring
 from sysinv.common import constants
 
 from sysinv.puppet import base
+from sysinv.helm import common
 
 
 class OpenstackBasePuppet(base.BasePuppet):
@@ -108,6 +109,10 @@ class OpenstackBasePuppet(base.BasePuppet):
     def _get_public_protocol(self):
         return 'https' if self._https_enabled() else 'http'
 
+    def _get_service_default_dns_name(self, service):
+        return "{}.{}.svc.{}".format(service, common.HELM_NS_OPENSTACK,
+                                     constants.DEFAULT_DNS_SERVICE_DOMAIN)
+
     def _get_private_protocol(self):
         return 'http'
 
@@ -155,6 +160,9 @@ class OpenstackBasePuppet(base.BasePuppet):
 
         return self._region_name()
 
+    def _get_swift_service_tenant_name(self):
+        return self._get_swift_service_project_name()
+
     def _get_service_tenant_name(self):
         return self._get_service_project_name()
 
@@ -183,6 +191,9 @@ class OpenstackBasePuppet(base.BasePuppet):
                 return service_config.capabilities.get(stype)
         return None
 
+    def _get_swift_service_user_domain_name(self):
+        return self._operator.keystone.get_swift_service_user_domain()
+
     def _get_service_user_domain_name(self):
         return self._operator.keystone.get_service_user_domain()