Merge remote-tracking branch 'gerrit/master' into f/centos8

Change-Id: I8aa34409313c0ea467c4d43b87b903bd9a948483
Signed-off-by: Shuicheng Lin <shuicheng.lin@intel.com>

ceph/ceph-manager/ceph-manager/ceph_manager/constants.py

@@ -91,3 +91,6 @@ SB_TIER_CEPH_POOLS = sysinv_constants.SB_TIER_CEPH_POOLS
 
 # System
 SYSTEM_MODE_SIMPLEX = sysinv_constants.SYSTEM_MODE_SIMPLEX
+
+# Port on which ceph manager and ceph-mgr listens
+CEPH_MGR_PORT = sysinv_constants.CEPH_MGR_PORT

ceph/ceph-manager/ceph-manager/ceph_manager/server.py

@@ -140,7 +140,7 @@ class Service(SysinvConductorUpgradeApi, service.Service):
                 topic=constants.SYSINV_CONDUCTOR_TOPIC))
 
         self.ceph_api = wrapper.CephWrapper(
-            endpoint='https://localhost:5001')
+            endpoint='http://localhost:{}'.format(constants.CEPH_MGR_PORT))
 
         # Get initial config from sysinv and send it to
         # services that need it before starting them

test-requirements.txt

@@ -1,4 +1,4 @@
 # hacking pulls in flake8
 hacking!=0.13.0,<0.14,>=0.12.0 # Apache-2.0
 bashate >= 0.2
-pylint
+pylint<2.5.0

tools/collector/centos/build_srpm.data

@@ -1,2 +1,2 @@
 SRC_DIR="scripts"
-TIS_PATCH_VER=31
+TIS_PATCH_VER=33

tools/collector/centos/collector.spec

@@ -1,4 +1,4 @@
-Summary: CGCS Platform Data Collection Scripts Package
+Summary: StarlingX Platform Data Collection Scripts Package
 Name: collector
 Version: 1.0
 Release: %{tis_patch_ver}%{?_tis_dist}
@@ -48,6 +48,7 @@ install -m 755 collect_crash.sh %{buildroot}%{_sysconfdir}/collect.d/collect_cra
 install -m 755 collect_ima.sh %{buildroot}%{_sysconfdir}/collect.d/collect_ima
 install -m 755 collect_fm.sh %{buildroot}%{_sysconfdir}/collect.d/collect_fm
 install -m 755 collect_containerization.sh %{buildroot}%{_sysconfdir}/collect.d/collect_containerization
+install -m 755 collect_dc.sh %{buildroot}%{_sysconfdir}/collect.d/collect_dc
 
 install -m 755 etc.exclude %{buildroot}%{_sysconfdir}/collect/etc.exclude
 install -m 755 run.exclude %{buildroot}%{_sysconfdir}/collect/run.exclude

tools/collector/scripts/collect_containerization.sh

@@ -14,7 +14,7 @@ source /usr/local/sbin/collect_utils
 SERVICE="containerization"
 LOGFILE="${extradir}/${SERVICE}.info"
 HELM_DIR="${extradir}/helm"
-ETCD_DB_FILE="${extradir}/etcd_database.json"
+ETCD_DB_FILE="${extradir}/etcd_database.dump"
 KUBE_CONFIG_FILE="/etc/kubernetes/admin.conf"
 KUBE_CONFIG="--kubeconfig ${KUBE_CONFIG_FILE}"
 echo    "${hostname}: Containerization Info ...: ${LOGFILE}"
@@ -82,9 +82,10 @@ if [ "$nodetype" = "controller" -a "${ACTIVE}" = true ] ; then
     delimiter ${LOGFILE} "${CMD}"
     ${CMD} 2>>${COLLECT_ERROR_LOG}
 
-    CMD="curl -L http://localhost:2379/v2/keys/?recursive=true -o ${ETCD_DB_FILE}"
+    export ETCDCTL_API=3
+    CMD="etcdctl --endpoints=localhost:2379 get / --prefix"
     delimiter ${LOGFILE} "${CMD}"
-    ${CMD} 2>>${COLLECT_ERROR_LOG}
+    ${CMD} 2>>${COLLECT_ERROR_LOG} >> ${ETCD_DB_FILE}
 fi
 
 exit 0

tools/collector/scripts/collect_dc.sh

@@ -0,0 +1,96 @@
+#! /bin/bash
+#
+# Copyright (c) 2020 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+
+# Loads Up Utilities and Commands Variables
+source /usr/local/sbin/collect_parms
+source /usr/local/sbin/collect_utils
+
+SERVICE="distributed_cloud"
+LOGFILE="${extradir}/${SERVICE}.info"
+RPMLOG="${extradir}/rpm.info"
+
+function is_active_controller {
+    active_controller=`sm-query service management-ip | grep "enabled-active"`
+    if [ -z "$active_controller" ] ; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+function is_distributed_cloud_env {
+    distributed_cloud=`sm-query service-group distributed-cloud-services | grep "active"`
+    if [ -z "$distributed_cloud" ] ; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+function is_subcloud {
+    subcloud=`cat /etc/platform/platform.conf | grep "distributed_cloud_role" | grep "subcloud"`
+    if [ -z "$subcloud" ] ; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+# Must be a distributed cloud environment
+is_distributed_cloud_env
+if [ "$?" = "0" ] ; then
+    exit 0
+fi
+
+###############################################################################
+# Only Controller
+###############################################################################
+if [ "$nodetype" = "controller" ] ; then
+
+    # Must be an active controller
+    is_active_controller
+    if [ "$?" = "0" ] ; then
+        exit 0
+    fi
+
+    echo "${hostname}: Distributed Cloud ..: ${LOGFILE}"
+
+    is_subcloud
+    if [ "$?" = "1" ] ; then
+        # Subcloud
+        echo "Distributed Cloud Role: Subcloud" >> ${LOGFILE}
+
+        delimiter ${LOGFILE} "Address Pool of System Controller"
+        # Prints the column names of the table
+        system addrpool-list --nowrap | head -3 2>>${COLLECT_ERROR_LOG} >> ${LOGFILE}
+        # Prints the System Controller's address pool
+        system addrpool-list --nowrap | grep "system-controller-subnet" 2>>${COLLECT_ERROR_LOG} >> ${LOGFILE}
+
+    else
+        # System Controller
+        echo "Distributed Cloud Role: System Controller" >> ${LOGFILE}
+
+        delimiter ${LOGFILE} "dcmanager alarm summary"
+        dcmanager alarm summary 2>>${COLLECT_ERROR_LOG} >> ${LOGFILE}
+
+        delimiter ${LOGFILE} "dcmanager subcloud list"
+        dcmanager subcloud list 2>>${COLLECT_ERROR_LOG} >> ${LOGFILE}
+
+        delimiter ${LOGFILE} "dcmanager subcloud-group list"
+        dcmanager subcloud-group list 2>>${COLLECT_ERROR_LOG} >> ${LOGFILE}
+
+        cp -a /opt/dc/ansible ${extradir}
+
+        delimiter ${LOGFILE} "find /opt/dc-vault -ls"
+        find /opt/dc-vault -ls 2>>${COLLECT_ERROR_LOG} >> ${LOGFILE}
+
+    fi
+
+fi
+
+exit 0

tools/engtools/hostdata-collectors/README

@@ -2,7 +2,7 @@ The Engineering tools is meant to be installed as a patch. Therefore, the RPM is
 of the build but is not included in the image. Assuming your development environment is fully set up,
 simply run patch-engtools.sh to generate the patch:
 
-In this directory ($MY_REPO/stx/middleware/util/recipes-common/engtools/hostdata-collectors),
+In this directory ($MY_REPO/stx/utilities/tools/engtools/hostdata-collectors),
 enter the command:
 >./patch-engtools.sh
 

tools/engtools/hostdata-collectors/centos/collect-engtools.spec

@@ -12,6 +12,8 @@ Source: %{name}-%{version}.tar.gz
 BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root
 BuildRequires: systemd
 Requires: iperf3
+Requires: python36-six
+Requires: python36-psutil
 
 %description
 This package contains data collection tools to monitor host performance.

tools/engtools/hostdata-collectors/patch-engtools.sh

@@ -14,7 +14,7 @@ if [ $? -ne 0 ]; then
     exit 1
 fi
 
-source ${MY_REPO}/stx/middleware/recipes-common/build-info/release-info.inc
+source ${MY_REPO}/stx/utilities/utilities/build-info/release-info.inc
 #TiS_REL="16.10"
 #PATCH_ID="ENGTOOLS-${TiS_REL}"
 PATCH_ID="ENGTOOLS-${PLATFORM_RELEASE}"
@@ -23,7 +23,7 @@ PWD=$(pwd)
 
 # Create CGCS Patch
 cd ${MY_WORKSPACE}
-PATCH_BUILD=${MY_REPO}/stx/stx-update/extras/scripts/patch_build.sh
+PATCH_BUILD=${MY_REPO}/stx/update/extras/scripts/patch_build.sh
 ${PATCH_BUILD} --id ${PATCH_ID} --reboot-required=N \
     --summary "System engineering data collection and analysis tools." \
     --desc    "System engineering data collection and analysis tools." \

tools/engtools/hostdata-collectors/scripts/buddyinfo.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 textwidth=79 autoindent
 
 """
@@ -8,7 +8,7 @@ Last author: lmwangi at gmail  com
 Displays the available memory fragments
 by querying /proc/buddyinfo
 Example:
-# python buddyinfo.py
+# python3 buddyinfo.py
 """
 import optparse
 import os

tools/engtools/hostdata-collectors/scripts/cfg/engtools.conf

@@ -63,7 +63,7 @@ FAST_POSTGRES_CONNECTIONS=N
 AUTO_DELETE_DB=N
 
 [ControllerServices]
-CONTROLLER_SERVICE_LIST=aodh-api aodh-listener aodh-notifier aodh-evaluator barbican-api barbican-keystone-listener barbican-worker beam.smp ceilometer-api ceilometer-collector ceilometer-agent-notification ceilometer-mem-db ceph-mon mgr-restful-plugin ceph-alarm-manager cinder-api cinder-volume cinder-scheduler glance-api glance-registry gnocchi-api gnocchi-metricd heat-api heat-engine heat-api-cfn heat-api-cloudwatch hbsAgent ironic-api ironic-conductor magnum-api magnum-conductor neutron-server nova-api nova-api-proxy nova-compute nova-scheduler nova-conductor nova-console-auth nova-novncproxy nova-placement-api panko-api sysinv-api sysinv-conductor postgres fmManager rabbitmq-server gunicorn postgres snmpd patch-alarm-manager lighttpd sw-patch-controller-daemon nfv-vim nfv-vim-api nfv-vim-webserver slapd mtcAgent guestAgent dcmanager-api dcmanager-manager dcorch-engine dcorch-patch-api-proxy dcorch-snmp dcorch-sysinv-api-proxy memcached influxd
+CONTROLLER_SERVICE_LIST=aodh-api aodh-listener aodh-notifier aodh-evaluator barbican-api barbican-keystone-listener barbican-worker beam.smp ceilometer-api ceilometer-collector ceilometer-agent-notification ceilometer-mem-db ceph-mon mgr-restful-plugin ceph-alarm-manager cinder-api cinder-volume cinder-scheduler glance-api glance-registry gnocchi-api gnocchi-metricd heat-api heat-engine heat-api-cfn heat-api-cloudwatch hbsAgent ironic-api ironic-conductor magnum-api magnum-conductor neutron-server nova-api nova-api-proxy nova-compute nova-scheduler nova-conductor nova-console-auth nova-novncproxy nova-placement-api panko-api sysinv-api sysinv-conductor postgres fmManager rabbitmq-server gunicorn postgres snmpd patch-alarm-manager lighttpd sw-patch-controller-daemon nfv-vim nfv-vim-api nfv-vim-webserver slapd mtcAgent guestAgent dcmanager-api dcmanager-manager dcorch-engine dcorch-patch-api-proxy dcorch-sysinv-api-proxy memcached influxd
 
 [ComputeServices]
 COMPUTE_SERVICE_LIST=nova-compute neutron-dhcp-agent neutron-metadata-agent neutron-sriov-nic-agent kvm libvirtd guestServer host_agent dmeventd virtlockd
@@ -87,7 +87,7 @@ OPEN_STACK_SERVICE_LIST=nova cinder aodh barbican ceilometer heat glance ceph ho
 SKIP_LIST=ps top sh <defunct> curl awk wc sleep lsof cut grep ip tail su
 
 [ExcludeList]
-EXCLUDE_LIST=python python2 bash perl sudo init
+EXCLUDE_LIST=python python2 python3 bash perl sudo init
 
 [ApiStatsConstantPorts]
 DB_PORT_NUMBER=5432

tools/engtools/hostdata-collectors/scripts/live_stream.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
 """
 Copyright (c) 2017 Wind River Systems, Inc.
@@ -1478,14 +1478,14 @@ if __name__ == "__main__":
             # use first interface if not specified in engtools.conf
             if external_if == "" or external_if is None:
                 p = Popen("ifconfig", shell=True, stdout=PIPE)
-                external_if = p.stdout.readline().split(":")[0]
+                external_if = p.stdout.readline().decode().split(":")[0]
                 p.kill()
             appendToFile("/tmp/livestream.log", "-External interface for {}: {}".format(node, external_if))
             # enable IP forwarding
             p = Popen("sysctl -w net.ipv4.ip_forward=1 > /dev/null", shell=True)
             p.communicate()
             p = Popen("iptables -t nat -L --line-numbers", shell=True, stdout=PIPE)
-            tmp = [line.strip("\n") for line in p.stdout.readlines()]
+            tmp = [line.decode().strip("\n") for line in p.stdout.readlines()]
             # entries need to be removed in reverse order
             for line in reversed(tmp):
                 formatted_line = " ".join(line.strip("\n").split()[1:])

utilities/logmgmt/scripts/etc/systemd/system/logmgmt.service

@@ -1,5 +1,5 @@
 [Unit]
-Description=Titanium Cloud Log Management
+Description=StarlingX Log Management
 After=network.target syslog-ng.service iscsid.service sw-patch.service
 Before=config.service pmon.service
 

utilities/namespace-utils/centos/namespace-utils.spec

@@ -11,7 +11,7 @@ URL: unknown
 Source0: %{name}-%{version}.tar.gz
 
 %description
-StarlingX Cloud namespace utilities
+StarlingX namespace utilities
 
 %define debug_package %{nil}
 

utilities/platform-util/platform-util/platform_util/license/license.py

@@ -1,30 +1,35 @@
 #
-# Copyright (c) 2017-2019 Wind River Systems, Inc.
+# Copyright (c) 2017-2020 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
-import logging
 import os
 from platform_util.license import exception
 import sys
 
-LOG = logging.getLogger(__name__)
 
-def verify_license(license_file):
+def verify_license(*args):
-    """Verify all features in a license file"""
+    """Verify the license file"""
-    if not os.path.isfile(license_file):
+    if not os.path.isfile(args[0]):
         raise exception.LicenseNotFound()
 
 
 def main():
-    if len(sys.argv) == 2:
+    # Pass the command arguments to verify_license
-        licensefile = sys.argv[1]
+    if len(sys.argv) < 2:
-    else:
+        print("Usage: verify-license <license file> [<optional_parameter>...]")
-        print("Usage: verify-license <license file>")
         exit(-1)
 
+    arg_list = []
+    for arg in sys.argv:
+        arg_list.append(arg)
+
+    # The arguments passed to verify_license from command line
+    # will not include sys.argv[0] which is the script name.
+    # Only the actual arguments: sys.argv[1] and onward will be passed,
+    # meaning license_file followed by optional attributes.
     try:
-        verify_license(licensefile)
+        verify_license(*arg_list[1:len(sys.argv)])
     except exception.InvalidLicenseType:
         exit(1)
     except exception.LicenseNotFound:

utilities/platform-util/scripts/opt-platform.service

@@ -1,5 +1,5 @@
 [Unit]
-Description=Titanium Cloud opt-platform mounter
+Description=StarlingX opt-platform mounter
 After=sw-patch.service
 
 [Service]

utilities/platform-util/scripts/patch-restart-processes

@@ -227,9 +227,6 @@ do
       "dcorch-engine")
          process_list=(${process_list[@]} "sm:dcorch-engine:dcorch-engine:controller:/var/run/resource-agents/dcorch-engine.pid:0")
          ;;
-      "dcorch-snmp")
-         process_list=(${process_list[@]} "sm:dcorch-snmp:dcorch-snmp:controller:/var/run/resource-agents/dcorch-snmp.pid:0")
-         ;;
       "dcorch-sysinv-api-proxy")
          process_list=(${process_list[@]} "sm:dcorch-sysinv-api-proxy:dcorch-sysinv-api-proxy:controller:/var/run/resource-agents/dcorch-sysinv-api-proxy.pid:0")
          ;;

utilities/stx-extensions/centos/tis-extensions.spec

@@ -7,7 +7,7 @@
 
 Name: stx-extensions
 Version: 1.0
-Summary: TIS Extensions to thirdparty pkgs
+Summary: StarlingX Extensions to thirdparty pkgs
 Release: %{tis_patch_ver}%{?_tis_dist}
 License: Apache-2.0
 Group: base
@@ -20,14 +20,14 @@ Source0: %{name}-%{version}.tar.gz
 Requires: systemd
 
 %description
-TIS Extensions to thirdparty pkgs
+StarlingX Extensions to thirdparty pkgs
 
 %package -n %{name}-controller
-Summary: TIS Extensions to thirdparty pkg on controller
+Summary: StarlingX Extensions to thirdparty pkg on controller
 Group: base
 
 %description -n %{name}-controller
-TIS Extensions to thirdparty pkgs on controller
+StarlingX Extensions to thirdparty pkgs on controller
 
 %define local_etc_systemd %{_sysconfdir}/systemd/system/
 %define local_etc_coredump %{_sysconfdir}/systemd/coredump.conf.d

utilities/worker-utils/worker-utils/affine-platform.sh.service

@@ -1,5 +1,5 @@
 [Unit]
-Description=Titanium Cloud Affine Platform
+Description=StarlingX Affine Platform
 After=syslog.service network.service dbus.service sw-patch.service
 Before=workerconfig.service