Remove wrs-configutilities SDK Module

Remove configutilities and move what is being used in other components
to controllerconfig.

Tested with a clean install on AIO-DX and running config_controller.

With the StarlingX move to supporting pure upstream OpenStack, the
majority of the SDK Modules are related to functionality no longer
supported. The remaining SDK Modules will be moved to StarlingX
documentation.

Story: 2005275
Task: 30262

Change-Id: Ie496548dfc6efee677a501c98c227c586df0a7d6
Signed-off-by: Kristine Bujold <kristine.bujold@windriver.com>

controllerconfig/centos/build_srpm.data

@@ -1,2 +1,2 @@
 SRC_DIR="controllerconfig"
-TIS_PATCH_VER=150
+TIS_PATCH_VER=151

controllerconfig/controllerconfig/controllerconfig/__init__.py

@@ -1,5 +1,34 @@
 #
-# Copyright (c) 2015 Wind River Systems, Inc.
+# Copyright (c) 2015-2019 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
+
+from controllerconfig.common.validator import validate  # noqa: F401
+from controllerconfig.common.configobjects import Network  # noqa: F401
+from controllerconfig.common.configobjects import DEFAULT_CONFIG  # noqa: F401
+from controllerconfig.common.configobjects import REGION_CONFIG  # noqa: F401
+from controllerconfig.common.configobjects import DEFAULT_NAMES  # noqa: F401
+from controllerconfig.common.configobjects import HP_NAMES  # noqa: F401
+from controllerconfig.common.configobjects import SUBCLOUD_CONFIG  # noqa: F401
+from controllerconfig.common.configobjects import MGMT_TYPE  # noqa: F401
+from controllerconfig.common.configobjects import INFRA_TYPE  # noqa: F401
+from controllerconfig.common.configobjects import OAM_TYPE  # noqa: F401
+from controllerconfig.common.configobjects import NETWORK_PREFIX_NAMES  # noqa: F401
+from controllerconfig.common.configobjects import HOST_XML_ATTRIBUTES  # noqa: F401
+from controllerconfig.common.configobjects import DEFAULT_DOMAIN_NAME  # noqa: F401
+from controllerconfig.common.exceptions import ConfigError  # noqa: F401
+from controllerconfig.common.exceptions import ConfigFail  # noqa: F401
+from controllerconfig.common.exceptions import ValidateFail  # noqa: F401
+from controllerconfig.utils import is_valid_vlan  # noqa: F401
+from controllerconfig.utils import is_mtu_valid  # noqa: F401
+from controllerconfig.utils import validate_network_str  # noqa: F401
+from controllerconfig.utils import validate_address_str  # noqa: F401
+from controllerconfig.utils import validate_address  # noqa: F401
+from controllerconfig.utils import is_valid_url  # noqa: F401
+from controllerconfig.utils import is_valid_domain_or_ip  # noqa: F401
+from controllerconfig.utils import ip_version_to_string  # noqa: F401
+from controllerconfig.utils import lag_mode_to_str  # noqa: F401
+from controllerconfig.utils import validate_openstack_password  # noqa: F401
+from controllerconfig.utils import validate_nameserver_address_str  # noqa: F401
+from controllerconfig.utils import extract_openstack_password_rules_from_file  # noqa: F401

controllerconfig/controllerconfig/controllerconfig/common/configobjects.py

@@ -0,0 +1,371 @@
+"""
+Copyright (c) 2015-2019 Wind River Systems, Inc.
+
+SPDX-License-Identifier: Apache-2.0
+
+"""
+
+from netaddr import IPRange
+from controllerconfig.common.exceptions import ConfigFail
+from controllerconfig.common.exceptions import ValidateFail
+from controllerconfig.utils import is_mtu_valid
+from controllerconfig.utils import is_valid_vlan
+from controllerconfig.utils import validate_network_str
+from controllerconfig.utils import validate_address_str
+
+DEFAULT_CONFIG = 0
+REGION_CONFIG = 1
+SUBCLOUD_CONFIG = 2
+
+MGMT_TYPE = 0
+INFRA_TYPE = 1
+OAM_TYPE = 2
+CLUSTER_TYPE = 3
+NETWORK_PREFIX_NAMES = [
+    ('MGMT', 'INFRA', 'OAM', 'CLUSTER'),
+    ('CLM', 'BLS', 'CAN', 'CLUSTER')
+]
+
+HOST_XML_ATTRIBUTES = ['hostname', 'personality', 'subfunctions',
+                       'mgmt_mac', 'mgmt_ip',
+                       'bm_ip', 'bm_type', 'bm_username',
+                       'bm_password', 'boot_device', 'rootfs_device',
+                       'install_output', 'console', 'vsc_controllers',
+                       'power_on', 'location']
+
+# Network naming types
+DEFAULT_NAMES = 0
+HP_NAMES = 1
+
+# well-known default domain name
+DEFAULT_DOMAIN_NAME = 'Default'
+
+
+class LogicalInterface(object):
+    """ Represents configuration for a logical interface.
+    """
+    def __init__(self):
+        self.name = None
+        self.mtu = None
+        self.lag_interface = False
+        self.lag_mode = None
+        self.ports = None
+
+    def parse_config(self, system_config, logical_interface):
+        # Ensure logical interface config is present
+        if not system_config.has_section(logical_interface):
+            raise ConfigFail("Missing config for logical interface %s." %
+                             logical_interface)
+        self.name = logical_interface
+
+        # Parse/validate the MTU
+        self.mtu = system_config.getint(logical_interface, 'INTERFACE_MTU')
+        if not is_mtu_valid(self.mtu):
+            raise ConfigFail("Invalid MTU value for %s. "
+                             "Valid values: 576 - 9216" % logical_interface)
+
+        # Parse the ports
+        self.ports = [_f for _f in
+                      [x.strip() for x in
+                       system_config.get(logical_interface,
+                                         'INTERFACE_PORTS').split(',')]
+                      if _f]
+
+        # Parse/validate the LAG config
+        lag_interface = system_config.get(logical_interface,
+                                          'LAG_INTERFACE')
+        if lag_interface.lower() == 'y':
+            self.lag_interface = True
+            if len(self.ports) != 2:
+                raise ConfigFail(
+                    "Invalid number of ports (%d) supplied for LAG "
+                    "interface %s" % (len(self.ports), logical_interface))
+            self.lag_mode = system_config.getint(logical_interface, 'LAG_MODE')
+            if self.lag_mode < 1 or self.lag_mode > 6:
+                raise ConfigFail(
+                    "Invalid LAG_MODE value of %d for %s. Valid values: 1-6" %
+                    (self.lag_mode, logical_interface))
+        elif lag_interface.lower() == 'n':
+            if len(self.ports) > 1:
+                raise ConfigFail(
+                    "More than one interface supplied for non-LAG "
+                    "interface %s" % logical_interface)
+            if len(self.ports) == 0:
+                raise ConfigFail(
+                    "No interfaces supplied for non-LAG "
+                    "interface %s" % logical_interface)
+        else:
+            raise ConfigFail(
+                "Invalid LAG_INTERFACE value of %s for %s. Valid values: "
+                "Y or N" % (lag_interface, logical_interface))
+
+
+class Network(object):
+    """ Represents configuration for a network.
+    """
+    def __init__(self):
+        self.vlan = None
+        self.cidr = None
+        self.multicast_cidr = None
+        self.start_address = None
+        self.end_address = None
+        self.start_end_in_config = False
+        self.floating_address = None
+        self.address_0 = None
+        self.address_1 = None
+        self.dynamic_allocation = False
+        self.gateway_address = None
+        self.logical_interface = None
+
+    def parse_config(self, system_config, config_type, network_type,
+                     min_addresses=0, multicast_addresses=0, optional=False,
+                     naming_type=DEFAULT_NAMES,
+                     logical_interface_required=True):
+        network_prefix = NETWORK_PREFIX_NAMES[naming_type][network_type]
+        network_name = network_prefix + '_NETWORK'
+
+        if naming_type == HP_NAMES:
+            attr_prefix = network_prefix + '_'
+        else:
+            attr_prefix = ''
+
+        # Ensure network config is present
+        if not system_config.has_section(network_name):
+            if not optional:
+                raise ConfigFail("Missing config for network %s." %
+                                 network_name)
+            else:
+                # Optional interface - just return
+                return
+
+        # Parse/validate the VLAN
+        if system_config.has_option(network_name, attr_prefix + 'VLAN'):
+            self.vlan = system_config.getint(network_name,
+                                             attr_prefix + 'VLAN')
+        if self.vlan:
+            if not is_valid_vlan(self.vlan):
+                raise ConfigFail(
+                    "Invalid %s value of %d for %s. Valid values: 1-4094" %
+                    (attr_prefix + 'VLAN', self.vlan, network_name))
+
+        # Parse/validate the cidr
+        cidr_str = system_config.get(network_name, attr_prefix + 'CIDR')
+        try:
+            self.cidr = validate_network_str(
+                cidr_str, min_addresses)
+        except ValidateFail as e:
+            raise ConfigFail(
+                "Invalid %s value of %s for %s.\nReason: %s" %
+                (attr_prefix + 'CIDR', cidr_str, network_name, e))
+
+        # Parse/validate the multicast subnet
+        if 0 < multicast_addresses and \
+                system_config.has_option(network_name,
+                                         attr_prefix + 'MULTICAST_CIDR'):
+            multicast_cidr_str = system_config.get(network_name, attr_prefix +
+                                                   'MULTICAST_CIDR')
+            try:
+                self.multicast_cidr = validate_network_str(
+                    multicast_cidr_str, multicast_addresses, multicast=True)
+            except ValidateFail as e:
+                raise ConfigFail(
+                    "Invalid %s value of %s for %s.\nReason: %s" %
+                    (attr_prefix + 'MULTICAST_CIDR', multicast_cidr_str,
+                     network_name, e))
+
+            if self.cidr.version != self.multicast_cidr.version:
+                raise ConfigFail(
+                    "Invalid %s value of %s for %s.  Multicast "
+                    "subnet and network IP families must be the same." %
+                    (attr_prefix + 'MULTICAST_CIDR', multicast_cidr_str,
+                     network_name))
+
+        # Parse/validate the hardwired controller addresses
+        floating_address_str = None
+        address_0_str = None
+        address_1_str = None
+
+        if min_addresses == 1:
+            if (system_config.has_option(
+                    network_name, attr_prefix + 'IP_FLOATING_ADDRESS') or
+                system_config.has_option(
+                    network_name, attr_prefix + 'IP_UNIT_0_ADDRESS') or
+                system_config.has_option(
+                    network_name, attr_prefix + 'IP_UNIT_1_ADDRESS') or
+                system_config.has_option(
+                    network_name, attr_prefix + 'IP_START_ADDRESS') or
+                system_config.has_option(
+                    network_name, attr_prefix + 'IP_END_ADDRESS')):
+                raise ConfigFail(
+                    "Only one IP address is required for OAM "
+                    "network, use 'IP_ADDRESS' to specify the OAM IP "
+                    "address")
+            floating_address_str = system_config.get(
+                network_name, attr_prefix + 'IP_ADDRESS')
+            try:
+                self.floating_address = validate_address_str(
+                    floating_address_str, self.cidr)
+            except ValidateFail as e:
+                    raise ConfigFail(
+                        "Invalid %s value of %s for %s.\nReason: %s" %
+                        (attr_prefix + 'IP_ADDRESS',
+                         floating_address_str, network_name, e))
+            self.address_0 = self.floating_address
+            self.address_1 = self.floating_address
+        else:
+            if system_config.has_option(
+                    network_name, attr_prefix + 'IP_FLOATING_ADDRESS'):
+                floating_address_str = system_config.get(
+                    network_name, attr_prefix + 'IP_FLOATING_ADDRESS')
+                try:
+                    self.floating_address = validate_address_str(
+                        floating_address_str, self.cidr)
+                except ValidateFail as e:
+                    raise ConfigFail(
+                        "Invalid %s value of %s for %s.\nReason: %s" %
+                        (attr_prefix + 'IP_FLOATING_ADDRESS',
+                         floating_address_str, network_name, e))
+
+            if system_config.has_option(
+                    network_name, attr_prefix + 'IP_UNIT_0_ADDRESS'):
+                address_0_str = system_config.get(
+                    network_name, attr_prefix + 'IP_UNIT_0_ADDRESS')
+                try:
+                    self.address_0 = validate_address_str(
+                        address_0_str, self.cidr)
+                except ValidateFail as e:
+                    raise ConfigFail(
+                        "Invalid %s value of %s for %s.\nReason: %s" %
+                        (attr_prefix + 'IP_UNIT_0_ADDRESS',
+                         address_0_str, network_name, e))
+
+            if system_config.has_option(
+                    network_name, attr_prefix + 'IP_UNIT_1_ADDRESS'):
+                address_1_str = system_config.get(
+                    network_name, attr_prefix + 'IP_UNIT_1_ADDRESS')
+                try:
+                    self.address_1 = validate_address_str(
+                        address_1_str, self.cidr)
+                except ValidateFail as e:
+                    raise ConfigFail(
+                        "Invalid %s value of %s for %s.\nReason: %s" %
+                        (attr_prefix + 'IP_UNIT_1_ADDRESS',
+                         address_1_str, network_name, e))
+
+            # Parse/validate the start/end addresses
+            start_address_str = None
+            end_address_str = None
+            if system_config.has_option(
+                    network_name, attr_prefix + 'IP_START_ADDRESS'):
+                start_address_str = system_config.get(
+                    network_name, attr_prefix + 'IP_START_ADDRESS')
+                try:
+                    self.start_address = validate_address_str(
+                        start_address_str, self.cidr)
+                except ValidateFail as e:
+                    raise ConfigFail(
+                        "Invalid %s value of %s for %s.\nReason: %s" %
+                        (attr_prefix + 'IP_START_ADDRESS',
+                         start_address_str, network_name, e))
+
+            if system_config.has_option(
+                    network_name, attr_prefix + 'IP_END_ADDRESS'):
+                end_address_str = system_config.get(
+                    network_name, attr_prefix + 'IP_END_ADDRESS')
+                try:
+                    self.end_address = validate_address_str(
+                        end_address_str, self.cidr)
+                except ValidateFail as e:
+                    raise ConfigFail(
+                        "Invalid %s value of %s for %s.\nReason: %s " %
+                        (attr_prefix + 'IP_END_ADDRESS',
+                         end_address_str, network_name, e))
+
+            if start_address_str or end_address_str:
+                if not end_address_str:
+                    raise ConfigFail("Missing attribute %s for %s_NETWORK" %
+                                     (attr_prefix + 'IP_END_ADDRESS',
+                                      network_name))
+                if not start_address_str:
+                    raise ConfigFail("Missing attribute %s for %s_NETWORK" %
+                                     (attr_prefix + 'IP_START_ADDRESS',
+                                      network_name))
+                if not self.start_address < self.end_address:
+                    raise ConfigFail(
+                        "Start address %s not less than end address %s for %s."
+                        % (str(self.start_address), str(self.end_address),
+                           network_name))
+                if not IPRange(start_address_str, end_address_str).size >= \
+                        min_addresses:
+                    raise ConfigFail("Address range for %s must contain at "
+                                     "least %d addresses." %
+                                     (network_name, min_addresses))
+                self.start_end_in_config = True
+
+            if floating_address_str or address_0_str or address_1_str:
+                if not floating_address_str:
+                    raise ConfigFail("Missing attribute %s for %s_NETWORK" %
+                                     (attr_prefix + 'IP_FLOATING_ADDRESS',
+                                      network_name))
+                if not address_0_str:
+                    raise ConfigFail("Missing attribute %s for %s_NETWORK" %
+                                     (attr_prefix + 'IP_UNIT_0_ADDRESS',
+                                      network_name))
+                if not address_1_str:
+                    raise ConfigFail("Missing attribute %s for %s_NETWORK" %
+                                     (attr_prefix + 'IP_UNIT_1_ADDRESS',
+                                      network_name))
+
+            if start_address_str and floating_address_str:
+                    raise ConfigFail("Overspecified network: Can only set %s "
+                                     "and %s OR %s, %s, and %s for "
+                                     "%s_NETWORK" %
+                                     (attr_prefix + 'IP_START_ADDRESS',
+                                      attr_prefix + 'IP_END_ADDRESS',
+                                      attr_prefix + 'IP_FLOATING_ADDRESS',
+                                      attr_prefix + 'IP_UNIT_0_ADDRESS',
+                                      attr_prefix + 'IP_UNIT_1_ADDRESS',
+                                      network_name))
+
+        if config_type == DEFAULT_CONFIG:
+            if not self.start_address:
+                self.start_address = self.cidr[2]
+            if not self.end_address:
+                self.end_address = self.cidr[-2]
+
+        # Parse/validate the dynamic IP address allocation
+        if system_config.has_option(network_name,
+                                    'DYNAMIC_ALLOCATION'):
+            dynamic_allocation = system_config.get(network_name,
+                                                   'DYNAMIC_ALLOCATION')
+            if dynamic_allocation.lower() == 'y':
+                self.dynamic_allocation = True
+            elif dynamic_allocation.lower() == 'n':
+                self.dynamic_allocation = False
+            else:
+                raise ConfigFail(
+                    "Invalid DYNAMIC_ALLOCATION value of %s for %s. "
+                    "Valid values: Y or N" %
+                    (dynamic_allocation, network_name))
+
+        # Parse/validate the gateway (optional)
+        if system_config.has_option(network_name, attr_prefix + 'GATEWAY'):
+            gateway_address_str = system_config.get(
+                network_name, attr_prefix + 'GATEWAY')
+            try:
+                self.gateway_address = validate_address_str(
+                    gateway_address_str, self.cidr)
+            except ValidateFail as e:
+                raise ConfigFail(
+                    "Invalid %s value of %s for %s.\nReason: %s" %
+                    (attr_prefix + 'GATEWAY',
+                     gateway_address_str, network_name, e))
+
+        # Parse/validate the logical interface
+        if logical_interface_required or system_config.has_option(
+                network_name, attr_prefix + 'LOGICAL_INTERFACE'):
+            logical_interface_name = system_config.get(
+                network_name, attr_prefix + 'LOGICAL_INTERFACE')
+            self.logical_interface = LogicalInterface()
+            self.logical_interface.parse_config(system_config,
+                                                logical_interface_name)

controllerconfig/controllerconfig/controllerconfig/common/crypt.py

@@ -0,0 +1,102 @@
+# Copyright 2011 OpenStack Foundation
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# Copyright (c) 2019 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+"""
+Routines for URL-safe encrypting/decrypting
+
+Cloned from git/glance/common
+"""
+
+import base64
+import os
+import random
+
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives.ciphers import algorithms
+from cryptography.hazmat.primitives.ciphers import Cipher
+from cryptography.hazmat.primitives.ciphers import modes
+from oslo_utils import encodeutils
+import six
+# NOTE(jokke): simplified transition to py3, behaves like py2 xrange
+from six.moves import range
+
+
+def urlsafe_encrypt(key, plaintext, blocksize=16):
+    """Encrypts plaintext.
+
+    Resulting ciphertext will contain URL-safe characters.
+    If plaintext is Unicode, encode it to UTF-8 before encryption.
+
+    :param key: AES secret key
+    :param plaintext: Input text to be encrypted
+    :param blocksize: Non-zero integer multiple of AES blocksize in bytes (16)
+    :returns: Resulting ciphertext
+    """
+
+    def pad(text):
+        """Pads text to be encrypted"""
+        pad_length = (blocksize - len(text) % blocksize)
+        # NOTE(rosmaita): I know this looks stupid, but we can't just
+        # use os.urandom() to get the bytes because we use char(0) as
+        # a delimiter
+        pad = b''.join(six.int2byte(random.SystemRandom().randint(1, 0xFF))
+                       for i in range(pad_length - 1))
+        # We use chr(0) as a delimiter between text and padding
+        return text + b'\0' + pad
+
+    plaintext = encodeutils.to_utf8(plaintext)
+    key = encodeutils.to_utf8(key)
+    # random initial 16 bytes for CBC
+    init_vector = os.urandom(16)
+    backend = default_backend()
+    cypher = Cipher(algorithms.AES(key), modes.CBC(init_vector),
+                    backend=backend)
+    encryptor = cypher.encryptor()
+    padded = encryptor.update(
+        pad(six.binary_type(plaintext))) + encryptor.finalize()
+    encoded = base64.urlsafe_b64encode(init_vector + padded)
+    if six.PY3:
+        encoded = encoded.decode('ascii')
+    return encoded
+
+
+def urlsafe_decrypt(key, ciphertext):
+    """Decrypts URL-safe base64 encoded ciphertext.
+
+    On Python 3, the result is decoded from UTF-8.
+
+    :param key: AES secret key
+    :param ciphertext: The encrypted text to decrypt
+
+    :returns: Resulting plaintext
+    """
+    # Cast from unicode
+    ciphertext = encodeutils.to_utf8(ciphertext)
+    key = encodeutils.to_utf8(key)
+    ciphertext = base64.urlsafe_b64decode(ciphertext)
+    backend = default_backend()
+    cypher = Cipher(algorithms.AES(key), modes.CBC(ciphertext[:16]),
+                    backend=backend)
+    decryptor = cypher.decryptor()
+    padded = decryptor.update(ciphertext[16:]) + decryptor.finalize()
+    text = padded[:padded.rfind(b'\0')]
+    if six.PY3:
+        text = text.decode('utf-8')
+    return text

controllerconfig/controllerconfig/controllerconfig/common/dcmanager.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2017 Wind River Systems, Inc.
+# Copyright (c) 2017-2019 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -11,7 +11,7 @@ DC Manager Interactions
 from controllerconfig.common import log
 
 from Crypto.Hash import MD5
-from configutilities.common import crypt
+from controllerconfig.common import crypt
 
 import json
 

controllerconfig/controllerconfig/controllerconfig/common/exceptions.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2014 Wind River Systems, Inc.
+# Copyright (c) 2014-2019 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -8,7 +8,26 @@
 Configuration Errors
 """
 
-from configutilities import ConfigError
+
+class ConfigError(Exception):
+    """Base class for configuration exceptions."""
+
+    def __init__(self, message=None):
+        self.message = message
+        super(ConfigError, self).__init__(message)
+
+    def __str__(self):
+        return self.message or ""
+
+
+class ConfigFail(ConfigError):
+    """General configuration error."""
+    pass
+
+
+class ValidateFail(ConfigError):
+    """Validation of data failed."""
+    pass
 
 
 class BackupFail(ConfigError):

controllerconfig/controllerconfig/controllerconfig/config_management.py

@@ -1,5 +1,5 @@
 """
-Copyright (c) 2017 Wind River Systems, Inc.
+Copyright (c) 2017-2019 Wind River Systems, Inc.
 
 SPDX-License-Identifier: Apache-2.0
 
@@ -13,8 +13,8 @@ import subprocess
 import sys
 import time
 
-import configutilities.common.exceptions as cexeptions
-import configutilities.common.utils as cutils
+import controllerconfig.common.exceptions as exeptions
+import controllerconfig.utils as utils
 from six.moves import input
 
 
@@ -109,9 +109,9 @@ def configure_management():
                 continue
             break
         except (netaddr.AddrFormatError, ValueError):
-            print ("Invalid CIDR - "
-                   "please enter a valid management IP address in "
-                   "CIDR notation.")
+            print("Invalid CIDR - "
+                  "please enter a valid management IP address in "
+                  "CIDR notation.")
 
     while True:
         user_input = input("Enter management gateway IP address [" +
@@ -127,18 +127,18 @@ def configure_management():
             management_gateway_address = ip_input
             break
         except (netaddr.AddrFormatError, ValueError):
-            print ("Invalid address - "
-                   "please enter a valid management gateway IP address")
+            print("Invalid address - "
+                  "please enter a valid management gateway IP address")
 
     min_addresses = 8
     while True:
         user_input = input("Enter System Controller subnet in "
                            "CIDR notation: ")
         try:
-            system_controller_subnet = cutils.validate_network_str(
+            system_controller_subnet = utils.validate_network_str(
                 user_input, min_addresses)
             break
-        except cexeptions.ValidateFail as e:
+        except exeptions.ValidateFail as e:
             print("{}".format(e))
 
     print("Disabling non-management interfaces... ", end=' ')

controllerconfig/controllerconfig/controllerconfig/configassistant.py

@@ -1,5 +1,5 @@
 """
-Copyright (c) 2014-2018 Wind River Systems, Inc.
+Copyright (c) 2014-2019 Wind River Systems, Inc.
 
 SPDX-License-Identifier: Apache-2.0
 
@@ -20,19 +20,19 @@ import textwrap
 import time
 
 import pyudev
-from configutilities import ConfigFail
-from configutilities import ValidateFail
-from configutilities import is_valid_vlan
-from configutilities import is_mtu_valid
-from configutilities import validate_network_str
-from configutilities import validate_address_str
-from configutilities import validate_address
-from configutilities import ip_version_to_string
-from configutilities import validate_nameserver_address_str
-from configutilities import is_valid_url
-from configutilities import is_valid_domain_or_ip
-from configutilities import validate_openstack_password
-from configutilities import DEFAULT_DOMAIN_NAME
+from controllerconfig import ConfigFail
+from controllerconfig import ValidateFail
+from controllerconfig import is_valid_vlan
+from controllerconfig import is_mtu_valid
+from controllerconfig import validate_network_str
+from controllerconfig import validate_address_str
+from controllerconfig import validate_address
+from controllerconfig import ip_version_to_string
+from controllerconfig import validate_nameserver_address_str
+from controllerconfig import is_valid_url
+from controllerconfig import is_valid_domain_or_ip
+from controllerconfig import validate_openstack_password
+from controllerconfig import DEFAULT_DOMAIN_NAME
 from netaddr import IPNetwork
 from netaddr import IPAddress
 from netaddr import IPRange

controllerconfig/controllerconfig/controllerconfig/regionconfig.py

@@ -1,5 +1,5 @@
 """
-Copyright (c) 2015-2018 Wind River Systems, Inc.
+Copyright (c) 2015-2019 Wind River Systems, Inc.
 
 SPDX-License-Identifier: Apache-2.0
 
@@ -19,16 +19,15 @@ from controllerconfig.common import constants
 from controllerconfig.common import log
 from controllerconfig.common import rest_api_utils as rutils
 from controllerconfig.common.exceptions import KeystoneFail
-from configutilities.common import utils as cutils
-from configutilities.common.configobjects import REGION_CONFIG
-from configutilities.common.configobjects import SUBCLOUD_CONFIG
-from configutilities import ConfigFail
+from controllerconfig.common.configobjects import REGION_CONFIG
+from controllerconfig.common.configobjects import SUBCLOUD_CONFIG
+from controllerconfig import ConfigFail
 from controllerconfig.configassistant import ConfigAssistant
 from netaddr import IPAddress
 from controllerconfig.systemconfig import parse_system_config
 from controllerconfig.systemconfig import configure_management_interface
 from controllerconfig.systemconfig import create_cgcs_config_file
-from configutilities import DEFAULT_DOMAIN_NAME
+from controllerconfig import DEFAULT_DOMAIN_NAME
 
 # Temporary file for building cgcs_config
 TEMP_CGCS_CONFIG_FILE = "/tmp/cgcs_config"
@@ -290,18 +289,18 @@ def validate_region_one_keystone_config(region_config, token, api_url, users,
 
     # Verify that region two endpoints & services match our requirements,
     # optionally creating missing entries
-    public_address = cutils.get_optional(region_config, 'CAN_NETWORK',
-                                         'CAN_IP_START_ADDRESS')
+    public_address = utils.get_optional(region_config, 'CAN_NETWORK',
+                                        'CAN_IP_START_ADDRESS')
     if not public_address:
-        public_address = cutils.get_optional(region_config, 'CAN_NETWORK',
-                                             'CAN_IP_FLOATING_ADDRESS')
+        public_address = utils.get_optional(region_config, 'CAN_NETWORK',
+                                            'CAN_IP_FLOATING_ADDRESS')
     if not public_address:
-        public_address = cutils.get_optional(region_config, 'OAM_NETWORK',
-                                             'IP_START_ADDRESS')
+        public_address = utils.get_optional(region_config, 'OAM_NETWORK',
+                                            'IP_START_ADDRESS')
     if not public_address:
         # AIO-SX configuration
-        public_address = cutils.get_optional(region_config, 'OAM_NETWORK',
-                                             'IP_ADDRESS')
+        public_address = utils.get_optional(region_config, 'OAM_NETWORK',
+                                            'IP_ADDRESS')
     if not public_address:
         public_address = region_config.get('OAM_NETWORK',
                                            'IP_FLOATING_ADDRESS')
@@ -313,17 +312,17 @@ def validate_region_one_keystone_config(region_config, token, api_url, users,
         internal_address = region_config.get('MGMT_NETWORK',
                                              'IP_START_ADDRESS')
 
-    internal_infra_address = cutils.get_optional(
+    internal_infra_address = utils.get_optional(
         region_config, 'BLS_NETWORK', 'BLS_IP_START_ADDRESS')
     if not internal_infra_address:
-        internal_infra_address = cutils.get_optional(
+        internal_infra_address = utils.get_optional(
             region_config, 'INFRA_NETWORK', 'IP_START_ADDRESS')
 
     for endpoint in expected_region_2_endpoints:
-        service_name = cutils.get_service(region_config, 'REGION_2_SERVICES',
-                                          endpoint[SERVICE_NAME])
-        service_type = cutils.get_service(region_config, 'REGION_2_SERVICES',
-                                          endpoint[SERVICE_TYPE])
+        service_name = utils.get_service(region_config, 'REGION_2_SERVICES',
+                                         endpoint[SERVICE_NAME])
+        service_type = utils.get_service(region_config, 'REGION_2_SERVICES',
+                                         endpoint[SERVICE_TYPE])
         service_id = services.get_service_id(service_name, service_type)
 
         expected_public_url = endpoint[PUBLIC_URL].format(public_address)

controllerconfig/controllerconfig/controllerconfig/systemconfig.py

@@ -1,5 +1,5 @@
 """
-Copyright (c) 2015-2017 Wind River Systems, Inc.
+Copyright (c) 2015-2019 Wind River Systems, Inc.
 
 SPDX-License-Identifier: Apache-2.0
 
@@ -18,16 +18,16 @@ from controllerconfig.common.exceptions import BackupFail
 from controllerconfig.common.exceptions import RestoreFail
 from controllerconfig.common.exceptions import UserQuit
 from controllerconfig.common.exceptions import CloneFail
-from configutilities import lag_mode_to_str
-from configutilities import Network
-from configutilities import validate
-from configutilities import ConfigFail
-from configutilities import DEFAULT_CONFIG
-from configutilities import REGION_CONFIG
-from configutilities import SUBCLOUD_CONFIG
-from configutilities import MGMT_TYPE
-from configutilities import HP_NAMES
-from configutilities import DEFAULT_NAMES
+from controllerconfig import lag_mode_to_str
+from controllerconfig import Network
+from controllerconfig import validate
+from controllerconfig import ConfigFail
+from controllerconfig import DEFAULT_CONFIG
+from controllerconfig import REGION_CONFIG
+from controllerconfig import SUBCLOUD_CONFIG
+from controllerconfig import MGMT_TYPE
+from controllerconfig import HP_NAMES
+from controllerconfig import DEFAULT_NAMES
 from controllerconfig.configassistant import ConfigAssistant
 from controllerconfig import backup_restore
 from controllerconfig import utils

controllerconfig/controllerconfig/controllerconfig/tests/test_region_config.py

@@ -1,5 +1,5 @@
 """
-Copyright (c) 2014-2017 Wind River Systems, Inc.
+Copyright (c) 2014-2019 Wind River Systems, Inc.
 
 SPDX-License-Identifier: Apache-2.0
 
@@ -17,9 +17,9 @@ import pytest
 import shutil
 import sys
 
-import configutilities.common.exceptions as exceptions
-from configutilities import REGION_CONFIG
-from configutilities import validate
+import controllerconfig.common.exceptions as exceptions
+from controllerconfig import REGION_CONFIG
+from controllerconfig import validate
 import controllerconfig.common.keystone as keystone
 from controllerconfig.tests import test_answerfile
 

controllerconfig/controllerconfig/controllerconfig/tests/test_system_config.py

@@ -1,5 +1,5 @@
 """
-Copyright (c) 2014, 2017 Wind River Systems, Inc.
+Copyright (c) 2014-2019 Wind River Systems, Inc.
 
 SPDX-License-Identifier: Apache-2.0
 
@@ -11,9 +11,9 @@ import os
 import pytest
 import sys
 
-import configutilities.common.exceptions as exceptions
-from configutilities import validate
-from configutilities import DEFAULT_CONFIG
+import controllerconfig.common.exceptions as exceptions
+from controllerconfig import validate
+from controllerconfig import DEFAULT_CONFIG
 
 sys.modules['fm_core'] = mock.Mock()
 

controllerconfig/controllerconfig/controllerconfig/upgrades/utils.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2016 Wind River Systems, Inc.
+# Copyright (c) 2016-2019 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -21,7 +21,7 @@ import yaml
 from tsconfig.tsconfig import SW_VERSION
 from tsconfig.tsconfig import PLATFORM_PATH
 
-from configutilities import DEFAULT_DOMAIN_NAME
+from controllerconfig import DEFAULT_DOMAIN_NAME
 from controllerconfig import utils as cutils
 from controllerconfig.common import log
 from controllerconfig.common import constants

controllerconfig/controllerconfig/controllerconfig/utils.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2014-2018 Wind River Systems, Inc.
+# Copyright (c) 2014-2019 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -19,13 +19,17 @@ import time
 import sys
 import yaml
 
+from six.moves import configparser
+import re
+import six
+
 import netaddr
 from tsconfig import tsconfig
-from configutilities.common.utils import is_valid_mac
 from sysinv.common import constants as sysinv_constants
 
 from controllerconfig.common import constants
 from controllerconfig.common import log
+from controllerconfig.common.exceptions import ValidateFail
 
 LOOPBACK_IFNAME = 'lo'
 
@@ -40,6 +44,21 @@ LOG = log.get_logger(__name__)
 
 DEVNULL = open(os.devnull, 'w')
 
+EXPECTED_SERVICE_NAME_AND_TYPE = (
+    {"KEYSTONE_SERVICE_NAME": "keystone",
+     "KEYSTONE_SERVICE_TYPE": "identity",
+     "SYSINV_SERVICE_NAME": "sysinv",
+     "SYSINV_SERVICE_TYPE": "platform",
+     "PATCHING_SERVICE_NAME": "patching",
+     "PATCHING_SERVICE_TYPE": "patching",
+     "NFV_SERVICE_NAME": "vim",
+     "NFV_SERVICE_TYPE": "nfv",
+     "FM_SERVICE_NAME": "fm",
+     "FM_SERVICE_TYPE": "faultmanagement",
+     "BARBICAN_SERVICE_NAME": "barbican",
+     "BARBICAN_SERVICE_TYPE": "key-manager",
+     })
+
 
 def filesystem_get_free_space(path):
     """ Get Free space of directory """
@@ -283,21 +302,6 @@ def validate_and_normalize_mac(address):
     return address.lower()
 
 
-def is_valid_ipv4(address):
-    """Verify that address represents a valid IPv4 address."""
-    try:
-        return netaddr.valid_ipv4(address)
-    except Exception:
-        return False
-
-
-def is_valid_ipv6(address):
-    try:
-        return netaddr.valid_ipv6(address)
-    except Exception:
-        return False
-
-
 def is_valid_ip(address):
     if not is_valid_ipv4(address):
         return is_valid_ipv6(address)
@@ -895,3 +899,318 @@ def is_ssh_parent():
             return False
     except subprocess.CalledProcessError:
         return False
+
+
+def is_valid_vlan(vlan):
+    """Determine whether vlan is valid."""
+    try:
+        if 0 < int(vlan) < 4095:
+            return True
+        else:
+            return False
+    except (ValueError, TypeError):
+        return False
+
+
+def is_mtu_valid(mtu):
+    """Determine whether a mtu is valid."""
+    try:
+        if int(mtu) < 576:
+            return False
+        elif int(mtu) > 9216:
+            return False
+        else:
+            return True
+    except (ValueError, TypeError):
+        return False
+
+
+def is_valid_hostname(hostname):
+    """Determine whether a hostname is valid as per RFC 1123."""
+
+    # Maximum length of 255
+    if not hostname or len(hostname) > 255:
+        return False
+    # Allow a single dot on the right hand side
+    if hostname[-1] == ".":
+        hostname = hostname[:-1]
+    # Create a regex to ensure:
+    # - hostname does not begin or end with a dash
+    # - each segment is 1 to 63 characters long
+    # - valid characters are A-Z (any case) and 0-9
+    valid_re = re.compile("(?!-)[A-Z\d-]{1,63}(?<!-)$", re.IGNORECASE)  # noqa pylint: disable=anomalous-backslash-in-string
+    return all(valid_re.match(x) for x in hostname.split("."))
+
+
+def is_valid_mac(mac):
+    """Verify the format of a MAC addres."""
+    if not mac:
+        return False
+    m = "[0-9a-f]{2}([-:])[0-9a-f]{2}(\\1[0-9a-f]{2}){4}$"
+    return isinstance(mac, six.string_types) and re.match(m, mac.lower())
+
+
+def validate_network_str(network_str, minimum_size,
+                         existing_networks=None, multicast=False):
+    """Determine whether a network is valid."""
+    try:
+        network = netaddr.IPNetwork(network_str)
+        if network.ip != network.network:
+            raise ValidateFail("Invalid network address")
+        elif network.size < minimum_size:
+            raise ValidateFail("Subnet too small - must have at least %d "
+                               "addresses" % minimum_size)
+        elif network.version == 6 and network.prefixlen < 64:
+            raise ValidateFail("IPv6 minimum prefix length is 64")
+        elif existing_networks:
+            if any(network.ip in subnet for subnet in existing_networks):
+                raise ValidateFail("Subnet overlaps with another "
+                                   "configured subnet")
+        elif multicast and not network.is_multicast():
+            raise ValidateFail("Invalid subnet - must be multicast")
+        return network
+    except netaddr.AddrFormatError:
+        raise ValidateFail(
+            "Invalid subnet - not a valid IP subnet")
+
+
+def is_valid_filename(filename):
+    return '\0' not in filename
+
+
+def is_valid_by_path(filename):
+    return "/dev/disk/by-path" in filename and "-part" not in filename
+
+
+def is_valid_url(url_str):
+    # Django URL validation patterns
+    r = re.compile(
+        r'^(?:http|ftp)s?://'  # http:// or https://
+        r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)'  # domain...
+        r'+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|'
+        r'localhost|'  # localhost...
+        r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})'  # ...or ip
+        r'(?::\d+)?'  # optional port
+        r'(?:/?|[/?]\S+)$', re.IGNORECASE)
+
+    url = r.match(url_str)
+    if url:
+        return True
+    else:
+        return False
+
+
+def is_valid_domain(url_str):
+    r = re.compile(
+        r'^(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)'  # domain...
+        r'+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|'
+        r'[A-Za-z0-9-_]*)'  # localhost, hostname
+        r'(?::\d+)?'  # optional port
+        r'(?:/?|[/?]\S+)$', re.IGNORECASE)
+
+    url = r.match(url_str)
+    if url:
+        return True
+    else:
+        return False
+
+
+def is_valid_ipv4(address):
+    """Verify that address represents a valid IPv4 address."""
+    try:
+        return netaddr.valid_ipv4(address)
+    except Exception:
+        return False
+
+
+def is_valid_ipv6(address):
+    try:
+        return netaddr.valid_ipv6(address)
+    except Exception:
+        return False
+
+
+def is_valid_domain_or_ip(url_str):
+    if url_str:
+        if is_valid_domain(url_str):
+            return True
+        ip_with_port = url_str.split(':')
+        if len(ip_with_port) <= 2:
+            # check ipv4 or ipv4 with port
+            return is_valid_ipv4(ip_with_port[0])
+        else:
+            # check ipv6 with port
+            if '[' in url_str:
+                try:
+                    bkt_idx = url_str.index(']')
+                    if bkt_idx + 1 == len(url_str):
+                        # brackets without port
+                        return False
+                    else:
+                        return is_valid_ipv6(url_str[1:bkt_idx])
+                except Exception:
+                    return False
+            else:
+                # check ipv6 without port
+                return is_valid_ipv6(url_str)
+    else:
+        return False
+
+
+def is_valid_bool_str(val):
+    """Check if the provided string is a valid bool string or not."""
+    boolstrs = ('true', 'false')
+    return str(val).lower() in boolstrs
+
+
+def validate_address_str(ip_address_str, network):
+    """Determine whether an address is valid."""
+    try:
+        ip_address = netaddr.IPAddress(ip_address_str)
+        if ip_address.version != network.version:
+            msg = ("Invalid IP version - must match network version " +
+                   ip_version_to_string(network.version))
+            raise ValidateFail(msg)
+        elif ip_address == network:
+            raise ValidateFail("Cannot use network address")
+        elif ip_address == network.broadcast:
+            raise ValidateFail("Cannot use broadcast address")
+        elif ip_address not in network:
+            raise ValidateFail(
+                "Address must be in subnet %s" % str(network))
+        return ip_address
+    except netaddr.AddrFormatError:
+        raise ValidateFail(
+            "Invalid address - not a valid IP address")
+
+
+def ip_version_to_string(ip_version):
+    """Determine whether a nameserver address is valid."""
+    if ip_version == 4:
+        return "IPv4"
+    elif ip_version == 6:
+        return "IPv6"
+    else:
+        return "IP"
+
+
+def validate_nameserver_address_str(ip_address_str, subnet_version=None):
+    """Determine whether a nameserver address is valid."""
+    try:
+        ip_address = netaddr.IPAddress(ip_address_str)
+        if subnet_version is not None and ip_address.version != subnet_version:
+            msg = ("Invalid IP version - must match OAM subnet version " +
+                   ip_version_to_string(subnet_version))
+            raise ValidateFail(msg)
+        return ip_address
+    except netaddr.AddrFormatError:
+        msg = "Invalid address - not a valid %s address" % \
+              ip_version_to_string(subnet_version)
+        raise ValidateFail(msg)
+
+
+def validate_address(ip_address, network):
+    """Determine whether an address is valid."""
+    if ip_address.version != network.version:
+            msg = ("Invalid IP version - must match network version " +
+                   ip_version_to_string(network.version))
+            raise ValidateFail(msg)
+    elif ip_address == network:
+        raise ValidateFail("Cannot use network address")
+    elif ip_address == network.broadcast:
+        raise ValidateFail("Cannot use broadcast address")
+    elif ip_address not in network:
+        raise ValidateFail("Address must be in subnet %s" % str(network))
+
+
+def check_network_overlap(new_network, configured_networks):
+    """ Validate that new_network does not overlap any configured_networks.
+    """
+    if any(new_network.ip in subnet for subnet in
+           configured_networks):
+        raise ValidateFail(
+            "Subnet %s overlaps with another configured subnet" % new_network)
+
+
+def validate_openstack_password(password, rules_file,
+                                section="security_compliance"):
+    try:
+        config = configparser.RawConfigParser()
+        parsed_config = config.read(rules_file)
+        if not parsed_config:
+            msg = ("Cannot parse rules file: %s" % rules_file)
+            raise Exception(msg)
+        if not config.has_section(section):
+            msg = ("Required section '%s' not found in rules file" % section)
+            raise Exception(msg)
+
+        password_regex = get_optional(config, section, 'password_regex')
+        password_regex_description = get_optional(config, section,
+                                                  'password_regex_description')
+
+        if not password_regex:
+            msg = ("Required option 'password_regex' not found in "
+                   "rule file: %s" % rules_file)
+            raise Exception(msg)
+        # Even if regex_description is not found, we will proceed
+        # and give a generic failure warning instead
+        if not password_regex_description:
+            password_regex_description = ("Password does not meet "
+                                          "complexity criteria")
+
+        if not isinstance(password, six.string_types):
+            msg = "Password must be a string type"
+            raise Exception(msg)
+        try:
+            # config parser would read in the string as a literal
+            # representation which would fail regex matching
+            password_regex = password_regex.strip('"')
+            if not re.match(password_regex, password):
+                return False, password_regex_description
+        except re.error:
+            msg = ("Unable to validate password due to invalid "
+                   "complexity criteria ('password_regex')")
+            raise Exception(msg)
+    except Exception:
+        raise Exception("Password validation failed")
+    return True, ""
+
+
+def extract_openstack_password_rules_from_file(
+        rules_file, section="security_compliance"):
+    try:
+        config = configparser.RawConfigParser()
+        parsed_config = config.read(rules_file)
+        if not parsed_config:
+            msg = ("Cannot parse rules file: %s" % rules_file)
+            raise Exception(msg)
+        if not config.has_section(section):
+            msg = ("Required section '%s' not found in rules file" % section)
+            raise Exception(msg)
+
+        rules = config.items(section)
+        if not rules:
+            msg = ("section '%s' contains no configuration options" % section)
+            raise Exception(msg)
+        return dict(rules)
+    except Exception:
+        raise Exception("Failed to extract password rules from file")
+
+
+def get_optional(conf, section, key):
+    if conf.has_option(section, key):
+        return conf.get(section, key)
+    return None
+
+
+def get_service(conf, section, key):
+    if key in EXPECTED_SERVICE_NAME_AND_TYPE:
+        if conf.has_option(section, key):
+            value = conf.get(section, key)
+            if value != EXPECTED_SERVICE_NAME_AND_TYPE[key]:
+                raise ValidateFail("Unsupported %s: %s " % (key, value))
+        else:
+            value = EXPECTED_SERVICE_NAME_AND_TYPE[key]
+        return value
+    else:
+        return conf.get(section, key)

controllerconfig/controllerconfig/tox.ini

@@ -14,7 +14,6 @@ whitelist_externals = find
 install_command = pip install  --no-cache-dir  -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=stable/pike} {opts} {packages}
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
-       -e{[tox]stxdir}/stx-config/configutilities/configutilities
        -e{[tox]stxdir}/stx-fault/fm-api
        -e{[tox]stxdir}/stx-update/tsconfig/tsconfig
        -e{[tox]stxdir}/stx-config/sysinv/sysinv/sysinv