Merge "Sync/rebuild for Dalmatian/Epoxy updates"

charmhelpers/contrib/network/ip.py

@@ -16,6 +16,7 @@ import glob
 import re
 import subprocess
 import socket
+import ssl
 
 from functools import partial
 
@@ -527,19 +528,56 @@ def get_hostname(address, fqdn=True):
         return result.split('.')[0]
 
 
-def port_has_listener(address, port):
+class SSLPortCheckInfo(object):
+
+    def __init__(self, key, cert, ca_cert, check_hostname=False):
+        self.key = key
+        self.cert = cert
+        self.ca_cert = ca_cert
+        # NOTE: by default we do not check hostname since the port check is
+        #       typically performed using 0.0.0.0 which will not match the
+        #       certificate. Hence the default for this is False.
+        self.check_hostname = check_hostname
+
+    @property
+    def ssl_context(self):
+        context = ssl.create_default_context()
+        context.check_hostname = self.check_hostname
+        context.load_cert_chain(self.cert, self.key)
+        context.load_verify_locations(self.ca_cert)
+        return context
+
+
+def port_has_listener(address, port, sslinfo=None):
     """
     Returns True if the address:port is open and being listened to,
-    else False.
+    else False. By default uses netcat to check ports but if sslinfo is
+    provided will use an SSL connection instead.
 
     @param address: an IP address or hostname
     @param port: integer port
+    @param sslinfo: optional SSLPortCheckInfo object.
+                    If provided, the check is performed using an ssl
+                    connection.
 
     Note calls 'zc' via a subprocess shell
     """
-    cmd = ['nc', '-z', address, str(port)]
-    result = subprocess.call(cmd)
-    return not (bool(result))
+    if not sslinfo:
+        cmd = ['nc', '-z', address, str(port)]
+        result = subprocess.call(cmd)
+        return not (bool(result))
+
+    try:
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) as sock:
+            ssock = sslinfo.ssl_context.wrap_socket(sock,
+                                                    server_hostname=address)
+            ssock.connect((address, port))
+            # this bit is crucial to ensure tls close_notify is sent
+            ssock.unwrap()
+
+        return True
+    except ConnectionRefusedError:
+        return False
 
 
 def assert_charm_supports_ipv6():

charmhelpers/contrib/openstack/utils.py

@@ -161,6 +161,9 @@ OPENSTACK_CODENAMES = OrderedDict([
     ('2022.2', 'zed'),
     ('2023.1', 'antelope'),
     ('2023.2', 'bobcat'),
+    ('2024.1', 'caracal'),
+    ('2024.2', 'dalmatian'),
+    ('2025.1', 'epoxy'),
 ])
 
 # The ugly duckling - must list releases oldest to newest
@@ -416,17 +419,6 @@ def get_os_version_codename(codename, version_map=OPENSTACK_CODENAMES,
     error_out(e)
 
 
-def get_os_version_codename_swift(codename):
-    '''Determine OpenStack version number of swift from codename.'''
-    # for k, v in six.iteritems(SWIFT_CODENAMES):
-    for k, v in SWIFT_CODENAMES.items():
-        if k == codename:
-            return v[-1]
-    e = 'Could not derive swift version for '\
-        'codename: %s' % codename
-    error_out(e)
-
-
 def get_swift_codename(version):
     '''Determine OpenStack codename that corresponds to swift version.'''
     codenames = [k for k, v in SWIFT_CODENAMES.items() if version in v]
@@ -585,7 +577,6 @@ def get_installed_os_version():
     return openstack_release().get('OPENSTACK_CODENAME')
 
 
-@cached
 def openstack_release():
     """Return /etc/os-release in a dict."""
     d = {}
@@ -847,14 +838,10 @@ def openstack_upgrade_available(package):
     if not cur_vers:
         # The package has not been installed yet do not attempt upgrade
         return False
-    if "swift" in package:
-        codename = get_os_codename_install_source(src)
-        avail_vers = get_os_version_codename_swift(codename)
-    else:
-        try:
-            avail_vers = get_os_version_install_source(src)
-        except Exception:
-            avail_vers = cur_vers
+    try:
+        avail_vers = get_os_version_install_source(src)
+    except Exception:
+        avail_vers = cur_vers
     apt.init()
     return apt.version_compare(avail_vers, cur_vers) >= 1
 
@@ -1222,12 +1209,14 @@ def _ows_check_services_running(services, ports):
     return ows_check_services_running(services, ports)
 
 
-def ows_check_services_running(services, ports):
+def ows_check_services_running(services, ports, ssl_check_info=None):
     """Check that the services that should be running are actually running
     and that any ports specified are being listened to.
 
     @param services: list of strings OR dictionary specifying services/ports
     @param ports: list of ports
+    @param ssl_check_info: SSLPortCheckInfo object. If provided, port checks
+                           will be done using an SSL connection.
     @returns state, message: strings or None, None
     """
     messages = []
@@ -1243,7 +1232,7 @@ def ows_check_services_running(services, ports):
         # also verify that the ports that should be open are open
         # NB, that ServiceManager objects only OPTIONALLY have ports
         map_not_open, ports_open = (
-            _check_listening_on_services_ports(services))
+            _check_listening_on_services_ports(services, ssl_check_info))
         if not all(ports_open):
             # find which service has missing ports. They are in service
             # order which makes it a bit easier.
@@ -1258,7 +1247,8 @@ def ows_check_services_running(services, ports):
 
     if ports is not None:
         # and we can also check ports which we don't know the service for
-        ports_open, ports_open_bools = _check_listening_on_ports_list(ports)
+        ports_open, ports_open_bools = \
+            _check_listening_on_ports_list(ports, ssl_check_info)
         if not all(ports_open_bools):
             messages.append(
                 "Ports which should be open, but are not: {}"
@@ -1317,7 +1307,8 @@ def _check_running_services(services):
     return list(zip(services, services_running)), services_running
 
 
-def _check_listening_on_services_ports(services, test=False):
+def _check_listening_on_services_ports(services, test=False,
+                                       ssl_check_info=None):
     """Check that the unit is actually listening (has the port open) on the
     ports that the service specifies are open. If test is True then the
     function returns the services with ports that are open rather than
@@ -1327,11 +1318,14 @@ def _check_listening_on_services_ports(services, test=False):
 
     @param services: OrderedDict(service: [port, ...], ...)
     @param test: default=False, if False, test for closed, otherwise open.
+    @param ssl_check_info: SSLPortCheckInfo object. If provided, port checks
+                           will be done using an SSL connection.
     @returns OrderedDict(service: [port-not-open, ...]...), [boolean]
     """
     test = not (not (test))  # ensure test is True or False
     all_ports = list(itertools.chain(*services.values()))
-    ports_states = [port_has_listener('0.0.0.0', p) for p in all_ports]
+    ports_states = [port_has_listener('0.0.0.0', p, ssl_check_info)
+                    for p in all_ports]
     map_ports = OrderedDict()
     matched_ports = [p for p, opened in zip(all_ports, ports_states)
                      if opened == test]  # essentially opened xor test
@@ -1342,16 +1336,19 @@ def _check_listening_on_services_ports(services, test=False):
     return map_ports, ports_states
 
 
-def _check_listening_on_ports_list(ports):
+def _check_listening_on_ports_list(ports, ssl_check_info=None):
     """Check that the ports list given are being listened to
 
     Returns a list of ports being listened to and a list of the
     booleans.
 
+    @param ssl_check_info: SSLPortCheckInfo object. If provided, port checks
+                           will be done using an SSL connection.
     @param ports: LIST of port numbers.
     @returns [(port_num, boolean), ...], [boolean]
     """
-    ports_open = [port_has_listener('0.0.0.0', p) for p in ports]
+    ports_open = [port_has_listener('0.0.0.0', p, ssl_check_info)
+                  for p in ports]
     return zip(ports, ports_open), ports_open
 
 

charmhelpers/contrib/storage/linux/ceph.py

@@ -158,15 +158,19 @@ def get_osd_settings(relation_name):
     return _order_dict_by_key(osd_settings)
 
 
-def send_application_name(relid=None):
+def send_application_name(relid=None, app_name=None):
     """Send the application name down the relation.
 
     :param relid: Relation id to set application name in.
     :type relid: str
+    :param app_name: Application name to send in the relation.
+    :type app_name: str
     """
+    if app_name is None:
+        app_name = application_name()
     relation_set(
         relation_id=relid,
-        relation_settings={'application-name': application_name()})
+        relation_settings={'application-name': app_name})
 
 
 def send_osd_settings():

charmhelpers/contrib/storage/linux/lvm.py

@@ -17,8 +17,6 @@ from subprocess import (
     CalledProcessError,
     check_call,
     check_output,
-    Popen,
-    PIPE,
 )
 
 
@@ -58,9 +56,7 @@ def remove_lvm_physical_volume(block_device):
 
     :param block_device: str: Full path of block device to scrub.
     '''
-    p = Popen(['pvremove', '-ff', block_device],
-              stdin=PIPE)
-    p.communicate(input='y\n')
+    check_call(['pvremove', '-ff', '--yes', block_device])
 
 
 def list_lvm_volume_group(block_device):

charmhelpers/core/hookenv.py

@@ -18,7 +18,10 @@
 #  Charm Helpers Developers <juju@lists.ubuntu.com>
 
 import copy
-from distutils.version import LooseVersion
+try:
+    from distutils.version import LooseVersion
+except ImportError:
+    from looseversion import LooseVersion
 from enum import Enum
 from functools import wraps
 from collections import namedtuple, UserDict

charmhelpers/core/host.py

@@ -256,8 +256,11 @@ def service_resume(service_name, init_dir="/etc/init",
     upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
     sysv_file = os.path.join(initd_dir, service_name)
     if init_is_systemd(service_name=service_name):
-        service('unmask', service_name)
-        service('enable', service_name)
+        if service('is-enabled', service_name):
+            log('service {} already enabled'.format(service_name), level=DEBUG)
+        else:
+            service('unmask', service_name)
+            service('enable', service_name)
     elif os.path.exists(upstart_file):
         override_path = os.path.join(
             init_dir, '{}.override'.format(service_name))

charmhelpers/core/unitdata.py

@@ -151,6 +151,7 @@ import contextlib
 import datetime
 import itertools
 import json
+import logging
 import os
 import pprint
 import sqlite3
@@ -521,6 +522,42 @@ _KV = None
 
 def kv():
     global _KV
+
+    # If we are running unit tests, it is useful to go into memory-backed KV store to
+    # avoid concurrency issues when running multiple tests. This is not a
+    # problem when juju is running normally.
+
+    env_var = os.environ.get("CHARM_HELPERS_TESTMODE", "auto").lower()
+    if env_var not in ["auto", "no", "yes"]:
+        logging.warning("Unknown value for CHARM_HELPERS_TESTMODE '%s'"
+                        ", assuming 'no'", env_var)
+        env_var = "no"
+
+    if env_var == "no":
+        in_memory_db = False
+    elif env_var == "yes":
+        in_memory_db = True
+    elif env_var == "auto":
+        # If UNIT_STATE_DB is set, respect this request
+        if "UNIT_STATE_DB" in os.environ:
+            in_memory_db = False
+        # Autodetect normal juju execution by looking for juju variables
+        elif "JUJU_CHARM_DIR" in os.environ or "JUJU_UNIT_NAME" in os.environ:
+            in_memory_db = False
+        else:
+            # We are probably running in unit test mode
+            logging.warning("Auto-detected unit test environment for KV store.")
+            in_memory_db = True
+    else:
+        # Help the linter realise that in_memory_db is always set
+        raise Exception("Cannot reach this line")
+
     if _KV is None:
-        _KV = Storage()
+        if in_memory_db:
+            _KV = Storage(":memory:")
+        else:
+            _KV = Storage()
+    else:
+        if in_memory_db and _KV.db_path != ":memory:":
+            logging.warning("Running with in_memory_db and KV is not set to :memory:")
     return _KV

charmhelpers/fetch/snap.py

@@ -52,7 +52,7 @@ def _snap_exec(commands):
     :param commands: List commands
     :return: Integer exit code
     """
-    assert type(commands) == list
+    assert isinstance(commands, list)
 
     retry_count = 0
     return_code = None

charmhelpers/fetch/ubuntu.py

@@ -246,6 +246,30 @@ CLOUD_ARCHIVE_POCKETS = {
     'bobcat/proposed': 'jammy-proposed/bobcat',
     'jammy-bobcat/proposed': 'jammy-proposed/bobcat',
     'jammy-proposed/bobcat': 'jammy-proposed/bobcat',
+    # caracal
+    'caracal': 'jammy-updates/caracal',
+    'jammy-caracal': 'jammy-updates/caracal',
+    'jammy-caracal/updates': 'jammy-updates/caracal',
+    'jammy-updates/caracal': 'jammy-updates/caracal',
+    'caracal/proposed': 'jammy-proposed/caracal',
+    'jammy-caracal/proposed': 'jammy-proposed/caracal',
+    'jammy-proposed/caracal': 'jammy-proposed/caracal',
+    # dalmatian
+    'dalmatian': 'noble-updates/dalmatian',
+    'noble-dalmatian': 'noble-updates/dalmatian',
+    'noble-dalmatian/updates': 'noble-updates/dalmatian',
+    'noble-updates/dalmatian': 'noble-updates/dalmatian',
+    'dalmatian/proposed': 'noble-proposed/dalmatian',
+    'noble-dalmatian/proposed': 'noble-proposed/dalmatian',
+    'noble-proposed/dalmatian': 'noble-proposed/dalmatian',
+    # epoxy
+    'epoxy': 'noble-updates/epoxy',
+    'noble-epoxy': 'noble-updates/epoxy',
+    'noble-epoxy/updates': 'noble-updates/epoxy',
+    'noble-updates/epoxy': 'noble-updates/epoxy',
+    'epoxy/proposed': 'noble-proposed/epoxy',
+    'noble-epoxy/proposed': 'noble-proposed/epoxy',
+    'noble-proposed/epoxy': 'noble-proposed/epoxy',
 
     # OVN
     'focal-ovn-22.03': 'focal-updates/ovn-22.03',
@@ -279,6 +303,9 @@ OPENSTACK_RELEASES = (
     'zed',
     'antelope',
     'bobcat',
+    'caracal',
+    'dalmatian',
+    'epoxy',
 )
 
 
@@ -308,6 +335,9 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
     ('kinetic', 'zed'),
     ('lunar', 'antelope'),
     ('mantic', 'bobcat'),
+    ('noble', 'caracal'),
+    ('oracular', 'dalmatian'),
+    ('plucky', 'epoxy'),
 ])
 
 

charmhelpers/osplatform.py

@@ -9,19 +9,13 @@ def get_platform():
     will be returned (which is the name of the module).
     This string is used to decide which platform module should be imported.
     """
-    # linux_distribution is deprecated and will be removed in Python 3.7
-    # Warnings *not* disabled, as we certainly need to fix this.
-    if hasattr(platform, 'linux_distribution'):
-        tuple_platform = platform.linux_distribution()
-        current_platform = tuple_platform[0]
-    else:
-        current_platform = _get_platform_from_fs()
+    current_platform = _get_current_platform()
 
     if "Ubuntu" in current_platform:
         return "ubuntu"
     elif "CentOS" in current_platform:
         return "centos"
-    elif "debian" in current_platform:
+    elif "debian" in current_platform or "Debian" in current_platform:
         # Stock Python does not detect Ubuntu and instead returns debian.
         # Or at least it does in some build environments like Travis CI
         return "ubuntu"
@@ -36,6 +30,24 @@ def get_platform():
                            .format(current_platform))
 
 
+def _get_current_platform():
+    """Return the current platform information for the OS.
+
+    Attempts to lookup linux distribution information from the platform
+    module for releases of python < 3.7. For newer versions of python,
+    the platform is determined from the /etc/os-release file.
+    """
+    # linux_distribution is deprecated and will be removed in Python 3.7
+    # Warnings *not* disabled, as we certainly need to fix this.
+    if hasattr(platform, 'linux_distribution'):
+        tuple_platform = platform.linux_distribution()
+        current_platform = tuple_platform[0]
+    else:
+        current_platform = _get_platform_from_fs()
+
+    return current_platform
+
+
 def _get_platform_from_fs():
     """Get Platform from /etc/os-release."""
     with open(os.path.join(os.sep, 'etc', 'os-release')) as fin: