Add package-upgrade action

The package-upgrade action performs package upgrades for the current
OpenStack release.

The code path used is similar to the openstack-upgrade action, with the
difference being that package-upgrade will not execute if an openstack
upgrade is available (based on the openstack-origin setting).

This change includes a charm-helpers sync.

Change-Id: Ifd99ea307a6e4d1d034d7c1e494e2cd8abd894e9

actions.yaml

@@ -39,6 +39,9 @@ openstack-upgrade:
   description: |
     Perform openstack upgrades. Config option action-managed-upgrade must be
     set to True.
+package-upgrade:
+  description: |
+    Perform package upgrades for the current OpenStack release.
 security-checklist:
   description: |
     Validate the running configuration against the OpenStack security guides

actions/package-upgrade

@@ -0,0 +1 @@
+package_upgrade.py

actions/package_upgrade.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+#
+# Copyright 2022 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+
+_path = os.path.dirname(os.path.realpath(__file__))
+_hooks = os.path.abspath(os.path.join(_path, '../hooks'))
+_root = os.path.abspath(os.path.join(_path, '..'))
+
+
+def _add_path(path):
+    if path not in sys.path:
+        sys.path.insert(1, path)
+
+
+_add_path(_hooks)
+_add_path(_root)
+
+from charmhelpers.contrib.openstack.utils import (
+    do_action_package_upgrade,
+)
+
+from keystone_utils import (
+    do_openstack_upgrade,
+    register_configs,
+)
+
+
+def package_upgrade():
+    """Perform package upgrade within the current OpenStack release.
+
+    In order to prevent this action from upgrading to a new release of
+    OpenStack, package upgrades are not run if a new OpenStack release is
+    available. See source of do_action_package_upgrade() for this check.
+
+    Upgrades packages and sets the corresponding action status as a result."""
+
+    if (do_action_package_upgrade('keystone',
+                                  do_openstack_upgrade,
+                                  register_configs())):
+        os.execl('./hooks/config-changed-postupgrade',
+                 'config-changed-postupgrade')
+
+
+if __name__ == '__main__':
+    package_upgrade()

charmhelpers/contrib/storage/linux/ceph.py

@@ -28,7 +28,6 @@ import os
 import shutil
 import json
 import time
-import uuid
 
 from subprocess import (
     check_call,
@@ -1677,6 +1676,10 @@ class CephBrokerRq(object):
     The API is versioned and defaults to version 1.
     """
 
+    # The below hash is the result of running
+    # `hashlib.sha1('[]'.encode()).hexdigest()`
+    EMPTY_LIST_SHA = '97d170e1550eee4afc0af065b78cda302a97674c'
+
     def __init__(self, api_version=1, request_id=None, raw_request_data=None):
         """Initialize CephBrokerRq object.
 
@@ -1685,8 +1688,12 @@ class CephBrokerRq(object):
 
         :param api_version: API version for request (default: 1).
         :type api_version: Optional[int]
-        :param request_id: Unique identifier for request.
+        :param request_id: Unique identifier for request. The identifier will
-                           (default: string representation of generated UUID)
+                           be updated as ops are added or removed from the
+                           broker request. This ensures that Ceph will
+                           correctly process requests where operations are
+                           added after the initial request is processed.
+                           (default: sha1 of operations)
         :type request_id: Optional[str]
         :param raw_request_data: JSON-encoded string to build request from.
         :type raw_request_data: Optional[str]
@@ -1695,16 +1702,20 @@ class CephBrokerRq(object):
         if raw_request_data:
             request_data = json.loads(raw_request_data)
             self.api_version = request_data['api-version']
-            self.request_id = request_data['request-id']
             self.set_ops(request_data['ops'])
+            self.request_id = request_data['request-id']
         else:
             self.api_version = api_version
             if request_id:
                 self.request_id = request_id
             else:
-                self.request_id = str(uuid.uuid1())
+                self.request_id = CephBrokerRq.EMPTY_LIST_SHA
             self.ops = []
 
+    def _hash_ops(self):
+        """Return the sha1 of the requested Broker ops."""
+        return hashlib.sha1(json.dumps(self.ops, sort_keys=True).encode()).hexdigest()
+
     def add_op(self, op):
         """Add an op if it is not already in the list.
 
@@ -1713,6 +1724,7 @@ class CephBrokerRq(object):
         """
         if op not in self.ops:
             self.ops.append(op)
+            self.request_id = self._hash_ops()
 
     def add_op_request_access_to_group(self, name, namespace=None,
                                        permission=None, key_name=None,
@@ -1991,6 +2003,7 @@ class CephBrokerRq(object):
         to allow comparisons to ensure validity.
         """
         self.ops = ops
+        self.request_id = self._hash_ops()
 
     @property
     def request(self):

charmhelpers/fetch/ubuntu.py

@@ -591,7 +591,7 @@ def _get_key_by_keyid(keyid):
     curl_cmd = ['curl', keyserver_url.format(keyid)]
     # use proxy server settings in order to retrieve the key
     return subprocess.check_output(curl_cmd,
-                                   env=env_proxy_settings(['https']))
+                                   env=env_proxy_settings(['https', 'no_proxy']))
 
 
 def _dearmor_gpg_key(key_asc):

charmhelpers/fetch/ubuntu_apt_pkg.py

@@ -122,13 +122,12 @@ class Cache(object):
         :raises: subprocess.CalledProcessError
         """
         pkgs = {}
-        cmd = ['dpkg-query', '--list']
+        cmd = [
+            'dpkg-query', '--show',
+            '--showformat',
+            r'${db:Status-Abbrev}\t${Package}\t${Version}\t${Architecture}\t${binary:Summary}\n'
+        ]
         cmd.extend(packages)
-        if locale.getlocale() == (None, None):
-            # subprocess calls out to locale.getpreferredencoding(False) to
-            # determine encoding.  Workaround for Trusty where the
-            # environment appears to not be set up correctly.
-            locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
         try:
             output = subprocess.check_output(cmd,
                                              stderr=subprocess.STDOUT,
@@ -140,24 +139,17 @@ class Cache(object):
             if cp.returncode != 1:
                 raise
             output = cp.output
-        headings = []
         for line in output.splitlines():
-            if line.startswith('||/'):
+            # only process lines for successfully installed packages
-                headings = line.split()
+            if not (line.startswith('ii ') or line.startswith('hi ')):
-                headings.pop(0)
                 continue
-            elif (line.startswith('|') or line.startswith('+') or
+            status, name, version, arch, desc = line.split('\t', 4)
-                  line.startswith('dpkg-query:')):
+            pkgs[name] = {
-                continue
+                'name': name,
-            else:
+                'version': version,
-                data = line.split(None, 4)
+                'architecture': arch,
-                status = data.pop(0)
+                'description': desc,
-                if status not in ('ii', 'hi'):
+            }
-                    continue
-                pkg = {}
-                pkg.update({k.lower(): v for k, v in zip(headings, data)})
-                if 'name' in pkg:
-                    pkgs.update({pkg['name']: pkg})
         return pkgs
 
     def _apt_cache_show(self, packages):

unit_tests/test_actions_package_upgrade.py

@@ -0,0 +1,69 @@
+# Copyright 2022 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from unittest.mock import patch
+import os
+
+os.environ['JUJU_UNIT_NAME'] = 'keystone'
+
+with patch('charmhelpers.contrib.openstack.utils'
+           '.snap_install_requested') as snap_install_requested:
+    snap_install_requested.return_value = False
+    import package_upgrade as package_upgrade
+
+from test_utils import (
+    CharmTestCase
+)
+
+TO_PATCH = [
+    'do_openstack_upgrade',
+    'os',
+]
+
+
+class TestKeystoneUpgradeActions(CharmTestCase):
+
+    def setUp(self):
+        super(TestKeystoneUpgradeActions, self).setUp(package_upgrade,
+                                                      TO_PATCH)
+
+    # NOTE(ajkavangh) patching charmhelpers here almost certainly means that
+    # these tests are in the wrong place and should be moved.  In general
+    # tests should only patch objects IN the file under test.  Anywhere else
+    # creates dependencies that make the code harder to maintain (e.g. here,
+    # changes to charmhelpers might break these tests).
+    @patch.object(package_upgrade, 'register_configs')
+    @patch('charmhelpers.contrib.openstack.utils.action_set')
+    @patch('charmhelpers.contrib.openstack.utils.openstack_upgrade_available')
+    def test_package_upgrade_success(self, upgrade_avail,
+                                     action_set, reg_config):
+        upgrade_avail.return_value = False
+
+        package_upgrade.package_upgrade()
+
+        self.assertTrue(self.do_openstack_upgrade.called)
+        self.os.execl.assert_called_with('./hooks/config-changed-postupgrade',
+                                         'config-changed-postupgrade')
+
+    @patch.object(package_upgrade, 'register_configs')
+    @patch('charmhelpers.contrib.openstack.utils.action_set')
+    @patch('charmhelpers.contrib.openstack.utils.openstack_upgrade_available')
+    def test_package_upgrade_fail(self, upgrade_avail,
+                                  action_set, reg_configs):
+        upgrade_avail.return_value = True
+
+        package_upgrade.package_upgrade()
+
+        self.assertFalse(self.do_openstack_upgrade.called)
+        self.assertFalse(self.os.execl.called)