Add package-upgrade action

The package-upgrade action is similar to the openstack-upgrade action
except that it performs package upgrades within the current openstack
release. If a new openstack release is available, the package-upgrade
action will not perform any upgrades.

Sync common files from release-tools, with modifications specific
to this layer.

Change-Id: I4eaee81e05d44d9649c37d397672d263bc2a1025

actions.yaml

@@ -2,3 +2,6 @@ openstack-upgrade:
   description: |
     Perform openstack upgrades. Config option action-managed-upgrade must be
     set to True.
+package-upgrade:
+  description: |
+    Perform package upgrades for the current OpenStack release.

actions/os_principle_actions.py

@@ -31,9 +31,22 @@ charms_openstack.bus.discover()
 
 
 def openstack_upgrade_action(*args):
-    """Run the resume action."""
+    """Run the openstack-upgrade action."""
     with charms_openstack.charm.provide_charm_instance() as charm_instance:
-        charm_instance.run_upgrade()
+        charm_instance.run_upgrade(upgrade_openstack=True)
+        charm_instance._assess_status()
+
+
+def package_upgrade_action(*args):
+    """Run the package-upgrade action."""
+    with charms_openstack.charm.provide_charm_instance() as charm_instance:
+        if charm_instance.openstack_upgrade_available(
+                charm_instance.release_pkg):
+            hookenv.action_set({'outcome':
+                                'upgrade skipped because an openstack upgrade '
+                                'is available'})
+            return
+        charm_instance.run_upgrade(upgrade_openstack=False)
         charm_instance._assess_status()
 
 
@@ -41,6 +54,7 @@ def openstack_upgrade_action(*args):
 # can map to a python function.
 ACTIONS = {
     "openstack-upgrade": openstack_upgrade_action,
+    "package-upgrade": package_upgrade_action,
 }
 
 

actions/package-upgrade

@@ -0,0 +1 @@
+os_principle_actions.py

pip.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+#
+# This file is managed centrally by release-tools and should not be modified
+# within individual charm repos.  See the 'global' dir contents for available
+# choices of tox.ini for OpenStack Charms:
+#     https://github.com/openstack-charmers/release-tools
+#
+# setuptools 58.0 dropped the support for use_2to3=true which is needed to
+# install blessings (an indirect dependency of charm-tools).
+#
+# More details on the beahvior of tox and virtualenv creation can be found at
+# https://github.com/tox-dev/tox/issues/448
+#
+# This script is wrapper to force the use of the pinned versions early in the
+# process when the virtualenv was created and upgraded before installing the
+# depedencies declared in the target.
+pip install 'pip<20.3' 'setuptools<50.0.0'
+pip "$@"

requirements.txt

@@ -9,11 +9,8 @@
 setuptools<50.0.0  # https://github.com/pypa/setuptools/commit/04e3df22df840c6bb244e9b27bc56750c44b7c85
 
 # Build requirements
-charm-tools>=2.4.4
-
-# Workaround until https://github.com/juju/charm-tools/pull/589 gets
-# published
-keyring<21
+cffi==1.14.6; python_version < '3.6'  # cffi 1.15.0 drops support for py35.
+charm-tools==2.8.3
 
 simplejson
 

test-requirements.txt

@@ -3,9 +3,9 @@
 # choices of *requirements.txt files for OpenStack Charms:
 #     https://github.com/openstack-charmers/release-tools
 #
+pyparsing<3.0.0  # aodhclient is pinned in zaza and needs pyparsing < 3.0.0, but cffi also needs it, so pin here.
+cffi==1.14.6; python_version < '3.6'  # cffi 1.15.0 drops support for py35.
 setuptools<50.0.0  # https://github.com/pypa/setuptools/commit/04e3df22df840c6bb244e9b27bc56750c44b7c85
-# Lint and unit test requirements
-flake8>=2.2.4
 
 stestr>=2.2.0
 
@@ -28,20 +28,22 @@ oslo.utils<=3.41.0;python_version<'3.6'
 requests>=2.18.4
 charms.reactive
 
-# Newer mock seems to have some syntax which is newer than python3.5 (e.g.
-# f'{something}'
-mock>=1.2,<4.0.0; python_version < '3.6'
-mock>=1.2; python_version >= '3.6'
-
 nose>=1.3.7
 coverage>=3.6
 git+https://github.com/openstack/charms.openstack.git#egg=charms.openstack
 #
 # Revisit for removal / mock improvement:
+#
+# NOTE(lourot): newer versions of cryptography require a Rust compiler to build,
+# see
+# * https://github.com/openstack-charmers/zaza/issues/421
+# * https://mail.python.org/pipermail/cryptography-dev/2021-January/001003.html
+#
 netifaces        # vault
 psycopg2-binary  # vault
 tenacity         # vault
-pbr              # vault
-cryptography     # vault, keystone-saml-mellon
+pbr==5.6.0       # vault
+cryptography<3.4 # vault, keystone-saml-mellon
 lxml             # keystone-saml-mellon
 hvac             # vault, barbican-vault
+psutil           # cinder-lvm

tox.ini

@@ -1,65 +1,129 @@
+# Source charm: ./tox.ini
+# This file is managed centrally by release-tools and should not be modified
+# within individual charm repos.  See the 'global' dir contents for available
+# choices of tox.ini for OpenStack Charms:
+#     https://github.com/openstack-charmers/release-tools
+
 [tox]
 skipsdist = True
-envlist = pep8,build
-toxworkdir = /tmp/tox
-# NOTE(beisner): Avoid build/test env pollution by not enabling sitepackages.
+envlist = pep8,py3
+# NOTE: Avoid build/test env pollution by not enabling sitepackages.
 sitepackages = False
-# NOTE(beisner): Avoid false positives by not skipping missing interpreters.
+# NOTE: Avoid false positives by not skipping missing interpreters.
 skip_missing_interpreters = False
+# NOTES:
+# * We avoid the new dependency resolver by pinning pip < 20.3, see
+#   https://github.com/pypa/pip/issues/9187
+# * Pinning dependencies requires tox >= 3.2.0, see
+#   https://tox.readthedocs.io/en/latest/config.html#conf-requires
+# * It is also necessary to pin virtualenv as a newer virtualenv would still
+#   lead to fetching the latest pip in the func* tox targets, see
+#   https://stackoverflow.com/a/38133283
+requires =
+  pip < 20.3
+  virtualenv < 20.0
+  setuptools<50.0.0
+
+# NOTE: https://wiki.canonical.com/engineering/OpenStack/InstallLatestToxOnOsci
+minversion = 3.18.0
 
 [testenv]
 setenv = VIRTUAL_ENV={envdir}
          PYTHONHASHSEED=0
          TERM=linux
-         JUJU_REPOSITORY={envdir}/tmp/build
-passenv = http_proxy https_proxy
+         LAYER_PATH={toxinidir}/layers
+         INTERFACE_PATH={toxinidir}/interfaces
+         JUJU_REPOSITORY={toxinidir}/build
+passenv = http_proxy https_proxy INTERFACE_PATH LAYER_PATH JUJU_REPOSITORY
 install_command =
-  pip install {opts} {packages}
+  {toxinidir}/pip.sh install {opts} {packages}
 deps =
     -r{toxinidir}/requirements.txt
-whitelist_externals = /bin/true /bin/echo /bin/mkdir /bin/ln
 
 [testenv:build]
 basepython = python3
-# ``charm build`` refuses to output to a subdirectory to the source tree
-# The gate check will look for and validate the built artifacts in the source
-# tree.
-# Build the artifats under /tmp and link back to source directory to alleviate.
-commands =
-  /bin/echo 'WARNING: *build* target is for testing only.'
-  /bin/mkdir -p {envdir}/tmp
-  charm-build --log-level DEBUG -o {envdir}/tmp/build .
-  /bin/ln -s {envdir}/tmp/build/builds {envdir}/tmp/build/trusty
-  /bin/ln -s {envdir}/tmp/build {toxinidir}/build
+# Nothing to charm-build as this is a layer without a src directory.
+commands = /bin/true
 
-[testenv:venv]
+[testenv:add-build-lock-file]
 basepython = python3
-commands = {posargs}
+# Nothing to charm-build as this is a layer without a src directory.
+commands = /bin/true
 
-[testenv:py34]
-basepython = python3.4
+[testenv:py3]
+basepython = python3
 deps = -r{toxinidir}/test-requirements.txt
-# TODO: Need to write unit tests then remove the following command.
+# TODO: Need to write unit tests then switch command to use stestr.
 commands = /bin/true
 
 [testenv:py35]
 basepython = python3.5
 deps = -r{toxinidir}/test-requirements.txt
-# TODO: Need to write unit tests then remove the following command.
+# TODO: Need to write unit tests then switch command to use stestr.
 commands = /bin/true
 
 [testenv:py36]
 basepython = python3.6
 deps = -r{toxinidir}/test-requirements.txt
-# TODO: Need to write unit tests then remove the following command.
+# TODO: Need to write unit tests then switch command to use stestr.
 commands = /bin/true
 
 [testenv:py37]
 basepython = python3.7
 deps = -r{toxinidir}/test-requirements.txt
-# TODO: Need to write unit tests then remove the following command.
+# TODO: Need to write unit tests then switch command to use stestr.
+commands = /bin/true
+
+[testenv:py38]
+basepython = python3.8
+deps = -r{toxinidir}/test-requirements.txt
+# TODO: Need to write unit tests then switch command to use stestr.
+commands = /bin/true
+
+[testenv:py39]
+basepython = python3.9
+deps = -r{toxinidir}/test-requirements.txt
+# TODO: Need to write unit tests then switch command to use stestr.
 commands = /bin/true
 
 [testenv:pep8]
 basepython = python3
-commands = /bin/true
+deps = flake8==3.9.2
+       charm-tools==2.8.3
+commands = flake8 {posargs}
+
+[testenv:cover]
+# Technique based heavily upon
+# https://github.com/openstack/nova/blob/master/tox.ini
+basepython = python3
+deps = -r{toxinidir}/requirements.txt
+       -r{toxinidir}/test-requirements.txt
+setenv =
+    {[testenv]setenv}
+    PYTHON=coverage run
+commands =
+    coverage erase
+    stestr run --slowest {posargs}
+    coverage combine
+    coverage html -d cover
+    coverage xml -o cover/coverage.xml
+    coverage report
+
+[coverage:run]
+branch = True
+concurrency = multiprocessing
+parallel = True
+source =
+    .
+omit =
+    .tox/*
+    */charmhelpers/*
+    unit_tests/*
+
+[testenv:venv]
+basepython = python3
+commands = {posargs}
+
+[flake8]
+# E402 ignore necessary for path append before sys module import in actions
+ignore = E402,W503,W504