openstack/charm-swift-storage
Code Issues Proposed changes

py3: Switch packages to Python 3 for train and later

Browse Source 
Switch package install to Python 3 for OpenStack Train and later.

When upgrading, remove any python-* packages that were explicitly
installated and then autoremove --purge any dependencies that are
no longer required.

This patch also includes the following related changes:
  * Use the common files package, swift, rather than python-swift
    when using a package name to determine release.

Also add OS_* to tox.ini to allow functional tests to execute.

Change-Id: I121af845bf11c22052479a497b196a4670021256
This commit is contained in:
Corey Bryant 2019-09-04 18:34:30 +00:00 committed by Sahid Orentino Ferdjaoui
parent 2fc678dcee
commit fb6e21b51f
5 changed files with 141 additions and 18 deletions

14
hooks/swift_storage_hooks.py

@ -39,9 +39,9 @@ _add_path(_root)
from lib.swift_storage_utils import (
PACKAGES,
RESTART_MAP,
SWIFT_SVCS,
determine_packages,
do_openstack_upgrade,
ensure_swift_directories,
fetch_swift_rings,
@ -103,6 +103,7 @@ from charmhelpers.contrib.openstack.utils import (
is_unit_paused_set,
set_unit_paused,
set_unit_upgrading,
get_os_codename_install_source,
)
from charmhelpers.contrib.network.ip import (
get_relation_ip,
@ -200,10 +201,13 @@ def initialize_ufw():
def install():
status_set('maintenance', 'Executing pre-install')
execd_preinstall()
configure_installation_source(config('openstack-origin'))
src = config('openstack-origin')
configure_installation_source(src)
status_set('maintenance', 'Installing apt packages')
apt_update()
apt_install(PACKAGES, fatal=True)
rel = get_os_codename_install_source(src)
pkgs = determine_packages(rel)
apt_install(pkgs, fatal=True)
initialize_ufw()
ensure_swift_directories()
@ -262,7 +266,9 @@ def install_vaultlocker():
@harden()
def upgrade_charm():
initialize_ufw()
apt_install(filter_installed_packages(PACKAGES), fatal=True)
rel = get_os_codename_install_source(config('openstack-origin'))
pkgs = determine_packages(rel)
apt_install(pkgs, fatal=True)
update_nrpe_config()
ensure_devs_tracked()

77
lib/swift_storage_utils.py

@ -24,7 +24,11 @@ from lib.swift_storage_context import (
from charmhelpers.fetch import (
apt_upgrade,
apt_update
apt_update,
apt_install,
apt_purge,
apt_autoremove,
filter_missing_packages,
)
from charmhelpers.core.unitdata import (
@ -57,6 +61,7 @@ from charmhelpers.core.hookenv import (
ingress_address,
storage_list,
storage_get,
status_set,
)
from charmhelpers.contrib.network import ufw
@ -77,6 +82,9 @@ from charmhelpers.contrib.openstack.utils import (
get_os_codename_install_source,
get_os_codename_package,
save_script_rc as _save_script_rc,
CompareOpenStackReleases,
reset_os_release,
os_release,
)
from charmhelpers.contrib.openstack import (
@ -93,9 +101,29 @@ import charmhelpers.contrib.openstack.vaultlocker as vaultlocker
from charmhelpers.core.unitdata import kv
PACKAGES = [
'swift', 'swift-account', 'swift-container', 'swift-object',
'xfsprogs', 'gdisk', 'lvm2', 'python-jinja2', 'python-psutil',
'gdisk',
'lvm2',
'swift',
'swift-account',
'swift-container',
'swift-object',
'python-jinja2',
'python-psutil',
'ufw',
'xfsprogs',
]
PY3_PACKAGES = [
'python3-jinja2',
'python3-psutil',
'python3-six',
'python3-swift',
]
PURGE_PACKAGES = [
'python-jinja2',
'python-psutil',
'python-swift',
]
VERSION_PACKAGE = 'swift-account'
@ -163,7 +191,7 @@ def ensure_swift_directories():
def register_configs():
release = get_os_codename_package('python-swift', fatal=False) or 'essex'
release = get_os_codename_package('swift', fatal=False) or 'essex'
configs = templating.OSConfigRenderer(templates_dir=TEMPLATES,
openstack_release=release)
configs.register('/etc/swift/swift.conf',
@ -181,6 +209,43 @@ def register_configs():
return configs
def determine_packages(release):
"""Determine what packages are needed for a given OpenStack release."""
cmp_openstack = CompareOpenStackReleases(release)
pkgs = PACKAGES[:]
if cmp_openstack >= 'train':
pkgs = [p for p in pkgs if not p.startswith('python-')]
pkgs.extend(PY3_PACKAGES)
return pkgs
def determine_purge_packages():
'''
Determine list of packages that where previously installed which are no
longer needed.
:returns: list of package names
'''
cmp_openstack = CompareOpenStackReleases(os_release('swift'))
if cmp_openstack >= 'train':
return PURGE_PACKAGES
return []
def remove_old_packages():
'''Purge any packages that need to be removed.
:returns: bool Whether packages were removed.
'''
installed_packages = filter_missing_packages(determine_purge_packages())
if installed_packages:
log('Removing apt packages')
status_set('maintenance', 'Removing apt packages')
apt_purge(installed_packages, fatal=True)
apt_autoremove(purge=True, fatal=True)
return bool(installed_packages)
def swift_init(target, action, fatal=False):
'''
Call swift-init on a specific target with given action, potentially
@ -204,6 +269,10 @@ def do_openstack_upgrade(configs):
]
apt_update()
apt_upgrade(options=dpkg_opts, fatal=True, dist=True)
reset_os_release()
apt_install(packages=determine_packages(new_os_rel),
options=dpkg_opts, fatal=True)
remove_old_packages()
configs.set_release(openstack_release=new_os_rel)
configs.write_all()
if not is_paused():

3
tox.ini

@ -19,8 +19,7 @@ setenv = VIRTUAL_ENV={envdir}
install_command =
pip install {opts} {packages}
commands = stestr run --slowest {posargs}
whitelist_externals = juju
passenv = HOME TERM AMULET_* CS_* OS_* TEST_*
passenv = HOME TERM AMULET_* CS_API_* OS_* TEST_*
[testenv:py3]
basepython = python3

9
unit_tests/test_swift_storage_relations.py

@ -169,11 +169,12 @@ class SwiftStorageRelationsTests(CharmTestCase):
@patch.object(hooks, 'add_ufw_gre_rule', lambda *args: None)
@patch.object(hooks, 'ensure_devs_tracked')
def test_upgrade_charm(self, mock_ensure_devs_tracked):
self.filter_installed_packages.return_value = [
'python-psutil']
hooks.upgrade_charm()
self.apt_install.assert_called_with([
'python-psutil'], fatal=True)
self.apt_install.assert_called_with(
['gdisk', 'lvm2', 'swift', 'swift-account',
'swift-container', 'swift-object', 'python-jinja2',
'python-psutil', 'ufw', 'xfsprogs'],
fatal=True)
self.assertTrue(self.update_nrpe_config.called)
self.assertTrue(mock_ensure_devs_tracked.called)

56
unit_tests/test_swift_storage_utils.py

@ -25,6 +25,7 @@ import lib.swift_storage_utils as swift_utils
TO_PATCH = [
'apt_update',
'apt_upgrade',
'apt_install',
'log',
'config',
'configure_installation_source',
@ -52,6 +53,8 @@ TO_PATCH = [
'relation_ids',
'vaultlocker',
'kv',
'reset_os_release',
'CompareOpenStackReleases',
]
@ -477,13 +480,15 @@ class SwiftStorageUtilsTests(CharmTestCase):
]
self.assertEqual(ex, configs.register.call_args_list)
def test_do_upgrade(self):
@patch.object(swift_utils, 'remove_old_packages')
def test_do_upgrade_queens(self, mock_remove_old_packages):
self.is_paused.return_value = False
self.test_config.set('openstack-origin', 'cloud:precise-grizzly')
self.get_os_codename_install_source.return_value = 'grizzly'
self.test_config.set('openstack-origin', 'cloud:bionic-queens')
self.get_os_codename_install_source.return_value = 'queens'
self.CompareOpenStackReleases.return_value = 'queens'
swift_utils.do_openstack_upgrade(MagicMock())
self.configure_installation_source.assert_called_with(
'cloud:precise-grizzly'
'cloud:bionic-queens'
)
dpkg_opts = [
'--option', 'Dpkg::Options::=--force-confnew',
@ -494,6 +499,49 @@ class SwiftStorageUtilsTests(CharmTestCase):
options=dpkg_opts,
fatal=True, dist=True
)
self.apt_install.assert_called_with(
options=dpkg_opts,
packages=['gdisk', 'lvm2', 'swift', 'swift-account',
'swift-container', 'swift-object', 'python-jinja2',
'python-psutil', 'ufw', 'xfsprogs'],
fatal=True
)
self.assertTrue(mock_remove_old_packages.called)
self.assertTrue(self.reset_os_release.called)
services = (swift_utils.ACCOUNT_SVCS + swift_utils.CONTAINER_SVCS +
swift_utils.OBJECT_SVCS)
for service in services:
self.assertIn(call(service), self.service_restart.call_args_list)
@patch.object(swift_utils, 'remove_old_packages')
def test_do_upgrade_train(self, mock_remove_old_packages):
self.is_paused.return_value = False
self.test_config.set('openstack-origin', 'cloud:bionic-train')
self.get_os_codename_install_source.return_value = 'train'
self.CompareOpenStackReleases.return_value = 'train'
swift_utils.do_openstack_upgrade(MagicMock())
self.configure_installation_source.assert_called_with(
'cloud:bionic-train'
)
dpkg_opts = [
'--option', 'Dpkg::Options::=--force-confnew',
'--option', 'Dpkg::Options::=--force-confdef',
]
self.assertTrue(self.apt_update.called)
self.apt_upgrade.assert_called_with(
options=dpkg_opts,
fatal=True, dist=True
)
self.apt_install.assert_called_with(
options=dpkg_opts,
packages=['gdisk', 'lvm2', 'swift', 'swift-account',
'swift-container', 'swift-object', 'ufw', 'xfsprogs',
'python3-jinja2', 'python3-psutil', 'python3-six',
'python3-swift'],
fatal=True
)
self.assertTrue(mock_remove_old_packages.called)
self.assertTrue(self.reset_os_release.called)
services = (swift_utils.ACCOUNT_SVCS + swift_utils.CONTAINER_SVCS +
swift_utils.OBJECT_SVCS)
for service in services: