Update Amulet defs, series metadata and c-h sync

- Sync charm helpers if applicable.

- Fix test executable hashbags for virtualenv prep.

- Add Yakkety-Newton Amulet test definitions.

- Prep Xenial-Ocata Amulet test definitions (not yet enabled).

- Prep Zesty-Ocata Amulet test definitions (not yet enabled).

- Remove Precise charm series metadata if present.

- Remove Precise Amulet test definitions if present.

Change-Id: Ic32eba6b28007101965bd2caec9ca6e5369ab379

hooks/charmhelpers/contrib/openstack/utils.py

@@ -109,7 +109,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
     ('wily', 'liberty'),
     ('xenial', 'mitaka'),
     ('yakkety', 'newton'),
-    ('zebra', 'ocata'),  # TODO: upload with real Z name
+    ('zesty', 'ocata'),
 ])
 
 
@@ -152,6 +152,8 @@ SWIFT_CODENAMES = OrderedDict([
         ['2.5.0', '2.6.0', '2.7.0']),
     ('newton',
         ['2.8.0', '2.9.0', '2.10.0']),
+    ('ocata',
+        ['2.11.0']),
 ])
 
 # >= Liberty version->codename mapping
@@ -410,14 +412,26 @@ def get_os_version_package(pkg, fatal=True):
 os_rel = None
 
 
-def os_release(package, base='essex'):
+def reset_os_release():
+    '''Unset the cached os_release version'''
+    global os_rel
+    os_rel = None
+
+
+def os_release(package, base='essex', reset_cache=False):
     '''
     Returns OpenStack release codename from a cached global.
+
+    If reset_cache then unset the cached os_release version and return the
+    freshly determined version.
+
     If the codename can not be determined from either an installed package or
     the installation source, the earliest release supported by the charm should
     be returned.
     '''
     global os_rel
+    if reset_cache:
+        reset_os_release()
     if os_rel:
         return os_rel
     os_rel = (git_os_codename_install_source(config('openstack-origin-git')) or
@@ -535,6 +549,9 @@ def configure_installation_source(rel):
             'newton': 'xenial-updates/newton',
             'newton/updates': 'xenial-updates/newton',
             'newton/proposed': 'xenial-proposed/newton',
+            'zesty': 'zesty-updates/ocata',
+            'zesty/updates': 'xenial-updates/ocata',
+            'zesty/proposed': 'xenial-proposed/ocata',
         }
 
         try:
@@ -668,6 +685,7 @@ def clean_storage(block_device):
     else:
         zap_disk(block_device)
 
+
 is_ip = ip.is_ip
 ns_query = ip.ns_query
 get_host_ip = ip.get_host_ip

hooks/charmhelpers/core/hookenv.py

@@ -332,6 +332,8 @@ def config(scope=None):
     config_cmd_line = ['config-get']
     if scope is not None:
         config_cmd_line.append(scope)
+    else:
+        config_cmd_line.append('--all')
     config_cmd_line.append('--format=json')
     try:
         config_data = json.loads(

hooks/charmhelpers/core/host.py

@@ -732,3 +732,20 @@ def get_total_ram():
                     assert unit == 'kB', 'Unknown unit'
                     return int(value) * 1024  # Classic, not KiB.
         raise NotImplementedError()
+
+
+UPSTART_CONTAINER_TYPE = '/run/container_type'
+
+
+def is_container():
+    """Determine whether unit is running in a container
+
+    @return: boolean indicating if unit is in a container
+    """
+    if init_is_systemd():
+        # Detect using systemd-detect-virt
+        return subprocess.call(['systemd-detect-virt',
+                                '--container']) == 0
+    else:
+        # Detect using upstart container file marker
+        return os.path.exists(UPSTART_CONTAINER_TYPE)

hooks/charmhelpers/core/kernel_factory/ubuntu.py

@@ -5,7 +5,7 @@ def persistent_modprobe(module):
     """Load a kernel module and configure for auto-load on reboot."""
     with open('/etc/modules', 'r+') as modules:
         if module not in modules.read():
-            modules.write(module)
+            modules.write(module + "\n")
 
 
 def update_initramfs(version='all'):

hooks/charmhelpers/fetch/ubuntu.py

@@ -105,6 +105,14 @@ CLOUD_ARCHIVE_POCKETS = {
     'newton/proposed': 'xenial-proposed/newton',
     'xenial-newton/proposed': 'xenial-proposed/newton',
     'xenial-proposed/newton': 'xenial-proposed/newton',
+    # Ocata
+    'ocata': 'xenial-updates/ocata',
+    'xenial-ocata': 'xenial-updates/ocata',
+    'xenial-ocata/updates': 'xenial-updates/ocata',
+    'xenial-updates/ocata': 'xenial-updates/ocata',
+    'ocata/proposed': 'xenial-proposed/ocata',
+    'xenial-ocata/proposed': 'xenial-proposed/ocata',
+    'xenial-ocata/newton': 'xenial-proposed/ocata',
 }
 
 APT_NO_LOCK = 100  # The return code for "couldn't acquire lock" in APT.

lib/ceph/__init__.py

@@ -23,6 +23,7 @@ import re
 import sys
 import errno
 import shutil
+import pyudev
 
 from charmhelpers.core import hookenv
 from charmhelpers.core.host import (
@@ -50,13 +51,15 @@ from charmhelpers.contrib.storage.linux.utils import (
     is_block_device,
     zap_disk,
     is_device_mounted)
+from charmhelpers.contrib.openstack.utils import (
+    get_os_codename_install_source)
 
 LEADER = 'leader'
 PEON = 'peon'
 QUORUM = [LEADER, PEON]
 
 PACKAGES = ['ceph', 'gdisk', 'ntp', 'btrfs-tools', 'python-ceph',
-            'radosgw', 'xfsprogs']
+            'radosgw', 'xfsprogs', 'python-pyudev']
 
 LinkSpeed = {
     "BASE_10": 10,
@@ -100,6 +103,23 @@ NETWORK_ADAPTER_SYSCTLS = {
 }
 
 
+def unmounted_disks():
+    """List of unmounted block devices on the current host."""
+    disks = []
+    context = pyudev.Context()
+    for device in context.list_devices(DEVTYPE='disk'):
+        if device['SUBSYSTEM'] == 'block':
+            matched = False
+            for block_type in [u'dm', u'loop', u'ram', u'nbd']:
+                if block_type in device.device_node:
+                    matched = True
+            if matched:
+                continue
+            disks.append(device.device_node)
+    log("Found disks: {}".format(disks))
+    return [disk for disk in disks if not is_device_mounted(disk)]
+
+
 def save_sysctls(sysctl_dict, save_location):
     """
     Persist the sysctls to the hard drive.
@@ -1344,7 +1364,7 @@ def roll_monitor_cluster(new_version, upgrade_key):
                           version=new_version)
         else:
             # Check if the previous node has finished
-            status_set('blocked',
+            status_set('waiting',
                        'Waiting on {} to finish upgrading'.format(
                            mon_sorted_list[position - 1]))
             wait_on_previous_node(upgrade_key=upgrade_key,
@@ -1361,11 +1381,10 @@ def roll_monitor_cluster(new_version, upgrade_key):
         status_set('blocked', 'failed to upgrade monitor')
 
 
-def upgrade_monitor():
+def upgrade_monitor(new_version):
     current_version = get_version()
     status_set("maintenance", "Upgrading monitor")
     log("Current ceph version is {}".format(current_version))
-    new_version = config('release-version')
     log("Upgrading to: {}".format(new_version))
 
     try:
@@ -1393,7 +1412,6 @@ def upgrade_monitor():
                 service_start('ceph-mon@{}'.format(mon_id))
         else:
             service_start('ceph-mon-all')
-        status_set("active", "")
     except subprocess.CalledProcessError as err:
         log("Stopping ceph and upgrading packages failed "
             "with message: {}".format(err.message))
@@ -1415,9 +1433,9 @@ def lock_and_roll(upgrade_key, service, my_name, version):
 
     # This should be quick
     if service == 'osd':
-        upgrade_osd()
+        upgrade_osd(version)
     elif service == 'mon':
-        upgrade_monitor()
+        upgrade_monitor(version)
     else:
         log("Unknown service {}.  Unable to upgrade".format(service),
             level=ERROR)
@@ -1541,11 +1559,10 @@ def roll_osd_cluster(new_version, upgrade_key):
         status_set('blocked', 'failed to upgrade osd')
 
 
-def upgrade_osd():
+def upgrade_osd(new_version):
     current_version = get_version()
     status_set("maintenance", "Upgrading osd")
     log("Current ceph version is {}".format(current_version))
-    new_version = config('release-version')
     log("Upgrading to: {}".format(new_version))
 
     try:
@@ -1578,3 +1595,58 @@ def upgrade_osd():
             "with message: {}".format(err.message))
         status_set("blocked", "Upgrade to {} failed".format(new_version))
         sys.exit(1)
+
+
+def list_pools(service):
+    """
+    This will list the current pools that Ceph has
+
+    :param service: String service id to run under
+    :return: list.  Returns a list of the ceph pools. Raises CalledProcessError
+    if the subprocess fails to run.
+    """
+    try:
+        pool_list = []
+        pools = subprocess.check_output(['rados', '--id', service, 'lspools'])
+        for pool in pools.splitlines():
+            pool_list.append(pool)
+        return pool_list
+    except subprocess.CalledProcessError as err:
+        log("rados lspools failed with error: {}".format(err.output))
+        raise
+
+
+# A dict of valid ceph upgrade paths.  Mapping is old -> new
+UPGRADE_PATHS = {
+    'firefly': 'hammer',
+    'hammer': 'jewel',
+}
+
+# Map UCA codenames to ceph codenames
+UCA_CODENAME_MAP = {
+    'icehouse': 'firefly',
+    'juno': 'firefly',
+    'kilo': 'hammer',
+    'liberty': 'hammer',
+    'mitaka': 'jewel',
+}
+
+
+def pretty_print_upgrade_paths():
+    '''Pretty print supported upgrade paths for ceph'''
+    lines = []
+    for key, value in UPGRADE_PATHS.iteritems():
+        lines.append("{} -> {}".format(key, value))
+    return lines
+
+
+def resolve_ceph_version(source):
+    '''
+    Resolves a version of ceph based on source configuration
+    based on Ubuntu Cloud Archive pockets.
+
+    @param: source: source configuration option of charm
+    @returns: ceph release codename or None if not resolvable
+    '''
+    os_release = get_os_codename_install_source(source)
+    return UCA_CODENAME_MAP.get(os_release)

metadata.yaml

@@ -12,7 +12,6 @@ tags:
 series:
   - xenial
   - trusty
-  - precise
   - yakkety
 peers:
   mon:

tests/basic_deployment.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 #
 # Copyright 2016 Canonical Ltd
 #
@@ -228,7 +228,7 @@ class CephBasicDeployment(OpenStackAmuletDeployment):
                                  'cinder-volume'],
         }
 
-        if self._get_openstack_release() < self.vivid_kilo:
+        if self._get_openstack_release() < self.xenial_mitaka:
             # For upstart systems only.  Ceph services under systemd
             # are checked by process name instead.
             ceph_services = [

tests/charmhelpers/contrib/amulet/utils.py

@@ -546,7 +546,7 @@ class AmuletUtils(object):
             raise if it is present.
         :returns: List of process IDs
         """
-        cmd = 'pidof -x {}'.format(process_name)
+        cmd = 'pidof -x "{}"'.format(process_name)
         if not expect_success:
             cmd += " || exit 0 && exit 1"
         output, code = sentry_unit.run(cmd)

tests/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -69,9 +69,9 @@ class OpenStackAmuletDeployment(AmuletDeployment):
 
         # Charms outside the ~openstack-charmers
         base_charms = {
-            'mysql': ['precise', 'trusty'],
-            'mongodb': ['precise', 'trusty'],
-            'nrpe': ['precise', 'trusty', 'wily', 'xenial'],
+            'mysql': ['trusty'],
+            'mongodb': ['trusty'],
+            'nrpe': ['trusty', 'xenial'],
         }
 
         for svc in other_services:
@@ -260,31 +260,20 @@ class OpenStackAmuletDeployment(AmuletDeployment):
            release.
            """
         # Must be ordered by OpenStack release (not by Ubuntu release):
-        (self.precise_essex, self.precise_folsom, self.precise_grizzly,
-         self.precise_havana, self.precise_icehouse,
-         self.trusty_icehouse, self.trusty_juno, self.utopic_juno,
-         self.trusty_kilo, self.vivid_kilo, self.trusty_liberty,
-         self.wily_liberty, self.trusty_mitaka,
-         self.xenial_mitaka, self.xenial_newton,
-         self.yakkety_newton) = range(16)
+        (self.trusty_icehouse, self.trusty_kilo, self.trusty_liberty,
+         self.trusty_mitaka, self.xenial_mitaka, self.xenial_newton,
+         self.yakkety_newton, self.xenial_ocata, self.zesty_ocata) = range(9)
 
         releases = {
-            ('precise', None): self.precise_essex,
-            ('precise', 'cloud:precise-folsom'): self.precise_folsom,
-            ('precise', 'cloud:precise-grizzly'): self.precise_grizzly,
-            ('precise', 'cloud:precise-havana'): self.precise_havana,
-            ('precise', 'cloud:precise-icehouse'): self.precise_icehouse,
             ('trusty', None): self.trusty_icehouse,
-            ('trusty', 'cloud:trusty-juno'): self.trusty_juno,
             ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo,
             ('trusty', 'cloud:trusty-liberty'): self.trusty_liberty,
             ('trusty', 'cloud:trusty-mitaka'): self.trusty_mitaka,
-            ('utopic', None): self.utopic_juno,
-            ('vivid', None): self.vivid_kilo,
-            ('wily', None): self.wily_liberty,
             ('xenial', None): self.xenial_mitaka,
             ('xenial', 'cloud:xenial-newton'): self.xenial_newton,
+            ('xenial', 'cloud:xenial-ocata'): self.xenial_ocata,
             ('yakkety', None): self.yakkety_newton,
+            ('zesty', None): self.zesty_ocata,
         }
         return releases[(self.series, self.openstack)]
 
@@ -294,16 +283,10 @@ class OpenStackAmuletDeployment(AmuletDeployment):
            Return a string representing the openstack release.
            """
         releases = OrderedDict([
-            ('precise', 'essex'),
-            ('quantal', 'folsom'),
-            ('raring', 'grizzly'),
-            ('saucy', 'havana'),
             ('trusty', 'icehouse'),
-            ('utopic', 'juno'),
-            ('vivid', 'kilo'),
-            ('wily', 'liberty'),
             ('xenial', 'mitaka'),
             ('yakkety', 'newton'),
+            ('zesty', 'ocata'),
         ])
         if self.openstack:
             os_origin = self.openstack.split(':')[1]

tests/charmhelpers/contrib/openstack/amulet/utils.py

@@ -28,6 +28,7 @@ import keystoneclient.v2_0 as keystone_client
 from keystoneclient.auth.identity import v3 as keystone_id_v3
 from keystoneclient import session as keystone_session
 from keystoneclient.v3 import client as keystone_client_v3
+from novaclient import exceptions
 
 import novaclient.client as nova_client
 import pika
@@ -377,6 +378,16 @@ class OpenStackAmuletUtils(AmuletUtils):
                                       tenant_name=tenant,
                                       auth_version='2.0')
 
+    def create_flavor(self, nova, name, ram, vcpus, disk, flavorid="auto",
+                      ephemeral=0, swap=0, rxtx_factor=1.0, is_public=True):
+        """Create the specified flavor."""
+        try:
+            nova.flavors.find(name=name)
+        except (exceptions.NotFound, exceptions.NoUniqueMatch):
+            self.log.debug('Creating flavor ({})'.format(name))
+            nova.flavors.create(name, ram, vcpus, disk, flavorid,
+                                ephemeral, swap, rxtx_factor, is_public)
+
     def create_cirros_image(self, glance, image_name):
         """Download the latest cirros image and upload it to glance,
         validate and return a resource pointer.

tests/gate-basic-trusty-icehouse

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 #
 # Copyright 2016 Canonical Ltd
 #

tests/gate-basic-trusty-kilo

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 #
 # Copyright 2016 Canonical Ltd
 #

tests/gate-basic-trusty-liberty

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 #
 # Copyright 2016 Canonical Ltd
 #

tests/gate-basic-trusty-mitaka

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 #
 # Copyright 2016 Canonical Ltd
 #

tests/gate-basic-xenial-mitaka

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 #
 # Copyright 2016 Canonical Ltd
 #

tests/gate-basic-xenial-newton

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 #
 # Copyright 2016 Canonical Ltd
 #

tests/gate-basic-xenial-ocata

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Amulet tests on a basic ceph deployment on xenial-ocata."""
+
+from basic_deployment import CephBasicDeployment
+
+if __name__ == '__main__':
+    deployment = CephBasicDeployment(series='xenial',
+                                     openstack='cloud:xenial-ocata',
+                                     source='cloud:xenial-updates/ocata')
+    deployment.run_tests()

tests/gate-basic-yakkety-newton

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 #
 # Copyright 2016 Canonical Ltd
 #

tests/gate-basic-zesty-ocata

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Amulet tests on a basic ceph deployment on zesty-ocata."""
+
+from basic_deployment import CephBasicDeployment
+
+if __name__ == '__main__':
+    deployment = CephBasicDeployment(series='zesty')
+    deployment.run_tests()