Update Amulet defs, series metadata and c-h sync

- Sync charm helpers if applicable.

- Fix test executable hashbangs for virtualenv prep.

- Add Yakkety-Newton Amulet test definitions.

- Prep Xenial-Ocata Amulet test definitions (not yet enabled).

- Prep Zesty-Ocata Amulet test definitions (not yet enabled).

- Remove Precise charm series metadata if present.

- Remove Precise Amulet test definitions if present.

Change-Id: Iebca0e30342d35e9c2344d9f1f527ca9c64129d0

hooks/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -69,9 +69,9 @@ class OpenStackAmuletDeployment(AmuletDeployment):
 
         # Charms outside the ~openstack-charmers
         base_charms = {
-            'mysql': ['precise', 'trusty'],
+            'mysql': ['trusty'],
-            'mongodb': ['precise', 'trusty'],
+            'mongodb': ['trusty'],
-            'nrpe': ['precise', 'trusty', 'wily', 'xenial'],
+            'nrpe': ['trusty', 'xenial'],
         }
 
         for svc in other_services:
@@ -260,31 +260,20 @@ class OpenStackAmuletDeployment(AmuletDeployment):
            release.
            """
         # Must be ordered by OpenStack release (not by Ubuntu release):
-        (self.precise_essex, self.precise_folsom, self.precise_grizzly,
+        (self.trusty_icehouse, self.trusty_kilo, self.trusty_liberty,
-         self.precise_havana, self.precise_icehouse,
+         self.trusty_mitaka, self.xenial_mitaka, self.xenial_newton,
-         self.trusty_icehouse, self.trusty_juno, self.utopic_juno,
+         self.yakkety_newton, self.xenial_ocata, self.zesty_ocata) = range(9)
-         self.trusty_kilo, self.vivid_kilo, self.trusty_liberty,
-         self.wily_liberty, self.trusty_mitaka,
-         self.xenial_mitaka, self.xenial_newton,
-         self.yakkety_newton) = range(16)
 
         releases = {
-            ('precise', None): self.precise_essex,
-            ('precise', 'cloud:precise-folsom'): self.precise_folsom,
-            ('precise', 'cloud:precise-grizzly'): self.precise_grizzly,
-            ('precise', 'cloud:precise-havana'): self.precise_havana,
-            ('precise', 'cloud:precise-icehouse'): self.precise_icehouse,
             ('trusty', None): self.trusty_icehouse,
-            ('trusty', 'cloud:trusty-juno'): self.trusty_juno,
             ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo,
             ('trusty', 'cloud:trusty-liberty'): self.trusty_liberty,
             ('trusty', 'cloud:trusty-mitaka'): self.trusty_mitaka,
-            ('utopic', None): self.utopic_juno,
-            ('vivid', None): self.vivid_kilo,
-            ('wily', None): self.wily_liberty,
             ('xenial', None): self.xenial_mitaka,
             ('xenial', 'cloud:xenial-newton'): self.xenial_newton,
+            ('xenial', 'cloud:xenial-ocata'): self.xenial_ocata,
             ('yakkety', None): self.yakkety_newton,
+            ('zesty', None): self.zesty_ocata,
         }
         return releases[(self.series, self.openstack)]
 
@@ -294,16 +283,10 @@ class OpenStackAmuletDeployment(AmuletDeployment):
            Return a string representing the openstack release.
            """
         releases = OrderedDict([
-            ('precise', 'essex'),
-            ('quantal', 'folsom'),
-            ('raring', 'grizzly'),
-            ('saucy', 'havana'),
             ('trusty', 'icehouse'),
-            ('utopic', 'juno'),
-            ('vivid', 'kilo'),
-            ('wily', 'liberty'),
             ('xenial', 'mitaka'),
             ('yakkety', 'newton'),
+            ('zesty', 'ocata'),
         ])
         if self.openstack:
             os_origin = self.openstack.split(':')[1]

hooks/charmhelpers/contrib/openstack/amulet/utils.py

@@ -28,6 +28,7 @@ import keystoneclient.v2_0 as keystone_client
 from keystoneclient.auth.identity import v3 as keystone_id_v3
 from keystoneclient import session as keystone_session
 from keystoneclient.v3 import client as keystone_client_v3
+from novaclient import exceptions
 
 import novaclient.client as nova_client
 import pika
@@ -377,6 +378,16 @@ class OpenStackAmuletUtils(AmuletUtils):
                                       tenant_name=tenant,
                                       auth_version='2.0')
 
+    def create_flavor(self, nova, name, ram, vcpus, disk, flavorid="auto",
+                      ephemeral=0, swap=0, rxtx_factor=1.0, is_public=True):
+        """Create the specified flavor."""
+        try:
+            nova.flavors.find(name=name)
+        except (exceptions.NotFound, exceptions.NoUniqueMatch):
+            self.log.debug('Creating flavor ({})'.format(name))
+            nova.flavors.create(name, ram, vcpus, disk, flavorid,
+                                ephemeral, swap, rxtx_factor, is_public)
+
     def create_cirros_image(self, glance, image_name):
         """Download the latest cirros image and upload it to glance,
         validate and return a resource pointer.

hooks/charmhelpers/contrib/openstack/context.py

@@ -344,6 +344,10 @@ class IdentityServiceContext(OSContextGenerator):
                              'auth_protocol': auth_protocol,
                              'api_version': api_version})
 
+                if float(api_version) > 2:
+                    ctxt.update({'admin_domain_name':
+                                 rdata.get('service_domain')})
+
                 if self.context_complete(ctxt):
                     # NOTE(jamespage) this is required for >= icehouse
                     # so a missing value just indicates keystone needs
@@ -644,7 +648,7 @@ class ApacheSSLContext(OSContextGenerator):
     service_namespace = None
 
     def enable_modules(self):
-        cmd = ['a2enmod', 'ssl', 'proxy', 'proxy_http']
+        cmd = ['a2enmod', 'ssl', 'proxy', 'proxy_http', 'headers']
         check_call(cmd)
 
     def configure_cert(self, cn=None):

hooks/charmhelpers/contrib/openstack/neutron.py

@@ -32,6 +32,7 @@ def headers_package():
     kver = check_output(['uname', '-r']).decode('UTF-8').strip()
     return 'linux-headers-%s' % kver
 
+
 QUANTUM_CONF_DIR = '/etc/quantum'
 
 
@@ -91,6 +92,7 @@ def quantum_plugins():
         }
     }
 
+
 NEUTRON_CONF_DIR = '/etc/neutron'
 
 

hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend

@@ -9,10 +9,13 @@ Listen {{ ext_port }}
     SSLProtocol +TLSv1 +TLSv1.1 +TLSv1.2
     SSLCipherSuite HIGH:!RC4:!MD5:!aNULL:!eNULL:!EXP:!LOW:!MEDIUM
     SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert_{{ endpoint }}
+    # See LP 1484489 - this is to support <= 2.4.7 and >= 2.4.8
+    SSLCertificateChainFile /etc/apache2/ssl/{{ namespace }}/cert_{{ endpoint }}
     SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key_{{ endpoint }}
     ProxyPass / http://localhost:{{ int }}/
     ProxyPassReverse / http://localhost:{{ int }}/
     ProxyPreserveHost on
+    RequestHeader set X-Forwarded-Proto "https"
 </VirtualHost>
 {% endfor -%}
 <Proxy *>

hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf

@@ -9,10 +9,13 @@ Listen {{ ext_port }}
     SSLProtocol +TLSv1 +TLSv1.1 +TLSv1.2
     SSLCipherSuite HIGH:!RC4:!MD5:!aNULL:!eNULL:!EXP:!LOW:!MEDIUM
     SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert_{{ endpoint }}
+    # See LP 1484489 - this is to support <= 2.4.7 and >= 2.4.8
+    SSLCertificateChainFile /etc/apache2/ssl/{{ namespace }}/cert_{{ endpoint }}
     SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key_{{ endpoint }}
     ProxyPass / http://localhost:{{ int }}/
     ProxyPassReverse / http://localhost:{{ int }}/
     ProxyPreserveHost on
+    RequestHeader set X-Forwarded-Proto "https"
 </VirtualHost>
 {% endfor -%}
 <Proxy *>

hooks/charmhelpers/contrib/openstack/templates/section-keystone-authtoken-mitaka

@@ -3,8 +3,13 @@
 auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}
 auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}
 auth_type = password
+{% if api_version == "3" -%}
+project_domain_name = {{ admin_domain_name }}
+user_domain_name = {{ admin_domain_name }}
+{% else -%}
 project_domain_name = default
 user_domain_name = default
+{% endif -%}
 project_name = {{ admin_tenant_name }}
 username = {{ admin_user }}
 password = {{ admin_password }}

hooks/charmhelpers/contrib/openstack/utils.py

@@ -109,7 +109,7 @@ UBUNTU_OPENSTACK_RELEASE = OrderedDict([
     ('wily', 'liberty'),
     ('xenial', 'mitaka'),
     ('yakkety', 'newton'),
-    ('zebra', 'ocata'),  # TODO: upload with real Z name
+    ('zesty', 'ocata'),
 ])
 
 
@@ -152,6 +152,8 @@ SWIFT_CODENAMES = OrderedDict([
         ['2.5.0', '2.6.0', '2.7.0']),
     ('newton',
         ['2.8.0', '2.9.0', '2.10.0']),
+    ('ocata',
+        ['2.11.0']),
 ])
 
 # >= Liberty version->codename mapping
@@ -547,6 +549,9 @@ def configure_installation_source(rel):
             'newton': 'xenial-updates/newton',
             'newton/updates': 'xenial-updates/newton',
             'newton/proposed': 'xenial-proposed/newton',
+            'zesty': 'zesty-updates/ocata',
+            'zesty/updates': 'xenial-updates/ocata',
+            'zesty/proposed': 'xenial-proposed/ocata',
         }
 
         try:
@@ -680,6 +685,7 @@ def clean_storage(block_device):
     else:
         zap_disk(block_device)
 
+
 is_ip = ip.is_ip
 ns_query = ip.ns_query
 get_host_ip = ip.get_host_ip

hooks/charmhelpers/core/hookenv.py

@@ -332,6 +332,8 @@ def config(scope=None):
     config_cmd_line = ['config-get']
     if scope is not None:
         config_cmd_line.append(scope)
+    else:
+        config_cmd_line.append('--all')
     config_cmd_line.append('--format=json')
     try:
         config_data = json.loads(

hooks/charmhelpers/core/host.py

@@ -732,3 +732,20 @@ def get_total_ram():
                     assert unit == 'kB', 'Unknown unit'
                     return int(value) * 1024  # Classic, not KiB.
         raise NotImplementedError()
+
+
+UPSTART_CONTAINER_TYPE = '/run/container_type'
+
+
+def is_container():
+    """Determine whether unit is running in a container
+
+    @return: boolean indicating if unit is in a container
+    """
+    if init_is_systemd():
+        # Detect using systemd-detect-virt
+        return subprocess.call(['systemd-detect-virt',
+                                '--container']) == 0
+    else:
+        # Detect using upstart container file marker
+        return os.path.exists(UPSTART_CONTAINER_TYPE)

hooks/charmhelpers/core/kernel_factory/ubuntu.py

@@ -5,7 +5,7 @@ def persistent_modprobe(module):
     """Load a kernel module and configure for auto-load on reboot."""
     with open('/etc/modules', 'r+') as modules:
         if module not in modules.read():
-            modules.write(module)
+            modules.write(module + "\n")
 
 
 def update_initramfs(version='all'):

hooks/charmhelpers/fetch/ubuntu.py

@@ -105,6 +105,14 @@ CLOUD_ARCHIVE_POCKETS = {
     'newton/proposed': 'xenial-proposed/newton',
     'xenial-newton/proposed': 'xenial-proposed/newton',
     'xenial-proposed/newton': 'xenial-proposed/newton',
+    # Ocata
+    'ocata': 'xenial-updates/ocata',
+    'xenial-ocata': 'xenial-updates/ocata',
+    'xenial-ocata/updates': 'xenial-updates/ocata',
+    'xenial-updates/ocata': 'xenial-updates/ocata',
+    'ocata/proposed': 'xenial-proposed/ocata',
+    'xenial-ocata/proposed': 'xenial-proposed/ocata',
+    'xenial-ocata/newton': 'xenial-proposed/ocata',
 }
 
 APT_NO_LOCK = 100  # The return code for "couldn't acquire lock" in APT.

metadata.yaml

@@ -18,7 +18,6 @@ tags:
 series:
   - xenial
   - trusty
-  - precise
   - yakkety
 extra-bindings:
   data:

tests/basic_deployment.py

@@ -234,7 +234,7 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment):
                             'neutron-metering-agent',
                             'neutron-plugin-openvswitch-agent']
 
-        if self._get_openstack_release() <= self.trusty_juno:
+        if self._get_openstack_release() <= self.trusty_icehouse:
             neutron_services.append('neutron-vpn-agent')
         if self._get_openstack_release() >= self.trusty_mitaka:
             # neutron-plugin-openvswitch-agent -> neutron-openvswitch-agent
@@ -633,9 +633,6 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment):
     def test_301_neutron_ml2_config(self):
         """Verify the data in the ml2 config file. This is only available
            since icehouse."""
-        u.log.debug('Checking neutron gateway ml2 config file data...')
-        if self._get_openstack_release() < self.precise_icehouse:
-            return
 
         unit = self.neutron_gateway_sentry
         if self._get_openstack_release() < self.trusty_mitaka:
@@ -774,9 +771,6 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment):
     def test_305_neutron_lbaas_agent_config(self):
         """Verify the data in the lbaas agent config file. This is only
            available since havana."""
-        u.log.debug('Checking neutron gateway lbaas config file data...')
-        if self._get_openstack_release() < self.precise_havana:
-            return
 
         unit = self.neutron_gateway_sentry
         conf = '/etc/neutron/lbaas_agent.ini'
@@ -1048,7 +1042,7 @@ class NeutronGatewayBasicDeployment(OpenStackAmuletDeployment):
             'neutron-openvswitch-agent': conf_file,
         }
 
-        if self._get_openstack_release() <= self.trusty_juno:
+        if self._get_openstack_release() <= self.trusty_icehouse:
             services.update({'neutron-vpn-agent': conf_file})
         if self._get_openstack_release() < self.xenial_newton:
             services.update({'neutron-lbaas-agent': conf_file})

tests/charmhelpers/contrib/amulet/utils.py

@@ -546,7 +546,7 @@ class AmuletUtils(object):
             raise if it is present.
         :returns: List of process IDs
         """
-        cmd = 'pidof -x {}'.format(process_name)
+        cmd = 'pidof -x "{}"'.format(process_name)
         if not expect_success:
             cmd += " || exit 0 && exit 1"
         output, code = sentry_unit.run(cmd)

tests/charmhelpers/contrib/openstack/amulet/deployment.py

@@ -69,9 +69,9 @@ class OpenStackAmuletDeployment(AmuletDeployment):
 
         # Charms outside the ~openstack-charmers
         base_charms = {
-            'mysql': ['precise', 'trusty'],
+            'mysql': ['trusty'],
-            'mongodb': ['precise', 'trusty'],
+            'mongodb': ['trusty'],
-            'nrpe': ['precise', 'trusty', 'wily', 'xenial'],
+            'nrpe': ['trusty', 'xenial'],
         }
 
         for svc in other_services:
@@ -260,31 +260,20 @@ class OpenStackAmuletDeployment(AmuletDeployment):
            release.
            """
         # Must be ordered by OpenStack release (not by Ubuntu release):
-        (self.precise_essex, self.precise_folsom, self.precise_grizzly,
+        (self.trusty_icehouse, self.trusty_kilo, self.trusty_liberty,
-         self.precise_havana, self.precise_icehouse,
+         self.trusty_mitaka, self.xenial_mitaka, self.xenial_newton,
-         self.trusty_icehouse, self.trusty_juno, self.utopic_juno,
+         self.yakkety_newton, self.xenial_ocata, self.zesty_ocata) = range(9)
-         self.trusty_kilo, self.vivid_kilo, self.trusty_liberty,
-         self.wily_liberty, self.trusty_mitaka,
-         self.xenial_mitaka, self.xenial_newton,
-         self.yakkety_newton) = range(16)
 
         releases = {
-            ('precise', None): self.precise_essex,
-            ('precise', 'cloud:precise-folsom'): self.precise_folsom,
-            ('precise', 'cloud:precise-grizzly'): self.precise_grizzly,
-            ('precise', 'cloud:precise-havana'): self.precise_havana,
-            ('precise', 'cloud:precise-icehouse'): self.precise_icehouse,
             ('trusty', None): self.trusty_icehouse,
-            ('trusty', 'cloud:trusty-juno'): self.trusty_juno,
             ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo,
             ('trusty', 'cloud:trusty-liberty'): self.trusty_liberty,
             ('trusty', 'cloud:trusty-mitaka'): self.trusty_mitaka,
-            ('utopic', None): self.utopic_juno,
-            ('vivid', None): self.vivid_kilo,
-            ('wily', None): self.wily_liberty,
             ('xenial', None): self.xenial_mitaka,
             ('xenial', 'cloud:xenial-newton'): self.xenial_newton,
+            ('xenial', 'cloud:xenial-ocata'): self.xenial_ocata,
             ('yakkety', None): self.yakkety_newton,
+            ('zesty', None): self.zesty_ocata,
         }
         return releases[(self.series, self.openstack)]
 
@@ -294,16 +283,10 @@ class OpenStackAmuletDeployment(AmuletDeployment):
            Return a string representing the openstack release.
            """
         releases = OrderedDict([
-            ('precise', 'essex'),
-            ('quantal', 'folsom'),
-            ('raring', 'grizzly'),
-            ('saucy', 'havana'),
             ('trusty', 'icehouse'),
-            ('utopic', 'juno'),
-            ('vivid', 'kilo'),
-            ('wily', 'liberty'),
             ('xenial', 'mitaka'),
             ('yakkety', 'newton'),
+            ('zesty', 'ocata'),
         ])
         if self.openstack:
             os_origin = self.openstack.split(':')[1]

tests/charmhelpers/contrib/openstack/amulet/utils.py

@@ -28,6 +28,7 @@ import keystoneclient.v2_0 as keystone_client
 from keystoneclient.auth.identity import v3 as keystone_id_v3
 from keystoneclient import session as keystone_session
 from keystoneclient.v3 import client as keystone_client_v3
+from novaclient import exceptions
 
 import novaclient.client as nova_client
 import pika
@@ -377,6 +378,16 @@ class OpenStackAmuletUtils(AmuletUtils):
                                       tenant_name=tenant,
                                       auth_version='2.0')
 
+    def create_flavor(self, nova, name, ram, vcpus, disk, flavorid="auto",
+                      ephemeral=0, swap=0, rxtx_factor=1.0, is_public=True):
+        """Create the specified flavor."""
+        try:
+            nova.flavors.find(name=name)
+        except (exceptions.NotFound, exceptions.NoUniqueMatch):
+            self.log.debug('Creating flavor ({})'.format(name))
+            nova.flavors.create(name, ram, vcpus, disk, flavorid,
+                                ephemeral, swap, rxtx_factor, is_public)
+
     def create_cirros_image(self, glance, image_name):
         """Download the latest cirros image and upload it to glance,
         validate and return a resource pointer.

tests/dfs-basic-trusty-icehouse

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
 """Amulet tests on a basic quantum-gateway git deployment on trusty-icehouse."""
 

tests/dfs-basic-trusty-kilo

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
 """Amulet tests on a basic neutron-gateway git deployment on trusty-kilo."""
 

tests/gate-basic-trusty-icehouse

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
 """Amulet tests on a basic quantum-gateway deployment on trusty-icehouse."""
 

tests/gate-basic-trusty-kilo

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
 """Amulet tests on a basic quantum-gateway deployment on trusty-kilo."""
 

tests/gate-basic-trusty-liberty

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
 """Amulet tests on a basic quantum-gateway deployment on trusty-liberty."""
 

tests/gate-basic-trusty-mitaka

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
 """Amulet tests on a basic quantum-gateway deployment on trusty-mitaka."""
 

tests/gate-basic-xenial-mitaka

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
 """Amulet tests on a basic quantum-gateway deployment on xenial-mitaka."""
 

tests/gate-basic-xenial-newton

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
 """Amulet tests on a basic quantum-gateway deployment on xenial-newton."""
 

tests/gate-basic-xenial-ocata

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+
+"""Amulet tests on a basic quantum-gateway deployment on xenial-ocata."""
+
+from basic_deployment import NeutronGatewayBasicDeployment
+
+if __name__ == '__main__':
+    deployment = NeutronGatewayBasicDeployment(series='xenial',
+                                        openstack='cloud:xenial-ocata',
+                                        source='cloud:xenial-updates/ocata')
+    deployment.run_tests()

tests/gate-basic-yakkety-newton

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
 """Amulet tests on a basic quantum-gateway deployment on yakkety-newton."""
 

tests/gate-basic-zesty-ocata

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+
+"""Amulet tests on a basic quantum-gateway deployment on zesty-ocata."""
+
+from basic_deployment import NeutronGatewayBasicDeployment
+
+if __name__ == '__main__':
+    deployment = NeutronGatewayBasicDeployment(series='zesty')
+    deployment.run_tests()