Use bundletester for amulet test execution

Switch to using bundletester for execution of functional tests,
leveraging tox to build out test virtualenvs.

Rename amulet tests inline with gate-* and dfs-* naming standards.

Update README to refer to functional testing section of the charm
guide.

Change-Id: I037529196125e507bb15c1b0a3681a7457a6e8d1

.gitignore

@@ -9,3 +9,5 @@ joined-string
 .unit-state.db
 trusty/**
 xenial/
+func-results.json
+.local

Makefile

@@ -10,8 +10,7 @@ test:
 
 functional_test:
 	@echo Starting Amulet tests...
-	@tests/setup/00-setup
-	@juju test -v -p AMULET_HTTP_PROXY,AMULET_OS_VIP --timeout 2700
+	@tox -e func27
 
 bin/charm_helpers_sync.py:
 	@mkdir -p bin

test-requirements.txt

@@ -7,3 +7,17 @@ flake8>=2.2.4,<=2.4.1
 os-testr>=0.4.1
 charm-tools>=2.0.0
 requests==2.6.0
+# BEGIN: Amulet OpenStack Charm Helper Requirements
+# Liberty client lower constraints
+amulet>=1.14.3,<2.0
+bundletester>=0.6.1,<1.0
+python-ceilometerclient>=1.5.0,<2.0
+python-cinderclient>=1.4.0,<2.0
+python-glanceclient>=1.1.0,<2.0
+python-heatclient>=0.8.0,<1.0
+python-novaclient>=2.30.1,<3.0
+python-openstackclient>=1.7.0,<2.0
+python-swiftclient>=2.6.0,<3.0
+pika>=0.10.0,<1.0
+distro-info
+# END: Amulet OpenStack Charm Helper Requirements

tests/016-basic-trusty-juno

@@ -1,25 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright 2016 Canonical Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#  http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Amulet tests on a basic keystone deployment on trusty-juno."""
-
-from basic_deployment import KeystoneBasicDeployment
-
-if __name__ == '__main__':
-    deployment = KeystoneBasicDeployment(series='trusty',
-                                         openstack='cloud:trusty-juno',
-                                         source='cloud:trusty-updates/juno')
-    deployment.run_tests()

tests/020-basic-wily-liberty

@@ -1,23 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright 2016 Canonical Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#  http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Amulet tests on a basic keystone deployment on wily-liberty."""
-
-from basic_deployment import KeystoneBasicDeployment
-
-if __name__ == '__main__':
-    deployment = KeystoneBasicDeployment(series='wily')
-    deployment.run_tests()

tests/051-basic-trusty-juno-git

@@ -1,26 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright 2016 Canonical Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#  http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Amulet tests on a basic keystone git deployment on trusty-juno."""
-
-from basic_deployment import KeystoneBasicDeployment
-
-if __name__ == '__main__':
-    deployment = KeystoneBasicDeployment(series='trusty',
-                                         openstack='cloud:trusty-juno',
-                                         source='cloud:trusty-updates/juno',
-                                         git=True)
-    deployment.run_tests()

tests/README

@@ -1,113 +0,0 @@
-This directory provides Amulet tests to verify basic deployment functionality
-from the perspective of this charm, its requirements and its features, as
-exercised in a subset of the full OpenStack deployment test bundle topology.
-
-Reference:  lp:openstack-charm-testing for full test bundles.
-
-A single topology and configuration is defined and deployed, once for each of
-the defined Ubuntu:OpenStack release combos.  The ongoing goal is for this
-charm to always possess tests and combo definitions for all currently-supported
-release combinations of U:OS.
-
-test_* methods are called in lexical sort order, as with most runners.  However,
-each individual test method should be idempotent and expected to pass regardless
-of run order or Ubuntu:OpenStack combo.  When writing or modifying tests,
-ensure that every individual test is not dependent on another test_ method.
-
-Test naming convention, purely for code organization purposes:
-    1xx service and endpoint checks
-    2xx relation checks
-    3xx config checks
-    4xx functional checks
-    9xx restarts, config changes, actions and other final checks
-
-In order to run tests, charm-tools and juju must be installed:
-    sudo add-apt-repository ppa:juju/stable
-    sudo apt-get update
-    sudo apt-get install charm-tools juju juju-deployer amulet
-
-Alternatively, tests may be exercised with proposed or development versions
-of juju and related tools:
-
-    # juju proposed version
-    sudo add-apt-repository ppa:juju/proposed
-    sudo apt-get update
-    sudo apt-get install charm-tools juju juju-deployer
-
-    # juju development version
-    sudo add-apt-repository ppa:juju/devel
-    sudo apt-get update
-    sudo apt-get install charm-tools juju juju-deployer
-
-Some tests may need to download files. If a web proxy server is required in
-the environment, the AMULET_HTTP_PROXY environment variable must be set and
-passed into the juju test command.  This is unrelated to juju's http proxy
-settings or behavior.
-
-The following examples demonstrate different ways that tests can be executed.
-All examples are run from the charm's root directory.
-
-  * To run all +x tests in the tests directory:
-
-      bzr branch lp:charms/trusty/foo
-      cd foo
-      make functional_test
-
-  * To run the tests against a specific release combo as defined in tests/:
-
-      bzr branch lp:charms/trusty/foo
-      cd foo
-      juju test -v -p AMULET_HTTP_PROXY 015-basic-trusty-icehouse
-
-  * To run tests and keep the juju environment deployed after a failure:
-
-      bzr branch lp:charms/trusty/foo
-      cd foo
-      juju test --set-e -v -p AMULET_HTTP_PROXY 015-basic-trusty-icehouse
-
-  * To re-run a test module against an already deployed environment (one
-    that was deployed by a previous call to 'juju test --set-e'):
-
-      ./tests/015-basic-trusty-icehouse
-
-  * Even with --set-e, `juju test` will tear down the deployment when all
-    tests pass. The following work flow may be more effective when
-    iterating on test writing.
-
-      bzr branch lp:charms/trusty/foo
-      cd foo
-      ./tests/setup/00-setup
-      juju bootstrap
-      ./tests/015-basic-trusty-icehouse
-      # make some changes, run tests again
-      ./tests/015-basic-trusty-icehouse
-      # make some changes, run tests again
-      ./tests/015-basic-trusty-icehouse
-
-  * There may be test definitions in the tests/ dir which are not set +x
-    executable.  This is generally true for deprecated releases, or for
-    upcoming releases which are not yet validated and enabled.  To enable
-    and run these tests:
-      bzr branch lp:charms/trusty/foo
-      cd foo
-      ls tests
-      chmod +x tests/017-basic-trusty-kilo
-      ./tests/setup/00-setup
-      juju bootstrap
-      ./tests/017-basic-trusty-kilo
-
-
-Additional notes:
-
-  * Use DEBUG to turn on debug logging, use ERROR otherwise.
-      u = OpenStackAmuletUtils(ERROR)
-      u = OpenStackAmuletUtils(DEBUG)
-
-  * To interact with the deployed environment:
-      export OS_USERNAME=admin
-      export OS_PASSWORD=openstack
-      export OS_TENANT_NAME=admin
-      export OS_REGION_NAME=RegionOne
-      export OS_AUTH_URL=${OS_AUTH_PROTOCOL:-http}://`juju-deployer -e trusty -f keystone`:5000/v2.0
-      keystone user-list
-      glance image-list

tests/README.md

@@ -0,0 +1,9 @@
+# Overview
+
+This directory provides Amulet tests to verify basic deployment functionality
+from the perspective of this charm, its requirements and its features, as
+exercised in a subset of the full OpenStack deployment test bundle topology.
+
+For full details on functional testing of OpenStack charms please refer to
+the [functional testing](http://docs.openstack.org/developer/charm-guide/testing.html#functional-testing)
+section of the OpenStack Charm Guide.

tests/basic_deployment.py

@@ -230,9 +230,9 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment):
     def _initialize_tests(self):
         """Perform final initialization before tests get run."""
         # Access the sentries for inspecting service units
-        self.mysql_sentry = self.d.sentry.unit['mysql/0']
-        self.keystone_sentry = self.d.sentry.unit['keystone/0']
-        self.cinder_sentry = self.d.sentry.unit['cinder/0']
+        self.mysql_sentry = self.d.sentry['mysql'][0]
+        self.keystone_sentry = self.d.sentry['keystone'][0]
+        self.cinder_sentry = self.d.sentry['cinder'][0]
         u.log.debug('openstack release val: {}'.format(
             self._get_openstack_release()))
         u.log.debug('openstack release str: {}'.format(
@@ -663,14 +663,12 @@ class KeystoneBasicDeployment(OpenStackAmuletDeployment):
 
     def test_901_pause_resume(self):
         """Test pause and resume actions."""
-        unit_name = "keystone/0"
-        unit = self.d.sentry.unit[unit_name]
         self._assert_services(should_run=True)
-        action_id = u.run_action(unit, "pause")
+        action_id = u.run_action(self.keystone_sentry, "pause")
         assert u.wait_on_action(action_id), "Pause action failed."
 
         self._assert_services(should_run=False)
 
-        action_id = u.run_action(unit, "resume")
+        action_id = u.run_action(self.keystone_sentry, "resume")
         assert u.wait_on_action(action_id), "Resume action failed"
         self._assert_services(should_run=True)

tests/setup/00-setup

@@ -1,17 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-sudo add-apt-repository --yes ppa:juju/stable
-sudo apt-get update --yes
-sudo apt-get install --yes amulet \
-                           distro-info-data \
-                           python-cinderclient \
-                           python-distro-info \
-                           python-glanceclient \
-                           python-heatclient \
-                           python-keystoneclient \
-                           python-neutronclient \
-                           python-novaclient \
-                           python-pika \
-                           python-swiftclient

tests/tests.yaml

@@ -1,21 +1,17 @@
-bootstrap: true
-reset: false
-virtualenv: true
-makefile:
-    - lint
-    - test
-sources:
-    - ppa:juju/stable
-packages:
-    - amulet
-    - distro-info-data
-    - python-ceilometerclient
-    - python-cinderclient
-    - python-distro-info
-    - python-glanceclient
-    - python-heatclient
-    - python-keystoneclient
-    - python-neutronclient
-    - python-novaclient
-    - python-pika
-    - python-swiftclient
+# Bootstrap the model if necessary.
+bootstrap: True
+# Re-use bootstrap node instead of destroying/re-bootstrapping.
+reset: True
+# Use tox/requirements to drive the venv instead of bundletester's venv feature.
+virtualenv: False
+# Leave makefile empty, otherwise unit/lint tests will rerun ahead of amulet.
+makefile: []
+# Do not specify juju PPA sources.  Juju is presumed to be pre-installed
+# and configured in all test runner environments.
+#sources:
+# Do not specify or rely on system packages.
+#packages:
+# Do not specify python packages here.  Use test-requirements.txt
+# and tox instead.  ie. The venv is constructed before bundletester
+# is invoked.
+#python-packages:

tox.ini

@@ -5,6 +5,7 @@ skipsdist = True
 [testenv]
 setenv = VIRTUAL_ENV={envdir}
          PYTHONHASHSEED=0
+passenv = HOME TERM
 install_command =
   pip install --allow-unverified python-apt {opts} {packages}
 commands = ostestr {posargs}
@@ -24,6 +25,50 @@ commands = flake8 {posargs} --exclude */charmhelpers hooks unit_tests tests
 [testenv:venv]
 commands = {posargs}
 
+[testenv:func27-noop]
+# DRY RUN - For Debug
+basepython = python2.7
+deps = -r{toxinidir}/requirements.txt
+       -r{toxinidir}/test-requirements.txt
+commands =
+    bundletester -vl DEBUG -r json -o func-results.json --test-pattern "gate-*" -n --no-destroy
+
+[testenv:func27]
+# Charm Functional Test
+# Run all gate tests which are +x (expected to always pass)
+basepython = python2.7
+deps = -r{toxinidir}/requirements.txt
+       -r{toxinidir}/test-requirements.txt
+commands =
+    bundletester -vl DEBUG -r json -o func-results.json --test-pattern "gate-*" --no-destroy
+
+[testenv:func27-smoke]
+# Charm Functional Test
+# Run a specific test as an Amulet smoke test (expected to always pass)
+basepython = python2.7
+deps = -r{toxinidir}/requirements.txt
+       -r{toxinidir}/test-requirements.txt
+commands =
+    bundletester -vl DEBUG -r json -o func-results.json gate-basic-xenial-mitaka --no-destroy
+
+[testenv:func27-dfs]
+# Charm Functional Test
+# Run all deploy-from-source tests which are +x (may not always pass!)
+basepython = python2.7
+deps = -r{toxinidir}/requirements.txt
+       -r{toxinidir}/test-requirements.txt
+commands =
+    bundletester -vl DEBUG -r json -o func-results.json --test-pattern "dfs-*" --no-destroy
+
+[testenv:func27-dev]
+# Charm Functional Test
+# Run all development test targets which are +x (may not always pass!)
+basepython = python2.7
+deps = -r{toxinidir}/requirements.txt
+       -r{toxinidir}/test-requirements.txt
+commands =
+    bundletester -vl DEBUG -r json -o func-results.json --test-pattern "dev-*" --no-destroy
+
 [flake8]
 ignore = E402,E226
 exclude = hooks/charmhelpers