Add basic gate functional testing jobs for aodh

This provides a way to run functional tests from the gate, initially
just a simple set of gabbi tests which exercise the API. Once this
has proven to be working we can add more.

The model here is a hybrid of what was started for ceilometer and
what was used in gnocchi with some adjustments to allow for some
local testing.

The tests produced by test_gabbi_live will only run if
AODH_SERVICE_URL is set to the alarming endpoint. A keystone auth
token is needed in AODH_SERVICE_TOKEN. post_test_hook.sh
makes this happen if we are in a devstack-gate environment.
Otherwise they must be set by the caller.

The expectation is that we'll stick more test in tests/functional as
we have time, energy and brains.

Note that the layout of the functional tests assumes that the tests
will skip (or not even generate tests) if a functional environment
is not in place, such as when the unit tests are run.

Change-Id: Ia40e9c6b956209ecd0086ca861a90ce0c92b7b10

aodh/tests/functional/gabbi/gabbits-live/alarming.yaml

@@ -0,0 +1,135 @@
+defaults:
+    request_headers:
+        x-auth-token: $ENVIRON['AODH_SERVICE_TOKEN']
+
+tests:
+    - name: list alarms none
+      desc: Lists alarms, none yet exist
+      url: /v2/alarms
+      method: GET
+      response_strings:
+          - "[]"
+
+    - name: try to PUT an alarm
+      desc: what does PUT do
+      url: /v2/alarms
+      method: PUT
+      request_headers:
+          content-type: application/json
+      data:
+          name: added_alarm_defaults2
+          type: threshold
+          threshold_rule:
+              meter_name: ameter
+              threshold: 300.0
+      status: 405
+      response_headers:
+          allow: GET, POST
+
+    - name: createAlarm
+      desc: Creates an alarm.
+      url: /v2/alarms
+      method: POST
+      request_headers:
+          content-type: application/json
+      data:
+          name: added_alarm_defaults
+          type: threshold
+          threshold_rule:
+              meter_name: ameter
+              threshold: 300.0
+      status: 201
+      response_headers:
+          location: /$SCHEME://$NETLOC/v2/alarms/
+          content-type: application/json; charset=UTF-8
+      response_json_paths:
+          $.severity: low
+          $.threshold_rule.threshold: 300.0
+          $.threshold_rule.comparison_operator: eq
+
+    - name: showAlarm
+      desc: Shows information for a specified alarm.
+      url: /v2/alarms/$RESPONSE['$.alarm_id']
+      method: GET
+      response_json_paths:
+          $.severity: low
+          $.alarm_id: $RESPONSE['$.alarm_id']
+          $.threshold_rule.threshold: 300.0
+          $.threshold_rule.comparison_operator: eq
+      response_headers:
+          content-type: application/json; charset=UTF-8
+
+    - name: updateAlarm
+      desc: Updates a specified alarm.
+      url: /v2/alarms/$RESPONSE['$.alarm_id']
+      method: PUT
+      request_headers:
+          content-type: application/json
+      data:
+          name: added_alarm_defaults
+          type: threshold
+          severity: moderate
+          threshold_rule:
+              meter_name: ameter
+              threshold: 200.0
+    # TODO(chdent): why do we have a response, why not status: 204?
+    # status: 204
+      response_json_paths:
+          $.threshold_rule.threshold: 200.0
+          $.severity: moderate
+          $.state: insufficient data
+
+    - name: showAlarmHistory
+      desc: Assembles the history for a specified alarm.
+      url: /v2/alarms/$RESPONSE['$.alarm_id']/history?q.field=type&q.op=eq&q.value=rule%20change
+      method: GET
+      response_json_paths:
+          $[0].type: rule change
+
+    - name: updateAlarmState
+      desc: Sets the state of a specified alarm.
+      url: /v2/alarms/$RESPONSE['$[0].alarm_id']/state
+      request_headers:
+          content-type: application/json
+      data: '"alarm"'
+      method: PUT
+    # TODO(chdent): really? Of what possible use is this?
+      response_json_paths:
+          $: alarm
+
+    # Get a list of alarms so we can extract an id for the next test
+    - name: list alarms for data
+      desc: Lists alarms, only one
+      url: /v2/alarms
+      method: GET
+      response_json_paths:
+          $[0].name: added_alarm_defaults
+
+    - name: showAlarmState
+      desc: Gets the state of a specified alarm.
+      url: /v2/alarms/$RESPONSE['$[0].alarm_id']/state
+      method: GET
+      response_headers:
+          content-type: application/json; charset=UTF-8
+      response_json_paths:
+          $: alarm
+
+    - name: list alarms one
+      desc: Lists alarms, only one
+      url: /v2/alarms
+      method: GET
+      response_json_paths:
+          $[0].name: added_alarm_defaults
+
+    - name: deleteAlarm
+      desc: Deletes a specified alarm.
+      url: /v2/alarms/$RESPONSE['$[0].alarm_id']
+      method: DELETE
+      status: 204
+
+    - name: list alarms none end
+      desc: Lists alarms, none now exist
+      url: /v2/alarms
+      method: GET
+      response_strings:
+          - "[]"

aodh/tests/functional/gabbi/test_gabbi_live.py

@@ -0,0 +1,46 @@
+#
+# Copyright 2015 Red Hat. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""A test module to exercise the Gnocchi API with gabbi."""
+
+import os
+
+from gabbi import driver
+import six.moves.urllib.parse as urlparse
+
+
+TESTS_DIR = 'gabbits-live'
+
+
+def load_tests(loader, tests, pattern):
+    """Provide a TestSuite to the discovery process."""
+    aodh_url = os.getenv('AODH_SERVICE_URL')
+    if aodh_url:
+        parsed_url = urlparse.urlsplit(aodh_url)
+        prefix = parsed_url.path.rstrip('/')  # turn it into a prefix
+
+        # NOTE(chdent): gabbi requires a port be passed or it will
+        # default to 8001, so we must dance a little dance to get
+        # the right ports. Probably gabbi needs to change.
+        # https://github.com/cdent/gabbi/issues/50
+        port = 443 if parsed_url.scheme == 'https' else 80
+        if parsed_url.port:
+            port = parsed_url.port
+
+        test_dir = os.path.join(os.path.dirname(__file__), TESTS_DIR)
+        return driver.build_tests(test_dir, loader,
+                                  host=parsed_url.hostname,
+                                  port=port,
+                                  prefix=prefix)

aodh/tests/functional/hooks/post_test_hook.sh

@@ -26,22 +26,32 @@ function generate_testr_results {
     fi
 }
 
-export AODH_DIR="$BASE/new/aodh"
+# If we're running in the gate find our keystone endpoint to give to
-
+# gabbi tests and do a chown. Otherwise the existing environment
-# Go to the aodh dir
+# should provide URL and TOKEN.
-cd $AODH_DIR
+if [ -f $BASE/new/devstack ]; then
-
+    export AODH_DIR="$BASE/new/aodh"
-sudo chown -R jenkins:stack $AODH_DIR
+    JENKINS_USER=jenkins
+    sudo chown -R jenkins:stack $AODH_DIR
+    source $BASE/new/devstack/openrc admin admin
+    openstack endpoint list
+    export AODH_SERVICE_URL=$(openstack endpoint show alarming -c publicurl -f value)
+    export AODH_SERVICE_TOKEN=$(openstack token issue -c id -f value)
+    # Go to the aodh dir
+    cd $AODH_DIR
+fi
 
 # Run tests
 echo "Running aodh functional test suite"
 set +e
 
-# NOTE(ityaptin) Expected a script param which contains a backend name
+# NOTE(ityaptin) Expect a script param which contains at least one backend name
-AODH_TEST_BACKEND="$1" sudo -E -H -u jenkins tox -efunctional
+AODH_TEST_BACKEND="${1:?test backend required}" sudo -E -H -u ${JENKINS_USER:-${USER}} tox -efunctional
 EXIT_CODE=$?
 set -e
 
 # Collect and parse result
-generate_testr_results
+if [ -n "$AODH_DIR" ]; then
+    generate_testr_results
+fi
 exit $EXIT_CODE

devstack/gate/gate_hook.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This script is executed inside gate_hook function in devstack gate.
+
+# A space separated lists of storage backends.
+STORAGE_DRIVERS="$1"
+
+ENABLED_SERVICES="key,aodi-api,aodh-notifier,aodh-evaluator"
+ENABLED_SERVICES+="ceilometer-acompute,ceilometer-acentral,ceilometer-anotification,"
+ENABLED_SERVICES+="ceilometer-collector,ceilometer-api,"
+
+export DEVSTACK_GATE_INSTALL_TESTONLY=1
+export DEVSTACK_GATE_NO_SERVICES=1
+export DEVSTACK_GATE_TEMPEST=0
+export DEVSTACK_GATE_EXERCISES=0
+export KEEP_LOCALRC=1
+
+# default to mysql
+case $STORAGE_DRIVER in
+    *postgresql*)
+        export DEVSTACK_GATE_POSTGRES=1
+        ;;
+esac
+
+export ENABLED_SERVICES
+
+$BASE/new/devstack-gate/devstack-vm-gate.sh

run-functional-tests.sh

@@ -1,9 +1,8 @@
 #!/bin/bash -x
 set -e
+
 # Use a mongodb backend by default
-
+if [ -z "$AODH_TEST_BACKEND" ]; then
-
-if [ -z $AODH_TEST_BACKEND ]; then
     AODH_TEST_BACKEND="mongodb"
 fi
 echo $AODH_TEST_BACKEND

tox.ini

@@ -36,6 +36,7 @@ commands =
 setenv = VIRTUAL_ENV={envdir}
          EVENTLET_NO_GREENDNS=yes
          OS_TEST_PATH=aodh/tests/functional/
+passenv = AODH_*
 commands =
    bash -x {toxinidir}/run-functional-tests.sh "{posargs}"