Added Docker Files to mimic third party API's

This change works towards making the API tests pass
against Mimic'd drivers so that real production accounts
do not need to be set up - namely:
- Mimic - Fastly
- Mimic - Keystone

It also makes the mimic fastly instance the default driver in /etc

Implements: bp/mimic-fastly

Change-Id: I35d36fbc2452fc4e2233c19abefd738c9975edd0

docker/api/poppy_sample_mocked.conf

@@ -0,0 +1,53 @@
+[DEFAULT]
+log_file = poppy.log
+log_config_append = /etc/logging.conf
+debug = True
+
+[drivers]
+transport = pecan
+manager = default
+dns = rackspace
+storage = cassandra
+providers = fastly
+
+[drivers:storage:cassandra]
+cluster = "cassandra"
+keyspace = poppy
+archive_on_delete = True
+
+[drivers:dns]
+retries = 5
+
+[drivers:distributed_task:taskflow]
+jobboard_backend_type = zookeeper
+persistent_backend_type = zookeeper
+jobboard_backend_host = "zookeeper"
+jobboard_backend_port = 2181
+persistent_backend_host = "zookeeper"
+persistent_backend_port = 2181
+
+[drivers:provider:fastly]
+apikey = "fake_key_for_fastly_mimic"
+scheme = "http"
+host = "mimic:8900"
+
+[drivers:dns:rackspace]
+auth_endpoint = http://mimic:8900/v2.0/tokens
+username = "mock_user"
+api_key = "mock_key"
+use_shards = True
+num_shards = 400
+shard_prefix = "cdn"
+shared_ssl_num_shards = 5
+shared_ssl_shard_prefix = "scdn"
+shared_ssl_domain_suffix = "secure.poppycdn.net"
+url = "poppycdn.net"
+url_404 = notfound.com
+email = "you@email.com"
+timeout = 30
+delay = 1
+
+[log_delivery]
+identity_url = http://mimic:8900/v2.0/tokens
+preferred_dcs = 'IAD'
+container_name = .CDN_ACCESS_LOGS

docker/compose/dependencies.yml

@@ -7,4 +7,9 @@ cassandra:
     image: library/cassandra
     ports:
      - 9160:9160
-     - 9042:9042
+     - 9042:9042
+
+mimic:
+    image: amitgandhinz/mimic
+    ports:
+     - 8900:8900

docker/compose/poppy_stack.yml

@@ -1,11 +1,3 @@
-poppy:
-    build: ../api/.
-    ports:
-        - "80:8080"
-    links:
-        - cassandra
-        - zookeeper
-
 zookeeper:
     extends:
         file: dependencies.yml
@@ -14,4 +6,12 @@ zookeeper:
 cassandra:
     extends:
         file: dependencies.yml
-        service: cassandra
+        service: cassandra
+
+poppy:
+    build: ../api/.
+    ports:
+        - "80:8080"
+    links:
+        - cassandra
+        - zookeeper

docker/compose/poppy_stack_mocked.yml

@@ -0,0 +1,24 @@
+zookeeper:
+    extends:
+        file: dependencies.yml
+        service: zookeeper
+
+cassandra:
+    extends:
+        file: dependencies.yml
+        service: cassandra
+
+mimic:
+    extends:
+        file: dependencies.yml
+        service: mimic
+
+poppy:
+    build: ../api/.
+    ports:
+        - "80:8080"
+    links:
+        - cassandra
+        - zookeeper
+        - mimic
+

docker/mimic/Dockerfile

@@ -13,7 +13,7 @@ RUN apt-get update && apt-get install -y \
 RUN /usr/bin/curl -s https://bootstrap.pypa.io/get-pip.py | python
 
 WORKDIR /home/source
-RUN git clone https://github.com/malini-kamalambal/mimic .
+RUN git clone https://github.com/rackerlabs/mimic .
 RUN pip install -r requirements.txt
 
 EXPOSE 8900

etc/poppy.conf

@@ -38,10 +38,10 @@ transport = pecan
 manager = default
 
 # Storage driver module (e.g., mongodb, sqlite, cassandra)
-storage = mockdb
+storage = cassandra
 
 # Provider modules list (a list of comma separated provider module list)
-providers = mock,fastly,akamai
+providers = fastly
 
 # DNS driver module (e.g. default, designate, rackspace)
 dns = default
@@ -80,9 +80,6 @@ replication_strategy = class:SimpleStrategy, replication_factor:1
 # Path to directory containing CQL migration scripts
 migrations_path = <poppy_code_path>/poppy/storage/cassandra/migrations
 
-[drivers:storage:mockdb]
-database = poppy
-
 [drivers:distributed_task:taskflow]
 jobboard_backend_type = zookeeper
 persistent_backend_type = zookeeper
@@ -119,8 +116,10 @@ default_cache_ttl = 86400
 
 [drivers:provider:fastly]
 apikey = "MYAPIKEY"
-scheme = "https"
-host = "api.fastly.com"
+# scheme = "https"
+# host = "api.fastly.com"
+scheme = "http"
+host = "dockerhost:8900/fastly"
 
 [drivers:provider:maxcdn]
 alias = "MYALIAS"

kill_poppy.sh

@@ -0,0 +1 @@
+ps -ef | grep [p]oppy- | awk -F ' ' '{print$2}' | xargs kill -9

run_poppy.sh

@@ -0,0 +1,92 @@
+#!/bin/bash
+DAEMONIZED=false
+WORKERS = 6
+
+for i in "$@"
+do
+  case $i in
+      -d|--daemonized)
+      DAEMONIZED=true
+      shift # past argument=value
+      ;;
+
+      -w=*|--workers=*)
+      WORKERS="${i#*=}"
+      shift # past argument=value
+      ;;
+
+      -?|--help)
+      echo "USAGE: ./run_poppy.sh -d -w=10"
+      echo "-d | --daemonized : run in daemonized mode"
+      echo "-w | --workers : the number of poppy-worker processes to spawn (defaults to 6)"
+      exit
+
+      shift
+      ;;
+
+      *)
+      echo "Invalid Options"
+      echo "Run ./run_poppy.sh --help for valid parameters."
+      exit
+              # unknown option
+      ;;
+  esac
+done
+
+pip install docker-compose
+
+
+# remove existing containers
+docker kill compose_cassandra_1
+docker kill compose_zookeeper_1
+docker rm compose_cassandra_1
+docker rm compose_zookeeper_1
+
+# start new containers
+docker-compose -f docker/compose/dependencies.yml up -d
+
+is_cassandra_ready() {
+    nc -z dockerhost 9042
+}
+
+is_zookeeper_ready() {
+    nc -z dockerhost 2181
+}
+
+# wait until cassandra is ready
+while ! is_cassandra_ready -eq 1
+do
+  echo "still trying to connect to cassandra"
+  sleep 1
+done
+echo "connected successfully to cassandra"
+
+
+# wait until zookeeper is ready
+while ! is_zookeeper_ready -eq 1
+do
+  echo "still trying to connect to zookeeper"
+  sleep 1
+done
+echo "connected successfully to zookeeper"
+
+# start the poppy-workers
+COUNTER=0
+while [ $COUNTER -lt $WORKERS ]; do
+  exec poppy-worker > /dev/null 2>&1 &
+  echo "poppy-worker spawned."
+  let COUNTER=COUNTER+1 
+done
+
+
+# start the poppy-server
+if $DAEMONIZED; then
+  exec poppy-server > /dev/null 2>&1 &
+  echo "poppy-server spawned."
+else
+  exec poppy-server
+fi
+
+
+
+echo "Poppy Server and Workers Started"

tests/api/README.rst

@@ -13,7 +13,7 @@ To run the tests
 
 1. Install the dependencies::
 
-    pip install -r requirements.txt
+    $ pip install -r requirements.txt
 
 2. Set the following environment variables::
 
@@ -21,17 +21,28 @@ To run the tests
     export CAFE_ROOT_LOG_PATH=~/.poppy/logs
     export CAFE_TEST_LOG_PATH=~/.poppy/logs
 
-3. The API tests require a running database (eg cassandra), in order to
+3. If you desire highlighting in the output, set the following environment variables::
+
+    export NOSE_WITH_OPENSTACK=1
+    export NOSE_OPENSTACK_COLOR=1
+    export NOSE_OPENSTACK_RED=0.05
+    export NOSE_OPENSTACK_YELLOW=0.025
+    export NOSE_OPENSTACK_SHOW_ELAPSED=1
+    export NOSE_OPENSTACK_STDOUT=1
+
+
+4. The API tests require a running database (eg cassandra) and zookeeper, in order to
 run via tox.
 
-4. Copy the api.conf file to the path set by CAFE_CONFIG_FILE_PATH::
+    $ ./run_poppy.sh
 
-    cp tests/etc/api.conf ~/.poppy/tests.conf
+5. Copy the api.conf file to the path set by CAFE_CONFIG_FILE_PATH::
 
-5. Once you are ready to run the tests::
+    $ cp tests/etc/api.conf ~/.poppy/tests.conf
 
-    cd tests/api
-    nosetests
+6. Once you are ready to run the tests::
+
+    $ nosetests api
 
 
 Tox Support
@@ -41,14 +52,24 @@ You can run tox using a docker container hosting Cassandra::
 
 Note - This will require docker (or boot2docker for MacOSX) to already be installed on the system.
 
-1. Update your `~/.poppy/tests.conf` to point to your docker cassandra container ip address.
+1. Update your `~/.poppy/tests.conf` to point to your docker cassandra/zookeeper container ip address.
 
 Example 1: Run all API tests against a docker hosted cassandra instance::
 
-    tox -e api
+    $ tox -e api
 
 Example 2: Run a particular API test function::
 
-    tox -e api api/services/test_services.py:TestCreateService -- -m test_create_service_positive
+    $ tox -e api api/services/test_services.py:TestCreateService -- -m test_create_service_positive
 
 
+Mimic Support
+-------------
+
+Occassionaly you want to test against a mock api rather than the real thing to get around rate limiting issues,
+and to get around having to create accounts with a certain provider.
+
+Mimic helps accomplish this goal for testing.
+
+1.  Run the mimic docker container (via ./run_poppy.sh) and point any remote api url in your test.conf file to your http://dockerhost:8900/mimic_service_name
+

tests/api/admin/test_akamai_SAN_domain_migration.py

@@ -26,6 +26,9 @@ class TestSanCertService(base.TestBase):
 
     def setUp(self):
         super(TestSanCertService, self).setUp()
+        if self.test_config.run_operator_tests is False:
+            self.skipTest(
+                'Test Operator Functions is disabled in configuration')
 
         self.service_name = self.generate_random_string(prefix='API-Test-')
         self.flavor_id = self.test_flavor
@@ -93,9 +96,10 @@ class TestSanCertService(base.TestBase):
         self.service_url = resp.headers["location"]
 
     def test_migrate(self):
+
         new_certs = self.akamai_config.san_certs
         new_certs_list = new_certs.split(',')
-        index = random.randint(0, len(new_certs_list)-1)
+        index = random.randint(0, len(new_certs_list) - 1)
         new_cert = new_certs_list[index]
 
         get_resp = self.client.get_service(location=self.service_url)
@@ -125,9 +129,10 @@ class TestSanCertService(base.TestBase):
         self.assertEqual(data, new_cert)
 
     def test_migrate_negative_invalid_projectid(self):
+
         new_certs = self.akamai_config.san_certs
         new_certs_list = new_certs.split(',')
-        index = random.randint(0, len(new_certs_list)-1)
+        index = random.randint(0, len(new_certs_list) - 1)
         new_cert = new_certs_list[index]
 
         get_resp = self.client.get_service(location=self.service_url)
@@ -141,9 +146,10 @@ class TestSanCertService(base.TestBase):
         self.assertEqual(resp.status_code, 404)
 
     def test_migrate_negative_invalid_serviceid(self):
+
         new_certs = self.akamai_config.san_certs
         new_certs_list = new_certs.split(',')
-        index = random.randint(0, len(new_certs_list)-1)
+        index = random.randint(0, len(new_certs_list) - 1)
         new_cert = new_certs_list[index]
 
         get_resp = self.client.get_service(location=self.service_url)
@@ -157,9 +163,10 @@ class TestSanCertService(base.TestBase):
         self.assertEqual(resp.status_code, 404)
 
     def test_migrate_negative_invalid_domain(self):
+
         new_certs = self.akamai_config.san_certs
         new_certs_list = new_certs.split(',')
-        index = random.randint(0, len(new_certs_list)-1)
+        index = random.randint(0, len(new_certs_list) - 1)
         new_cert = new_certs_list[index]
 
         get_resp = self.client.get_service(location=self.service_url)
@@ -185,6 +192,10 @@ class TestSanCertServiceWithLogDelivery(base.TestBase):
     def setUp(self):
         super(TestSanCertServiceWithLogDelivery, self).setUp()
 
+        if self.test_config.run_operator_tests is False:
+            self.skipTest(
+                'Test Operator Functions is disabled in configuration')
+
         self.service_name = self.generate_random_string(prefix='API-Test-')
         self.flavor_id = self.test_flavor
 
@@ -256,7 +267,7 @@ class TestSanCertServiceWithLogDelivery(base.TestBase):
     def test_migrate(self):
         new_certs = self.akamai_config.san_certs
         new_certs_list = new_certs.split(',')
-        index = random.randint(0, len(new_certs_list)-1)
+        index = random.randint(0, len(new_certs_list) - 1)
         new_cert = new_certs_list[index]
         get_resp = self.client.get_service(location=self.service_url)
         get_resp_body = get_resp.json()

tests/api/admin/test_get_service_by_domain.py

@@ -28,6 +28,10 @@ class TestGetServiceByDomain(base.TestBase):
     def setUp(self):
         super(TestGetServiceByDomain, self).setUp()
 
+        if self.test_config.run_operator_tests is False:
+            self.skipTest(
+                'Test Operator Functions is disabled in configuration')
+
         self.service_name = self.generate_random_string(prefix='api-test')
         self.flavor_id = self.test_flavor
 
@@ -141,6 +145,10 @@ class TestGetServiceByDomain(base.TestBase):
         self.assertEqual(api_resp2.status_code, 200)
 
     def test_negative_get_by_non_existing_domain(self):
+        if self.test_config.run_operator_tests is False:
+            self.skipTest(
+                'Test Operator Functions is disabled in configuration')
+
         domain_name = self.domain_list[0]['domain'] + str(uuid.uuid1()) + \
             ".com"
         resp = self.operator_client.admin_get_service_by_domain_name(
@@ -180,6 +188,10 @@ class TestGetServiceBySharedDomain(base.TestBase):
     def setUp(self):
         super(TestGetServiceBySharedDomain, self).setUp()
 
+        if self.test_config.run_operator_tests is False:
+            self.skipTest(
+                'Test Operator Functions is disabled in configuration')
+
         self.service_name = self.generate_random_string(prefix='API-Test-')
         self.flavor_id = self.test_flavor
 
@@ -247,6 +259,7 @@ class TestGetServiceBySharedDomain(base.TestBase):
         self.service_url = resp.headers["location"]
 
     def test_get_service_by_domain(self):
+
         get_resp = self.client.get_service(self.service_url)
         resp_body = get_resp.json()
         domain = resp_body['domains'][0]['domain']
@@ -281,6 +294,11 @@ class TestGetServiceBySANCertDomain(base.TestBase):
 
     def setUp(self):
         super(TestGetServiceBySANCertDomain, self).setUp()
+
+        if self.test_config.run_operator_tests is False:
+            self.skipTest(
+                'Test Operator Functions is disabled in configuration')
+
         self.service_name = self.generate_random_string(prefix='API-Test-')
         self.flavor_id = self.test_flavor
 
@@ -348,6 +366,7 @@ class TestGetServiceBySANCertDomain(base.TestBase):
         self.service_url = resp.headers["location"]
 
     def test_get_service_by_domain(self):
+
         get_resp = self.client.get_service(self.service_url)
         resp_body = get_resp.json()
         domain = resp_body['domains'][0]['domain']
@@ -382,6 +401,11 @@ class TestGetServiceByCustomCertDomain(base.TestBase):
 
     def setUp(self):
         super(TestGetServiceByCustomCertDomain, self).setUp()
+
+        if self.test_config.run_operator_tests is False:
+            self.skipTest(
+                'Test Operator Functions is disabled in configuration')
+
         self.service_name = self.generate_random_string(prefix='API-Test-')
         self.flavor_id = self.test_flavor
 
@@ -449,6 +473,7 @@ class TestGetServiceByCustomCertDomain(base.TestBase):
         self.service_url = resp.headers["location"]
 
     def test_get_service_by_domain(self):
+
         get_resp = self.client.get_service(self.service_url)
         resp_body = get_resp.json()
         domain = resp_body['domains'][0]['domain']

tests/api/admin/test_perform_action.py

@@ -25,6 +25,11 @@ class TestHttpService(base.TestBase):
 
     def setUp(self):
         super(TestHttpService, self).setUp()
+
+        if self.test_config.run_operator_tests is False:
+            self.skipTest(
+                'Test Operator Functions is disabled in configuration')
+
         self.service_name = self.generate_random_string(prefix='API-Test-')
         self.flavor_id = self.test_flavor
 
@@ -198,6 +203,11 @@ class TestSharedCertService(base.TestBase):
 
     def setUp(self):
         super(TestSharedCertService, self).setUp()
+
+        if self.test_config.run_operator_tests is False:
+            self.skipTest(
+                'Test Operator Functions is disabled in configuration')
+
         self.service_name = self.generate_random_string(prefix='API-Test-')
         self.flavor_id = self.test_flavor
 
@@ -352,6 +362,11 @@ class TestSanCertService(base.TestBase):
 
     def setUp(self):
         super(TestSanCertService, self).setUp()
+
+        if self.test_config.run_operator_tests is False:
+            self.skipTest(
+                'Test Operator Functions is disabled in configuration')
+
         self.service_name = self.generate_random_string(prefix='API-Test-')
         self.flavor_id = self.test_flavor
 
@@ -506,6 +521,11 @@ class TestCustomCertService(base.TestBase):
 
     def setUp(self):
         super(TestCustomCertService, self).setUp()
+
+        if self.test_config.run_operator_tests is False:
+            self.skipTest(
+                'Test Operator Functions is disabled in configuration')
+
         self.service_name = self.generate_random_string(prefix='API-Test-')
         self.flavor_id = self.test_flavor
 
@@ -660,6 +680,11 @@ class TestHttpServiceWithLogDelivery(base.TestBase):
 
     def setUp(self):
         super(TestHttpServiceWithLogDelivery, self).setUp()
+
+        if self.test_config.run_operator_tests is False:
+            self.skipTest(
+                'Test Operator Functions is disabled in configuration')
+
         self.service_name = self.generate_random_string(prefix='API-Test-')
         self.flavor_id = self.test_flavor
 

tests/api/base.py

@@ -25,6 +25,7 @@ from tests.api.utils import config
 
 
 class TestBase(fixtures.BaseTestFixture):
+
     """Child class of fixtures.BaseTestFixture for testing CDN.
 
     Inherit from this and write your test methods. If the child class defines
@@ -92,16 +93,17 @@ class TestBase(fixtures.BaseTestFixture):
                     operator_project_id
             else:
                 cls.operator_url = cls.config.base_url + '/v1.0'
+
             cls.operator_client = client.PoppyClient(
                 cls.operator_url, operator_auth_token, operator_project_id,
                 serialize_format='json',
                 deserialize_format='json')
 
-        cls.dns_config = config.DNSConfig()
-        cls.dns_client = client.DNSClient(cls.dns_config.dns_username,
-                                          cls.dns_config.dns_api_key)
+            cls.dns_config = config.DNSConfig()
+            cls.dns_client = client.DNSClient(cls.dns_config.dns_username,
+                                              cls.dns_config.dns_api_key)
 
-        cls.akamai_config = config.AkamaiConfig()
+            cls.akamai_config = config.AkamaiConfig()
 
     def generate_random_string(self, prefix='API-Tests', length=12):
         """Generates a random string of given prefix & length"""

tests/api/services/test_services.py

@@ -357,7 +357,9 @@ class TestServiceActions(base.TestBase):
 
         self.service_url = resp.headers["location"]
         self.client.wait_for_service_status(
-            location=self.service_url, status='deployed')
+            location=self.service_url,
+            status='deployed',
+            abort_on_status='failed')
 
     @attrib.attr('smoke')
     def test_delete_service(self):

tests/api/utils/client.py

@@ -58,6 +58,7 @@ class AuthClient(client.HTTPClient):
 
 
 class DNSClient(client.HTTPClient):
+
     def __init__(self, username, api_key):
         super(DNSClient, self).__init__()
 
@@ -339,6 +340,8 @@ class PoppyClient(client.AutoMarshallingHTTPClient):
                     # this is for debugging purpose,
                     # will be removed later, so simply use print
                     print(body.get('errors', []))
+                    assert False, ("Aborted on status {0}").format(
+                        current_status)
                     return service
 
             current_time = int(time.time())
@@ -366,6 +369,8 @@ class PoppyClient(client.AutoMarshallingHTTPClient):
                     # this is for debugging purpose,
                     # will be removed later, so simply use print
                     print(resp.get('errors', []))
+                    assert False, ("Aborted on status {0}").format(
+                        current_status)
                     return resp
 
             current_time = int(time.time())

tox.ini

@@ -66,9 +66,6 @@ import_exceptions = poppy.openstack.common.gettextutils._
 [testenv:api]
 deps = -r{toxinidir}/requirements/requirements.txt
        -r{toxinidir}/tests/test-requirements.txt
-whitelist_externals =
-        docker
-        sleep
 setenv = CAFE_CONFIG_FILE_PATH={homedir}/.poppy/tests.conf
          CAFE_ROOT_LOG_PATH={homedir}/.poppy/logs
          CAFE_TEST_LOG_PATH={homedir}/.poppy/logs
@@ -82,9 +79,6 @@ setenv = CAFE_CONFIG_FILE_PATH={homedir}/.poppy/tests.conf
 
 commands =
     pip install git+https://github.com/stackforge/opencafe.git#egg=cafe
-    pip install -U fig
+    ./run_poppy.sh -d
 
-    fig -f docker/fig/fig_cassandra.yml up -d
-    sleep 5
-    poppy-server --daemon
-    nosetests {posargs:--nologcapture}
+    nosetests api --nologcapture