Merge "Added Docker Files to mimic third party API's"

docker/api/poppy_sample_mocked.conf

@@ -0,0 +1,53 @@
+[DEFAULT]
+log_file = poppy.log
+log_config_append = /etc/logging.conf
+debug = True
+
+[drivers]
+transport = pecan
+manager = default
+dns = rackspace
+storage = cassandra
+providers = fastly
+
+[drivers:storage:cassandra]
+cluster = "cassandra"
+keyspace = poppy
+archive_on_delete = True
+
+[drivers:dns]
+retries = 5
+
+[drivers:distributed_task:taskflow]
+jobboard_backend_type = zookeeper
+persistent_backend_type = zookeeper
+jobboard_backend_host = "zookeeper"
+jobboard_backend_port = 2181
+persistent_backend_host = "zookeeper"
+persistent_backend_port = 2181
+
+[drivers:provider:fastly]
+apikey = "fake_key_for_fastly_mimic"
+scheme = "http"
+host = "mimic:8900"
+
+[drivers:dns:rackspace]
+auth_endpoint = http://mimic:8900/v2.0/tokens
+username = "mock_user"
+api_key = "mock_key"
+use_shards = True
+num_shards = 400
+shard_prefix = "cdn"
+shared_ssl_num_shards = 5
+shared_ssl_shard_prefix = "scdn"
+shared_ssl_domain_suffix = "secure.poppycdn.net"
+url = "poppycdn.net"
+url_404 = notfound.com
+email = "you@email.com"
+timeout = 30
+delay = 1
+
+[log_delivery]
+identity_url = http://mimic:8900/v2.0/tokens
+preferred_dcs = 'IAD'
+container_name = .CDN_ACCESS_LOGS

docker/compose/dependencies.yml

@@ -7,4 +7,9 @@ cassandra:
     image: library/cassandra
     ports:
      - 9160:9160
-     - 9042:9042
+     - 9042:9042
+
+mimic:
+    image: amitgandhinz/mimic
+    ports:
+     - 8900:8900

docker/compose/poppy_stack.yml

@@ -1,11 +1,3 @@
-poppy:
-    build: ../api/.
-    ports:
-        - "80:8080"
-    links:
-        - cassandra
-        - zookeeper
-
 zookeeper:
     extends:
         file: dependencies.yml
@@ -14,4 +6,12 @@ zookeeper:
 cassandra:
     extends:
         file: dependencies.yml
-        service: cassandra
+        service: cassandra
+
+poppy:
+    build: ../api/.
+    ports:
+        - "80:8080"
+    links:
+        - cassandra
+        - zookeeper

docker/compose/poppy_stack_mocked.yml

@@ -0,0 +1,24 @@
+zookeeper:
+    extends:
+        file: dependencies.yml
+        service: zookeeper
+
+cassandra:
+    extends:
+        file: dependencies.yml
+        service: cassandra
+
+mimic:
+    extends:
+        file: dependencies.yml
+        service: mimic
+
+poppy:
+    build: ../api/.
+    ports:
+        - "80:8080"
+    links:
+        - cassandra
+        - zookeeper
+        - mimic
+

docker/mimic/Dockerfile

@@ -13,7 +13,7 @@ RUN apt-get update && apt-get install -y \
 RUN /usr/bin/curl -s https://bootstrap.pypa.io/get-pip.py | python
 
 WORKDIR /home/source
-RUN git clone https://github.com/malini-kamalambal/mimic .
+RUN git clone https://github.com/rackerlabs/mimic .
 RUN pip install -r requirements.txt
 
 EXPOSE 8900

etc/poppy.conf

@@ -38,10 +38,10 @@ transport = pecan
 manager = default
 
 # Storage driver module (e.g., mongodb, sqlite, cassandra)
-storage = mockdb
+storage = cassandra
 
 # Provider modules list (a list of comma separated provider module list)
-providers = mock,fastly,akamai
+providers = fastly
 
 # DNS driver module (e.g. default, designate, rackspace)
 dns = default
@@ -80,9 +80,6 @@ replication_strategy = class:SimpleStrategy, replication_factor:1
 # Path to directory containing CQL migration scripts
 migrations_path = <poppy_code_path>/poppy/storage/cassandra/migrations
 
-[drivers:storage:mockdb]
-database = poppy
-
 [drivers:distributed_task:taskflow]
 jobboard_backend_type = zookeeper
 persistent_backend_type = zookeeper
@@ -119,8 +116,10 @@ default_cache_ttl = 86400
 
 [drivers:provider:fastly]
 apikey = "MYAPIKEY"
-scheme = "https"
-host = "api.fastly.com"
+# scheme = "https"
+# host = "api.fastly.com"
+scheme = "http"
+host = "dockerhost:8900/fastly"
 
 [drivers:provider:maxcdn]
 alias = "MYALIAS"

kill_poppy.sh

@@ -0,0 +1 @@
+ps -ef | grep [p]oppy- | awk -F ' ' '{print$2}' | xargs kill -9

run_poppy.sh

@@ -0,0 +1,92 @@
+#!/bin/bash
+DAEMONIZED=false
+WORKERS = 6
+
+for i in "$@"
+do
+  case $i in
+      -d|--daemonized)
+      DAEMONIZED=true
+      shift # past argument=value
+      ;;
+
+      -w=*|--workers=*)
+      WORKERS="${i#*=}"
+      shift # past argument=value
+      ;;
+
+      -?|--help)
+      echo "USAGE: ./run_poppy.sh -d -w=10"
+      echo "-d | --daemonized : run in daemonized mode"
+      echo "-w | --workers : the number of poppy-worker processes to spawn (defaults to 6)"
+      exit
+
+      shift
+      ;;
+
+      *)
+      echo "Invalid Options"
+      echo "Run ./run_poppy.sh --help for valid parameters."
+      exit
+              # unknown option
+      ;;
+  esac
+done
+
+pip install docker-compose
+
+
+# remove existing containers
+docker kill compose_cassandra_1
+docker kill compose_zookeeper_1
+docker rm compose_cassandra_1
+docker rm compose_zookeeper_1
+
+# start new containers
+docker-compose -f docker/compose/dependencies.yml up -d
+
+is_cassandra_ready() {
+    nc -z dockerhost 9042
+}
+
+is_zookeeper_ready() {
+    nc -z dockerhost 2181
+}
+
+# wait until cassandra is ready
+while ! is_cassandra_ready -eq 1
+do
+  echo "still trying to connect to cassandra"
+  sleep 1
+done
+echo "connected successfully to cassandra"
+
+
+# wait until zookeeper is ready
+while ! is_zookeeper_ready -eq 1
+do
+  echo "still trying to connect to zookeeper"
+  sleep 1
+done
+echo "connected successfully to zookeeper"
+
+# start the poppy-workers
+COUNTER=0
+while [ $COUNTER -lt $WORKERS ]; do
+  exec poppy-worker > /dev/null 2>&1 &
+  echo "poppy-worker spawned."
+  let COUNTER=COUNTER+1 
+done
+
+
+# start the poppy-server
+if $DAEMONIZED; then
+  exec poppy-server > /dev/null 2>&1 &
+  echo "poppy-server spawned."
+else
+  exec poppy-server
+fi
+
+
+
+echo "Poppy Server and Workers Started"

tests/api/README.rst

@@ -13,7 +13,7 @@ To run the tests
 
 1. Install the dependencies::
 
-    pip install -r requirements.txt
+    $ pip install -r requirements.txt
 
 2. Set the following environment variables::
 
@@ -21,17 +21,28 @@ To run the tests
     export CAFE_ROOT_LOG_PATH=~/.poppy/logs
     export CAFE_TEST_LOG_PATH=~/.poppy/logs
 
-3. The API tests require a running database (eg cassandra), in order to
+3. If you desire highlighting in the output, set the following environment variables::
+
+    export NOSE_WITH_OPENSTACK=1
+    export NOSE_OPENSTACK_COLOR=1
+    export NOSE_OPENSTACK_RED=0.05
+    export NOSE_OPENSTACK_YELLOW=0.025
+    export NOSE_OPENSTACK_SHOW_ELAPSED=1
+    export NOSE_OPENSTACK_STDOUT=1
+
+
+4. The API tests require a running database (eg cassandra) and zookeeper, in order to
 run via tox.
 
-4. Copy the api.conf file to the path set by CAFE_CONFIG_FILE_PATH::
+    $ ./run_poppy.sh
 
-    cp tests/etc/api.conf ~/.poppy/tests.conf
+5. Copy the api.conf file to the path set by CAFE_CONFIG_FILE_PATH::
 
-5. Once you are ready to run the tests::
+    $ cp tests/etc/api.conf ~/.poppy/tests.conf
 
-    cd tests/api
-    nosetests
+6. Once you are ready to run the tests::
+
+    $ nosetests api
 
 
 Tox Support
@@ -41,14 +52,24 @@ You can run tox using a docker container hosting Cassandra::
 
 Note - This will require docker (or boot2docker for MacOSX) to already be installed on the system.
 
-1. Update your `~/.poppy/tests.conf` to point to your docker cassandra container ip address.
+1. Update your `~/.poppy/tests.conf` to point to your docker cassandra/zookeeper container ip address.
 
 Example 1: Run all API tests against a docker hosted cassandra instance::
 
-    tox -e api
+    $ tox -e api
 
 Example 2: Run a particular API test function::
 
-    tox -e api api/services/test_services.py:TestCreateService -- -m test_create_service_positive
+    $ tox -e api api/services/test_services.py:TestCreateService -- -m test_create_service_positive
 
 
+Mimic Support
+-------------
+
+Occassionaly you want to test against a mock api rather than the real thing to get around rate limiting issues,
+and to get around having to create accounts with a certain provider.
+
+Mimic helps accomplish this goal for testing.
+
+1.  Run the mimic docker container (via ./run_poppy.sh) and point any remote api url in your test.conf file to your http://dockerhost:8900/mimic_service_name
+

tests/api/admin/test_akamai_SAN_domain_migration.py

@@ -26,6 +26,9 @@ class TestSanCertService(base.TestBase):
 
     def setUp(self):
         super(TestSanCertService, self).setUp()
+        if self.test_config.run_operator_tests is False:
+            self.skipTest(
+                'Test Operator Functions is disabled in configuration')
 
         self.service_name = self.generate_random_string(prefix='API-Test-')
         self.flavor_id = self.test_flavor
@@ -93,9 +96,10 @@ class TestSanCertService(base.TestBase):
         self.service_url = resp.headers["location"]
 
     def test_migrate(self):
+
         new_certs = self.akamai_config.san_certs
         new_certs_list = new_certs.split(',')
-        index = random.randint(0, len(new_certs_list)-1)
+        index = random.randint(0, len(new_certs_list) - 1)
         new_cert = new_certs_list[index]
 
         get_resp = self.client.get_service(location=self.service_url)
@@ -125,9 +129,10 @@ class TestSanCertService(base.TestBase):
         self.assertEqual(data, new_cert)
 
     def test_migrate_negative_invalid_projectid(self):
+
         new_certs = self.akamai_config.san_certs
         new_certs_list = new_certs.split(',')
-        index = random.randint(0, len(new_certs_list)-1)
+        index = random.randint(0, len(new_certs_list) - 1)
         new_cert = new_certs_list[index]
 
         get_resp = self.client.get_service(location=self.service_url)
@@ -141,9 +146,10 @@ class TestSanCertService(base.TestBase):
         self.assertEqual(resp.status_code, 404)
 
     def test_migrate_negative_invalid_serviceid(self):
+
         new_certs = self.akamai_config.san_certs
         new_certs_list = new_certs.split(',')
-        index = random.randint(0, len(new_certs_list)-1)
+        index = random.randint(0, len(new_certs_list) - 1)
         new_cert = new_certs_list[index]
 
         get_resp = self.client.get_service(location=self.service_url)
@@ -157,9 +163,10 @@ class TestSanCertService(base.TestBase):
         self.assertEqual(resp.status_code, 404)
 
     def test_migrate_negative_invalid_domain(self):
+
         new_certs = self.akamai_config.san_certs
         new_certs_list = new_certs.split(',')
-        index = random.randint(0, len(new_certs_list)-1)
+        index = random.randint(0, len(new_certs_list) - 1)
         new_cert = new_certs_list[index]
 
         get_resp = self.client.get_service(location=self.service_url)
@@ -185,6 +192,10 @@ class TestSanCertServiceWithLogDelivery(base.TestBase):
     def setUp(self):
         super(TestSanCertServiceWithLogDelivery, self).setUp()
 
+        if self.test_config.run_operator_tests is False:
+            self.skipTest(
+                'Test Operator Functions is disabled in configuration')
+
         self.service_name = self.generate_random_string(prefix='API-Test-')
         self.flavor_id = self.test_flavor
 
@@ -256,7 +267,7 @@ class TestSanCertServiceWithLogDelivery(base.TestBase):
     def test_migrate(self):
         new_certs = self.akamai_config.san_certs
         new_certs_list = new_certs.split(',')
-        index = random.randint(0, len(new_certs_list)-1)
+        index = random.randint(0, len(new_certs_list) - 1)
         new_cert = new_certs_list[index]
         get_resp = self.client.get_service(location=self.service_url)
         get_resp_body = get_resp.json()

tests/api/admin/test_get_service_by_domain.py

@@ -28,6 +28,10 @@ class TestGetServiceByDomain(base.TestBase):
     def setUp(self):
         super(TestGetServiceByDomain, self).setUp()
 
+        if self.test_config.run_operator_tests is False:
+            self.skipTest(
+                'Test Operator Functions is disabled in configuration')
+
         self.service_name = self.generate_random_string(prefix='api-test')
         self.flavor_id = self.test_flavor
 
@@ -141,6 +145,10 @@ class TestGetServiceByDomain(base.TestBase):
         self.assertEqual(api_resp2.status_code, 200)
 
     def test_negative_get_by_non_existing_domain(self):
+        if self.test_config.run_operator_tests is False:
+            self.skipTest(
+                'Test Operator Functions is disabled in configuration')
+
         domain_name = self.domain_list[0]['domain'] + str(uuid.uuid1()) + \
             ".com"
         resp = self.operator_client.admin_get_service_by_domain_name(
@@ -179,6 +187,10 @@ class TestGetServiceBySharedDomain(base.TestBase):
     def setUp(self):
         super(TestGetServiceBySharedDomain, self).setUp()
 
+        if self.test_config.run_operator_tests is False:
+            self.skipTest(
+                'Test Operator Functions is disabled in configuration')
+
         self.service_name = self.generate_random_string(prefix='API-Test-')
         self.flavor_id = self.test_flavor
 
@@ -246,6 +258,7 @@ class TestGetServiceBySharedDomain(base.TestBase):
         self.service_url = resp.headers["location"]
 
     def test_get_service_by_domain(self):
+
         get_resp = self.client.get_service(self.service_url)
         resp_body = get_resp.json()
         domain = resp_body['domains'][0]['domain']
@@ -280,6 +293,11 @@ class TestGetServiceBySANCertDomain(base.TestBase):
 
     def setUp(self):
         super(TestGetServiceBySANCertDomain, self).setUp()
+
+        if self.test_config.run_operator_tests is False:
+            self.skipTest(
+                'Test Operator Functions is disabled in configuration')
+
         self.service_name = self.generate_random_string(prefix='API-Test-')
         self.flavor_id = self.test_flavor
 
@@ -347,6 +365,7 @@ class TestGetServiceBySANCertDomain(base.TestBase):
         self.service_url = resp.headers["location"]
 
     def test_get_service_by_domain(self):
+
         get_resp = self.client.get_service(self.service_url)
         resp_body = get_resp.json()
         domain = resp_body['domains'][0]['domain']
@@ -381,6 +400,11 @@ class TestGetServiceByCustomCertDomain(base.TestBase):
 
     def setUp(self):
         super(TestGetServiceByCustomCertDomain, self).setUp()
+
+        if self.test_config.run_operator_tests is False:
+            self.skipTest(
+                'Test Operator Functions is disabled in configuration')
+
         self.service_name = self.generate_random_string(prefix='API-Test-')
         self.flavor_id = self.test_flavor
 
@@ -448,6 +472,7 @@ class TestGetServiceByCustomCertDomain(base.TestBase):
         self.service_url = resp.headers["location"]
 
     def test_get_service_by_domain(self):
+
         get_resp = self.client.get_service(self.service_url)
         resp_body = get_resp.json()
         domain = resp_body['domains'][0]['domain']

tests/api/admin/test_perform_action.py

@@ -25,6 +25,11 @@ class TestHttpService(base.TestBase):
 
     def setUp(self):
         super(TestHttpService, self).setUp()
+
+        if self.test_config.run_operator_tests is False:
+            self.skipTest(
+                'Test Operator Functions is disabled in configuration')
+
         self.service_name = self.generate_random_string(prefix='API-Test-')
         self.flavor_id = self.test_flavor
 
@@ -198,6 +203,11 @@ class TestSharedCertService(base.TestBase):
 
     def setUp(self):
         super(TestSharedCertService, self).setUp()
+
+        if self.test_config.run_operator_tests is False:
+            self.skipTest(
+                'Test Operator Functions is disabled in configuration')
+
         self.service_name = self.generate_random_string(prefix='API-Test-')
         self.flavor_id = self.test_flavor
 
@@ -352,6 +362,11 @@ class TestSanCertService(base.TestBase):
 
     def setUp(self):
         super(TestSanCertService, self).setUp()
+
+        if self.test_config.run_operator_tests is False:
+            self.skipTest(
+                'Test Operator Functions is disabled in configuration')
+
         self.service_name = self.generate_random_string(prefix='API-Test-')
         self.flavor_id = self.test_flavor
 
@@ -506,6 +521,11 @@ class TestCustomCertService(base.TestBase):
 
     def setUp(self):
         super(TestCustomCertService, self).setUp()
+
+        if self.test_config.run_operator_tests is False:
+            self.skipTest(
+                'Test Operator Functions is disabled in configuration')
+
         self.service_name = self.generate_random_string(prefix='API-Test-')
         self.flavor_id = self.test_flavor
 
@@ -660,6 +680,11 @@ class TestHttpServiceWithLogDelivery(base.TestBase):
 
     def setUp(self):
         super(TestHttpServiceWithLogDelivery, self).setUp()
+
+        if self.test_config.run_operator_tests is False:
+            self.skipTest(
+                'Test Operator Functions is disabled in configuration')
+
         self.service_name = self.generate_random_string(prefix='API-Test-')
         self.flavor_id = self.test_flavor
 

tests/api/base.py

@@ -25,6 +25,7 @@ from tests.api.utils import config
 
 
 class TestBase(fixtures.BaseTestFixture):
+
     """Child class of fixtures.BaseTestFixture for testing CDN.
 
     Inherit from this and write your test methods. If the child class defines
@@ -92,16 +93,17 @@ class TestBase(fixtures.BaseTestFixture):
                     operator_project_id
             else:
                 cls.operator_url = cls.config.base_url + '/v1.0'
+
             cls.operator_client = client.PoppyClient(
                 cls.operator_url, operator_auth_token, operator_project_id,
                 serialize_format='json',
                 deserialize_format='json')
 
-        cls.dns_config = config.DNSConfig()
-        cls.dns_client = client.DNSClient(cls.dns_config.dns_username,
-                                          cls.dns_config.dns_api_key)
+            cls.dns_config = config.DNSConfig()
+            cls.dns_client = client.DNSClient(cls.dns_config.dns_username,
+                                              cls.dns_config.dns_api_key)
 
-        cls.akamai_config = config.AkamaiConfig()
+            cls.akamai_config = config.AkamaiConfig()
 
     def generate_random_string(self, prefix='API-Tests', length=12):
         """Generates a random string of given prefix & length"""

tests/api/services/test_services.py

@@ -357,7 +357,9 @@ class TestServiceActions(base.TestBase):
 
         self.service_url = resp.headers["location"]
         self.client.wait_for_service_status(
-            location=self.service_url, status='deployed')
+            location=self.service_url,
+            status='deployed',
+            abort_on_status='failed')
 
     @attrib.attr('smoke')
     def test_delete_service(self):

tests/api/utils/client.py

@@ -58,6 +58,7 @@ class AuthClient(client.HTTPClient):
 
 
 class DNSClient(client.HTTPClient):
+
     def __init__(self, username, api_key):
         super(DNSClient, self).__init__()
 
@@ -339,6 +340,8 @@ class PoppyClient(client.AutoMarshallingHTTPClient):
                     # this is for debugging purpose,
                     # will be removed later, so simply use print
                     print(body.get('errors', []))
+                    assert False, ("Aborted on status {0}").format(
+                        current_status)
                     return service
 
             current_time = int(time.time())
@@ -366,6 +369,8 @@ class PoppyClient(client.AutoMarshallingHTTPClient):
                     # this is for debugging purpose,
                     # will be removed later, so simply use print
                     print(resp.get('errors', []))
+                    assert False, ("Aborted on status {0}").format(
+                        current_status)
                     return resp
 
             current_time = int(time.time())

tox.ini

@@ -66,9 +66,6 @@ import_exceptions = poppy.openstack.common.gettextutils._
 [testenv:api]
 deps = -r{toxinidir}/requirements/requirements.txt
        -r{toxinidir}/tests/test-requirements.txt
-whitelist_externals =
-        docker
-        sleep
 setenv = CAFE_CONFIG_FILE_PATH={homedir}/.poppy/tests.conf
          CAFE_ROOT_LOG_PATH={homedir}/.poppy/logs
          CAFE_TEST_LOG_PATH={homedir}/.poppy/logs
@@ -82,9 +79,6 @@ setenv = CAFE_CONFIG_FILE_PATH={homedir}/.poppy/tests.conf
 
 commands =
     pip install git+https://github.com/stackforge/opencafe.git#egg=cafe
-    pip install -U fig
+    ./run_poppy.sh -d
 
-    fig -f docker/fig/fig_cassandra.yml up -d
-    sleep 5
-    poppy-server --daemon
-    nosetests {posargs:--nologcapture}
+    nosetests api --nologcapture