diff --git a/docker/api/poppy_sample_mocked.conf b/docker/api/poppy_sample_mocked.conf new file mode 100644 index 00000000..837c6fa5 --- /dev/null +++ b/docker/api/poppy_sample_mocked.conf @@ -0,0 +1,53 @@ +[DEFAULT] +log_file = poppy.log +log_config_append = /etc/logging.conf +debug = True + +[drivers] +transport = pecan +manager = default +dns = rackspace +storage = cassandra +providers = fastly + +[drivers:storage:cassandra] +cluster = "cassandra" +keyspace = poppy +archive_on_delete = True + +[drivers:dns] +retries = 5 + +[drivers:distributed_task:taskflow] +jobboard_backend_type = zookeeper +persistent_backend_type = zookeeper +jobboard_backend_host = "zookeeper" +jobboard_backend_port = 2181 +persistent_backend_host = "zookeeper" +persistent_backend_port = 2181 + +[drivers:provider:fastly] +apikey = "fake_key_for_fastly_mimic" +scheme = "http" +host = "mimic:8900" + +[drivers:dns:rackspace] +auth_endpoint = http://mimic:8900/v2.0/tokens +username = "mock_user" +api_key = "mock_key" +use_shards = True +num_shards = 400 +shard_prefix = "cdn" +shared_ssl_num_shards = 5 +shared_ssl_shard_prefix = "scdn" +shared_ssl_domain_suffix = "secure.poppycdn.net" +url = "poppycdn.net" +url_404 = notfound.com +email = "you@email.com" +timeout = 30 +delay = 1 + +[log_delivery] +identity_url = http://mimic:8900/v2.0/tokens +preferred_dcs = 'IAD' +container_name = .CDN_ACCESS_LOGS \ No newline at end of file diff --git a/docker/compose/dependencies.yml b/docker/compose/dependencies.yml index 201e21ce..a26ecc3b 100644 --- a/docker/compose/dependencies.yml +++ b/docker/compose/dependencies.yml @@ -7,4 +7,9 @@ cassandra: image: library/cassandra ports: - 9160:9160 - - 9042:9042 \ No newline at end of file + - 9042:9042 + +mimic: + image: amitgandhinz/mimic + ports: + - 8900:8900 \ No newline at end of file diff --git a/docker/compose/poppy_stack.yml b/docker/compose/poppy_stack.yml index 72532ceb..b8b22427 100644 --- a/docker/compose/poppy_stack.yml +++ b/docker/compose/poppy_stack.yml @@ -1,11 +1,3 @@ -poppy: - build: ../api/. - ports: - - "80:8080" - links: - - cassandra - - zookeeper - zookeeper: extends: file: dependencies.yml @@ -14,4 +6,12 @@ zookeeper: cassandra: extends: file: dependencies.yml - service: cassandra \ No newline at end of file + service: cassandra + +poppy: + build: ../api/. + ports: + - "80:8080" + links: + - cassandra + - zookeeper \ No newline at end of file diff --git a/docker/compose/poppy_stack_mocked.yml b/docker/compose/poppy_stack_mocked.yml new file mode 100644 index 00000000..e498a724 --- /dev/null +++ b/docker/compose/poppy_stack_mocked.yml @@ -0,0 +1,24 @@ +zookeeper: + extends: + file: dependencies.yml + service: zookeeper + +cassandra: + extends: + file: dependencies.yml + service: cassandra + +mimic: + extends: + file: dependencies.yml + service: mimic + +poppy: + build: ../api/. + ports: + - "80:8080" + links: + - cassandra + - zookeeper + - mimic + diff --git a/docker/mimic/Dockerfile b/docker/mimic/Dockerfile index 411816ca..a5b8acc8 100644 --- a/docker/mimic/Dockerfile +++ b/docker/mimic/Dockerfile @@ -13,7 +13,7 @@ RUN apt-get update && apt-get install -y \ RUN /usr/bin/curl -s https://bootstrap.pypa.io/get-pip.py | python WORKDIR /home/source -RUN git clone https://github.com/malini-kamalambal/mimic . +RUN git clone https://github.com/rackerlabs/mimic . RUN pip install -r requirements.txt EXPOSE 8900 diff --git a/etc/poppy.conf b/etc/poppy.conf index 5e4b1e4f..3d27ff4b 100644 --- a/etc/poppy.conf +++ b/etc/poppy.conf @@ -38,10 +38,10 @@ transport = pecan manager = default # Storage driver module (e.g., mongodb, sqlite, cassandra) -storage = mockdb +storage = cassandra # Provider modules list (a list of comma separated provider module list) -providers = mock,fastly,akamai +providers = fastly # DNS driver module (e.g. default, designate, rackspace) dns = default @@ -80,9 +80,6 @@ replication_strategy = class:SimpleStrategy, replication_factor:1 # Path to directory containing CQL migration scripts migrations_path = /poppy/storage/cassandra/migrations -[drivers:storage:mockdb] -database = poppy - [drivers:distributed_task:taskflow] jobboard_backend_type = zookeeper persistent_backend_type = zookeeper @@ -119,8 +116,10 @@ default_cache_ttl = 86400 [drivers:provider:fastly] apikey = "MYAPIKEY" -scheme = "https" -host = "api.fastly.com" +# scheme = "https" +# host = "api.fastly.com" +scheme = "http" +host = "dockerhost:8900/fastly" [drivers:provider:maxcdn] alias = "MYALIAS" diff --git a/kill_poppy.sh b/kill_poppy.sh new file mode 100755 index 00000000..b80f9126 --- /dev/null +++ b/kill_poppy.sh @@ -0,0 +1 @@ +ps -ef | grep [p]oppy- | awk -F ' ' '{print$2}' | xargs kill -9 \ No newline at end of file diff --git a/run_poppy.sh b/run_poppy.sh new file mode 100755 index 00000000..35aa53e9 --- /dev/null +++ b/run_poppy.sh @@ -0,0 +1,92 @@ +#!/bin/bash +DAEMONIZED=false +WORKERS = 6 + +for i in "$@" +do + case $i in + -d|--daemonized) + DAEMONIZED=true + shift # past argument=value + ;; + + -w=*|--workers=*) + WORKERS="${i#*=}" + shift # past argument=value + ;; + + -?|--help) + echo "USAGE: ./run_poppy.sh -d -w=10" + echo "-d | --daemonized : run in daemonized mode" + echo "-w | --workers : the number of poppy-worker processes to spawn (defaults to 6)" + exit + + shift + ;; + + *) + echo "Invalid Options" + echo "Run ./run_poppy.sh --help for valid parameters." + exit + # unknown option + ;; + esac +done + +pip install docker-compose + + +# remove existing containers +docker kill compose_cassandra_1 +docker kill compose_zookeeper_1 +docker rm compose_cassandra_1 +docker rm compose_zookeeper_1 + +# start new containers +docker-compose -f docker/compose/dependencies.yml up -d + +is_cassandra_ready() { + nc -z dockerhost 9042 +} + +is_zookeeper_ready() { + nc -z dockerhost 2181 +} + +# wait until cassandra is ready +while ! is_cassandra_ready -eq 1 +do + echo "still trying to connect to cassandra" + sleep 1 +done +echo "connected successfully to cassandra" + + +# wait until zookeeper is ready +while ! is_zookeeper_ready -eq 1 +do + echo "still trying to connect to zookeeper" + sleep 1 +done +echo "connected successfully to zookeeper" + +# start the poppy-workers +COUNTER=0 +while [ $COUNTER -lt $WORKERS ]; do + exec poppy-worker > /dev/null 2>&1 & + echo "poppy-worker spawned." + let COUNTER=COUNTER+1 +done + + +# start the poppy-server +if $DAEMONIZED; then + exec poppy-server > /dev/null 2>&1 & + echo "poppy-server spawned." +else + exec poppy-server +fi + + + +echo "Poppy Server and Workers Started" \ No newline at end of file diff --git a/tests/api/README.rst b/tests/api/README.rst index 44878f32..5ba0b0db 100644 --- a/tests/api/README.rst +++ b/tests/api/README.rst @@ -13,7 +13,7 @@ To run the tests 1. Install the dependencies:: - pip install -r requirements.txt + $ pip install -r requirements.txt 2. Set the following environment variables:: @@ -21,17 +21,28 @@ To run the tests export CAFE_ROOT_LOG_PATH=~/.poppy/logs export CAFE_TEST_LOG_PATH=~/.poppy/logs -3. The API tests require a running database (eg cassandra), in order to +3. If you desire highlighting in the output, set the following environment variables:: + + export NOSE_WITH_OPENSTACK=1 + export NOSE_OPENSTACK_COLOR=1 + export NOSE_OPENSTACK_RED=0.05 + export NOSE_OPENSTACK_YELLOW=0.025 + export NOSE_OPENSTACK_SHOW_ELAPSED=1 + export NOSE_OPENSTACK_STDOUT=1 + + +4. The API tests require a running database (eg cassandra) and zookeeper, in order to run via tox. -4. Copy the api.conf file to the path set by CAFE_CONFIG_FILE_PATH:: + $ ./run_poppy.sh - cp tests/etc/api.conf ~/.poppy/tests.conf +5. Copy the api.conf file to the path set by CAFE_CONFIG_FILE_PATH:: -5. Once you are ready to run the tests:: + $ cp tests/etc/api.conf ~/.poppy/tests.conf - cd tests/api - nosetests +6. Once you are ready to run the tests:: + + $ nosetests api Tox Support @@ -41,14 +52,24 @@ You can run tox using a docker container hosting Cassandra:: Note - This will require docker (or boot2docker for MacOSX) to already be installed on the system. -1. Update your `~/.poppy/tests.conf` to point to your docker cassandra container ip address. +1. Update your `~/.poppy/tests.conf` to point to your docker cassandra/zookeeper container ip address. Example 1: Run all API tests against a docker hosted cassandra instance:: - tox -e api + $ tox -e api Example 2: Run a particular API test function:: - tox -e api api/services/test_services.py:TestCreateService -- -m test_create_service_positive + $ tox -e api api/services/test_services.py:TestCreateService -- -m test_create_service_positive +Mimic Support +------------- + +Occassionaly you want to test against a mock api rather than the real thing to get around rate limiting issues, +and to get around having to create accounts with a certain provider. + +Mimic helps accomplish this goal for testing. + +1. Run the mimic docker container (via ./run_poppy.sh) and point any remote api url in your test.conf file to your http://dockerhost:8900/mimic_service_name + diff --git a/tests/api/admin/test_akamai_SAN_domain_migration.py b/tests/api/admin/test_akamai_SAN_domain_migration.py index e6376f3e..2948672d 100644 --- a/tests/api/admin/test_akamai_SAN_domain_migration.py +++ b/tests/api/admin/test_akamai_SAN_domain_migration.py @@ -26,6 +26,9 @@ class TestSanCertService(base.TestBase): def setUp(self): super(TestSanCertService, self).setUp() + if self.test_config.run_operator_tests is False: + self.skipTest( + 'Test Operator Functions is disabled in configuration') self.service_name = self.generate_random_string(prefix='API-Test-') self.flavor_id = self.test_flavor @@ -93,9 +96,10 @@ class TestSanCertService(base.TestBase): self.service_url = resp.headers["location"] def test_migrate(self): + new_certs = self.akamai_config.san_certs new_certs_list = new_certs.split(',') - index = random.randint(0, len(new_certs_list)-1) + index = random.randint(0, len(new_certs_list) - 1) new_cert = new_certs_list[index] get_resp = self.client.get_service(location=self.service_url) @@ -125,9 +129,10 @@ class TestSanCertService(base.TestBase): self.assertEqual(data, new_cert) def test_migrate_negative_invalid_projectid(self): + new_certs = self.akamai_config.san_certs new_certs_list = new_certs.split(',') - index = random.randint(0, len(new_certs_list)-1) + index = random.randint(0, len(new_certs_list) - 1) new_cert = new_certs_list[index] get_resp = self.client.get_service(location=self.service_url) @@ -141,9 +146,10 @@ class TestSanCertService(base.TestBase): self.assertEqual(resp.status_code, 404) def test_migrate_negative_invalid_serviceid(self): + new_certs = self.akamai_config.san_certs new_certs_list = new_certs.split(',') - index = random.randint(0, len(new_certs_list)-1) + index = random.randint(0, len(new_certs_list) - 1) new_cert = new_certs_list[index] get_resp = self.client.get_service(location=self.service_url) @@ -157,9 +163,10 @@ class TestSanCertService(base.TestBase): self.assertEqual(resp.status_code, 404) def test_migrate_negative_invalid_domain(self): + new_certs = self.akamai_config.san_certs new_certs_list = new_certs.split(',') - index = random.randint(0, len(new_certs_list)-1) + index = random.randint(0, len(new_certs_list) - 1) new_cert = new_certs_list[index] get_resp = self.client.get_service(location=self.service_url) @@ -185,6 +192,10 @@ class TestSanCertServiceWithLogDelivery(base.TestBase): def setUp(self): super(TestSanCertServiceWithLogDelivery, self).setUp() + if self.test_config.run_operator_tests is False: + self.skipTest( + 'Test Operator Functions is disabled in configuration') + self.service_name = self.generate_random_string(prefix='API-Test-') self.flavor_id = self.test_flavor @@ -256,7 +267,7 @@ class TestSanCertServiceWithLogDelivery(base.TestBase): def test_migrate(self): new_certs = self.akamai_config.san_certs new_certs_list = new_certs.split(',') - index = random.randint(0, len(new_certs_list)-1) + index = random.randint(0, len(new_certs_list) - 1) new_cert = new_certs_list[index] get_resp = self.client.get_service(location=self.service_url) get_resp_body = get_resp.json() diff --git a/tests/api/admin/test_get_service_by_domain.py b/tests/api/admin/test_get_service_by_domain.py index e07c1c2d..9e9abb85 100644 --- a/tests/api/admin/test_get_service_by_domain.py +++ b/tests/api/admin/test_get_service_by_domain.py @@ -28,6 +28,10 @@ class TestGetServiceByDomain(base.TestBase): def setUp(self): super(TestGetServiceByDomain, self).setUp() + if self.test_config.run_operator_tests is False: + self.skipTest( + 'Test Operator Functions is disabled in configuration') + self.service_name = self.generate_random_string(prefix='api-test') self.flavor_id = self.test_flavor @@ -141,6 +145,10 @@ class TestGetServiceByDomain(base.TestBase): self.assertEqual(api_resp2.status_code, 200) def test_negative_get_by_non_existing_domain(self): + if self.test_config.run_operator_tests is False: + self.skipTest( + 'Test Operator Functions is disabled in configuration') + domain_name = self.domain_list[0]['domain'] + str(uuid.uuid1()) + \ ".com" resp = self.operator_client.admin_get_service_by_domain_name( @@ -179,6 +187,10 @@ class TestGetServiceBySharedDomain(base.TestBase): def setUp(self): super(TestGetServiceBySharedDomain, self).setUp() + if self.test_config.run_operator_tests is False: + self.skipTest( + 'Test Operator Functions is disabled in configuration') + self.service_name = self.generate_random_string(prefix='API-Test-') self.flavor_id = self.test_flavor @@ -246,6 +258,7 @@ class TestGetServiceBySharedDomain(base.TestBase): self.service_url = resp.headers["location"] def test_get_service_by_domain(self): + get_resp = self.client.get_service(self.service_url) resp_body = get_resp.json() domain = resp_body['domains'][0]['domain'] @@ -280,6 +293,11 @@ class TestGetServiceBySANCertDomain(base.TestBase): def setUp(self): super(TestGetServiceBySANCertDomain, self).setUp() + + if self.test_config.run_operator_tests is False: + self.skipTest( + 'Test Operator Functions is disabled in configuration') + self.service_name = self.generate_random_string(prefix='API-Test-') self.flavor_id = self.test_flavor @@ -347,6 +365,7 @@ class TestGetServiceBySANCertDomain(base.TestBase): self.service_url = resp.headers["location"] def test_get_service_by_domain(self): + get_resp = self.client.get_service(self.service_url) resp_body = get_resp.json() domain = resp_body['domains'][0]['domain'] @@ -381,6 +400,11 @@ class TestGetServiceByCustomCertDomain(base.TestBase): def setUp(self): super(TestGetServiceByCustomCertDomain, self).setUp() + + if self.test_config.run_operator_tests is False: + self.skipTest( + 'Test Operator Functions is disabled in configuration') + self.service_name = self.generate_random_string(prefix='API-Test-') self.flavor_id = self.test_flavor @@ -448,6 +472,7 @@ class TestGetServiceByCustomCertDomain(base.TestBase): self.service_url = resp.headers["location"] def test_get_service_by_domain(self): + get_resp = self.client.get_service(self.service_url) resp_body = get_resp.json() domain = resp_body['domains'][0]['domain'] diff --git a/tests/api/admin/test_perform_action.py b/tests/api/admin/test_perform_action.py index ca38bde1..a590d438 100644 --- a/tests/api/admin/test_perform_action.py +++ b/tests/api/admin/test_perform_action.py @@ -25,6 +25,11 @@ class TestHttpService(base.TestBase): def setUp(self): super(TestHttpService, self).setUp() + + if self.test_config.run_operator_tests is False: + self.skipTest( + 'Test Operator Functions is disabled in configuration') + self.service_name = self.generate_random_string(prefix='API-Test-') self.flavor_id = self.test_flavor @@ -198,6 +203,11 @@ class TestSharedCertService(base.TestBase): def setUp(self): super(TestSharedCertService, self).setUp() + + if self.test_config.run_operator_tests is False: + self.skipTest( + 'Test Operator Functions is disabled in configuration') + self.service_name = self.generate_random_string(prefix='API-Test-') self.flavor_id = self.test_flavor @@ -352,6 +362,11 @@ class TestSanCertService(base.TestBase): def setUp(self): super(TestSanCertService, self).setUp() + + if self.test_config.run_operator_tests is False: + self.skipTest( + 'Test Operator Functions is disabled in configuration') + self.service_name = self.generate_random_string(prefix='API-Test-') self.flavor_id = self.test_flavor @@ -506,6 +521,11 @@ class TestCustomCertService(base.TestBase): def setUp(self): super(TestCustomCertService, self).setUp() + + if self.test_config.run_operator_tests is False: + self.skipTest( + 'Test Operator Functions is disabled in configuration') + self.service_name = self.generate_random_string(prefix='API-Test-') self.flavor_id = self.test_flavor @@ -660,6 +680,11 @@ class TestHttpServiceWithLogDelivery(base.TestBase): def setUp(self): super(TestHttpServiceWithLogDelivery, self).setUp() + + if self.test_config.run_operator_tests is False: + self.skipTest( + 'Test Operator Functions is disabled in configuration') + self.service_name = self.generate_random_string(prefix='API-Test-') self.flavor_id = self.test_flavor diff --git a/tests/api/base.py b/tests/api/base.py index 0b8a0d3c..27f55fdb 100644 --- a/tests/api/base.py +++ b/tests/api/base.py @@ -25,6 +25,7 @@ from tests.api.utils import config class TestBase(fixtures.BaseTestFixture): + """Child class of fixtures.BaseTestFixture for testing CDN. Inherit from this and write your test methods. If the child class defines @@ -92,16 +93,17 @@ class TestBase(fixtures.BaseTestFixture): operator_project_id else: cls.operator_url = cls.config.base_url + '/v1.0' + cls.operator_client = client.PoppyClient( cls.operator_url, operator_auth_token, operator_project_id, serialize_format='json', deserialize_format='json') - cls.dns_config = config.DNSConfig() - cls.dns_client = client.DNSClient(cls.dns_config.dns_username, - cls.dns_config.dns_api_key) + cls.dns_config = config.DNSConfig() + cls.dns_client = client.DNSClient(cls.dns_config.dns_username, + cls.dns_config.dns_api_key) - cls.akamai_config = config.AkamaiConfig() + cls.akamai_config = config.AkamaiConfig() def generate_random_string(self, prefix='API-Tests', length=12): """Generates a random string of given prefix & length""" diff --git a/tests/api/services/test_services.py b/tests/api/services/test_services.py index 12b81b95..a8d309bb 100644 --- a/tests/api/services/test_services.py +++ b/tests/api/services/test_services.py @@ -357,7 +357,9 @@ class TestServiceActions(base.TestBase): self.service_url = resp.headers["location"] self.client.wait_for_service_status( - location=self.service_url, status='deployed') + location=self.service_url, + status='deployed', + abort_on_status='failed') @attrib.attr('smoke') def test_delete_service(self): diff --git a/tests/api/utils/client.py b/tests/api/utils/client.py index 628aef10..6562370b 100644 --- a/tests/api/utils/client.py +++ b/tests/api/utils/client.py @@ -58,6 +58,7 @@ class AuthClient(client.HTTPClient): class DNSClient(client.HTTPClient): + def __init__(self, username, api_key): super(DNSClient, self).__init__() @@ -339,6 +340,8 @@ class PoppyClient(client.AutoMarshallingHTTPClient): # this is for debugging purpose, # will be removed later, so simply use print print(body.get('errors', [])) + assert False, ("Aborted on status {0}").format( + current_status) return service current_time = int(time.time()) @@ -366,6 +369,8 @@ class PoppyClient(client.AutoMarshallingHTTPClient): # this is for debugging purpose, # will be removed later, so simply use print print(resp.get('errors', [])) + assert False, ("Aborted on status {0}").format( + current_status) return resp current_time = int(time.time()) diff --git a/tox.ini b/tox.ini index 0648ff7d..b88b8cd4 100644 --- a/tox.ini +++ b/tox.ini @@ -66,9 +66,6 @@ import_exceptions = poppy.openstack.common.gettextutils._ [testenv:api] deps = -r{toxinidir}/requirements/requirements.txt -r{toxinidir}/tests/test-requirements.txt -whitelist_externals = - docker - sleep setenv = CAFE_CONFIG_FILE_PATH={homedir}/.poppy/tests.conf CAFE_ROOT_LOG_PATH={homedir}/.poppy/logs CAFE_TEST_LOG_PATH={homedir}/.poppy/logs @@ -82,9 +79,6 @@ setenv = CAFE_CONFIG_FILE_PATH={homedir}/.poppy/tests.conf commands = pip install git+https://github.com/stackforge/opencafe.git#egg=cafe - pip install -U fig + ./run_poppy.sh -d - fig -f docker/fig/fig_cassandra.yml up -d - sleep 5 - poppy-server --daemon - nosetests {posargs:--nologcapture} + nosetests api --nologcapture