From 0d08dfa66e37fa32b859580808723a5001724deb Mon Sep 17 00:00:00 2001 From: amitgandhinz Date: Tue, 21 Jul 2015 17:52:30 -0400 Subject: [PATCH] Added Docker Files to mimic third party API's This change works towards making the API tests pass against Mimic'd drivers so that real production accounts do not need to be set up - namely: - Mimic - Fastly - Mimic - Keystone It also makes the mimic fastly instance the default driver in /etc Implements: bp/mimic-fastly Change-Id: I35d36fbc2452fc4e2233c19abefd738c9975edd0 --- docker/api/poppy_sample_mocked.conf | 53 +++++++++++ docker/compose/dependencies.yml | 7 +- docker/compose/poppy_stack.yml | 18 ++-- docker/compose/poppy_stack_mocked.yml | 24 +++++ docker/mimic/Dockerfile | 2 +- etc/poppy.conf | 13 ++- kill_poppy.sh | 1 + run_poppy.sh | 92 +++++++++++++++++++ tests/api/README.rst | 41 +++++++-- .../admin/test_akamai_SAN_domain_migration.py | 21 ++++- tests/api/admin/test_get_service_by_domain.py | 25 +++++ tests/api/admin/test_perform_action.py | 25 +++++ tests/api/base.py | 10 +- tests/api/services/test_services.py | 4 +- tests/api/utils/client.py | 5 + tox.ini | 10 +- 16 files changed, 305 insertions(+), 46 deletions(-) create mode 100644 docker/api/poppy_sample_mocked.conf create mode 100644 docker/compose/poppy_stack_mocked.yml create mode 100755 kill_poppy.sh create mode 100755 run_poppy.sh diff --git a/docker/api/poppy_sample_mocked.conf b/docker/api/poppy_sample_mocked.conf new file mode 100644 index 00000000..837c6fa5 --- /dev/null +++ b/docker/api/poppy_sample_mocked.conf @@ -0,0 +1,53 @@ +[DEFAULT] +log_file = poppy.log +log_config_append = /etc/logging.conf +debug = True + +[drivers] +transport = pecan +manager = default +dns = rackspace +storage = cassandra +providers = fastly + +[drivers:storage:cassandra] +cluster = "cassandra" +keyspace = poppy +archive_on_delete = True + +[drivers:dns] +retries = 5 + +[drivers:distributed_task:taskflow] +jobboard_backend_type = zookeeper +persistent_backend_type = zookeeper +jobboard_backend_host = "zookeeper" +jobboard_backend_port = 2181 +persistent_backend_host = "zookeeper" +persistent_backend_port = 2181 + +[drivers:provider:fastly] +apikey = "fake_key_for_fastly_mimic" +scheme = "http" +host = "mimic:8900" + +[drivers:dns:rackspace] +auth_endpoint = http://mimic:8900/v2.0/tokens +username = "mock_user" +api_key = "mock_key" +use_shards = True +num_shards = 400 +shard_prefix = "cdn" +shared_ssl_num_shards = 5 +shared_ssl_shard_prefix = "scdn" +shared_ssl_domain_suffix = "secure.poppycdn.net" +url = "poppycdn.net" +url_404 = notfound.com +email = "you@email.com" +timeout = 30 +delay = 1 + +[log_delivery] +identity_url = http://mimic:8900/v2.0/tokens +preferred_dcs = 'IAD' +container_name = .CDN_ACCESS_LOGS \ No newline at end of file diff --git a/docker/compose/dependencies.yml b/docker/compose/dependencies.yml index 201e21ce..a26ecc3b 100644 --- a/docker/compose/dependencies.yml +++ b/docker/compose/dependencies.yml @@ -7,4 +7,9 @@ cassandra: image: library/cassandra ports: - 9160:9160 - - 9042:9042 \ No newline at end of file + - 9042:9042 + +mimic: + image: amitgandhinz/mimic + ports: + - 8900:8900 \ No newline at end of file diff --git a/docker/compose/poppy_stack.yml b/docker/compose/poppy_stack.yml index 72532ceb..b8b22427 100644 --- a/docker/compose/poppy_stack.yml +++ b/docker/compose/poppy_stack.yml @@ -1,11 +1,3 @@ -poppy: - build: ../api/. - ports: - - "80:8080" - links: - - cassandra - - zookeeper - zookeeper: extends: file: dependencies.yml @@ -14,4 +6,12 @@ zookeeper: cassandra: extends: file: dependencies.yml - service: cassandra \ No newline at end of file + service: cassandra + +poppy: + build: ../api/. + ports: + - "80:8080" + links: + - cassandra + - zookeeper \ No newline at end of file diff --git a/docker/compose/poppy_stack_mocked.yml b/docker/compose/poppy_stack_mocked.yml new file mode 100644 index 00000000..e498a724 --- /dev/null +++ b/docker/compose/poppy_stack_mocked.yml @@ -0,0 +1,24 @@ +zookeeper: + extends: + file: dependencies.yml + service: zookeeper + +cassandra: + extends: + file: dependencies.yml + service: cassandra + +mimic: + extends: + file: dependencies.yml + service: mimic + +poppy: + build: ../api/. + ports: + - "80:8080" + links: + - cassandra + - zookeeper + - mimic + diff --git a/docker/mimic/Dockerfile b/docker/mimic/Dockerfile index 411816ca..a5b8acc8 100644 --- a/docker/mimic/Dockerfile +++ b/docker/mimic/Dockerfile @@ -13,7 +13,7 @@ RUN apt-get update && apt-get install -y \ RUN /usr/bin/curl -s https://bootstrap.pypa.io/get-pip.py | python WORKDIR /home/source -RUN git clone https://github.com/malini-kamalambal/mimic . +RUN git clone https://github.com/rackerlabs/mimic . RUN pip install -r requirements.txt EXPOSE 8900 diff --git a/etc/poppy.conf b/etc/poppy.conf index 5e4b1e4f..3d27ff4b 100644 --- a/etc/poppy.conf +++ b/etc/poppy.conf @@ -38,10 +38,10 @@ transport = pecan manager = default # Storage driver module (e.g., mongodb, sqlite, cassandra) -storage = mockdb +storage = cassandra # Provider modules list (a list of comma separated provider module list) -providers = mock,fastly,akamai +providers = fastly # DNS driver module (e.g. default, designate, rackspace) dns = default @@ -80,9 +80,6 @@ replication_strategy = class:SimpleStrategy, replication_factor:1 # Path to directory containing CQL migration scripts migrations_path = /poppy/storage/cassandra/migrations -[drivers:storage:mockdb] -database = poppy - [drivers:distributed_task:taskflow] jobboard_backend_type = zookeeper persistent_backend_type = zookeeper @@ -119,8 +116,10 @@ default_cache_ttl = 86400 [drivers:provider:fastly] apikey = "MYAPIKEY" -scheme = "https" -host = "api.fastly.com" +# scheme = "https" +# host = "api.fastly.com" +scheme = "http" +host = "dockerhost:8900/fastly" [drivers:provider:maxcdn] alias = "MYALIAS" diff --git a/kill_poppy.sh b/kill_poppy.sh new file mode 100755 index 00000000..b80f9126 --- /dev/null +++ b/kill_poppy.sh @@ -0,0 +1 @@ +ps -ef | grep [p]oppy- | awk -F ' ' '{print$2}' | xargs kill -9 \ No newline at end of file diff --git a/run_poppy.sh b/run_poppy.sh new file mode 100755 index 00000000..35aa53e9 --- /dev/null +++ b/run_poppy.sh @@ -0,0 +1,92 @@ +#!/bin/bash +DAEMONIZED=false +WORKERS = 6 + +for i in "$@" +do + case $i in + -d|--daemonized) + DAEMONIZED=true + shift # past argument=value + ;; + + -w=*|--workers=*) + WORKERS="${i#*=}" + shift # past argument=value + ;; + + -?|--help) + echo "USAGE: ./run_poppy.sh -d -w=10" + echo "-d | --daemonized : run in daemonized mode" + echo "-w | --workers : the number of poppy-worker processes to spawn (defaults to 6)" + exit + + shift + ;; + + *) + echo "Invalid Options" + echo "Run ./run_poppy.sh --help for valid parameters." + exit + # unknown option + ;; + esac +done + +pip install docker-compose + + +# remove existing containers +docker kill compose_cassandra_1 +docker kill compose_zookeeper_1 +docker rm compose_cassandra_1 +docker rm compose_zookeeper_1 + +# start new containers +docker-compose -f docker/compose/dependencies.yml up -d + +is_cassandra_ready() { + nc -z dockerhost 9042 +} + +is_zookeeper_ready() { + nc -z dockerhost 2181 +} + +# wait until cassandra is ready +while ! is_cassandra_ready -eq 1 +do + echo "still trying to connect to cassandra" + sleep 1 +done +echo "connected successfully to cassandra" + + +# wait until zookeeper is ready +while ! is_zookeeper_ready -eq 1 +do + echo "still trying to connect to zookeeper" + sleep 1 +done +echo "connected successfully to zookeeper" + +# start the poppy-workers +COUNTER=0 +while [ $COUNTER -lt $WORKERS ]; do + exec poppy-worker > /dev/null 2>&1 & + echo "poppy-worker spawned." + let COUNTER=COUNTER+1 +done + + +# start the poppy-server +if $DAEMONIZED; then + exec poppy-server > /dev/null 2>&1 & + echo "poppy-server spawned." +else + exec poppy-server +fi + + + +echo "Poppy Server and Workers Started" \ No newline at end of file diff --git a/tests/api/README.rst b/tests/api/README.rst index 44878f32..5ba0b0db 100644 --- a/tests/api/README.rst +++ b/tests/api/README.rst @@ -13,7 +13,7 @@ To run the tests 1. Install the dependencies:: - pip install -r requirements.txt + $ pip install -r requirements.txt 2. Set the following environment variables:: @@ -21,17 +21,28 @@ To run the tests export CAFE_ROOT_LOG_PATH=~/.poppy/logs export CAFE_TEST_LOG_PATH=~/.poppy/logs -3. The API tests require a running database (eg cassandra), in order to +3. If you desire highlighting in the output, set the following environment variables:: + + export NOSE_WITH_OPENSTACK=1 + export NOSE_OPENSTACK_COLOR=1 + export NOSE_OPENSTACK_RED=0.05 + export NOSE_OPENSTACK_YELLOW=0.025 + export NOSE_OPENSTACK_SHOW_ELAPSED=1 + export NOSE_OPENSTACK_STDOUT=1 + + +4. The API tests require a running database (eg cassandra) and zookeeper, in order to run via tox. -4. Copy the api.conf file to the path set by CAFE_CONFIG_FILE_PATH:: + $ ./run_poppy.sh - cp tests/etc/api.conf ~/.poppy/tests.conf +5. Copy the api.conf file to the path set by CAFE_CONFIG_FILE_PATH:: -5. Once you are ready to run the tests:: + $ cp tests/etc/api.conf ~/.poppy/tests.conf - cd tests/api - nosetests +6. Once you are ready to run the tests:: + + $ nosetests api Tox Support @@ -41,14 +52,24 @@ You can run tox using a docker container hosting Cassandra:: Note - This will require docker (or boot2docker for MacOSX) to already be installed on the system. -1. Update your `~/.poppy/tests.conf` to point to your docker cassandra container ip address. +1. Update your `~/.poppy/tests.conf` to point to your docker cassandra/zookeeper container ip address. Example 1: Run all API tests against a docker hosted cassandra instance:: - tox -e api + $ tox -e api Example 2: Run a particular API test function:: - tox -e api api/services/test_services.py:TestCreateService -- -m test_create_service_positive + $ tox -e api api/services/test_services.py:TestCreateService -- -m test_create_service_positive +Mimic Support +------------- + +Occassionaly you want to test against a mock api rather than the real thing to get around rate limiting issues, +and to get around having to create accounts with a certain provider. + +Mimic helps accomplish this goal for testing. + +1. Run the mimic docker container (via ./run_poppy.sh) and point any remote api url in your test.conf file to your http://dockerhost:8900/mimic_service_name + diff --git a/tests/api/admin/test_akamai_SAN_domain_migration.py b/tests/api/admin/test_akamai_SAN_domain_migration.py index e6376f3e..2948672d 100644 --- a/tests/api/admin/test_akamai_SAN_domain_migration.py +++ b/tests/api/admin/test_akamai_SAN_domain_migration.py @@ -26,6 +26,9 @@ class TestSanCertService(base.TestBase): def setUp(self): super(TestSanCertService, self).setUp() + if self.test_config.run_operator_tests is False: + self.skipTest( + 'Test Operator Functions is disabled in configuration') self.service_name = self.generate_random_string(prefix='API-Test-') self.flavor_id = self.test_flavor @@ -93,9 +96,10 @@ class TestSanCertService(base.TestBase): self.service_url = resp.headers["location"] def test_migrate(self): + new_certs = self.akamai_config.san_certs new_certs_list = new_certs.split(',') - index = random.randint(0, len(new_certs_list)-1) + index = random.randint(0, len(new_certs_list) - 1) new_cert = new_certs_list[index] get_resp = self.client.get_service(location=self.service_url) @@ -125,9 +129,10 @@ class TestSanCertService(base.TestBase): self.assertEqual(data, new_cert) def test_migrate_negative_invalid_projectid(self): + new_certs = self.akamai_config.san_certs new_certs_list = new_certs.split(',') - index = random.randint(0, len(new_certs_list)-1) + index = random.randint(0, len(new_certs_list) - 1) new_cert = new_certs_list[index] get_resp = self.client.get_service(location=self.service_url) @@ -141,9 +146,10 @@ class TestSanCertService(base.TestBase): self.assertEqual(resp.status_code, 404) def test_migrate_negative_invalid_serviceid(self): + new_certs = self.akamai_config.san_certs new_certs_list = new_certs.split(',') - index = random.randint(0, len(new_certs_list)-1) + index = random.randint(0, len(new_certs_list) - 1) new_cert = new_certs_list[index] get_resp = self.client.get_service(location=self.service_url) @@ -157,9 +163,10 @@ class TestSanCertService(base.TestBase): self.assertEqual(resp.status_code, 404) def test_migrate_negative_invalid_domain(self): + new_certs = self.akamai_config.san_certs new_certs_list = new_certs.split(',') - index = random.randint(0, len(new_certs_list)-1) + index = random.randint(0, len(new_certs_list) - 1) new_cert = new_certs_list[index] get_resp = self.client.get_service(location=self.service_url) @@ -185,6 +192,10 @@ class TestSanCertServiceWithLogDelivery(base.TestBase): def setUp(self): super(TestSanCertServiceWithLogDelivery, self).setUp() + if self.test_config.run_operator_tests is False: + self.skipTest( + 'Test Operator Functions is disabled in configuration') + self.service_name = self.generate_random_string(prefix='API-Test-') self.flavor_id = self.test_flavor @@ -256,7 +267,7 @@ class TestSanCertServiceWithLogDelivery(base.TestBase): def test_migrate(self): new_certs = self.akamai_config.san_certs new_certs_list = new_certs.split(',') - index = random.randint(0, len(new_certs_list)-1) + index = random.randint(0, len(new_certs_list) - 1) new_cert = new_certs_list[index] get_resp = self.client.get_service(location=self.service_url) get_resp_body = get_resp.json() diff --git a/tests/api/admin/test_get_service_by_domain.py b/tests/api/admin/test_get_service_by_domain.py index 7a2b6b62..7cab16ca 100644 --- a/tests/api/admin/test_get_service_by_domain.py +++ b/tests/api/admin/test_get_service_by_domain.py @@ -28,6 +28,10 @@ class TestGetServiceByDomain(base.TestBase): def setUp(self): super(TestGetServiceByDomain, self).setUp() + if self.test_config.run_operator_tests is False: + self.skipTest( + 'Test Operator Functions is disabled in configuration') + self.service_name = self.generate_random_string(prefix='api-test') self.flavor_id = self.test_flavor @@ -141,6 +145,10 @@ class TestGetServiceByDomain(base.TestBase): self.assertEqual(api_resp2.status_code, 200) def test_negative_get_by_non_existing_domain(self): + if self.test_config.run_operator_tests is False: + self.skipTest( + 'Test Operator Functions is disabled in configuration') + domain_name = self.domain_list[0]['domain'] + str(uuid.uuid1()) + \ ".com" resp = self.operator_client.admin_get_service_by_domain_name( @@ -180,6 +188,10 @@ class TestGetServiceBySharedDomain(base.TestBase): def setUp(self): super(TestGetServiceBySharedDomain, self).setUp() + if self.test_config.run_operator_tests is False: + self.skipTest( + 'Test Operator Functions is disabled in configuration') + self.service_name = self.generate_random_string(prefix='API-Test-') self.flavor_id = self.test_flavor @@ -247,6 +259,7 @@ class TestGetServiceBySharedDomain(base.TestBase): self.service_url = resp.headers["location"] def test_get_service_by_domain(self): + get_resp = self.client.get_service(self.service_url) resp_body = get_resp.json() domain = resp_body['domains'][0]['domain'] @@ -281,6 +294,11 @@ class TestGetServiceBySANCertDomain(base.TestBase): def setUp(self): super(TestGetServiceBySANCertDomain, self).setUp() + + if self.test_config.run_operator_tests is False: + self.skipTest( + 'Test Operator Functions is disabled in configuration') + self.service_name = self.generate_random_string(prefix='API-Test-') self.flavor_id = self.test_flavor @@ -348,6 +366,7 @@ class TestGetServiceBySANCertDomain(base.TestBase): self.service_url = resp.headers["location"] def test_get_service_by_domain(self): + get_resp = self.client.get_service(self.service_url) resp_body = get_resp.json() domain = resp_body['domains'][0]['domain'] @@ -382,6 +401,11 @@ class TestGetServiceByCustomCertDomain(base.TestBase): def setUp(self): super(TestGetServiceByCustomCertDomain, self).setUp() + + if self.test_config.run_operator_tests is False: + self.skipTest( + 'Test Operator Functions is disabled in configuration') + self.service_name = self.generate_random_string(prefix='API-Test-') self.flavor_id = self.test_flavor @@ -449,6 +473,7 @@ class TestGetServiceByCustomCertDomain(base.TestBase): self.service_url = resp.headers["location"] def test_get_service_by_domain(self): + get_resp = self.client.get_service(self.service_url) resp_body = get_resp.json() domain = resp_body['domains'][0]['domain'] diff --git a/tests/api/admin/test_perform_action.py b/tests/api/admin/test_perform_action.py index ca38bde1..a590d438 100644 --- a/tests/api/admin/test_perform_action.py +++ b/tests/api/admin/test_perform_action.py @@ -25,6 +25,11 @@ class TestHttpService(base.TestBase): def setUp(self): super(TestHttpService, self).setUp() + + if self.test_config.run_operator_tests is False: + self.skipTest( + 'Test Operator Functions is disabled in configuration') + self.service_name = self.generate_random_string(prefix='API-Test-') self.flavor_id = self.test_flavor @@ -198,6 +203,11 @@ class TestSharedCertService(base.TestBase): def setUp(self): super(TestSharedCertService, self).setUp() + + if self.test_config.run_operator_tests is False: + self.skipTest( + 'Test Operator Functions is disabled in configuration') + self.service_name = self.generate_random_string(prefix='API-Test-') self.flavor_id = self.test_flavor @@ -352,6 +362,11 @@ class TestSanCertService(base.TestBase): def setUp(self): super(TestSanCertService, self).setUp() + + if self.test_config.run_operator_tests is False: + self.skipTest( + 'Test Operator Functions is disabled in configuration') + self.service_name = self.generate_random_string(prefix='API-Test-') self.flavor_id = self.test_flavor @@ -506,6 +521,11 @@ class TestCustomCertService(base.TestBase): def setUp(self): super(TestCustomCertService, self).setUp() + + if self.test_config.run_operator_tests is False: + self.skipTest( + 'Test Operator Functions is disabled in configuration') + self.service_name = self.generate_random_string(prefix='API-Test-') self.flavor_id = self.test_flavor @@ -660,6 +680,11 @@ class TestHttpServiceWithLogDelivery(base.TestBase): def setUp(self): super(TestHttpServiceWithLogDelivery, self).setUp() + + if self.test_config.run_operator_tests is False: + self.skipTest( + 'Test Operator Functions is disabled in configuration') + self.service_name = self.generate_random_string(prefix='API-Test-') self.flavor_id = self.test_flavor diff --git a/tests/api/base.py b/tests/api/base.py index 0b8a0d3c..27f55fdb 100644 --- a/tests/api/base.py +++ b/tests/api/base.py @@ -25,6 +25,7 @@ from tests.api.utils import config class TestBase(fixtures.BaseTestFixture): + """Child class of fixtures.BaseTestFixture for testing CDN. Inherit from this and write your test methods. If the child class defines @@ -92,16 +93,17 @@ class TestBase(fixtures.BaseTestFixture): operator_project_id else: cls.operator_url = cls.config.base_url + '/v1.0' + cls.operator_client = client.PoppyClient( cls.operator_url, operator_auth_token, operator_project_id, serialize_format='json', deserialize_format='json') - cls.dns_config = config.DNSConfig() - cls.dns_client = client.DNSClient(cls.dns_config.dns_username, - cls.dns_config.dns_api_key) + cls.dns_config = config.DNSConfig() + cls.dns_client = client.DNSClient(cls.dns_config.dns_username, + cls.dns_config.dns_api_key) - cls.akamai_config = config.AkamaiConfig() + cls.akamai_config = config.AkamaiConfig() def generate_random_string(self, prefix='API-Tests', length=12): """Generates a random string of given prefix & length""" diff --git a/tests/api/services/test_services.py b/tests/api/services/test_services.py index 12b81b95..a8d309bb 100644 --- a/tests/api/services/test_services.py +++ b/tests/api/services/test_services.py @@ -357,7 +357,9 @@ class TestServiceActions(base.TestBase): self.service_url = resp.headers["location"] self.client.wait_for_service_status( - location=self.service_url, status='deployed') + location=self.service_url, + status='deployed', + abort_on_status='failed') @attrib.attr('smoke') def test_delete_service(self): diff --git a/tests/api/utils/client.py b/tests/api/utils/client.py index 628aef10..6562370b 100644 --- a/tests/api/utils/client.py +++ b/tests/api/utils/client.py @@ -58,6 +58,7 @@ class AuthClient(client.HTTPClient): class DNSClient(client.HTTPClient): + def __init__(self, username, api_key): super(DNSClient, self).__init__() @@ -339,6 +340,8 @@ class PoppyClient(client.AutoMarshallingHTTPClient): # this is for debugging purpose, # will be removed later, so simply use print print(body.get('errors', [])) + assert False, ("Aborted on status {0}").format( + current_status) return service current_time = int(time.time()) @@ -366,6 +369,8 @@ class PoppyClient(client.AutoMarshallingHTTPClient): # this is for debugging purpose, # will be removed later, so simply use print print(resp.get('errors', [])) + assert False, ("Aborted on status {0}").format( + current_status) return resp current_time = int(time.time()) diff --git a/tox.ini b/tox.ini index 0648ff7d..b88b8cd4 100644 --- a/tox.ini +++ b/tox.ini @@ -66,9 +66,6 @@ import_exceptions = poppy.openstack.common.gettextutils._ [testenv:api] deps = -r{toxinidir}/requirements/requirements.txt -r{toxinidir}/tests/test-requirements.txt -whitelist_externals = - docker - sleep setenv = CAFE_CONFIG_FILE_PATH={homedir}/.poppy/tests.conf CAFE_ROOT_LOG_PATH={homedir}/.poppy/logs CAFE_TEST_LOG_PATH={homedir}/.poppy/logs @@ -82,9 +79,6 @@ setenv = CAFE_CONFIG_FILE_PATH={homedir}/.poppy/tests.conf commands = pip install git+https://github.com/stackforge/opencafe.git#egg=cafe - pip install -U fig + ./run_poppy.sh -d - fig -f docker/fig/fig_cassandra.yml up -d - sleep 5 - poppy-server --daemon - nosetests {posargs:--nologcapture} + nosetests api --nologcapture