diff --git a/etc/poppy.conf b/etc/poppy.conf index 529a75c1..334671df 100644 --- a/etc/poppy.conf +++ b/etc/poppy.conf @@ -120,6 +120,7 @@ username = "" api_key = "" use_shards = True num_shards = 400 +;records_limit = 400 shard_prefix = "cdn" shared_ssl_num_shards = 5 shared_ssl_shard_prefix = "scdn" diff --git a/poppy/dns/base/services.py b/poppy/dns/base/services.py index f54ec8d0..3275d6d7 100644 --- a/poppy/dns/base/services.py +++ b/poppy/dns/base/services.py @@ -69,3 +69,11 @@ class ServicesControllerBase(controller.DNSControllerBase): """ raise NotImplementedError + + def is_shard_full(self, shard_name): + """Check if shard can accept new records + + :raises NotImplementedError + """ + + raise NotImplementedError diff --git a/poppy/dns/default/services.py b/poppy/dns/default/services.py index 7d3070d8..b4a997ad 100644 --- a/poppy/dns/default/services.py +++ b/poppy/dns/default/services.py @@ -90,3 +90,6 @@ class ServicesController(base.ServicesBase): random.shuffle(shard_ids) for shard in shard_ids: yield 'scdn{0}.secure.defaultcdn.com'.format(shard) + + def is_shard_full(self, shard_name): + return False diff --git a/poppy/dns/rackspace/driver.py b/poppy/dns/rackspace/driver.py index f57b5391..45f357b2 100644 --- a/poppy/dns/rackspace/driver.py +++ b/poppy/dns/rackspace/driver.py @@ -32,6 +32,8 @@ RACKSPACE_OPTIONS = [ cfg.BoolOpt('sharding_enabled', default=True, help='Enable Sharding?'), cfg.IntOpt('num_shards', default=10, help='Number of Shards to use'), + cfg.IntOpt('records_limit', default=400, + help='Number of records per domain.'), cfg.StrOpt('shard_prefix', default='cdn', help='The shard prefix to use'), cfg.IntOpt('shared_ssl_num_shards', default=5, help='Number of Shards ' diff --git a/poppy/dns/rackspace/services.py b/poppy/dns/rackspace/services.py index 917a16ff..57e0e3e0 100644 --- a/poppy/dns/rackspace/services.py +++ b/poppy/dns/rackspace/services.py @@ -799,3 +799,33 @@ class ServicesController(base.ServicesBase): def modify_cname(self, access_url, new_cert): self._change_cname_record(access_url=access_url, target_url=new_cert, shared_ssl_flag=False) + + def is_shard_full(self, shard_name): + count = 0 + try: + shard_domain = self.client.find(name=shard_name) + except exc.NotFound: + LOG.error("Shards not configured properly, could not find {0}.") + return True + + records = shard_domain.list_records(limit=100) + count += len(records) + + # Loop until all records are printed + while True: + try: + records = self.client.list_records_next_page() + count += len(records) + except exc.NoMoreResults: + break + + LOG.info( + "There were a total of {0} record(s) for {1}.".format( + count, + shard_name + )) + + if count >= self._driver.rackdns_conf.records_limit: + return True + else: + return False diff --git a/poppy/manager/default/services.py b/poppy/manager/default/services.py index 63f257c6..b50dd8d4 100644 --- a/poppy/manager/default/services.py +++ b/poppy/manager/default/services.py @@ -685,14 +685,19 @@ class DefaultServicesController(base.ServicesController): uuid_store[domain_name] = \ self.dns_controller.generate_shared_ssl_domain_suffix() setattr(self, store, uuid_store) - return '.'.join([domain_name, - next(uuid_store[domain_name])]) + + shard = next(uuid_store[domain_name]) + while self.dns_controller.is_shard_full(shard): + LOG.info( + "Skipped shard {0} because it's at maximum capacity." + .format(shard)) + shard = next(uuid_store[domain_name]) + + return '.'.join([domain_name, shard]) except StopIteration: delattr(self, store) - raise errors.SharedShardsExhausted('Domain {0} ' - 'has already ' - 'been ' - 'taken'.format(domain_name)) + raise errors.SharedShardsExhausted( + 'Domain {0} has already been taken'.format(domain_name)) def _pick_shared_ssl_domain(self, domain, service_id, store): shared_ssl_domain = self._generate_shared_ssl_domain( @@ -707,7 +712,7 @@ class DefaultServicesController(base.ServicesController): return shared_ssl_domain def _shard_retry(self, project_id, service_obj, store=None): - # deal with shared ssl domains + # deal with shared ssl domains try: for domain in service_obj.domains: if domain.protocol == 'https' \ diff --git a/poppy/transport/validators/helpers.py b/poppy/transport/validators/helpers.py index db847867..f3a97bf5 100644 --- a/poppy/transport/validators/helpers.py +++ b/poppy/transport/validators/helpers.py @@ -28,12 +28,12 @@ import uuid import jsonschema import pecan +import tld as tld_helper from poppy.common import util from poppy.transport.validators import root_domain_regexes as regexes from poppy.transport.validators.stoplight import decorators from poppy.transport.validators.stoplight import exceptions -from tld import get_tld def req_accepts_json_pecan(request, desired_content_type='application/json'): @@ -143,8 +143,10 @@ def is_valid_tld(domain_name): status = whois.whois(domain_name)['status'] if status is not None or status != '': url = 'https://{domain}' - tld_obj = get_tld(url.format(domain=domain_name), - as_object=True) + tld_obj = tld_helper.get_tld( + url.format(domain=domain_name), + as_object=True + ) tld = tld_obj.suffix try: dns.resolver.query(tld + '.', 'SOA') diff --git a/tests/unit/distributed_task/utils/test_utils.py b/tests/unit/distributed_task/utils/test_utils.py index a177f2a8..5987079c 100644 --- a/tests/unit/distributed_task/utils/test_utils.py +++ b/tests/unit/distributed_task/utils/test_utils.py @@ -15,6 +15,8 @@ """Unittests for TaskFlow distributed_task driver implementation.""" +import mock + from poppy.distributed_task.utils import memoized_controllers from tests.unit import base @@ -24,6 +26,14 @@ class TestMemoizeUtils(base.TestCase): def setUp(self): super(TestMemoizeUtils, self).setUp() + rax_dns_set_credentials = mock.patch('pyrax.set_credentials') + rax_dns_set_credentials.start() + self.addCleanup(rax_dns_set_credentials.stop) + + rax_dns = mock.patch('pyrax.cloud_dns') + rax_dns.start() + self.addCleanup(rax_dns.stop) + def test_memoization_service_controller(self): service_controller_first = \ memoized_controllers.task_controllers('poppy') diff --git a/tests/unit/dns/rackspace/test_services.py b/tests/unit/dns/rackspace/test_services.py index 708c8a9d..7631faf2 100644 --- a/tests/unit/dns/rackspace/test_services.py +++ b/tests/unit/dns/rackspace/test_services.py @@ -34,6 +34,8 @@ RACKSPACE_OPTIONS = [ cfg.BoolOpt('sharding_enabled', default=True, help='Enable Sharding?'), cfg.IntOpt('num_shards', default=500, help='Number of Shards to use'), + cfg.IntOpt('records_limit', default=400, + help='Number of records per domain.'), cfg.StrOpt('shard_prefix', default='cdn', help='The shard prefix to use'), cfg.StrOpt('url', default='mycdn.com', @@ -1047,3 +1049,38 @@ class TestServicesUpdate(base.TestCase): responder_disable = self.controller.disable(self.service_old) # TODO(isaacm): Add assertions on the returned object self.assertIsNotNone(responder_disable) + + def test_is_shard_full_shard_not_found(self): + self.client.find.side_effect = exc.NotFound(404) + + self.assertTrue(self.controller.is_shard_full('shard_name')) + + def test_is_shard_full_false(self): + find_mock = mock.Mock() + find_mock.list_records.return_value = range(100) + self.client.find.return_value = find_mock + + self.client.list_records_next_page.side_effect = exc.NoMoreResults + + self.assertFalse(self.controller.is_shard_full('shard_name')) + + def test_is_shard_full_true(self): + find_mock = mock.Mock() + find_mock.list_records.return_value = range(600) + self.client.find.return_value = find_mock + + self.client.list_records_next_page.side_effect = exc.NoMoreResults + + self.assertTrue(self.controller.is_shard_full('shard_name')) + + def test_is_shard_full_paginate_true(self): + find_mock = mock.Mock() + find_mock.list_records.return_value = range(300) + self.client.find.return_value = find_mock + + self.client.list_records_next_page.side_effect = [ + range(300), + exc.NoMoreResults, + ] + + self.assertTrue(self.controller.is_shard_full('shard_name')) diff --git a/tests/unit/manager/default/test_services.py b/tests/unit/manager/default/test_services.py index edb398a3..2b3ddce4 100644 --- a/tests/unit/manager/default/test_services.py +++ b/tests/unit/manager/default/test_services.py @@ -116,6 +116,14 @@ class DefaultManagerServiceTests(base.TestCase): # in the reverse order of the arguments present super(DefaultManagerServiceTests, self).setUp() + tld_patcher = mock.patch('tld.get_tld') + tld_patcher.start() + self.addCleanup(tld_patcher.stop) + + dns_resolver_patcher = mock.patch('dns.resolver') + dns_resolver_patcher.start() + self.addCleanup(dns_resolver_patcher.stop) + self.context = context.RequestContext() # create mocked config and driver conf = cfg.ConfigOpts() diff --git a/tests/unit/manager/default/test_ssl_certificate.py b/tests/unit/manager/default/test_ssl_certificate.py index 88a88951..aed0451f 100644 --- a/tests/unit/manager/default/test_ssl_certificate.py +++ b/tests/unit/manager/default/test_ssl_certificate.py @@ -40,6 +40,14 @@ class DefaultSSLCertificateControllerTests(base.TestCase): super(DefaultSSLCertificateControllerTests, self).setUp() + tld_patcher = mock.patch('tld.get_tld') + tld_patcher.start() + self.addCleanup(tld_patcher.stop) + + dns_resolver_patcher = mock.patch('dns.resolver') + dns_resolver_patcher.start() + self.addCleanup(dns_resolver_patcher.stop) + conf = cfg.ConfigOpts() self.provider_mocks = {