x
/
poppy
Code Issues Proposed changes

fix: retry on shared ssl shards sequentially. 

- Add a generator to yield shards, each time we are asked
  to generate a shared ssl suffix.

- check for domain existence, playing through the shards.

Change-Id: Ic7e06636324c40d1373193ebbea42638652fe2a3
This commit is contained in:
Sriram Madapusi Vasudevan 2015-10-01 10:46:03 -04:00
parent 0d361d1980
commit a32a46314d
11 changed files with 289 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
poppy/common/errors.py
View File

@ -56,3 +56,8 @@ class InvalidServiceState(Exception):
class ServicesOverLimit(Exception):
"""Raised when number of services is above a permissible threshold."""
class SharedShardsExhausted(Exception):
"""Raised when all shared ssl shards are occupied for a given domain."""

4
poppy/dns/default/services.py
View File

@ -22,6 +22,7 @@ class ServicesController(base.ServicesBase):
super(ServicesController, self).__init__(driver)
self.driver = driver
self.shared_ssl_shards = 5
def update(self, service_old, service_updates, responders):
"""Default DNS update.
@ -84,4 +85,5 @@ class ServicesController(base.ServicesBase):
to be used with manager for shared ssl feature
"""
return 'scdn023.secure.defaultcdn.com'
for shard in range(self.shared_ssl_shards):
yield 'scdn{0}.secure.defaultcdn.com'.format(shard)

12
poppy/dns/rackspace/services.py
View File

@ -192,11 +192,11 @@ class ServicesController(base.ServicesBase):
"""
if num_shards == 0:
# shard disabled, just use the suffix
return suffix
yield suffix
else:
# shard enabled, randomly select a shard
shard_id = random.randint(1, num_shards)
return '{0}{1}.{2}'.format(shard_prefix, shard_id, suffix)
# shard enabled, iterate through shards
for shard_id in range(1, num_shards):
yield '{0}{1}.{2}'.format(shard_prefix, shard_id, suffix)
def generate_shared_ssl_domain_suffix(self):
"""Rackespace DNS scheme to generate a shared ssl domain suffix,
@ -205,11 +205,13 @@ class ServicesController(base.ServicesBase):
:return A string of shared ssl domain name
"""
return self._generate_sharded_domain_name(
shared_ssl_domain_name = self._generate_sharded_domain_name(
self._driver.rackdns_conf.shared_ssl_shard_prefix,
self._driver.rackdns_conf.shared_ssl_num_shards,
self._driver.rackdns_conf.shared_ssl_domain_suffix)
return shared_ssl_domain_name
def create(self, responders):
"""Create CNAME record for a service.

99
poppy/manager/default/services.py
View File

@ -15,6 +15,7 @@
import json
import random
import uuid
import jsonpatch
@ -173,18 +174,20 @@ class DefaultServicesController(base.ServicesController):
'Limit of {0} '
'reached!'.format(service_limit))
# deal with shared ssl domains
for domain in service_obj.domains:
if domain.protocol == 'https' and domain.certificate == 'shared':
domain.domain = self._generate_shared_ssl_domain(
domain.domain
)
if any([domain for domain in service_obj.domains
if domain.certificate == "shared"]):
try:
store = str(uuid.uuid4()).replace('-', '_')
service_obj = self._shard_retry(project_id,
service_obj,
store=store)
except errors.SharedShardsExhausted as e:
raise e
except ValueError as e:
raise e
try:
self.storage_controller.create(
project_id,
service_obj)
# ValueError will be raised if the service has already existed
self.storage_controller.create(project_id,
service_obj)
except ValueError as e:
raise e
@ -261,8 +264,9 @@ class DefaultServicesController(base.ServicesController):
service_new_json['service_id'] = service_old.service_id
service_new = service.Service.init_from_dict(project_id,
service_new_json)
store = str(uuid.uuid4()).replace('-', '_')
# fixing the old and new shared ssl domains in service_new
for domain in service_new.domains:
if domain.protocol == 'https' and domain.certificate == 'shared':
customer_domain = domain.domain.split('.')[0]
@ -270,12 +274,14 @@ class DefaultServicesController(base.ServicesController):
if customer_domain in existing_shared_domains:
domain.domain = existing_shared_domains[customer_domain]
else:
domain.domain = self._generate_shared_ssl_domain(
domain.domain
)
domain.domain = self._pick_shared_ssl_domain(
customer_domain,
service_new.service_id,
store)
if hasattr(self, store):
delattr(self, store)
# check if the service domain names already exist
# existing ones doesnot count!
# existing ones does not count!
for d in service_new.domains:
if self.storage_controller.domain_exists_elsewhere(
d.domain,
@ -443,10 +449,61 @@ class DefaultServicesController(base.ServicesController):
return
def _generate_shared_ssl_domain(self, domain_name):
shared_ssl_domain_suffix = (
self.dns_controller.generate_shared_ssl_domain_suffix())
return '.'.join([domain_name, shared_ssl_domain_suffix])
def _generate_shared_ssl_domain(self, domain_name, store):
try:
if not hasattr(self, store):
gen_store = {
domain_name:
self.dns_controller.generate_shared_ssl_domain_suffix()
}
setattr(self, store, gen_store)
uuid_store = getattr(self, store)
if domain_name not in uuid_store:
uuid_store[domain_name] = \
self.dns_controller.generate_shared_ssl_domain_suffix()
setattr(self, store, uuid_store)
return '.'.join([domain_name,
next(uuid_store[domain_name])])
except StopIteration:
delattr(self, store)
raise errors.SharedShardsExhausted('Domain {0} '
'has already '
'been '
'taken'.format(domain_name))
def _pick_shared_ssl_domain(self, domain, service_id, store):
shared_ssl_domain = self._generate_shared_ssl_domain(
domain,
store)
while self.storage_controller.domain_exists_elsewhere(
shared_ssl_domain,
service_id):
shared_ssl_domain = self._generate_shared_ssl_domain(
domain,
store)
return shared_ssl_domain
def _shard_retry(self, project_id, service_obj, store=None):
# deal with shared ssl domains
try:
for domain in service_obj.domains:
if domain.protocol == 'https' \
and domain.certificate == 'shared':
shared_domain = domain.domain.split('.')[0]
shared_ssl_domain = self._pick_shared_ssl_domain(
shared_domain,
service_obj.service_id,
store)
domain.domain = shared_ssl_domain
if hasattr(self, store):
delattr(self, store)
return service_obj
except ValueError as e:
if hasattr(self, store):
delattr(self, store)
raise ValueError(str(e))
def migrate_domain(self, project_id, service_id, domain_name, new_cert):
dns_controller = self.dns_controller

12
poppy/storage/mockdb/services.py
View File

@ -91,7 +91,17 @@ class ServicesController(base.ServicesController):
def update(self, project_id, service_id, service_json):
# update configuration in storage
return ''
if service_json.service_id in self.created_service_ids \
and service_json.service_id == service_id:
if not any([domain_obj for domain_obj in service_json.domains
if self.domain_exists_elsewhere(domain_obj,
service_id)]):
self.created_services[service_id] = service_json.to_dict()
for domain_obj in service_json.domains:
if domain_obj not in self.claimed_domains:
self.claimed_domains.append(domain_obj.domain)
else:
raise ValueError("Domain has already been taken")
def update_state(self, project_id, service_id, state):
"""update_state

6
poppy/transport/pecan/controllers/v1/services.py
View File

@ -189,6 +189,9 @@ class ServicesController(base.Controller, hooks.HookController):
self.auth_token,
service_json_dict)
service_id = service_obj.service_id
except errors.SharedShardsExhausted as e:
# domain - shared domains exhausted
pecan.abort(400, detail=str(e))
except LookupError as e: # error handler for no flavor
pecan.abort(400, detail=str(e))
except ValueError as e: # error handler for existing service name
@ -249,6 +252,9 @@ class ServicesController(base.Controller, hooks.HookController):
pecan.abort(404, detail=str(e))
except errors.ServiceStatusNeitherDeployedNorFailed as e:
pecan.abort(400, detail=str(e))
except errors.SharedShardsExhausted as e:
# domain - shared domains exhausted
pecan.abort(400, detail=str(e))
except Exception as e:
pecan.abort(400, detail=util.help_escape(str(e)))

1
tests/api/base.py
View File

@ -100,6 +100,7 @@ class TestBase(fixtures.BaseTestFixture):
deserialize_format='json')
cls.dns_config = config.DNSConfig()
cls.shared_ssl_num_shards = cls.dns_config.shared_ssl_num_shards
cls.dns_client = client.DNSClient(cls.dns_config.dns_username,
cls.dns_config.dns_api_key)

110
tests/api/services/test_services.py
View File

@ -15,6 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import six
import time
import urlparse
@ -316,6 +317,115 @@ class TestListServices(base.TestBase):
super(TestListServices, self).tearDown()
@ddt.ddt
class TestSharedShardServices(base.TestBase):
"""Tests for Shared Services with Shard Selection."""
def _create_shared_service(self, domain, wait_for_deploy=False):
service_name = str(uuid.uuid1())
self.domain_list = [{"domain": domain, "certificate": "shared",
"protocol": "https"}]
self.origin_list = [{"origin": self.generate_random_string(
prefix='api-test-origin') + '.com', "port": 80, "ssl": False,
"hostheadertype": "custom", "hostheadervalue":
"www.customweb.com"}]
self.caching_list = [{"name": "default", "ttl": 3600},
{"name": "home", "ttl": 1200,
"rules": [{"name": "index",
"request_url": "/index.htm"}]}]
self.log_delivery = {"enabled": False}
resp = self.client.create_service(service_name=service_name,
domain_list=self.domain_list,
origin_list=self.origin_list,
caching_list=self.caching_list,
flavor_id=self.flavor_id,
log_delivery=self.log_delivery)
if wait_for_deploy:
self.client.wait_for_service_status(
location=resp.headers['Location'],
status='deployed',
abort_on_status='failed',
retry_interval=self.test_config.status_check_retry_interval,
retry_timeout=self.test_config.status_check_retry_timeout)
return resp
def setUp(self):
super(TestSharedShardServices, self).setUp()
self.service_list = []
self.flavor_id = self.test_flavor
def test_create_shared_services_beyond_shard_limit(self):
self.skipTest('See https://bugs.launchpad.net/poppy/+bug/1486103')
domain = self.generate_random_string(prefix="shared")
for _ in range(self.shared_ssl_num_shards):
resp = self._create_shared_service(domain=domain)
# self.assertEqual(resp.status_code, 202)
self.service_list.append(resp.headers['Location'])
resp = self._create_shared_service(domain=domain)
self.assertEqual(resp.status_code, 400)
def test_patch_add_shard_domains_beyond_shard_limit(self):
self.skipTest('See https://bugs.launchpad.net/poppy/+bug/1486103')
shared_domain = self.generate_random_string(prefix="sharedcheck")
patch_body = ([{
'op': 'add',
'path': '/domains/-',
'value': {
'domain': shared_domain,
'protocol': 'https',
'certificate': 'shared'
}
}])
for _ in range(self.shared_ssl_num_shards):
create_resp = self._create_shared_service(
domain=self.generate_random_string(prefix=""),
wait_for_deploy=True)
self.assertEqual(create_resp.status_code, 202)
self.service_list.append(create_resp.headers['Location'])
self.client.patch_service(
create_resp.headers['Location'],
request_body=patch_body)
self.client.wait_for_service_status(
location=create_resp.headers['Location'],
status='deployed',
abort_on_status='failed',
retry_interval=self.test_config.status_check_retry_interval,
retry_timeout=self.test_config.status_check_retry_timeout)
# NOTE(TheSriram): Now with shards exhausted, create a new service
# and patch it.
create_resp = self._create_shared_service(
domain=self.generate_random_string(prefix=""),
wait_for_deploy=True)
self.service_list.append(create_resp.headers['Location'])
patch_resp = self.client.patch_service(
create_resp.headers['location'],
request_body=json.dumps(patch_body))
self.assertEqual(patch_resp.status_code, 400)
def tearDown(self):
for service in self.service_list:
self.client.delete_service(location=service)
if self.test_config.generate_flavors:
self.client.delete_flavor(flavor_id=self.flavor_id)
super(TestSharedShardServices, self).tearDown()
@ddt.ddt
class TestServiceActions(base.TestBase):

5
tests/api/utils/config.py
View File

@ -107,6 +107,11 @@ class DNSConfig(data_interfaces.ConfigSectionInterface):
"""The url for customers to CNAME to."""
return self.get('dns_url_suffix')
@property
def shared_ssl_num_shards(self):
"""The number of shared ssl shards."""
return int(self.get('shared_ssl_num_shards'))
class AuthConfig(data_interfaces.ConfigSectionInterface):
"""Defines the auth config values."""

1
tests/etc/api.conf
View File

@ -40,6 +40,7 @@ dns_url_suffix={the suffix for the DNS shards}
[provider_akamai]
access_url_suffix = edgekey.net
san_certs = secure1.san2.xyzcdn.com,domain2.san3.xyz.com,cert5.san9.abc.com
shared_ssl_num_shards = 5
[provider_1]
api_key=INSERT_YOUR_API_KEY

67
tests/functional/transport/pecan/controllers/test_services.py
View File

@ -25,6 +25,7 @@ import ddt
from oslo_config import cfg
import pecan
from poppy.dns.default.services import ServicesController
from poppy.transport.pecan.controllers import base as c_base
from tests.functional.transport.pecan import base
@ -231,7 +232,7 @@ class ServiceControllerTest(base.FunctionalTest):
'protocol': 'https',
'certificate': 'shared'}
],
"flavor_id": "mock",
"flavor_id": self.flavor_id,
"origins": [
{
"origin": "mocksite.com",
@ -249,15 +250,72 @@ class ServiceControllerTest(base.FunctionalTest):
})
self.assertEqual(202, response.status_code)
# This is mocksite2
# NOTE (TheSriram): One shard is already taken, since we have a
# service created with it. Create n-1 services with n being
# total number of shared shards
class MockDriver(object):
def __init__(self):
super(MockDriver, self).__init__()
def dns_name(self):
return 'mock_dns'
total_shards = ServicesController(
driver=MockDriver()).shared_ssl_shards
for shard_id in range(total_shards - 1):
service_json = {
"name": "mocksite.com",
"domains": [
{"domain": "mocksiteshard{0}".format(shard_id),
'protocol': 'https',
'certificate': 'shared'}
],
"flavor_id": self.flavor_id,
"origins": [
{
"origin": "mocksite.com",
"port": 443,
"ssl": True
}
]
}
response = self.app.post('/v1.0/services',
params=json.dumps(service_json),
headers={
'Content-Type': 'application/json',
'X-Project-ID': self.project_id
})
self.assertEqual(202, response.status_code)
response = self.app.patch(response.location,
params=json.dumps([{
"op": "add",
"path": "/domains/-",
"value": {
"domain": "mocksite",
"protocol": "https",
"certificate": "shared"}
}]),
headers={'Content-Type':
'application/json',
'X-Project-ID':
self.project_id
}, expect_errors=True)
self.assertEqual(202, response.status_code)
# NOTE(TheSriram): Now create another service, and patch it.
# with all shards Exhausted, this will result in a 400
service_json = {
"name": "mocksite.com",
"domains": [
{"domain": "mocksite2",
{"domain": "mocksitefinal",
'protocol': 'https',
'certificate': 'shared'}
],
"flavor_id": "mock",
"flavor_id": self.flavor_id,
"origins": [
{
"origin": "mocksite.com",
@ -274,7 +332,6 @@ class ServiceControllerTest(base.FunctionalTest):
'X-Project-ID': self.project_id
})
self.assertEqual(202, response.status_code)
response = self.app.patch(response.location,
params=json.dumps([{
"op": "add",