x/poppy
Code Issues Proposed changes

IP restriction

Browse Source 
Implements blueprint: ip-restrictions

Change-Id: Idfc9f1789d0dcbed0e6094a47e886acabeb5d19f
This commit is contained in:
tonytan4ever 2015-03-11 12:04:41 -04:00
parent e18e44e2cb
commit d7fe8fdbf9
17 changed files with 462 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
poppy/model/helpers/rule.py
View File

@ -21,7 +21,7 @@ class Rule(common.DictSerializableModel):
def __init__(self, name=None,
referrer=None, http_host=None, client_ip=None,
http_method=None, request_url="/*"):
geography=None, http_method=None, request_url="/*"):
self._name = name
self._request_url = request_url
@ -33,6 +33,10 @@ class Rule(common.DictSerializableModel):
self._client_ip = client_ip
if http_method:
self._http_method = http_method
if geography:
self._geography = geography
if request_url:
self._request_url = request_url
@property
def name(self):
@ -79,3 +83,12 @@ class Rule(common.DictSerializableModel):
@request_url.setter
def request_url(self, value):
self._request_url = value
@property
def geography(self):
"""http_host."""
return self._geography
@geography.setter
def geography(self, value):
self._geography = value

117
poppy/provider/akamai/services.py
View File

@ -134,7 +134,6 @@ class ServiceController(base.ServiceBase):
def update(self, provider_service_id,
service_obj):
try:
# depending on domains field presented or not, do PUT/POST
# and depending on origins field presented or not, set behavior on
@ -541,28 +540,78 @@ class ServiceController(base.ServiceBase):
'/' + rule_entry.request_url)
# restriction implementation for akamai
# for each restriction rule
# restriction entities include: referrer, geography, client_ip
restriction_entities = ['referrer', 'client_ip']
class entityRequestUrlMappingList(dict):
'''A dictionary with a name attribute'''
def __init__(self, name, orig_dict):
self.name = name
self.update(orig_dict)
# classify restriction/rules based on their white/black-list
white_list_entities = entityRequestUrlMappingList(
'whitelist',
{entity: {} for entity
in restriction_entities})
black_list_entities = entityRequestUrlMappingList(
'blacklist',
{entity: {} for entity
in restriction_entities})
for restriction_rule in restriction_rules:
entity_rule_mapping = {}
if restriction_rule.type == 'whitelist':
entity_rule_mapping = white_list_entities
elif restriction_rule.type == 'blacklist':
entity_rule_mapping = black_list_entities
for rule_entry in restriction_rule.rules:
if rule_entry.referrer is not None:
# classify rules based on their entities, then request_urls
if getattr(rule_entry, "referrer", None) is not None:
if (rule_entry.request_url not in
entity_rule_mapping['referrer']):
entity_rule_mapping['referrer'][rule_entry.request_url]\
= [rule_entry]
else:
entity_rule_mapping['referrer'][rule_entry.request_url]\
.append(rule_entry)
elif getattr(rule_entry, "client_ip", None) is not None:
if (rule_entry.request_url not in
entity_rule_mapping['client_ip']):
entity_rule_mapping['client_ip'][rule_entry.request_url]\
= [rule_entry]
else:
entity_rule_mapping['client_ip'][rule_entry.request_url]\
.append(rule_entry)
for entity_request_url_rule_mapping in [white_list_entities,
black_list_entities]:
for entity in entity_request_url_rule_mapping:
for request_url in entity_request_url_rule_mapping[entity]:
found_match = False
referrer_whitelist_value = ' '.join(
['*%s*' % referrer
for referrer
in rule_entry.referrer.split(' ')
])
# need to write up a function gets the value of behavior
behavior_name = self._get_behavior_name(
entity, entity_request_url_rule_mapping.name)
behavior_value = self._get_behavior_value(
entity,
entity_request_url_rule_mapping[entity][request_url])
behavior_dict = {
'name': behavior_name,
'value': behavior_value
}
# if we have a matches rule already
for rule in rules_list:
for match in rule['matches']:
if rule_entry.request_url == match['value']:
if request_url == match['value']:
# we found an existing matching rule.
# add the cache behavior to it
# add the whitelist/blacklist behavior to it
found_match = True
rule['behaviors'].append({
'name': 'referer-whitelist',
'value': referrer_whitelist_value
})
rule['behaviors'].append(behavior_dict)
# if there is no matches entry yet for this rule
if not found_match:
@ -576,17 +625,45 @@ class ServiceController(base.ServiceBase):
if rule_entry.request_url is not None:
match_rule = {
'name': 'url-wildcard',
'value': rule_entry.request_url
'value': request_url
}
rule_dict_template['matches'].append(match_rule)
rule_dict_template['behaviors'].append({
'name': 'referer-whitelist',
'value': referrer_whitelist_value
})
rule_dict_template['behaviors'].append(
behavior_dict)
rules_list.append(rule_dict_template)
# end loop - restriction_rule.rules
# end lop - restriction_rules
# end loop - request url
# end loop - entity
def _get_behavior_name(self, entity, entity_restriction_type):
prefix = suffix = None
if entity == 'referrer':
prefix = 'referer'
elif entity == 'client_ip':
prefix = 'ip'
if entity_restriction_type == 'whitelist':
suffix = 'whitelist'
elif entity_restriction_type == 'blacklist':
suffix = 'blacklist'
return '-'.join([prefix, suffix])
def _get_behavior_value(self, entity, rule_entries):
if entity == 'referrer':
return ' '.join(
['*%s*' % referrer
for rule_entry
in rule_entries
for referrer
in rule_entry.referrer.split()
])
elif entity == 'client_ip':
return ' '.join(
['%s' % rule_entry.client_ip
for rule_entry
in rule_entries
])
def _process_caching_rules(self, caching_rules, rules_list):
# akamai requires all caching rules to start with '/'

3
poppy/storage/cassandra/services.py
View File

@ -640,8 +640,11 @@ class ServicesController(base.ServicesController):
restrictions = [restriction.Restriction(
r.get('name'),
r.get('type'),
[rule.Rule(r_rule.get('name'),
referrer=r_rule.get('referrer'),
client_ip=r_rule.get('client_ip'),
geography=r_rule.get('geography'),
request_url=r_rule.get('request_url', "/*") or "/*")
for r_rule in r['rules']])
for r in restrictions]

1
poppy/transport/pecan/models/request/rule.py
View File

@ -24,4 +24,5 @@ def load_from_json(json_data):
res.http_method = json_data.get('http_method', None)
res.client_ip = json_data.get('client_ip', None)
res.request_url = json_data.get('request_url', None)
res.geography = json_data.get('geography', None)
return res

3
poppy/transport/pecan/models/response/rule.py
View File

@ -29,7 +29,8 @@ class Model(collections.OrderedDict):
self['name'] = util.help_escape(rule.name)
self['request_url'] = util.help_escape(rule.request_url)
for attr_name in ['http_host', 'http_method',
'client_ip', 'referrer']:
'client_ip', 'request_url',
'referrer', 'geography']:
attr = getattr(rule, attr_name, None)
if attr is not None:
self[attr_name] = util.help_escape(attr)

17
poppy/transport/validators/helpers.py
View File

@ -224,19 +224,7 @@ def is_valid_service_configuration(service, schema):
else:
domains.append(domain_value)
# 4. referrer restriction paths must be unique
if 'restrictions' in service:
restriction_paths = []
for restriction in service['restrictions']:
if 'rules' in restriction:
for rule in restriction['rules']:
if 'referrer' in rule:
request_url = rule.get('request_url', '/*')
if request_url in restriction_paths:
raise exceptions.ValidationFailed(
'Referrer - the request_url must be unique')
else:
restriction_paths.append(request_url)
# We allow multiple restrictions rules on the same path
# 5. domains protocols must be of the same type, and domains protocols must
# match the description (ssl/port) of the origin
@ -312,9 +300,10 @@ def is_valid_service_configuration(service, schema):
}
blacklist_restriction_entities = {
}
if 'restrictions' in service:
for restriction in service['restrictions']:
if restriction.get('type', 'whitelist') == 'blacklist':
if restriction.get('type', 'blacklist') == 'blacklist':
for rule in restriction['rules']:
entity = None
request_url = '/*'

65
poppy/transport/validators/schemas/service.py
View File

@ -432,17 +432,66 @@ class ServiceSchema(schema_base.SchemaBase):
'type': 'array',
'required': True,
'items': {
'type': [
{
'type': 'object',
'properties': {
'name': {
"properties": {
"name": {
"type": "string",
},
"referrer": {
"type": "string",
'required': True,
},
'request_url': {
'type': 'string',
'minLength': 1,
'maxLength': 256},
'referrer': {
'minLength': 2,
'maxLength': 100
}
},
"additionalProperties": False,
}, {
'type': 'object',
"properties": {
"name": {
"type": "string",
},
"geography": {
"type": "string",
'required': True,
},
'request_url': {
'type': 'string',
'minLength': 3,
'maxLength': 1024}
}},
'minLength': 2,
'maxLength': 100
}
},
"additionalProperties": False
}, {
'type': 'object',
"properties": {
"name": {
"type": "string",
},
"client_ip": {
"type": "string",
'pattern': re.compile(
"^\d{1,3}\.\d{1,3}"
"\.\d{1,3}"
"\.\d{1,3}$"
),
'required': True,
},
'request_url': {
'type': 'string',
'minLength': 2,
'maxLength': 100
}
},
"additionalProperties": False
}
]
}
}},
},
},

107
tests/api/services/data_create_service.json
View File

@ -38,12 +38,19 @@
}
],
"restrictions_list": [
{"name": "test",
{
"name": "test",
"type": "whitelist",
"rules": [
{
"name": "only me",
"referrer": "www.mywebsite.com",
"request_url" : "/*"
},
{
"name": "only 1234",
"client_ip": "1.2.3.4",
"request_url" : "/*"
}
]}
],
@ -435,6 +442,104 @@
}
]}
]
},
"ip_restriction": {
"name": "my_service_name",
"domain_list": [{"domain": "mywebsite.com", "protocol": "http"},
{"domain": "blog.mywebsite.com", "protocol": "http"}],
"origin_list": [{
"origin": "mywebsite1.com",
"port": 80,
"ssl": false,
"rules": [
{
"name" : "default",
"request_url" : "/*"
}]
}],
"caching_list": [
{
"name": "default",
"ttl": 3600,
"rules": [
{
"name" : "default",
"request_url" : "/*"
}
]
},
{
"name": "home",
"ttl": 1200,
"rules": [
{
"name" : "index",
"request_url" : "/index.htm"
}
]
}
],
"restrictions_list": [
{
"name": "test",
"type": "whitelist",
"rules": [
{
"name": "only 1234",
"client_ip": "1.2.3.4",
"request_url" : "/*"
}
]}
]
},
"ip_restriction_blacklist": {
"name": "my_service_name",
"domain_list": [{"domain": "mywebsite.com", "protocol": "http"},
{"domain": "blog.mywebsite.com", "protocol": "http"}],
"origin_list": [{
"origin": "mywebsite1.com",
"port": 80,
"ssl": false,
"rules": [
{
"name" : "default",
"request_url" : "/*"
}]
}],
"caching_list": [
{
"name": "default",
"ttl": 3600,
"rules": [
{
"name" : "default",
"request_url" : "/*"
}
]
},
{
"name": "home",
"ttl": 1200,
"rules": [
{
"name" : "index",
"request_url" : "/index.htm"
}
]
}
],
"restrictions_list": [
{
"name": "test",
"type": "blacklist",
"rules": [
{
"name": "only 1234",
"client_ip": "1.2.3.4",
"request_url" : "/*"
}
]}
]
}
}

37
tests/api/services/data_create_service_negative.json
View File

@ -121,6 +121,7 @@
"request_url" : "/index.htm"}]}],
"restrictions_list": [
{"name": "test",
"type": "whitelist",
"rules": [{"name": "only me",
"referrer": "www.mywebsite.com"}]}
]
@ -459,6 +460,21 @@
"referrer": "www.mywebsite.com"}]}
]
},
"restrictions_no_name": {
"service_name": "my_service_name",
"domain_list": [{"domain": "mywebsite.com", "protocol": "http"},
{"domain": "blog.mywebsite.com", "protocol": "http"}],
"origin_list": [{"origin": "mywebsite1.com",
"port": 443,
"ssl": false}],
"caching_list": [{"name": "default", "ttl": 3600},
{"name": "home",
"ttl": 3600,
"rules": [{"name" : "index",
"request_url" : "/index.htm"}]}],
"restrictions_list": [{
"rules": [{"referrer": "www.abc.com"}]}]
},
"restrictions_no_rules": {
"service_name": "my_service_name",
"domain_list": [{"domain": "mywebsite.com", "protocol": "http"},
@ -473,7 +489,7 @@
"request_url" : "/index.htm"}]}],
"restrictions_list": [{"name":"default"}]
},
"restrictions_rule_no_name": {
"restrictions_invalid_ip_address": {
"service_name": "my_service_name",
"domain_list": [{"domain": "mywebsite.com", "protocol": "http"},
{"domain": "blog.mywebsite.com", "protocol": "http"}],
@ -486,7 +502,24 @@
"rules": [{"name" : "index",
"request_url" : "/index.htm"}]}],
"restrictions_list": [{"name":"default",
"rules": [{}]}]
"rules": [{"client_ip": "1.2.3"}]}]
},
"restrictions_black_and_whitelist_ip": {
"service_name": "my_service_name",
"domain_list": [{"domain": "mywebsite.com", "protocol": "http"},
{"domain": "blog.mywebsite.com", "protocol": "http"}],
"origin_list": [{"origin": "mywebsite1.com",
"port": 443,
"ssl": false}],
"caching_list": [{"name": "default", "ttl": 3600},
{"name": "home",
"ttl": 3600,
"rules": [{"name" : "index",
"request_url" : "/index.htm"}]}],
"restrictions_list": [{"name":"default",
"rules": [{"client_ip": "1.2.3.4"}]},{
"name":"default-blacklist", "type": "blacklist",
"rules": [{"client_ip": "5.6.7.8"}]}]
},
"origin_invalid_port": {
"service_name": "origin_invalid_port",

50
tests/api/services/data_patch_service.json
View File

@ -198,6 +198,7 @@
{"op": "add",
"path": "/restrictions/-",
"value": {"name": "restricted",
"type": "whitelist",
"rules": [{"name": "rule1", "referrer": "www.thesite.com", "request_url": "/images/*"}]
}}
],
@ -205,11 +206,13 @@
{"op": "add",
"path": "/restrictions/-",
"value": {"name": "restricted",
"type": "whitelist",
"rules": [{"name": "rule1", "referrer": "www.thesite.com", "request_url": "/images2/*"}]
}},
{"op": "replace",
"path": "/restrictions/0",
"value": {"name": "new_restricted",
"type": "whitelist",
"rules": [{"name": "rule1", "referrer": "www.thesite.com", "request_url": "/*"}]
}}
],
@ -233,6 +236,7 @@
{"op": "add",
"path": "/restrictions/-",
"value": {"name": "r",
"type": "whitelist",
"rules": [{"name": "rule1", "referrer": "www.thesite.com", "request_url": "/images/*"}]
}
}
@ -276,6 +280,7 @@
{"op": "add",
"path": "/restrictions/-",
"value": {"name": "restricted",
"type": "whitelist",
"rules": [{"name": "r", "referrer": "www.thesite.com", "request_url": "/images/*"}]
}
}
@ -284,6 +289,7 @@
{"op": "add",
"path": "/restrictions/-",
"value": {"name": "restricted",
"type": "whitelist",
"rules": [{"name": "rule1", "referrer": "a.co", "request_url":"/images/*"}]
}
}
@ -309,5 +315,47 @@
"value": {"origin": "1.1.1.1", "port": 443, "ssl": true,
"rules": [{"name" : "origin", "request_url" : "/origin2.htm"}], "hostheadertype": "custom", "hostheadervalue": "www.customweb.com"}}
],
"empty_list": []
"empty_list": [],
"replace_change_restricted_ip_to_another":[
{"op": "replace",
"path": "/restrictions/0",
"value": {"name": "website only",
"type": "whitelist",
"rules": [{"name": "mywebsite.com",
"referrer": "www.mywebsite.com",
"request_url": "/*"
}, {"name": "myrestricted-ip",
"client_ip": "5.6.7.8",
"request_url": "/*"}
]}}
],
"replace_remove_all_restrictions":[
{"op": "remove",
"path": "/restrictions/0"}
],
"replace_remove_restricted_ip":[
{"op": "replace",
"path": "/restrictions/0",
"value": {"name": "website only",
"type": "whitelist",
"rules": [{"name": "mywebsite.com",
"referrer": "www.mywebsite.com",
"request_url": "/*"
}]}}
],
"replace_add_new_restricted_ip":[
{"op": "replace",
"path": "/restrictions/0",
"value": {"name": "website only",
"type": "whitelist",
"rules": [{"name": "mywebsite.com",
"referrer": "www.mywebsite.com",
"request_url": "/*"
}, {"name": "myrestricted-ip",
"client_ip": "1.2.3.4",
"request_url": "/*"}
,{"name": "myrestricted-ip-2",
"client_ip": "5.6.7.8",
"request_url": "/*"}]}}
]
}

26
tests/api/services/data_patch_service_negative.json
View File

@ -203,5 +203,31 @@
"path": "/domains/-",
"value": {"domain": "replaceme.com", "protocol": "https",
"certificate": "shared"}}
],
"add_restrictions_with_invalid_ip": [
{"op": "add",
"path": "/restrictions/-",
"value": {"name": "restricted-ip2",
"rules": [{"name": "rule1", "client_ip": "13.4.5"}]
}
}
],
"add_restrictions_with_conflict_blacklist_referrer": [
{"op": "add",
"path": "/restrictions/-",
"value": {"name": "restricted-blacklist",
"type": "blacklist",
"rules": [{"name": "rule1", "referrer": "myblacklist.com"}]
}
}
],
"add_restrictions_with_conflict_blacklist": [
{"op": "add",
"path": "/restrictions/-",
"value": {"name": "restricted-blacklist",
"type": "blacklist",
"rules": [{"name": "rule1", "client_ip": "5.6.7.8"}]
}
}
]
}

6
tests/functional/transport/pecan/controllers/data_create_service.json
View File

@ -26,7 +26,11 @@
"rules": [
{
"name": "mocksite.com",
"http_host": "www.mocksite.com"
"referrer": "www.mocksite.com"
},
{
"name": "us-website-only",
"geography": "USA"
}
]
}

2
tests/functional/transport/pecan/controllers/data_create_service_duplicate.json
View File

@ -26,7 +26,7 @@
"rules": [
{
"name": "mocksite.com",
"http_host": "www.mocksite.com"
"referrer": "www.mocksite.com"
}
]
}

4
tests/functional/transport/pecan/controllers/test_services.py
View File

@ -96,7 +96,7 @@ class ServiceControllerTest(base.FunctionalTest):
"rules": [
{
"name": "mocksite.com",
"http_host": "www.mocksite.com"
"referrer": "www.mocksite.com"
}
]
}
@ -461,7 +461,7 @@ class ServiceControllerTest1(base.FunctionalTest):
"rules": [
{
"name": "mocksite.com",
"http_host": "www.mocksite.com"
"referrer": "www.mocksite.com"
}
]
}

5
tests/unit/model/helpers/test_rule.py
View File

@ -30,6 +30,7 @@ class TestRule(base.TestCase):
client_ip = '192.168.1.1'
http_method = 'POST'
request_url = '/index.html'
geography = 'USA'
myrule = rule.Rule(name)
@ -53,3 +54,7 @@ class TestRule(base.TestCase):
# request_url
myrule.request_url = request_url
self.assertEqual(myrule.request_url, request_url)
# geography
myrule.geography = geography
self.assertEqual(myrule.geography, geography)

46
tests/unit/provider/akamai/data_service.json
View File

@ -61,6 +61,7 @@
"restrictions": [
{
"name": "website only",
"type": "whitelist",
"rules": [
{
"name": "mocksite.com",
@ -69,11 +70,16 @@
]
},
{
"name": "graphic only",
"name": "graphic and US only",
"type": "whitelist",
"rules": [
{
"name": "mockgraphicsite.com",
"referrer": "www.mocksitegraphic.com"
},
{
"name": "US geo restriction",
"geography": "USA"
}
]
}
@ -221,5 +227,43 @@
}
],
"flavor_id" : "standard"
},
"ip_restrictions_whitelist": {
"name" : "mysite.com",
"domains": [
{"domain": "parsely.sage.com"}
],
"origins": [
{"origin": "mockdomain-text.com", "ssl": false, "port": 80,
"rules": [{"name": "global", "request_url": "/*"},
{"name": "text", "request_url": "/text"}]}
],
"restrictions": [
{
"name": "only from ip 1234",
"type": "blacklist",
"rules": [
{
"name": "only ip",
"client_ip": "1.2.3.4"
}
]
}
],
"caching": [
{"name": "default", "ttl": 1200 },
{"name": "img-only",
"ttl": 1800,
"rules": [
{ "name": "jpeg-rules",
"request_url": "/*.jpeg"
},
{ "name": "gif-rules",
"request_url": "/*.gif"
}
]
}
],
"flavor_id" : "standard"
}
}

8
tests/unit/provider/akamai/data_update_service.json
View File

@ -44,6 +44,7 @@
"restrictions": [
{
"name": "website only",
"type": "whitelist",
"rules": [
{
"name": "mocksite.com",
@ -52,11 +53,16 @@
]
},
{
"name": "graphic only",
"name": "graphic and US only",
"type": "whitelist",
"rules": [
{
"name": "mockgraphicsite.com",
"referrer": "www.mocksitegraphic.com"
},
{
"name": "US geo restriction",
"geography": "USA"
}
]
}