From b47bcf19e41e862ca84d77a7b8843f836e084b6a Mon Sep 17 00:00:00 2001 From: John Dickinson Date: Wed, 9 May 2012 10:15:23 -0500 Subject: [PATCH] removed cname lookup middleware The code has moved to https://github.com/notmyname/swift-cnamelookup. For current users of cname lookup, this will require installing the new package and changing the "use" line of the cname lookup conf section's to: [filter:cname_lookup] use = egg:swift_cnamelookup#swift_cnamelookup And then 'swift-init proxy reload'. Change-Id: If622486ddb04a53251244c9840aa3cfe72168fc5 --- doc/manpages/proxy-server.conf.5 | 27 --- doc/source/misc.rst | 7 - etc/proxy-server.conf-sample | 11 -- locale/swift.pot | 10 -- setup.py | 1 - swift/common/middleware/cname_lookup.py | 152 ---------------- .../common/middleware/test_cname_lookup.py | 166 ------------------ 7 files changed, 374 deletions(-) delete mode 100644 swift/common/middleware/cname_lookup.py delete mode 100644 test/unit/common/middleware/test_cname_lookup.py diff --git a/doc/manpages/proxy-server.conf.5 b/doc/manpages/proxy-server.conf.5 index bd4bfc7922..9b651dd5fd 100644 --- a/doc/manpages/proxy-server.conf.5 +++ b/doc/manpages/proxy-server.conf.5 @@ -228,33 +228,6 @@ Enables the ability to log request headers. The default is False. -.RS 0 -.IP "\fB[filter:cname_lookup]\fR" -.RE - -Note: this middleware requires python-dnspython - -.RS 3 -.IP \fBuse\fR -Entry point for paste.deploy for the cname_lookup middleware. This is the reference to the installed python egg. -The default is \fBegg:swift#cname_lookup\fR. -.IP "\fBset log_name\fR" -Label used when logging. The default is cname_lookup. -.IP "\fBset log_facility\fR" -Syslog log facility. The default is LOG_LOCAL0. -.IP "\fBset log_level\fR " -Logging level. The default is INFO. -.IP "\fBset log_headers\fR" -Enables the ability to log request headers. The default is False. -.IP \fBstorage_domain\fR -The domain to be used by the middleware. -.IP \fBlookup_depth\fR -How deep in the CNAME chain to look for something that matches the storage domain. -The default is 1. -.RE - - - .RS 0 .IP "\fB[filter:name_check]\fR" .RE diff --git a/doc/source/misc.rst b/doc/source/misc.rst index e4e7358c44..388a3bf956 100644 --- a/doc/source/misc.rst +++ b/doc/source/misc.rst @@ -139,10 +139,3 @@ Swift3 .. automodule:: swift.common.middleware.swift3 :members: :show-inheritance: - -CNAME Lookup -============ - -.. automodule:: swift.common.middleware.cname_lookup - :members: - :show-inheritance: diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 5fa33e66e9..366462148f 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -138,17 +138,6 @@ use = egg:swift#catch_errors # set log_level = INFO # set log_headers = False -[filter:cname_lookup] -# Note: this middleware requires python-dnspython -use = egg:swift#cname_lookup -# You can override the default log routing for this filter here: -# set log_name = cname_lookup -# set log_facility = LOG_LOCAL0 -# set log_level = INFO -# set log_headers = False -# storage_domain = example.com -# lookup_depth = 1 - # Note: Just needs to be placed before the proxy-server in the pipeline. [filter:name_check] use = egg:swift#name_check diff --git a/locale/swift.pot b/locale/swift.pot index 2f27d97be3..5471ae507f 100644 --- a/locale/swift.pot +++ b/locale/swift.pot @@ -401,16 +401,6 @@ msgstr "" msgid "Error: %s" msgstr "" -#: swift/common/middleware/cname_lookup.py:91 -#, python-format -msgid "Mapped %(given_domain)s to %(found_domain)s" -msgstr "" - -#: swift/common/middleware/cname_lookup.py:102 -#, python-format -msgid "Following CNAME chain for %(given_domain)s to %(found_domain)s" -msgstr "" - #: swift/common/middleware/swauth.py:635 #, python-format msgid "" diff --git a/setup.py b/setup.py index 291fd788da..5782cc473d 100644 --- a/setup.py +++ b/setup.py @@ -81,7 +81,6 @@ setup( 'paste.filter_factory': [ 'healthcheck=swift.common.middleware.healthcheck:filter_factory', 'memcache=swift.common.middleware.memcache:filter_factory', - 'cname_lookup=swift.common.middleware.cname_lookup:filter_factory', 'catch_errors=swift.common.middleware.catch_errors:filter_factory', 'swift3=swift.common.middleware.swift3:filter_factory', 'tempauth=swift.common.middleware.tempauth:filter_factory', diff --git a/swift/common/middleware/cname_lookup.py b/swift/common/middleware/cname_lookup.py deleted file mode 100644 index ae55f60569..0000000000 --- a/swift/common/middleware/cname_lookup.py +++ /dev/null @@ -1,152 +0,0 @@ -# Copyright (c) 2010-2012 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -""" -CNAME Lookup Middleware - -Middleware that translates an unknown domain in the host header to -something that ends with the configured storage_domain by looking up -the given domain's CNAME record in DNS. - -This middleware will continue to follow a CNAME chain in DNS until it finds -a record ending in the configured storage domain or it reaches the configured -maximum lookup depth. If a match is found, the environment's Host header is -rewritten and the request is passed further down the WSGI chain. -""" - -from webob import Request -from webob.exc import HTTPBadRequest -try: - import dns.resolver - from dns.exception import DNSException - from dns.resolver import NXDOMAIN, NoAnswer -except ImportError: - # catch this to allow docs to be built without the dependency - MODULE_DEPENDENCY_MET = False -else: # executed if the try block finishes with no errors - MODULE_DEPENDENCY_MET = True - -from swift.common.utils import cache_from_env, get_logger - - -def lookup_cname(domain): # pragma: no cover - """ - Given a domain, returns its DNS CNAME mapping and DNS ttl. - - :param domain: domain to query on - :returns: (ttl, result) - """ - try: - answer = dns.resolver.query(domain, 'CNAME').rrset - ttl = answer.ttl - result = answer.items[0].to_text() - result = result.rstrip('.') - return ttl, result - except (DNSException, NXDOMAIN, NoAnswer): - return 0, None - - -class CNAMELookupMiddleware(object): - """ - CNAME Lookup Middleware - - See above for a full description. - - :param app: The next WSGI filter or app in the paste.deploy - chain. - :param conf: The configuration dict for the middleware. - """ - - def __init__(self, app, conf): - if not MODULE_DEPENDENCY_MET: - # reraise the exception if the dependency wasn't met - raise ImportError('dnspython is required for this module') - self.app = app - self.storage_domain = conf.get('storage_domain', 'example.com') - if self.storage_domain and self.storage_domain[0] != '.': - self.storage_domain = '.' + self.storage_domain - self.lookup_depth = int(conf.get('lookup_depth', '1')) - self.memcache = None - self.logger = get_logger(conf, log_route='cname-lookup') - - def __call__(self, env, start_response): - if not self.storage_domain: - return self.app(env, start_response) - given_domain = env['HTTP_HOST'] - port = '' - if ':' in given_domain: - given_domain, port = given_domain.rsplit(':', 1) - if given_domain == self.storage_domain[1:]: # strip initial '.' - return self.app(env, start_response) - a_domain = given_domain - if not a_domain.endswith(self.storage_domain): - if self.memcache is None: - self.memcache = cache_from_env(env) - error = True - for tries in xrange(self.lookup_depth): - found_domain = None - if self.memcache: - memcache_key = ''.join(['cname-', a_domain]) - found_domain = self.memcache.get(memcache_key) - if not found_domain: - ttl, found_domain = lookup_cname(a_domain) - if self.memcache: - memcache_key = ''.join(['cname-', given_domain]) - self.memcache.set(memcache_key, found_domain, - timeout=ttl) - if found_domain is None or found_domain == a_domain: - # no CNAME records or we're at the last lookup - error = True - found_domain = None - break - elif found_domain.endswith(self.storage_domain): - # Found it! - self.logger.info( - _('Mapped %(given_domain)s to %(found_domain)s') % - {'given_domain': given_domain, - 'found_domain': found_domain}) - if port: - env['HTTP_HOST'] = ':'.join([found_domain, port]) - else: - env['HTTP_HOST'] = found_domain - error = False - break - else: - # try one more deep in the chain - self.logger.debug(_('Following CNAME chain for ' \ - '%(given_domain)s to %(found_domain)s') % - {'given_domain': given_domain, - 'found_domain': found_domain}) - a_domain = found_domain - if error: - if found_domain: - msg = 'CNAME lookup failed after %d tries' % \ - self.lookup_depth - else: - msg = 'CNAME lookup failed to resolve to a valid domain' - resp = HTTPBadRequest(request=Request(env), body=msg, - content_type='text/plain') - return resp(env, start_response) - return self.app(env, start_response) - - -def filter_factory(global_conf, **local_conf): # pragma: no cover - conf = global_conf.copy() - conf.update(local_conf) - - def cname_filter(app): - return CNAMELookupMiddleware(app, conf) - return cname_filter diff --git a/test/unit/common/middleware/test_cname_lookup.py b/test/unit/common/middleware/test_cname_lookup.py deleted file mode 100644 index 12ab7ad1f1..0000000000 --- a/test/unit/common/middleware/test_cname_lookup.py +++ /dev/null @@ -1,166 +0,0 @@ -# Copyright (c) 2010-2012 OpenStack, LLC. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest -from nose import SkipTest - -from webob import Request - -try: - # this test requires the dnspython package to be installed - import dns.resolver -except ImportError: - skip = True -else: # executed if the try has no errors - skip = False -from swift.common.middleware import cname_lookup - -class FakeApp(object): - - def __call__(self, env, start_response): - return "FAKE APP" - - -def start_response(*args): - pass - - -class TestCNAMELookup(unittest.TestCase): - - def setUp(self): - if skip: - raise SkipTest - self.app = cname_lookup.CNAMELookupMiddleware(FakeApp(), - {'lookup_depth': 2}) - - def test_passthrough(self): - - def my_lookup(d): - return 0, d - cname_lookup.lookup_cname = my_lookup - - req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, - headers={'Host': 'foo.example.com'}) - resp = self.app(req.environ, start_response) - self.assertEquals(resp, 'FAKE APP') - req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, - headers={'Host': 'foo.example.com:8080'}) - resp = self.app(req.environ, start_response) - self.assertEquals(resp, 'FAKE APP') - - def test_good_lookup(self): - req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, - headers={'Host': 'mysite.com'}) - - def my_lookup(d): - return 0, '%s.example.com' % d - cname_lookup.lookup_cname = my_lookup - - resp = self.app(req.environ, start_response) - self.assertEquals(resp, 'FAKE APP') - req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, - headers={'Host': 'mysite.com:8080'}) - resp = self.app(req.environ, start_response) - self.assertEquals(resp, 'FAKE APP') - - def test_lookup_chain_too_long(self): - req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, - headers={'Host': 'mysite.com'}) - - def my_lookup(d): - if d == 'mysite.com': - site = 'level1.foo.com' - elif d == 'level1.foo.com': - site = 'level2.foo.com' - elif d == 'level2.foo.com': - site = 'bar.example.com' - return 0, site - cname_lookup.lookup_cname = my_lookup - - resp = self.app(req.environ, start_response) - self.assertEquals(resp, ['CNAME lookup failed after 2 tries']) - - def test_lookup_chain_bad_target(self): - req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, - headers={'Host': 'mysite.com'}) - - def my_lookup(d): - return 0, 'some.invalid.site.com' - cname_lookup.lookup_cname = my_lookup - - resp = self.app(req.environ, start_response) - self.assertEquals(resp, - ['CNAME lookup failed to resolve to a valid domain']) - - def test_something_weird(self): - req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, - headers={'Host': 'mysite.com'}) - - def my_lookup(d): - return 0, None - cname_lookup.lookup_cname = my_lookup - - resp = self.app(req.environ, start_response) - self.assertEquals(resp, - ['CNAME lookup failed to resolve to a valid domain']) - - def test_with_memcache(self): - def my_lookup(d): - return 0, '%s.example.com' % d - cname_lookup.lookup_cname = my_lookup - class memcache_stub(object): - def __init__(self): - self.cache = {} - def get(self, key): - return self.cache.get(key, None) - def set(self, key, value, *a, **kw): - self.cache[key] = value - memcache = memcache_stub() - req = Request.blank('/', environ={'REQUEST_METHOD': 'GET', - 'swift.cache': memcache}, - headers={'Host': 'mysite.com'}) - resp = self.app(req.environ, start_response) - self.assertEquals(resp, 'FAKE APP') - req = Request.blank('/', environ={'REQUEST_METHOD': 'GET', - 'swift.cache': memcache}, - headers={'Host': 'mysite.com'}) - resp = self.app(req.environ, start_response) - self.assertEquals(resp, 'FAKE APP') - - def test_cname_matching_ending_not_domain(self): - req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, - headers={'Host': 'foo.com'}) - - def my_lookup(d): - return 0, 'c.aexample.com' - cname_lookup.lookup_cname = my_lookup - - resp = self.app(req.environ, start_response) - self.assertEquals(resp, - ['CNAME lookup failed to resolve to a valid domain']) - - def test_cname_configured_with_empty_storage_domain(self): - app = cname_lookup.CNAMELookupMiddleware(FakeApp(), - {'storage_domain': '', - 'lookup_depth': 2}) - req = Request.blank('/', environ={'REQUEST_METHOD': 'GET'}, - headers={'Host': 'c.a.example.com'}) - - def my_lookup(d): - return 0, None - cname_lookup.lookup_cname = my_lookup - - resp = app(req.environ, start_response) - self.assertEquals(resp, 'FAKE APP')