From c3cef9207b9477f604bedc5780fae60210a60bc5 Mon Sep 17 00:00:00 2001 From: gholt Date: Sun, 5 Jun 2011 23:22:35 +0000 Subject: [PATCH 01/18] Adding account_autocreate mode and refactoring TRUE_VALUES --- doc/source/deployment_guide.rst | 4 ++++ etc/proxy-server.conf-sample | 3 +++ swift/common/bench.py | 4 ++-- swift/common/daemon.py | 3 ++- swift/common/middleware/staticweb.py | 2 +- swift/common/utils.py | 2 +- swift/proxy/server.py | 23 ++++++++++++++++++----- swift/stats/log_uploader.py | 2 +- 8 files changed, 32 insertions(+), 11 deletions(-) diff --git a/doc/source/deployment_guide.rst b/doc/source/deployment_guide.rst index 52a4f80f6e..04b99fa1f6 100644 --- a/doc/source/deployment_guide.rst +++ b/doc/source/deployment_guide.rst @@ -547,6 +547,10 @@ error_suppression_limit 10 Error count to consider a node error limited allow_account_management false Whether account PUTs and DELETEs are even callable +account_autocreate false If set to 'true' authorized + accounts that do not yet exist + within the Swift cluster will + be automatically created. ============================ =============== ============================= [tempauth] diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index fef0e81fa0..496eb4aea1 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -40,6 +40,9 @@ use = egg:swift#proxy # If set to 'true' any authorized user may create and delete accounts; if # 'false' no one, even authorized, can. # allow_account_management = false +# If set to 'true' authorized accounts that do not yet exist within the Swift +# cluster will be automatically created. +# account_autocreate = false [filter:tempauth] use = egg:swift#tempauth diff --git a/swift/common/bench.py b/swift/common/bench.py index 28d8c7e8d9..51e39f793d 100644 --- a/swift/common/bench.py +++ b/swift/common/bench.py @@ -43,7 +43,7 @@ class Bench(object): self.user = conf.user self.key = conf.key self.auth_url = conf.auth - self.use_proxy = conf.use_proxy in TRUE_VALUES + self.use_proxy = conf.use_proxy.lower() in TRUE_VALUES if self.use_proxy: url, token = client.get_auth(self.auth_url, self.user, self.key) self.token = token @@ -125,7 +125,7 @@ class BenchController(object): self.logger = logger self.conf = conf self.names = [] - self.delete = conf.delete in TRUE_VALUES + self.delete = conf.delete.lower() in TRUE_VALUES self.gets = int(conf.num_gets) def run(self): diff --git a/swift/common/daemon.py b/swift/common/daemon.py index 96914d9510..abcc8dea1e 100644 --- a/swift/common/daemon.py +++ b/swift/common/daemon.py @@ -75,7 +75,8 @@ def run_daemon(klass, conf_file, section_name='', once=False, **kwargs): log_name=kwargs.get('log_name')) # once on command line (i.e. daemonize=false) will over-ride config - once = once or conf.get('daemonize', 'true') not in utils.TRUE_VALUES + once = once or \ + conf.get('daemonize', 'true').lower() not in utils.TRUE_VALUES # pre-configure logger if 'logger' in kwargs: diff --git a/swift/common/middleware/staticweb.py b/swift/common/middleware/staticweb.py index 8e58ad5068..81225a90ea 100644 --- a/swift/common/middleware/staticweb.py +++ b/swift/common/middleware/staticweb.py @@ -270,7 +270,7 @@ class StaticWeb(object): :param start_response: The original WSGI start_response hook. :param prefix: Any prefix desired for the container listing. """ - if self._listings not in TRUE_VALUES: + if self._listings.lower() not in TRUE_VALUES: resp = HTTPNotFound()(env, self._start_response) return self._error_response(resp, env, start_response) tmp_env = self._get_escalated_env(env) diff --git a/swift/common/utils.py b/swift/common/utils.py index 4ee57db8f7..ac18331703 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -72,7 +72,7 @@ if hash_conf.read('/etc/swift/swift.conf'): pass # Used when reading config values -TRUE_VALUES = set(('true', '1', 'yes', 'True', 'Yes', 'on', 'On', 't', 'y')) +TRUE_VALUES = set(('true', '1', 'yes', 'on', 't', 'y')) def validate_configuration(): diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 3300e7a384..01e33d8498 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -41,8 +41,8 @@ from webob.exc import HTTPBadRequest, HTTPMethodNotAllowed, \ from webob import Request, Response from swift.common.ring import Ring -from swift.common.utils import get_logger, normalize_timestamp, split_path, \ - cache_from_env, ContextPool +from swift.common.utils import cache_from_env, ContextPool, get_logger, \ + normalize_timestamp, split_path, TRUE_VALUES from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_metadata, check_object_creation, \ check_utf8, CONTAINER_LISTING_LIMIT, MAX_ACCOUNT_NAME_LENGTH, \ @@ -353,7 +353,7 @@ class Controller(object): result_code = self.app.memcache.get(cache_key) if result_code == 200: return partition, nodes - elif result_code == 404: + elif result_code == 404 and not self.app.account_autocreate: return None, None result_code = 0 attempts_left = self.app.account_ring.replica_count @@ -386,6 +386,17 @@ class Controller(object): except (Exception, TimeoutError): self.exception_occurred(node, _('Account'), _('Trying to get account info for %s') % path) + if result_code == 404: + if self.app.account_autocreate: + if len(account) > MAX_ACCOUNT_NAME_LENGTH: + return None, None + headers = {'X-Timestamp': normalize_timestamp(time.time()), + 'x-trans-id': self.trans_id} + resp = self.make_requests(Request.blank('/v1' + path), + self.app.account_ring, partition, 'PUT', + path, [headers] * len(nodes)) + if resp.status_int // 100 == 2: + result_code = 200 if self.app.memcache and result_code in (200, 404): if result_code == 200: cache_timeout = self.app.recheck_account_existence @@ -1391,7 +1402,7 @@ class BaseApplication(object): self.put_queue_depth = int(conf.get('put_queue_depth', 10)) self.object_chunk_size = int(conf.get('object_chunk_size', 65536)) self.client_chunk_size = int(conf.get('client_chunk_size', 65536)) - self.log_headers = conf.get('log_headers') == 'True' + self.log_headers = conf.get('log_headers', 'no').lower() in TRUE_VALUES self.error_suppression_interval = \ int(conf.get('error_suppression_interval', 60)) self.error_suppression_limit = \ @@ -1401,7 +1412,7 @@ class BaseApplication(object): self.recheck_account_existence = \ int(conf.get('recheck_account_existence', 60)) self.allow_account_management = \ - conf.get('allow_account_management', 'false').lower() == 'true' + conf.get('allow_account_management', 'no').lower() in TRUE_VALUES self.resellers_conf = ConfigParser() self.resellers_conf.read(os.path.join(swift_dir, 'resellers.conf')) self.object_ring = object_ring or \ @@ -1413,6 +1424,8 @@ class BaseApplication(object): self.memcache = memcache mimetypes.init(mimetypes.knownfiles + [os.path.join(swift_dir, 'mime.types')]) + self.account_autocreate = \ + conf.get('account_autocreate', 'no').lower() in TRUE_VALUES def get_controller(self, path): """ diff --git a/swift/stats/log_uploader.py b/swift/stats/log_uploader.py index 6051107a86..ea51061d54 100644 --- a/swift/stats/log_uploader.py +++ b/swift/stats/log_uploader.py @@ -69,7 +69,7 @@ class LogUploader(Daemon): self.internal_proxy = InternalProxy(proxy_server_conf) self.new_log_cutoff = int(cutoff or uploader_conf.get('new_log_cutoff', '7200')) - self.unlink_log = uploader_conf.get('unlink_log', 'True').lower() in \ + self.unlink_log = uploader_conf.get('unlink_log', 'true').lower() in \ utils.TRUE_VALUES self.filename_pattern = regex or \ uploader_conf.get('source_filename_pattern', From e34385f516b6b581695dfad754d2907028e7adeb Mon Sep 17 00:00:00 2001 From: gholt Date: Sun, 5 Jun 2011 23:44:39 +0000 Subject: [PATCH 02/18] Only autocreate accounts for certain operations --- swift/proxy/server.py | 22 +++++++++++++--------- test/unit/proxy/test_server.py | 6 +++--- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 01e33d8498..a863b8a1af 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -338,7 +338,7 @@ class Controller(object): node['errors'] = self.app.error_suppression_limit + 1 node['last_error'] = time.time() - def account_info(self, account): + def account_info(self, account, autocreate=False): """ Get account information, and also verify that the account exists. @@ -353,7 +353,7 @@ class Controller(object): result_code = self.app.memcache.get(cache_key) if result_code == 200: return partition, nodes - elif result_code == 404 and not self.app.account_autocreate: + elif result_code == 404 and not autocreate: return None, None result_code = 0 attempts_left = self.app.account_ring.replica_count @@ -387,7 +387,7 @@ class Controller(object): self.exception_occurred(node, _('Account'), _('Trying to get account info for %s') % path) if result_code == 404: - if self.app.account_autocreate: + if autocreate: if len(account) > MAX_ACCOUNT_NAME_LENGTH: return None, None headers = {'X-Timestamp': normalize_timestamp(time.time()), @@ -408,7 +408,7 @@ class Controller(object): return partition, nodes return None, None - def container_info(self, account, container): + def container_info(self, account, container, account_autocreate=False): """ Get container information and thusly verify container existance. This will also make a call to account_info to verify that the @@ -434,7 +434,7 @@ class Controller(object): return partition, nodes, read_acl, write_acl elif status == 404: return None, None, None, None - if not self.account_info(account)[1]: + if not self.account_info(account, autocreate=account_autocreate)[1]: return None, None, None, None result_code = 0 read_acl = None @@ -865,7 +865,8 @@ class ObjectController(Controller): if error_response: return error_response container_partition, containers, _junk, req.acl = \ - self.container_info(self.account_name, self.container_name) + self.container_info(self.account_name, self.container_name, + account_autocreate=self.app.account_autocreate) if 'swift.authorize' in req.environ: aresp = req.environ['swift.authorize'](req) if aresp: @@ -922,7 +923,8 @@ class ObjectController(Controller): def PUT(self, req): """HTTP PUT request handler.""" container_partition, containers, _junk, req.acl = \ - self.container_info(self.account_name, self.container_name) + self.container_info(self.account_name, self.container_name, + account_autocreate=self.app.account_autocreate) if 'swift.authorize' in req.environ: aresp = req.environ['swift.authorize'](req) if aresp: @@ -1230,7 +1232,8 @@ class ContainerController(Controller): resp.body = 'Container name length of %d longer than %d' % \ (len(self.container_name), MAX_CONTAINER_NAME_LENGTH) return resp - account_partition, accounts = self.account_info(self.account_name) + account_partition, accounts = self.account_info(self.account_name, + autocreate=self.app.account_autocreate) if not accounts: return HTTPNotFound(request=req) container_partition, containers = self.app.container_ring.get_nodes( @@ -1260,7 +1263,8 @@ class ContainerController(Controller): self.clean_acls(req) or check_metadata(req, 'container') if error_response: return error_response - account_partition, accounts = self.account_info(self.account_name) + account_partition, accounts = self.account_info(self.account_name, + autocreate=self.app.account_autocreate) if not accounts: return HTTPNotFound(request=req) container_partition, containers = self.app.container_ring.get_nodes( diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index fe2d1ca01e..1ea65b4943 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -406,7 +406,7 @@ class TestController(unittest.TestCase): self.assertEqual(write_acl, ret[3]) def test_container_info_invalid_account(self): - def account_info(self, account): + def account_info(self, account, autocreate=False): return None, None with save_globals(): @@ -417,7 +417,7 @@ class TestController(unittest.TestCase): # tests if 200 is cached and used def test_container_info_200(self): - def account_info(self, account): + def account_info(self, account, autocreate=False): return True, True with save_globals(): @@ -443,7 +443,7 @@ class TestController(unittest.TestCase): # tests if 404 is cached and used def test_container_info_404(self): - def account_info(self, account): + def account_info(self, account, autocreate=False): return True, True with save_globals(): From 34e121200ab253bbe79a72066baf1377b4795996 Mon Sep 17 00:00:00 2001 From: David Goetz Date: Tue, 7 Jun 2011 17:32:59 -0700 Subject: [PATCH 03/18] Adding the collection of specified metadata keys, unit tests working --- swift/common/db.py | 31 +++++++++---- swift/stats/db_stats_collector.py | 34 +++++++++++--- test/unit/stats/test_db_stats_collector.py | 53 +++++++++++++++++++--- 3 files changed, 96 insertions(+), 22 deletions(-) diff --git a/swift/common/db.py b/swift/common/db.py index 67913ca94e..219c097c5b 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -879,14 +879,16 @@ class ContainerBroker(DatabaseBroker): return (row['object_count'] in (None, '', 0, '0')) and \ (float(row['delete_timestamp']) > float(row['put_timestamp'])) - def get_info(self): + def get_info(self, include_metadata=False): """ Get global data for the container. - :returns: sqlite.row of (account, container, created_at, put_timestamp, + :returns: dict with keys: account, container, created_at, put_timestamp, delete_timestamp, object_count, bytes_used, reported_put_timestamp, reported_delete_timestamp, - reported_object_count, reported_bytes_used, hash, id) + reported_object_count, reported_bytes_used, hash, id + If include_metadata is set, metadata is included as a key + pointing to a dict of tuples of the metadata """ try: self._commit_puts() @@ -894,13 +896,24 @@ class ContainerBroker(DatabaseBroker): if not self.stale_reads_ok: raise with self.get() as conn: - return conn.execute(''' + metadata = '' + if include_metadata: + metadata = ', metadata' + data = conn.execute(''' SELECT account, container, created_at, put_timestamp, delete_timestamp, object_count, bytes_used, reported_put_timestamp, reported_delete_timestamp, reported_object_count, reported_bytes_used, hash, id + %s FROM container_stat - ''').fetchone() + ''' % metadata).fetchone() + data = dict(data) + if include_metadata: + try: + data['metadata'] = json.loads(data.get('metadata','')) + except ValueError: + data['metadata'] = {} + return data def reported(self, put_timestamp, delete_timestamp, object_count, bytes_used): @@ -1394,9 +1407,9 @@ class AccountBroker(DatabaseBroker): """ Get global data for the account. - :returns: sqlite.row of (account, created_at, put_timestamp, + :returns: dict with keys: account, created_at, put_timestamp, delete_timestamp, container_count, object_count, - bytes_used, hash, id) + bytes_used, hash, id """ try: self._commit_puts() @@ -1404,11 +1417,11 @@ class AccountBroker(DatabaseBroker): if not self.stale_reads_ok: raise with self.get() as conn: - return conn.execute(''' + return dict(conn.execute(''' SELECT account, created_at, put_timestamp, delete_timestamp, container_count, object_count, bytes_used, hash, id FROM account_stat - ''').fetchone() + ''').fetchone()) def list_containers_iter(self, limit, marker, end_marker, prefix, delimiter): diff --git a/swift/stats/db_stats_collector.py b/swift/stats/db_stats_collector.py index 04968f181f..c8faa66779 100644 --- a/swift/stats/db_stats_collector.py +++ b/swift/stats/db_stats_collector.py @@ -60,6 +60,9 @@ class DatabaseStatsCollector(Daemon): def get_data(self): raise Exception('Not Implemented') + def get_header(self): + raise Exception('Not Implemented') + def find_and_process(self): src_filename = time.strftime(self.filename_format) working_dir = os.path.join(self.target_dir, @@ -70,6 +73,7 @@ class DatabaseStatsCollector(Daemon): hasher = hashlib.md5() try: with open(tmp_filename, 'wb') as statfile: + statfile.write(self.get_header()) for device in os.listdir(self.devices): if self.mount_check and not check_mount(self.devices, device): @@ -122,6 +126,8 @@ class AccountStatsCollector(DatabaseStatsCollector): info['bytes_used']) return line_data + def get_header(self): + return '' class ContainerStatsCollector(DatabaseStatsCollector): """ @@ -133,20 +139,36 @@ class ContainerStatsCollector(DatabaseStatsCollector): super(ContainerStatsCollector, self).__init__(stats_conf, 'container', container_server_data_dir, 'container-stats-%Y%m%d%H_') + self.metadata_keys = [mkey.strip() for mkey in + stats_conf.get('metadata_keys', '').split(',') if mkey.strip()] + + def get_header(self): + header = 'Account Hash, Container Name, Object Count, Bytes Used' + if self.metadata_keys: + xtra_headers = ','.join(self.metadata_keys) + header += ',%s' % xtra_headers + header += '\n' + return header def get_data(self, db_path): """ Data for generated csv has the following columns: Account Hash, Container Name, Object Count, Bytes Used + This will just collect whether or not the metadata is set + using a 1 or ''. """ line_data = None broker = ContainerBroker(db_path) if not broker.is_deleted(): - info = broker.get_info() + info = broker.get_info(include_metadata=bool(self.metadata_keys)) encoded_container_name = urllib.quote(info['container']) - line_data = '"%s","%s",%d,%d\n' % ( - info['account'], - encoded_container_name, - info['object_count'], - info['bytes_used']) + line_data = '"%s","%s",%d,%d' % ( + info['account'], encoded_container_name, + info['object_count'], info['bytes_used']) + if self.metadata_keys: + metadata_results = ','.join( + [info['metadata'].get(mkey) and '1' or '' + for mkey in self.metadata_keys]) + line_data += ',%s' % metadata_results + line_data += '\n' return line_data diff --git a/test/unit/stats/test_db_stats_collector.py b/test/unit/stats/test_db_stats_collector.py index 2721614e9f..7836351a79 100644 --- a/test/unit/stats/test_db_stats_collector.py +++ b/test/unit/stats/test_db_stats_collector.py @@ -66,6 +66,17 @@ class TestDbStats(unittest.TestCase): info = stat.get_data("%s/con.db" % self.containers) self.assertEquals('''"test_acc","test_con",1,10\n''', info) + def test_container_stat_get_metadata(self): + stat = db_stats_collector.ContainerStatsCollector(self.conf) + container_db = ContainerBroker("%s/con.db" % self.containers, + account='test_acc', container='test_con') + container_db.initialize() + container_db.put_object('test_obj', time.time(), 10, 'text', 'faketag') + info = stat.get_data("%s/con.db" % self.containers) + self.assertEquals('''"test_acc","test_con",1,10\n''', info) + container_db.update_metadata({'test1': ('val',1000)}) + + def _gen_account_stat(self): stat = db_stats_collector.AccountStatsCollector(self.conf) output_data = set() @@ -83,20 +94,30 @@ class TestDbStats(unittest.TestCase): self.assertEqual(len(output_data), 10) return stat, output_data - def _gen_container_stat(self): + def _gen_container_stat(self, set_metadata=False): + if set_metadata: + self.conf['metadata_keys'] = 'test1,test2' stat = db_stats_collector.ContainerStatsCollector(self.conf) output_data = set() for i in range(10): - account_db = ContainerBroker( + cont_db = ContainerBroker( "%s/container-stats-201001010%s-%s.db" % (self.containers, i, uuid.uuid4().hex), account='test_acc_%s' % i, container='test_con') - account_db.initialize() - account_db.put_object('test_obj', time.time(), 10, 'text', - 'faketag') + cont_db.initialize() + cont_db.put_object('test_obj', time.time(), 10, 'text', 'faketag') + metadata_output = '' + if set_metadata: + if i%2: + cont_db.update_metadata({'test1': (55,100)}) + metadata_output = ',1,' + else: + cont_db.update_metadata({'test2': (55,100)}) + metadata_output = ',,1' # this will "commit" the data - account_db.get_info() - output_data.add('''"test_acc_%s","test_con",1,10''' % i), + cont_db.get_info() + output_data.add('''"test_acc_%s","test_con",1,10%s''' % + (i, metadata_output)) self.assertEqual(len(output_data), 10) return stat, output_data @@ -112,6 +133,21 @@ class TestDbStats(unittest.TestCase): self.assertEqual(len(output_data), 0) + def test_account_stat_run_once_container_metadata(self): + + stat, output_data = self._gen_container_stat(set_metadata=True) + stat.run_once() + stat_file = os.listdir(self.log_dir)[0] + with open(os.path.join(self.log_dir, stat_file)) as stat_handle: + headers = stat_handle.readline() + self.assert_(headers.startswith('Account Hash, Container Name,')) + for i in range(10): + data = stat_handle.readline() + output_data.discard(data.strip()) + + self.assertEqual(len(output_data), 0) + + def test_account_stat_run_once_both(self): acc_stat, acc_output_data = self._gen_account_stat() con_stat, con_output_data = self._gen_container_stat() @@ -128,6 +164,8 @@ class TestDbStats(unittest.TestCase): con_stat.run_once() stat_file = [f for f in os.listdir(self.log_dir) if f != stat_file][0] with open(os.path.join(self.log_dir, stat_file)) as stat_handle: + headers = stat_handle.readline() + self.assert_(headers.startswith('Account Hash, Container Name,')) for i in range(10): data = stat_handle.readline() con_output_data.discard(data.strip()) @@ -144,6 +182,7 @@ class TestDbStats(unittest.TestCase): db_stat = db_stats_collector.DatabaseStatsCollector(self.conf, 'account', 'test_dir', 'stats-%Y%m%d%H_') self.assertRaises(Exception, db_stat.get_data) + self.assertRaises(Exception, db_stat.get_header) def test_not_not_mounted(self): self.conf['mount_check'] = 'true' From 68e5719a81374e38ef866330a5b93e32b7efa218 Mon Sep 17 00:00:00 2001 From: David Goetz Date: Wed, 8 Jun 2011 08:55:14 -0700 Subject: [PATCH 04/18] fixing case thing and adding docs --- doc/source/overview_stats.rst | 11 ++++++----- etc/log-processor.conf-sample | 1 + swift/stats/db_stats_collector.py | 6 ++++-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/doc/source/overview_stats.rst b/doc/source/overview_stats.rst index 04d2299e79..3043b57ece 100644 --- a/doc/source/overview_stats.rst +++ b/doc/source/overview_stats.rst @@ -19,11 +19,11 @@ the proxy log output to an hourly log file. For example, a proxy request that is made on August 4, 2010 at 12:37 gets logged in a file named 2010080412. This allows easy log rotation and easy per-hour log processing. -****************** -Account stats logs -****************** +********************************* +Account / Container DB stats logs +********************************* -Account stats logs are generated by a stats system process. +DB stats logs are generated by a stats system process. swift-account-stats-logger runs on each account server (via cron) and walks the filesystem looking for account databases. When an account database is found, the logger selects the account hash, bytes_used, container_count, and @@ -34,7 +34,8 @@ runs the account stats logger every hour. Therefore, in a cluster of ten account servers, ten csv files are produced every hour. Also, every account will have one entry for every replica in the system. On average, there will be three copies of each account in the aggregate of all account stat csv files -created in one system-wide run. +created in one system-wide run. The swift-container-stats-logger runs in a +similar fashion, scanning the container dbs. ---------------------- Log Processing plugins diff --git a/etc/log-processor.conf-sample b/etc/log-processor.conf-sample index f014168a90..350ae73010 100644 --- a/etc/log-processor.conf-sample +++ b/etc/log-processor.conf-sample @@ -54,3 +54,4 @@ processable = false # devices = /srv/node # mount_check = true # user = swift +# metadata_keys = comma separated list of user metadata keys to be collected diff --git a/swift/stats/db_stats_collector.py b/swift/stats/db_stats_collector.py index c8faa66779..e6edf47002 100644 --- a/swift/stats/db_stats_collector.py +++ b/swift/stats/db_stats_collector.py @@ -139,8 +139,10 @@ class ContainerStatsCollector(DatabaseStatsCollector): super(ContainerStatsCollector, self).__init__(stats_conf, 'container', container_server_data_dir, 'container-stats-%Y%m%d%H_') - self.metadata_keys = [mkey.strip() for mkey in - stats_conf.get('metadata_keys', '').split(',') if mkey.strip()] + # webob calls title on all the header keys + self.metadata_keys = ['X-Container-Meta-%s' % mkey.strip().title() + for mkey in stats_conf.get('metadata_keys', '').split(',') + if mkey.strip()] def get_header(self): header = 'Account Hash, Container Name, Object Count, Bytes Used' From 533946c2c8be9db55c6b5728c016b2c7495ec6c6 Mon Sep 17 00:00:00 2001 From: David Goetz Date: Wed, 8 Jun 2011 09:19:55 -0700 Subject: [PATCH 05/18] fix unittest --- test/unit/stats/test_db_stats_collector.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/unit/stats/test_db_stats_collector.py b/test/unit/stats/test_db_stats_collector.py index 7836351a79..01ae8bd5b2 100644 --- a/test/unit/stats/test_db_stats_collector.py +++ b/test/unit/stats/test_db_stats_collector.py @@ -97,6 +97,7 @@ class TestDbStats(unittest.TestCase): def _gen_container_stat(self, set_metadata=False): if set_metadata: self.conf['metadata_keys'] = 'test1,test2' + # webob runs title on all headers stat = db_stats_collector.ContainerStatsCollector(self.conf) output_data = set() for i in range(10): @@ -109,10 +110,10 @@ class TestDbStats(unittest.TestCase): metadata_output = '' if set_metadata: if i%2: - cont_db.update_metadata({'test1': (55,100)}) + cont_db.update_metadata({'X-Container-Meta-Test1': (55,1)}) metadata_output = ',1,' else: - cont_db.update_metadata({'test2': (55,100)}) + cont_db.update_metadata({'X-Container-Meta-Test2': (55,2)}) metadata_output = ',,1' # this will "commit" the data cont_db.get_info() From 625c1452006f1d48f3ce77f50c224ff92df277cd Mon Sep 17 00:00:00 2001 From: David Goetz Date: Wed, 8 Jun 2011 09:24:44 -0700 Subject: [PATCH 06/18] pep8 --- swift/common/db.py | 6 +++--- swift/stats/db_stats_collector.py | 3 ++- test/unit/stats/test_db_stats_collector.py | 10 ++++------ 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/swift/common/db.py b/swift/common/db.py index 219c097c5b..783e00dd5c 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -883,8 +883,8 @@ class ContainerBroker(DatabaseBroker): """ Get global data for the container. - :returns: dict with keys: account, container, created_at, put_timestamp, - delete_timestamp, object_count, bytes_used, + :returns: dict with keys: account, container, created_at, + put_timestamp, delete_timestamp, object_count, bytes_used, reported_put_timestamp, reported_delete_timestamp, reported_object_count, reported_bytes_used, hash, id If include_metadata is set, metadata is included as a key @@ -910,7 +910,7 @@ class ContainerBroker(DatabaseBroker): data = dict(data) if include_metadata: try: - data['metadata'] = json.loads(data.get('metadata','')) + data['metadata'] = json.loads(data.get('metadata', '')) except ValueError: data['metadata'] = {} return data diff --git a/swift/stats/db_stats_collector.py b/swift/stats/db_stats_collector.py index e6edf47002..f46e12f2c0 100644 --- a/swift/stats/db_stats_collector.py +++ b/swift/stats/db_stats_collector.py @@ -129,6 +129,7 @@ class AccountStatsCollector(DatabaseStatsCollector): def get_header(self): return '' + class ContainerStatsCollector(DatabaseStatsCollector): """ Extract storage stats from container databases on the container @@ -168,7 +169,7 @@ class ContainerStatsCollector(DatabaseStatsCollector): info['account'], encoded_container_name, info['object_count'], info['bytes_used']) if self.metadata_keys: - metadata_results = ','.join( + metadata_results = ','.join( [info['metadata'].get(mkey) and '1' or '' for mkey in self.metadata_keys]) line_data += ',%s' % metadata_results diff --git a/test/unit/stats/test_db_stats_collector.py b/test/unit/stats/test_db_stats_collector.py index 01ae8bd5b2..3e3cc522d0 100644 --- a/test/unit/stats/test_db_stats_collector.py +++ b/test/unit/stats/test_db_stats_collector.py @@ -74,8 +74,7 @@ class TestDbStats(unittest.TestCase): container_db.put_object('test_obj', time.time(), 10, 'text', 'faketag') info = stat.get_data("%s/con.db" % self.containers) self.assertEquals('''"test_acc","test_con",1,10\n''', info) - container_db.update_metadata({'test1': ('val',1000)}) - + container_db.update_metadata({'test1': ('val', 1000)}) def _gen_account_stat(self): stat = db_stats_collector.AccountStatsCollector(self.conf) @@ -109,11 +108,11 @@ class TestDbStats(unittest.TestCase): cont_db.put_object('test_obj', time.time(), 10, 'text', 'faketag') metadata_output = '' if set_metadata: - if i%2: - cont_db.update_metadata({'X-Container-Meta-Test1': (55,1)}) + if i % 2: + cont_db.update_metadata({'X-Container-Meta-Test1': (5, 1)}) metadata_output = ',1,' else: - cont_db.update_metadata({'X-Container-Meta-Test2': (55,2)}) + cont_db.update_metadata({'X-Container-Meta-Test2': (7, 2)}) metadata_output = ',,1' # this will "commit" the data cont_db.get_info() @@ -148,7 +147,6 @@ class TestDbStats(unittest.TestCase): self.assertEqual(len(output_data), 0) - def test_account_stat_run_once_both(self): acc_stat, acc_output_data = self._gen_account_stat() con_stat, con_output_data = self._gen_container_stat() From c5aeca412c81ea3c618ce945277b119254c01415 Mon Sep 17 00:00:00 2001 From: David Goetz Date: Wed, 8 Jun 2011 09:42:18 -0700 Subject: [PATCH 07/18] clean headers --- swift/stats/db_stats_collector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swift/stats/db_stats_collector.py b/swift/stats/db_stats_collector.py index f46e12f2c0..f65c20c3e2 100644 --- a/swift/stats/db_stats_collector.py +++ b/swift/stats/db_stats_collector.py @@ -146,7 +146,7 @@ class ContainerStatsCollector(DatabaseStatsCollector): if mkey.strip()] def get_header(self): - header = 'Account Hash, Container Name, Object Count, Bytes Used' + header = 'Account Hash,Container Name,Object Count,Bytes Used' if self.metadata_keys: xtra_headers = ','.join(self.metadata_keys) header += ',%s' % xtra_headers From aa91a5a6dab71c7d642645ce278b9abf364471d8 Mon Sep 17 00:00:00 2001 From: David Goetz Date: Wed, 8 Jun 2011 09:44:03 -0700 Subject: [PATCH 08/18] clean headers in unit tests... --- test/unit/stats/test_db_stats_collector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unit/stats/test_db_stats_collector.py b/test/unit/stats/test_db_stats_collector.py index 3e3cc522d0..d336016e77 100644 --- a/test/unit/stats/test_db_stats_collector.py +++ b/test/unit/stats/test_db_stats_collector.py @@ -140,7 +140,7 @@ class TestDbStats(unittest.TestCase): stat_file = os.listdir(self.log_dir)[0] with open(os.path.join(self.log_dir, stat_file)) as stat_handle: headers = stat_handle.readline() - self.assert_(headers.startswith('Account Hash, Container Name,')) + self.assert_(headers.startswith('Account Hash,Container Name,')) for i in range(10): data = stat_handle.readline() output_data.discard(data.strip()) @@ -164,7 +164,7 @@ class TestDbStats(unittest.TestCase): stat_file = [f for f in os.listdir(self.log_dir) if f != stat_file][0] with open(os.path.join(self.log_dir, stat_file)) as stat_handle: headers = stat_handle.readline() - self.assert_(headers.startswith('Account Hash, Container Name,')) + self.assert_(headers.startswith('Account Hash,Container Name,')) for i in range(10): data = stat_handle.readline() con_output_data.discard(data.strip()) From bb48838404b887063d86c4f480c67b274c57f788 Mon Sep 17 00:00:00 2001 From: gholt Date: Fri, 10 Jun 2011 15:55:25 +0000 Subject: [PATCH 09/18] Adding some tests --- swift/proxy/server.py | 21 ++++++++++----------- test/unit/common/test_utils.py | 4 ++++ test/unit/proxy/test_server.py | 20 ++++++++++++++++++++ 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/swift/proxy/server.py b/swift/proxy/server.py index a863b8a1af..2943c6a93e 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -386,17 +386,16 @@ class Controller(object): except (Exception, TimeoutError): self.exception_occurred(node, _('Account'), _('Trying to get account info for %s') % path) - if result_code == 404: - if autocreate: - if len(account) > MAX_ACCOUNT_NAME_LENGTH: - return None, None - headers = {'X-Timestamp': normalize_timestamp(time.time()), - 'x-trans-id': self.trans_id} - resp = self.make_requests(Request.blank('/v1' + path), - self.app.account_ring, partition, 'PUT', - path, [headers] * len(nodes)) - if resp.status_int // 100 == 2: - result_code = 200 + if result_code == 404 and autocreate: + if len(account) > MAX_ACCOUNT_NAME_LENGTH: + return None, None + headers = {'X-Timestamp': normalize_timestamp(time.time()), + 'X-Trans-Id': self.trans_id} + resp = self.make_requests(Request.blank('/v1' + path), + self.app.account_ring, partition, 'PUT', + path, [headers] * len(nodes)) + if resp.status_int // 100 == 2: + result_code = 200 if self.app.memcache and result_code in (200, 404): if result_code == 200: cache_timeout = self.app.recheck_account_existence diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 323f60672e..67a6a1fbf2 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -768,6 +768,10 @@ log_name = yarr''' self.assertEquals(utils.human_readable(1237940039285380274899124224), '1024Yi') + def test_TRUE_VALUES(self): + for v in utils.TRUE_VALUES: + self.assertEquals(v, v.lower()) + if __name__ == '__main__': unittest.main() diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 1ea65b4943..afdea73fd2 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -393,6 +393,26 @@ class TestController(unittest.TestCase): test(404, 507, 503) test(503, 503, 503) + def test_account_info_account_autocreate(self): + with save_globals(): + proxy_server.http_connect = \ + fake_http_connect(404, 404, 404, 201, 201, 201) + partition, nodes = \ + self.controller.account_info(self.account, autocreate=False) + self.check_account_info_return(partition, nodes, is_none=True) + + proxy_server.http_connect = \ + fake_http_connect(404, 404, 404, 201, 201, 201) + partition, nodes = \ + self.controller.account_info(self.account) + self.check_account_info_return(partition, nodes, is_none=True) + + proxy_server.http_connect = \ + fake_http_connect(404, 404, 404, 201, 201, 201) + partition, nodes = \ + self.controller.account_info(self.account, autocreate=True) + self.check_account_info_return(partition, nodes) + def check_container_info_return(self, ret, is_none=False): if is_none: partition, nodes, read_acl, write_acl = None, None, None, None From c5aafe4992efea8c95cc09e05fdb89288eafb6bb Mon Sep 17 00:00:00 2001 From: David Goetz Date: Fri, 10 Jun 2011 08:59:34 -0700 Subject: [PATCH 10/18] handling no metadata column --- swift/common/db.py | 26 ++++++---- swift/stats/db_stats_collector.py | 4 +- test/unit/stats/test_db_stats_collector.py | 55 ++++++++++++++++++++-- 3 files changed, 70 insertions(+), 15 deletions(-) diff --git a/swift/common/db.py b/swift/common/db.py index 783e00dd5c..8683d4e088 100644 --- a/swift/common/db.py +++ b/swift/common/db.py @@ -899,14 +899,24 @@ class ContainerBroker(DatabaseBroker): metadata = '' if include_metadata: metadata = ', metadata' - data = conn.execute(''' - SELECT account, container, created_at, put_timestamp, - delete_timestamp, object_count, bytes_used, - reported_put_timestamp, reported_delete_timestamp, - reported_object_count, reported_bytes_used, hash, id - %s - FROM container_stat - ''' % metadata).fetchone() + try: + data = conn.execute(''' + SELECT account, container, created_at, put_timestamp, + delete_timestamp, object_count, bytes_used, + reported_put_timestamp, reported_delete_timestamp, + reported_object_count, reported_bytes_used, hash, id + %s + FROM container_stat + ''' % metadata).fetchone() + except sqlite3.OperationalError, err: + if 'no such column: metadata' not in str(err): + raise + data = conn.execute(''' + SELECT account, container, created_at, put_timestamp, + delete_timestamp, object_count, bytes_used, + reported_put_timestamp, reported_delete_timestamp, + reported_object_count, reported_bytes_used, hash, id + FROM container_stat''').fetchone() data = dict(data) if include_metadata: try: diff --git a/swift/stats/db_stats_collector.py b/swift/stats/db_stats_collector.py index f65c20c3e2..95efaa8597 100644 --- a/swift/stats/db_stats_collector.py +++ b/swift/stats/db_stats_collector.py @@ -58,10 +58,10 @@ class DatabaseStatsCollector(Daemon): (self.stats_type, (time.time() - start) / 60)) def get_data(self): - raise Exception('Not Implemented') + raise NotImplementedError('Subclasses must override') def get_header(self): - raise Exception('Not Implemented') + raise NotImplementedError('Subclasses must override') def find_and_process(self): src_filename = time.strftime(self.filename_format) diff --git a/test/unit/stats/test_db_stats_collector.py b/test/unit/stats/test_db_stats_collector.py index d336016e77..3c4949aff5 100644 --- a/test/unit/stats/test_db_stats_collector.py +++ b/test/unit/stats/test_db_stats_collector.py @@ -93,7 +93,32 @@ class TestDbStats(unittest.TestCase): self.assertEqual(len(output_data), 10) return stat, output_data - def _gen_container_stat(self, set_metadata=False): + def _drop_metadata_col(self, broker, acc_name): + broker.conn.execute('''drop table container_stat''') + broker.conn.executescript(""" + CREATE TABLE container_stat ( + account TEXT DEFAULT '%s', + container TEXT DEFAULT 'test_con', + created_at TEXT, + put_timestamp TEXT DEFAULT '0', + delete_timestamp TEXT DEFAULT '0', + object_count INTEGER, + bytes_used INTEGER, + reported_put_timestamp TEXT DEFAULT '0', + reported_delete_timestamp TEXT DEFAULT '0', + reported_object_count INTEGER DEFAULT 0, + reported_bytes_used INTEGER DEFAULT 0, + hash TEXT default '00000000000000000000000000000000', + id TEXT, + status TEXT DEFAULT '', + status_changed_at TEXT DEFAULT '0' + ); + + INSERT INTO container_stat (object_count, bytes_used) + VALUES (1, 10); + """ % acc_name) + + def _gen_container_stat(self, set_metadata=False, drop_metadata=False): if set_metadata: self.conf['metadata_keys'] = 'test1,test2' # webob runs title on all headers @@ -116,8 +141,13 @@ class TestDbStats(unittest.TestCase): metadata_output = ',,1' # this will "commit" the data cont_db.get_info() - output_data.add('''"test_acc_%s","test_con",1,10%s''' % - (i, metadata_output)) + if drop_metadata: + output_data.add('''"test_acc_%s","test_con",1,10,,''' % i) + else: + output_data.add('''"test_acc_%s","test_con",1,10%s''' % + (i, metadata_output)) + if drop_metadata: + self._drop_metadata_col(cont_db, 'test_acc_%s' % i) self.assertEqual(len(output_data), 10) return stat, output_data @@ -147,6 +177,21 @@ class TestDbStats(unittest.TestCase): self.assertEqual(len(output_data), 0) + def test_account_stat_run_once_container_no_metadata(self): + + stat, output_data = self._gen_container_stat(set_metadata=True, + drop_metadata=True) + stat.run_once() + stat_file = os.listdir(self.log_dir)[0] + with open(os.path.join(self.log_dir, stat_file)) as stat_handle: + headers = stat_handle.readline() + self.assert_(headers.startswith('Account Hash,Container Name,')) + for i in range(10): + data = stat_handle.readline() + output_data.discard(data.strip()) + + self.assertEqual(len(output_data), 0) + def test_account_stat_run_once_both(self): acc_stat, acc_output_data = self._gen_account_stat() con_stat, con_output_data = self._gen_container_stat() @@ -180,8 +225,8 @@ class TestDbStats(unittest.TestCase): def test_not_implemented(self): db_stat = db_stats_collector.DatabaseStatsCollector(self.conf, 'account', 'test_dir', 'stats-%Y%m%d%H_') - self.assertRaises(Exception, db_stat.get_data) - self.assertRaises(Exception, db_stat.get_header) + self.assertRaises(NotImplementedError, db_stat.get_data) + self.assertRaises(NotImplementedError, db_stat.get_header) def test_not_not_mounted(self): self.conf['mount_check'] = 'true' From 818c4faa7368d0d25250e855173fa4016544bf1f Mon Sep 17 00:00:00 2001 From: gholt Date: Fri, 10 Jun 2011 16:56:53 +0000 Subject: [PATCH 11/18] Made failed account autocreate raise exception to cause 5xx. --- swift/proxy/server.py | 5 +++-- test/unit/proxy/test_server.py | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 2943c6a93e..8a451200cf 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -394,8 +394,9 @@ class Controller(object): resp = self.make_requests(Request.blank('/v1' + path), self.app.account_ring, partition, 'PUT', path, [headers] * len(nodes)) - if resp.status_int // 100 == 2: - result_code = 200 + if resp.status_int // 100 != 2: + raise Exception('Could not autocreate account %r' % path) + result_code = 200 if self.app.memcache and result_code in (200, 404): if result_code == 200: cache_timeout = self.app.recheck_account_existence diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index afdea73fd2..eb09ad34dc 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -395,24 +395,46 @@ class TestController(unittest.TestCase): def test_account_info_account_autocreate(self): with save_globals(): + self.memcache.store = {} proxy_server.http_connect = \ fake_http_connect(404, 404, 404, 201, 201, 201) partition, nodes = \ self.controller.account_info(self.account, autocreate=False) self.check_account_info_return(partition, nodes, is_none=True) + self.memcache.store = {} proxy_server.http_connect = \ fake_http_connect(404, 404, 404, 201, 201, 201) partition, nodes = \ self.controller.account_info(self.account) self.check_account_info_return(partition, nodes, is_none=True) + self.memcache.store = {} proxy_server.http_connect = \ fake_http_connect(404, 404, 404, 201, 201, 201) partition, nodes = \ self.controller.account_info(self.account, autocreate=True) self.check_account_info_return(partition, nodes) + self.memcache.store = {} + proxy_server.http_connect = \ + fake_http_connect(404, 404, 404, 503, 201, 201) + partition, nodes = \ + self.controller.account_info(self.account, autocreate=True) + self.check_account_info_return(partition, nodes) + + self.memcache.store = {} + proxy_server.http_connect = \ + fake_http_connect(404, 404, 404, 503, 201, 503) + exc = None + try: + partition, nodes = \ + self.controller.account_info(self.account, autocreate=True) + except Exception, err: + exc = err + self.assertEquals(str(exc), + "Could not autocreate account '/some_account'") + def check_container_info_return(self, ret, is_none=False): if is_none: partition, nodes, read_acl, write_acl = None, None, None, None From f68232bf89e0940ce7cac3bede38b4b9dc57f12b Mon Sep 17 00:00:00 2001 From: gholt Date: Fri, 10 Jun 2011 18:36:02 +0000 Subject: [PATCH 12/18] Fixed account and container listings to return with charset=utf-8 instead of charset=utf8 --- swift/account/server.py | 2 +- swift/container/server.py | 2 +- test/unit/account/test_server.py | 3 +++ test/unit/container/test_server.py | 3 +++ 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/swift/account/server.py b/swift/account/server.py index d51f9f27a6..fd839a4909 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -244,7 +244,7 @@ class AccountController(object): account_list = '\n'.join(r[0] for r in account_list) + '\n' ret = Response(body=account_list, request=req, headers=resp_headers) ret.content_type = out_content_type - ret.charset = 'utf8' + ret.charset = 'utf-8' return ret def REPLICATE(self, req): diff --git a/swift/container/server.py b/swift/container/server.py index dfdc884a6d..bc3856d18e 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -334,7 +334,7 @@ class ContainerController(object): container_list = '\n'.join(r[0] for r in container_list) + '\n' ret = Response(body=container_list, request=req, headers=resp_headers) ret.content_type = out_content_type - ret.charset = 'utf8' + ret.charset = 'utf-8' return ret def REPLICATE(self, req): diff --git a/test/unit/account/test_server.py b/test/unit/account/test_server.py index 16800ca165..238b7f3d18 100644 --- a/test/unit/account/test_server.py +++ b/test/unit/account/test_server.py @@ -388,6 +388,7 @@ class TestAccountController(unittest.TestCase): self.assertEquals(resp.status_int, 200) self.assertEquals(resp.body.strip().split('\n'), ['c1', 'c2']) self.assertEquals(resp.content_type, 'text/plain') + self.assertEquals(resp.charset, 'utf-8') def test_GET_with_containers_json(self): req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', @@ -436,6 +437,7 @@ class TestAccountController(unittest.TestCase): [{'count': 1, 'bytes': 2, 'name': 'c1'}, {'count': 3, 'bytes': 4, 'name': 'c2'}]) self.assertEquals(resp.content_type, 'application/json') + self.assertEquals(resp.charset, 'utf-8') def test_GET_with_containers_xml(self): req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', @@ -529,6 +531,7 @@ class TestAccountController(unittest.TestCase): self.assertEquals(node.firstChild.nodeValue, '3') node = [n for n in container if n.nodeName == 'bytes'][0] self.assertEquals(node.firstChild.nodeValue, '4') + self.assertEquals(resp.charset, 'utf-8') def test_GET_limit_marker_plain(self): req = Request.blank('/sda1/p/a', environ={'REQUEST_METHOD': 'PUT', diff --git a/test/unit/container/test_server.py b/test/unit/container/test_server.py index 5127fe093c..117a0dccd5 100644 --- a/test/unit/container/test_server.py +++ b/test/unit/container/test_server.py @@ -514,6 +514,7 @@ class TestContainerController(unittest.TestCase): resp = self.controller.GET(req) self.assertEquals(resp.content_type, 'application/json') self.assertEquals(eval(resp.body), json_body) + self.assertEquals(resp.charset, 'utf-8') for accept in ('application/json', 'application/json;q=1.0,*/*;q=0.9', '*/*;q=0.9,application/json;q=1.0', 'application/*'): @@ -552,6 +553,7 @@ class TestContainerController(unittest.TestCase): resp = self.controller.GET(req) self.assertEquals(resp.content_type, 'text/plain') self.assertEquals(resp.body, plain_body) + self.assertEquals(resp.charset, 'utf-8') for accept in ('', 'text/plain', 'application/xml;q=0.8,*/*;q=0.9', '*/*;q=0.9,application/xml;q=0.8', '*/*', @@ -609,6 +611,7 @@ class TestContainerController(unittest.TestCase): resp = self.controller.GET(req) self.assertEquals(resp.content_type, 'application/xml') self.assertEquals(resp.body, xml_body) + self.assertEquals(resp.charset, 'utf-8') for xml_accept in ('application/xml', 'application/xml;q=1.0,*/*;q=0.9', '*/*;q=0.9,application/xml;q=1.0', 'application/xml,text/xml'): From bdd2d9968081e2ff3d30340cb61cac5079603f94 Mon Sep 17 00:00:00 2001 From: gholt Date: Fri, 10 Jun 2011 18:49:41 +0000 Subject: [PATCH 13/18] Updated functests to test for utf-8 --- test/functional/tests.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/functional/tests.py b/test/functional/tests.py index 59dcf38960..8c513490d2 100644 --- a/test/functional/tests.py +++ b/test/functional/tests.py @@ -227,10 +227,10 @@ class TestAccount(Base): headers = dict(self.env.conn.response.getheaders()) if format == 'json': self.assertEquals(headers['content-type'], - 'application/json; charset=utf8') + 'application/json; charset=utf-8') elif format == 'xml': self.assertEquals(headers['content-type'], - 'application/xml; charset=utf8') + 'application/xml; charset=utf-8') def testListingLimit(self): limit = 10000 @@ -1355,10 +1355,10 @@ class TestFile(Base): headers = dict(self.env.conn.response.getheaders()) if format == 'json': self.assertEquals(headers['content-type'], - 'application/json; charset=utf8') + 'application/json; charset=utf-8') elif format == 'xml': self.assertEquals(headers['content-type'], - 'application/xml; charset=utf8') + 'application/xml; charset=utf-8') lm_diff = max([f['last_modified'] for f in files]) - \ min([f['last_modified'] for f in files]) From 7b6b85aef818e64875d510edc72ab78a58e41b97 Mon Sep 17 00:00:00 2001 From: gholt Date: Fri, 10 Jun 2011 19:33:53 +0000 Subject: [PATCH 14/18] Update staticweb generated test data to match utf-8 update --- test/unit/common/middleware/test_staticweb.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/unit/common/middleware/test_staticweb.py b/test/unit/common/middleware/test_staticweb.py index 7b5385da71..55ff3959fa 100644 --- a/test/unit/common/middleware/test_staticweb.py +++ b/test/unit/common/middleware/test_staticweb.py @@ -187,7 +187,7 @@ class FakeApp(object): headers.update({'X-Container-Object-Count': '11', 'X-Container-Bytes-Used': '73741', 'X-Container-Read': '.r:*', - 'Content-Type': 'application/json; charset=utf8'}) + 'Content-Type': 'application/json; charset=utf-8'}) body = ''' [{"name":"subdir/1.txt", "hash":"5f595114a4b3077edfac792c61ca4fe4", "bytes":20, @@ -204,14 +204,14 @@ class FakeApp(object): headers.update({'X-Container-Object-Count': '11', 'X-Container-Bytes-Used': '73741', 'X-Container-Read': '.r:*', - 'Content-Type': 'application/json; charset=utf8'}) + 'Content-Type': 'application/json; charset=utf-8'}) body = '[]' elif env['PATH_INFO'] == '/v1/a/c3' and env['QUERY_STRING'] == \ 'limit=1&format=json&delimiter=/&limit=1&prefix=subdirz/': headers.update({'X-Container-Object-Count': '11', 'X-Container-Bytes-Used': '73741', 'X-Container-Read': '.r:*', - 'Content-Type': 'application/json; charset=utf8'}) + 'Content-Type': 'application/json; charset=utf-8'}) body = ''' [{"name":"subdirz/1.txt", "hash":"5f595114a4b3077edfac792c61ca4fe4", "bytes":20, @@ -224,7 +224,7 @@ class FakeApp(object): 'X-Container-Bytes-Used': '73741', 'X-Container-Read': '.r:*', 'X-Container-Web-Listings': 't', - 'Content-Type': 'application/json; charset=utf8'}) + 'Content-Type': 'application/json; charset=utf-8'}) body = ''' [{"name":"subdir/1.txt", "hash":"5f595114a4b3077edfac792c61ca4fe4", "bytes":20, @@ -236,7 +236,7 @@ class FakeApp(object): elif 'format=json' in env['QUERY_STRING']: headers.update({'X-Container-Object-Count': '11', 'X-Container-Bytes-Used': '73741', - 'Content-Type': 'application/json; charset=utf8'}) + 'Content-Type': 'application/json; charset=utf-8'}) body = ''' [{"name":"401error.html", "hash":"893f8d80692a4d3875b45be8f152ad18", "bytes":110, @@ -283,7 +283,7 @@ class FakeApp(object): else: headers.update({'X-Container-Object-Count': '11', 'X-Container-Bytes-Used': '73741', - 'Content-Type': 'text/plain; charset=utf8'}) + 'Content-Type': 'text/plain; charset=utf-8'}) body = '\n'.join(['401error.html', '404error.html', 'index.html', 'listing.css', 'one.txt', 'subdir/1.txt', 'subdir/2.txt', 'subdir/omgomg.txt', 'subdir2', From 1a586323825ca681a7c6739bc6f6fd9131932a2c Mon Sep 17 00:00:00 2001 From: gholt Date: Sat, 11 Jun 2011 04:57:04 +0000 Subject: [PATCH 15/18] Fixed so account GETs and HEADs autocreate the account when account_autocreate = true --- swift/proxy/server.py | 36 ++++++++++++++++++++++++++++++++-- test/unit/proxy/test_server.py | 30 ++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 8a451200cf..581eef21c3 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -1319,8 +1319,26 @@ class AccountController(Controller): def GETorHEAD(self, req): """Handler for HTTP GET/HEAD requests.""" partition, nodes = self.app.account_ring.get_nodes(self.account_name) - return self.GETorHEAD_base(req, _('Account'), partition, nodes, + resp = self.GETorHEAD_base(req, _('Account'), partition, nodes, req.path_info.rstrip('/'), self.app.account_ring.replica_count) + if resp.status_int == 404 and self.app.account_autocreate: + if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH: + resp = HTTPBadRequest(request=req) + resp.body = 'Account name length of %d longer than %d' % \ + (len(self.account_name), MAX_ACCOUNT_NAME_LENGTH) + return resp + headers = {'X-Timestamp': normalize_timestamp(time.time()), + 'X-Trans-Id': self.trans_id} + resp = self.make_requests( + Request.blank('/v1/' + self.account_name), + self.app.account_ring, partition, 'PUT', + '/' + self.account_name, [headers] * len(nodes)) + if resp.status_int // 100 != 2: + raise Exception('Could not autocreate account %r' % + self.account_name) + resp = self.GETorHEAD_base(req, _('Account'), partition, nodes, + req.path_info.rstrip('/'), self.app.account_ring.replica_count) + return resp @public def PUT(self, req): @@ -1360,9 +1378,23 @@ class AccountController(Controller): if value[0].lower().startswith('x-account-meta-')) if self.app.memcache: self.app.memcache.delete('account%s' % req.path_info.rstrip('/')) - return self.make_requests(req, self.app.account_ring, + resp = self.make_requests(req, self.app.account_ring, account_partition, 'POST', req.path_info, [headers] * len(accounts)) + if resp.status_int == 404 and self.app.account_autocreate: + if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH: + resp = HTTPBadRequest(request=req) + resp.body = 'Account name length of %d longer than %d' % \ + (len(self.account_name), MAX_ACCOUNT_NAME_LENGTH) + return resp + resp = self.make_requests( + Request.blank('/v1/' + self.account_name), + self.app.account_ring, account_partition, 'PUT', + '/' + self.account_name, [headers] * len(accounts)) + if resp.status_int // 100 != 2: + raise Exception('Could not autocreate account %r' % + self.account_name) + return resp @public def DELETE(self, req): diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index eb09ad34dc..b525ea9037 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -3171,6 +3171,16 @@ class TestAccountController(unittest.TestCase): self.app.memcache = FakeMemcacheReturnsNone() self.assert_status_map(controller.GET, (404, 404, 404), 404) + def test_GET_autocreate(self): + with save_globals(): + controller = proxy_server.AccountController(self.app, 'account') + self.app.memcache = FakeMemcacheReturnsNone() + self.assert_status_map(controller.GET, + (404, 404, 404, 201, 201, 201, 204), 404) + controller.app.account_autocreate = True + self.assert_status_map(controller.GET, + (404, 404, 404, 201, 201, 201, 204), 204) + def test_HEAD(self): with save_globals(): controller = proxy_server.AccountController(self.app, 'account') @@ -3189,6 +3199,26 @@ class TestAccountController(unittest.TestCase): self.assert_status_map(controller.HEAD, (404, 503, 503), 503) self.assert_status_map(controller.HEAD, (404, 204, 503), 204) + def test_HEAD_autocreate(self): + with save_globals(): + controller = proxy_server.AccountController(self.app, 'account') + self.app.memcache = FakeMemcacheReturnsNone() + self.assert_status_map(controller.HEAD, + (404, 404, 404, 201, 201, 201, 204), 404) + controller.app.account_autocreate = True + self.assert_status_map(controller.HEAD, + (404, 404, 404, 201, 201, 201, 204), 204) + + def test_POST_autocreate(self): + with save_globals(): + controller = proxy_server.AccountController(self.app, 'account') + self.app.memcache = FakeMemcacheReturnsNone() + self.assert_status_map(controller.POST, + (404, 404, 404, 201, 201, 201), 404) + controller.app.account_autocreate = True + self.assert_status_map(controller.POST, + (404, 404, 404, 201, 201, 201), 201) + def test_connection_refused(self): self.app.account_ring.get_nodes('account') for dev in self.app.account_ring.devs.values(): From 0eeeae664e0ff805d5e3c6ac7bdf4394cdc96d85 Mon Sep 17 00:00:00 2001 From: John Dickinson Date: Tue, 14 Jun 2011 11:04:06 -0500 Subject: [PATCH 16/18] renamed st to swift --- bin/{st => swift} | 0 doc/source/development_saio.rst | 2 +- doc/source/howto_installmultinode.rst | 30 +++++++++++++-------------- doc/source/overview_large_objects.rst | 20 +++++++++--------- setup.py | 2 +- swift/common/middleware/staticweb.py | 12 +++++------ 6 files changed, 33 insertions(+), 33 deletions(-) rename bin/{st => swift} (100%) diff --git a/bin/st b/bin/swift similarity index 100% rename from bin/st rename to bin/swift diff --git a/doc/source/development_saio.rst b/doc/source/development_saio.rst index bbce6fdeb0..a6dc0060b6 100644 --- a/doc/source/development_saio.rst +++ b/doc/source/development_saio.rst @@ -625,7 +625,7 @@ Setting up scripts for running Swift #. `recreateaccounts` #. Get an `X-Storage-Url` and `X-Auth-Token`: ``curl -v -H 'X-Storage-User: test:tester' -H 'X-Storage-Pass: testing' http://127.0.0.1:8080/auth/v1.0`` #. Check that you can GET account: ``curl -v -H 'X-Auth-Token: ' `` - #. Check that `st` works: `st -A http://127.0.0.1:8080/auth/v1.0 -U test:tester -K testing stat` + #. Check that `swift` works: `swift -A http://127.0.0.1:8080/auth/v1.0 -U test:tester -K testing stat` #. `cp ~/swift/trunk/test/functional/sample.conf /etc/swift/func_test.conf` #. `cd ~/swift/trunk; ./.functests` (Note: functional tests will first delete everything in the configured accounts.) diff --git a/doc/source/howto_installmultinode.rst b/doc/source/howto_installmultinode.rst index 6f10e30757..abe7647db8 100644 --- a/doc/source/howto_installmultinode.rst +++ b/doc/source/howto_installmultinode.rst @@ -372,34 +372,34 @@ You run these commands from the Proxy node. curl -k -v -H 'X-Auth-Token: ' -#. Check that ``st`` works (at this point, expect zero containers, zero objects, and zero bytes):: +#. Check that ``swift`` works (at this point, expect zero containers, zero objects, and zero bytes):: - st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass stat + swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass stat -#. Use ``st`` to upload a few files named 'bigfile[1-2].tgz' to a container named 'myfiles':: +#. Use ``swift`` to upload a few files named 'bigfile[1-2].tgz' to a container named 'myfiles':: - st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload myfiles bigfile1.tgz - st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload myfiles bigfile2.tgz + swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload myfiles bigfile1.tgz + swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload myfiles bigfile2.tgz -#. Use ``st`` to download all files from the 'myfiles' container:: +#. Use ``swift`` to download all files from the 'myfiles' container:: - st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass download myfiles + swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass download myfiles -#. Use ``st`` to save a backup of your builder files to a container named 'builders'. Very important not to lose your builders!:: +#. Use ``swift`` to save a backup of your builder files to a container named 'builders'. Very important not to lose your builders!:: - st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload builders /etc/swift/*.builder + swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass upload builders /etc/swift/*.builder -#. Use ``st`` to list your containers:: +#. Use ``swift`` to list your containers:: - st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass list + swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass list -#. Use ``st`` to list the contents of your 'builders' container:: +#. Use ``swift`` to list the contents of your 'builders' container:: - st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass list builders + swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass list builders -#. Use ``st`` to download all files from the 'builders' container:: +#. Use ``swift`` to download all files from the 'builders' container:: - st -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass download builders + swift -A https://$PROXY_LOCAL_NET_IP:8080/auth/v1.0 -U system:root -K testpass download builders .. _add-proxy-server: diff --git a/doc/source/overview_large_objects.rst b/doc/source/overview_large_objects.rst index 01f4990732..923afb872a 100644 --- a/doc/source/overview_large_objects.rst +++ b/doc/source/overview_large_objects.rst @@ -14,24 +14,24 @@ concatenated as a single object. This also offers much greater upload speed with the possibility of parallel uploads of the segments. ---------------------------------- -Using ``st`` for Segmented Objects +Using ``swift`` for Segmented Objects ---------------------------------- -The quickest way to try out this feature is use the included ``st`` Swift Tool. +The quickest way to try out this feature is use the included ``swift`` Swift Tool. You can use the ``-S`` option to specify the segment size to use when splitting a large file. For example:: - st upload test_container -S 1073741824 large_file + swift upload test_container -S 1073741824 large_file This would split the large_file into 1G segments and begin uploading those -segments in parallel. Once all the segments have been uploaded, ``st`` will +segments in parallel. Once all the segments have been uploaded, ``swift`` will then create the manifest file so the segments can be downloaded as one. -So now, the following ``st`` command would download the entire large object:: +So now, the following ``swift`` command would download the entire large object:: - st download test_container large_file + swift download test_container large_file -``st`` uses a strict convention for its segmented object support. In the above +``swift`` uses a strict convention for its segmented object support. In the above example it will upload all the segments into a second container named test_container_segments. These segments will have names like large_file/1290206778.25/21474836480/00000000, @@ -43,7 +43,7 @@ the segment name format of /// is so that an upload of a new file with the same name won't overwrite the contents of the first until the last moment when the manifest file is updated. -``st`` will manage these segment files for you, deleting old segments on +``swift`` will manage these segment files for you, deleting old segments on deletes and overwrites, etc. You can override this behavior with the ``--leave-segments`` option if desired; this is useful if you want to have multiple versions of the same large object available. @@ -53,14 +53,14 @@ Direct API ---------- You can also work with the segments and manifests directly with HTTP requests -instead of having ``st`` do that for you. You can just upload the segments like +instead of having ``swift`` do that for you. You can just upload the segments like you would any other object and the manifest is just a zero-byte file with an extra ``X-Object-Manifest`` header. All the object segments need to be in the same container, have a common object name prefix, and their names sort in the order they should be concatenated. They don't have to be in the same container as the manifest file will be, which -is useful to keep container listings clean as explained above with ``st``. +is useful to keep container listings clean as explained above with ``swift``. The manifest file is simply a zero-byte file with the extra ``X-Object-Manifest: /`` header, where ```` is diff --git a/setup.py b/setup.py index 49b6ed278f..1d12283511 100644 --- a/setup.py +++ b/setup.py @@ -76,7 +76,7 @@ setup( ], install_requires=[], # removed for better compat scripts=[ - 'bin/st', 'bin/swift-account-auditor', + 'bin/swift', 'bin/swift-account-auditor', 'bin/swift-account-audit', 'bin/swift-account-reaper', 'bin/swift-account-replicator', 'bin/swift-account-server', 'bin/swift-container-auditor', diff --git a/swift/common/middleware/staticweb.py b/swift/common/middleware/staticweb.py index 81225a90ea..f04d4a7d55 100644 --- a/swift/common/middleware/staticweb.py +++ b/swift/common/middleware/staticweb.py @@ -74,36 +74,36 @@ the .../listing.css style sheet. If you "view source" in your browser on a listing page, you will see the well defined document structure that can be styled. -Example usage of this middleware via ``st``: +Example usage of this middleware via ``swift``: Make the container publicly readable:: - st post -r '.r:*' container + swift post -r '.r:*' container You should be able to get objects directly, but no index.html resolution or listings. Set an index file directive:: - st post -m 'web-index:index.html' container + swift post -m 'web-index:index.html' container You should be able to hit paths that have an index.html without needing to type the index.html part. Turn on listings:: - st post -m 'web-listings: true' container + swift post -m 'web-listings: true' container Now you should see object listings for paths and pseudo paths that have no index.html. Enable a custom listings style sheet:: - st post -m 'web-listings-css:listings.css' container + swift post -m 'web-listings-css:listings.css' container Set an error file:: - st post -m 'web-error:error.html' container + swift post -m 'web-error:error.html' container Now 401's should load 401error.html, 404's should load 404error.html, etc. """ From c4f0b5561ae048a406c6bd3b4310d5795de5ae88 Mon Sep 17 00:00:00 2001 From: John Dickinson Date: Tue, 14 Jun 2011 11:29:22 -0500 Subject: [PATCH 17/18] updated changelog with 1.4.1 changes --- CHANGELOG | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index ff74a19d57..c2e56f2edd 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,19 @@ +swift (1.4.1) + + * st renamed to swift + + * swauth was separated froms swift. It is now its own project and can be + found at https://github.com/gholt/swauth. + + * tempauth middleware added as an extremely limited auth system for dev + work. + + * Account and container listings now properly labeled UTF-8 (previously the + label was "utf8"). + + * Accounts are auto-created if an auth token is valid when the + account_autocreate proxy config parameter is set to true. + swift (1.4.0) * swift-bench now cleans up containers it creates. From 28f74380cafc542d0c1866276e1b54f4475293e7 Mon Sep 17 00:00:00 2001 From: John Dickinson Date: Tue, 14 Jun 2011 11:39:30 -0500 Subject: [PATCH 18/18] bumped version to 1.4.2 --- swift/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swift/__init__.py b/swift/__init__.py index c26a554fbd..36791d4794 100644 --- a/swift/__init__.py +++ b/swift/__init__.py @@ -14,7 +14,7 @@ class Version(object): return '%s-dev' % (self.canonical_version,) -_version = Version('1.4.1', False) +_version = Version('1.4.2', False) __version__ = _version.pretty_version __canonical_version__ = _version.canonical_version