From 4601548dabdec0a4dc89cefba11e963217255be3 Mon Sep 17 00:00:00 2001 From: Clay Gerrard Date: Tue, 31 Dec 2019 13:48:13 -0600 Subject: [PATCH] Deprecate per-service auto_create_account_prefix If we move it to constraints it's more globally accessible in our code, but more importantly it's more obvious to ops that everything breaks if you try to mis-configure different values per-service. Change-Id: Ib8f7d08bc48da12be5671abe91a17ae2b49ecfee --- doc/manpages/account-server.conf.5 | 4 +- doc/manpages/container-server.conf.5 | 6 +-- doc/manpages/object-expirer.conf.5 | 4 +- doc/manpages/object-server.conf.5 | 6 +-- doc/manpages/proxy-server.conf.5 | 5 +- doc/manpages/swift.conf.5 | 4 ++ doc/saio/swift/object-expirer.conf | 1 - doc/source/deployment_guide.rst | 49 ++++++++--------- etc/account-server.conf-sample | 2 - etc/container-server.conf-sample | 5 -- etc/object-expirer.conf-sample | 1 - etc/object-server.conf-sample | 3 -- etc/proxy-server.conf-sample | 16 +++--- etc/swift.conf-sample | 5 ++ swift/account/server.py | 17 ++++-- swift/common/constraints.py | 13 +++-- swift/common/internal_client.py | 3 +- swift/common/utils.py | 3 +- swift/common/wsgi.py | 3 ++ swift/container/server.py | 16 ++++-- swift/container/sharder.py | 16 ++++-- swift/obj/expirer.py | 16 +++++- swift/obj/server.py | 16 ++++-- swift/proxy/controllers/base.py | 3 +- swift/proxy/server.py | 19 +++++-- .../test_container_merge_policy_index.py | 13 ++--- test/unit/account/test_server.py | 19 ++++++- test/unit/common/test_constraints.py | 9 +++- test/unit/common/test_internal_client.py | 29 +++++++---- test/unit/container/test_server.py | 15 ++++-- test/unit/container/test_sharder.py | 50 +++++++++++------- test/unit/obj/test_expirer.py | 14 +++++ test/unit/obj/test_server.py | 24 ++++++++- test/unit/proxy/test_server.py | 52 ++++++++++++++----- 34 files changed, 323 insertions(+), 138 deletions(-) diff --git a/doc/manpages/account-server.conf.5 b/doc/manpages/account-server.conf.5 index f48d231a22..dd43ee0464 100644 --- a/doc/manpages/account-server.conf.5 +++ b/doc/manpages/account-server.conf.5 @@ -179,8 +179,8 @@ Logging level. The default is INFO. Enables request logging. The default is True. .IP "\fBset log_address\fR Logging address. The default is /dev/log. -.IP "\fBauto_create_account_prefix\fR -The default is ".". +.IP "\fBauto_create_account_prefix [deprecated]\fR" +The default is ".". Should be configured in swift.conf instead. .IP "\fBreplication_server\fR Configure parameter for creating specific server. To handle all verbs, including replication verbs, do not specify diff --git a/doc/manpages/container-server.conf.5 b/doc/manpages/container-server.conf.5 index 1ca31e03fa..d0b1778cc5 100644 --- a/doc/manpages/container-server.conf.5 +++ b/doc/manpages/container-server.conf.5 @@ -191,8 +191,8 @@ Request timeout to external services. The default is 3 seconds. Connection timeout to external services. The default is 0.5 seconds. .IP \fBallow_versions\fR The default is false. -.IP \fBauto_create_account_prefix\fR -The default is '.'. +.IP "\fBauto_create_account_prefix [deprecated]\fR" +The default is '.'. Should be configured in swift.conf instead. .IP \fBreplication_server\fR Configure parameter for creating specific server. To handle all verbs, including replication verbs, do not specify @@ -362,7 +362,7 @@ Request timeout to external services. The default is 3 seconds. Connection timeout to external services. The default is 0.5 seconds. .IP \fBcontainers_per_second\fR Maximum containers updated per second. Should be tuned according to individual system specs. 0 is unlimited. The default is 50. -.IP \fBslowdown\fR +.IP "\fBslowdown [deprecated]\fR" Slowdown will sleep that amount between containers. The default is 0.01 seconds. Deprecated in favor of containers_per_second .IP \fBaccount_suppression_time\fR Seconds to suppress updating an account that has generated an error. The default is 60 seconds. diff --git a/doc/manpages/object-expirer.conf.5 b/doc/manpages/object-expirer.conf.5 index 910b8ab0ab..2ee94ec85e 100644 --- a/doc/manpages/object-expirer.conf.5 +++ b/doc/manpages/object-expirer.conf.5 @@ -202,8 +202,8 @@ This is normally \fBegg:swift#proxy_logging\fR. See proxy-server.conf-sample for .RS 3 .IP \fBinterval\fR Replaces run_pause with the more standard "interval", which means the replicator won't pause unless it takes less than the interval set. The default is 300. -.IP "\fBauto_create_account_prefix\fR -The default is ".". +.IP "\fBauto_create_account_prefix [deprecated]\fR" +The default is ".". Should be configured in swift.conf instead. .IP \fBexpiring_objects_account_name\fR The default is 'expiring_objects'. .IP \fBreport_interval\fR diff --git a/doc/manpages/object-server.conf.5 b/doc/manpages/object-server.conf.5 index 23ba8aef77..3402e025d9 100644 --- a/doc/manpages/object-server.conf.5 +++ b/doc/manpages/object-server.conf.5 @@ -216,8 +216,8 @@ On PUTs, sync data every n MB. The default is 512. Comma separated list of headers that can be set in metadata on an object. This list is in addition to X-Object-Meta-* headers and cannot include Content-Type, etag, Content-Length, or deleted. The default is 'Content-Disposition, Content-Encoding, X-Delete-At, X-Object-Manifest, X-Static-Large-Object, Cache-Control, Content-Language, Expires, X-Robots-Tag'. -.IP "\fBauto_create_account_prefix\fR" -The default is '.'. +.IP "\fBauto_create_account_prefix [deprecated]\fR" +The default is '.'. Should be configured in swift.conf instead. .IP "\fBreplication_server\fR" Configure parameter for creating specific server To handle all verbs, including replication verbs, do not specify @@ -504,7 +504,7 @@ Number of updater workers to spawn. The default is 1. Request timeout to external services. The default is 10 seconds. .IP \fBobjects_per_second\fR Maximum objects updated per second. Should be tuned according to individual system specs. 0 is unlimited. The default is 50. -.IP \fBslowdown\fR +.IP "\fBslowdown [deprecated]\fR" Slowdown will sleep that amount between objects. The default is 0.01 seconds. Deprecated in favor of objects_per_second. .IP "\fBrecon_cache_path\fR" The recon_cache_path simply sets the directory where stats for a few items will be stored. diff --git a/doc/manpages/proxy-server.conf.5 b/doc/manpages/proxy-server.conf.5 index 1fea62bc7a..b23a3a72cb 100644 --- a/doc/manpages/proxy-server.conf.5 +++ b/doc/manpages/proxy-server.conf.5 @@ -1050,8 +1050,9 @@ is false. .IP \fBaccount_autocreate\fR If set to 'true' authorized accounts that do not yet exist within the Swift cluster will be automatically created. The default is set to false. -.IP \fBauto_create_account_prefix\fR -Prefix used when automatically creating accounts. The default is '.'. +.IP "\fBauto_create_account_prefix [deprecated]\fR" +Prefix used when automatically creating accounts. The default is '.'. Should +be configured in swift.conf instead. .IP \fBmax_containers_per_account\fR If set to a positive value, trying to create a container when the account already has at least this maximum containers will result in a 403 Forbidden. diff --git a/doc/manpages/swift.conf.5 b/doc/manpages/swift.conf.5 index bf0521995f..87659b1757 100644 --- a/doc/manpages/swift.conf.5 +++ b/doc/manpages/swift.conf.5 @@ -201,6 +201,10 @@ Use a comma-separated list in case of multiple allowed versions, for example valid_api_versions = v0,v1,v2. This is only enforced for account, container and object requests. The allowed api versions are by default excluded from /info. +.IP "\fBauto_create_account_prefix\fR" +auto_create_account_prefix specifies the prefix for system accounts, such as +those used by the object-expirer, and container-sharder. +Default is ".". diff --git a/doc/saio/swift/object-expirer.conf b/doc/saio/swift/object-expirer.conf index f19b09b46d..58c85d2843 100644 --- a/doc/saio/swift/object-expirer.conf +++ b/doc/saio/swift/object-expirer.conf @@ -25,7 +25,6 @@ log_level = INFO [object-expirer] interval = 300 -# auto_create_account_prefix = . # report_interval = 300 # concurrency is the level of concurrency to use to do the work, this value # must be set to at least 1 diff --git a/doc/source/deployment_guide.rst b/doc/source/deployment_guide.rst index 6c132dd1e0..d94e393049 100644 --- a/doc/source/deployment_guide.rst +++ b/doc/source/deployment_guide.rst @@ -383,26 +383,28 @@ An example of common configuration file can be found at etc/swift.conf-sample The following configuration options are available: -=================== ========== ============================================= -Option Default Description -------------------- ---------- --------------------------------------------- -max_header_size 8192 max_header_size is the max number of bytes in - the utf8 encoding of each header. Using 8192 - as default because eventlet use 8192 as max - size of header line. This value may need to - be increased when using identity v3 API - tokens including more than 7 catalog entries. - See also include_service_catalog in - proxy-server.conf-sample (documented in - overview_auth.rst). -extra_header_count 0 By default the maximum number of allowed - headers depends on the number of max - allowed metadata settings plus a default - value of 32 for regular http headers. - If for some reason this is not enough (custom - middleware for example) it can be increased - with the extra_header_count constraint. -=================== ========== ============================================= +========================== ========== ============================================= +Option Default Description +-------------------------- ---------- --------------------------------------------- +max_header_size 8192 max_header_size is the max number of bytes in + the utf8 encoding of each header. Using 8192 + as default because eventlet use 8192 as max + size of header line. This value may need to + be increased when using identity v3 API + tokens including more than 7 catalog entries. + See also include_service_catalog in + proxy-server.conf-sample (documented in + overview_auth.rst). +extra_header_count 0 By default the maximum number of allowed + headers depends on the number of max + allowed metadata settings plus a default + value of 32 for regular http headers. + If for some reason this is not enough (custom + middleware for example) it can be increased + with the extra_header_count constraint. +auto_create_account_prefix . Prefix used when automatically creating + accounts. +========================== ========== ============================================= --------------------------- Object Server Configuration @@ -600,8 +602,6 @@ allowed_headers Content-Disposition, Comma separated list o Content-Language, Expires, X-Robots-Tag -auto_create_account_prefix . Prefix used when automatically - creating accounts. replication_server Configure parameter for creating specific server. To handle all verbs, including replication verbs, do not @@ -1017,8 +1017,6 @@ log_address /dev/log Logging directory interval 300 Time in seconds to wait between expirer passes report_interval 300 Frequency of status logs in seconds. -auto_create_account_prefix . Prefix used when automatically - creating accounts. concurrency 1 Level of concurrency to use to do the work, this value must be set to at least 1 expiring_objects_account_name expiring_objects name for legacy expirer task queue @@ -1196,7 +1194,6 @@ set log_address /dev/log Logging directory node_timeout 3 Request timeout to external services conn_timeout 0.5 Connection timeout to external services allow_versions false Enable/Disable object versioning feature -auto_create_account_prefix . Prefix used when automatically replication_server Configure parameter for creating specific server. To handle all verbs, including replication verbs, do not @@ -1551,8 +1548,6 @@ set log_level INFO Logging level set log_requests True Whether or not to log each request set log_address /dev/log Logging directory -auto_create_account_prefix . Prefix used when automatically - creating accounts. replication_server Configure parameter for creating specific server. To handle all verbs, including replication verbs, do not diff --git a/etc/account-server.conf-sample b/etc/account-server.conf-sample index bfaac62918..5f08fc6cfa 100644 --- a/etc/account-server.conf-sample +++ b/etc/account-server.conf-sample @@ -91,8 +91,6 @@ use = egg:swift#account # set log_requests = true # set log_address = /dev/log # -# auto_create_account_prefix = . -# # Configure parameter for creating specific server # To handle all verbs, including replication verbs, do not specify # "replication_server" (this is the default). To only handle replication, diff --git a/etc/container-server.conf-sample b/etc/container-server.conf-sample index c6cef31f8f..33098ee5f3 100644 --- a/etc/container-server.conf-sample +++ b/etc/container-server.conf-sample @@ -84,10 +84,6 @@ bind_port = 6201 # Work only with ionice_class. # ionice_class = # ionice_priority = -# -# The prefix used for hidden auto-created accounts, for example accounts in -# which shard containers are created. Defaults to '.'. -# auto_create_account_prefix = . [pipeline:main] pipeline = healthcheck recon container-server @@ -104,7 +100,6 @@ use = egg:swift#container # node_timeout = 3 # conn_timeout = 0.5 # allow_versions = false -# auto_create_account_prefix = . # # Configure parameter for creating specific server # To handle all verbs, including replication verbs, do not specify diff --git a/etc/object-expirer.conf-sample b/etc/object-expirer.conf-sample index 80557a08f2..4356baf504 100644 --- a/etc/object-expirer.conf-sample +++ b/etc/object-expirer.conf-sample @@ -39,7 +39,6 @@ [object-expirer] # interval = 300 -# auto_create_account_prefix = . # expiring_objects_account_name = expiring_objects # report_interval = 300 # diff --git a/etc/object-server.conf-sample b/etc/object-server.conf-sample index 04a7be6872..cbef1be1bf 100644 --- a/etc/object-server.conf-sample +++ b/etc/object-server.conf-sample @@ -133,9 +133,6 @@ use = egg:swift#object # This list is in addition to X-Object-Meta-* headers and cannot include # Content-Type, etag, Content-Length, or deleted # allowed_headers = Content-Disposition, Content-Encoding, X-Delete-At, X-Object-Manifest, X-Static-Large-Object, Cache-Control, Content-Language, Expires, X-Robots-Tag -# -# auto_create_account_prefix = . -# # The number of threads in eventlet's thread pool. Most IO will occur # in the object server's main thread, but certain "heavy" IO diff --git a/etc/proxy-server.conf-sample b/etc/proxy-server.conf-sample index 6e4d369f39..94905d46ec 100644 --- a/etc/proxy-server.conf-sample +++ b/etc/proxy-server.conf-sample @@ -16,12 +16,13 @@ bind_port = 8080 # # Allows the ability to withhold sections from showing up in the public calls # to /info. You can withhold subsections by separating the dict level with a -# ".". The following would cause the sections 'container_quotas' and 'tempurl' -# to not be listed, and the key max_failed_deletes would be removed from -# bulk_delete. Default value is 'swift.valid_api_versions' which allows all -# registered features to be listed via HTTP GET /info except -# swift.valid_api_versions information -# disallowed_sections = swift.valid_api_versions, container_quotas, tempurl +# ".". Default value is 'swift.valid_api_versions, swift.auto_create_account_prefix' +# which allows all registered features to be listed via HTTP GET /info except +# swift.valid_api_versions and swift.auto_create_account_prefix information. +# As an example, the following would cause the sections 'container_quotas' and +# 'tempurl' to not be listed, and the key max_failed_deletes would be removed from +# bulk_delete. +# disallowed_sections = swift.valid_api_versions, container_quotas, tempurl, bulk_delete.max_failed_deletes # Use an integer to override the number of pre-forked processes that will # accept connections. Should default to the number of effective cpu @@ -185,9 +186,6 @@ use = egg:swift#proxy # Comma separated list of Host headers to which the proxy will deny requests. # deny_host_headers = # -# Prefix used when automatically creating accounts. -# auto_create_account_prefix = . -# # During GET and HEAD requests, storage nodes can be chosen at random # (shuffle), by using timing measurements (timing), or by using an explicit # region/zone match (affinity). Using timing measurements may allow for lower diff --git a/etc/swift.conf-sample b/etc/swift.conf-sample index f28c020fb2..2a1ceaf372 100644 --- a/etc/swift.conf-sample +++ b/etc/swift.conf-sample @@ -194,3 +194,8 @@ aliases = yellow, orange # api versions are by default excluded from /info. # valid_api_versions = v1,v1.0 + +# The prefix used for hidden auto-created accounts, for example accounts in +# which shard containers are created. It defaults to '.'; don't change it. + +# auto_create_account_prefix = . diff --git a/swift/account/server.py b/swift/account/server.py index a02d66fee5..fb316a9cc1 100644 --- a/swift/account/server.py +++ b/swift/account/server.py @@ -32,7 +32,8 @@ from swift.common.utils import get_logger, hash_path, public, \ Timestamp, storage_directory, config_true_value, \ timing_stats, replication, get_log_line, \ config_fallocate_value, fs_has_free_space -from swift.common.constraints import valid_timestamp, check_utf8, check_drive +from swift.common.constraints import valid_timestamp, check_utf8, \ + check_drive, AUTO_CREATE_ACCOUNT_PREFIX from swift.common import constraints from swift.common.db_replicator import ReplicatorRpc from swift.common.base_storage_server import BaseStorageServer @@ -85,8 +86,18 @@ class AccountController(BaseStorageServer): self.replicator_rpc = ReplicatorRpc(self.root, DATADIR, AccountBroker, self.mount_check, logger=self.logger) - self.auto_create_account_prefix = \ - conf.get('auto_create_account_prefix') or '.' + if conf.get('auto_create_account_prefix'): + self.logger.warning('Option auto_create_account_prefix is ' + 'deprecated. Configure ' + 'auto_create_account_prefix under the ' + 'swift-constraints section of ' + 'swift.conf. This option will ' + 'be ignored in a future release.') + self.auto_create_account_prefix = \ + conf['auto_create_account_prefix'] + else: + self.auto_create_account_prefix = AUTO_CREATE_ACCOUNT_PREFIX + swift.common.db.DB_PREALLOCATION = \ config_true_value(conf.get('db_preallocation', 'f')) swift.common.db.QUERY_LOGGING = \ diff --git a/swift/common/constraints.py b/swift/common/constraints.py index dd2e75a8f0..f572107b84 100644 --- a/swift/common/constraints.py +++ b/swift/common/constraints.py @@ -39,6 +39,7 @@ MAX_ACCOUNT_NAME_LENGTH = 256 MAX_CONTAINER_NAME_LENGTH = 256 VALID_API_VERSIONS = ["v1", "v1.0"] EXTRA_HEADER_COUNT = 0 +AUTO_CREATE_ACCOUNT_PREFIX = '.' # If adding an entry to DEFAULT_CONSTRAINTS, note that # these constraints are automatically published by the @@ -58,6 +59,7 @@ DEFAULT_CONSTRAINTS = { 'max_container_name_length': MAX_CONTAINER_NAME_LENGTH, 'valid_api_versions': VALID_API_VERSIONS, 'extra_header_count': EXTRA_HEADER_COUNT, + 'auto_create_account_prefix': AUTO_CREATE_ACCOUNT_PREFIX, } SWIFT_CONSTRAINTS_LOADED = False @@ -76,7 +78,7 @@ def reload_constraints(): constraints_conf = ConfigParser() if constraints_conf.read(utils.SWIFT_CONF_FILE): SWIFT_CONSTRAINTS_LOADED = True - for name in DEFAULT_CONSTRAINTS: + for name, default in DEFAULT_CONSTRAINTS.items(): try: value = constraints_conf.get('swift-constraints', name) except NoOptionError: @@ -85,9 +87,12 @@ def reload_constraints(): # We are never going to find the section for another option break else: - try: - value = int(value) - except ValueError: + if isinstance(default, int): + value = int(value) # Go ahead and let it error + elif isinstance(default, str): + pass # No translation needed, I guess + else: + # Hope we want a list! value = utils.list_from_csv(value) OVERRIDE_CONSTRAINTS[name] = value for name, default in DEFAULT_CONSTRAINTS.items(): diff --git a/swift/common/internal_client.py b/swift/common/internal_client.py index 851282bd6f..04114ef59d 100644 --- a/swift/common/internal_client.py +++ b/swift/common/internal_client.py @@ -25,6 +25,7 @@ import zlib from time import gmtime, strftime, time from zlib import compressobj +from swift.common.constraints import AUTO_CREATE_ACCOUNT_PREFIX from swift.common.exceptions import ClientException from swift.common.http import (HTTP_NOT_FOUND, HTTP_MULTIPLE_CHOICES, is_server_error) @@ -158,7 +159,7 @@ class InternalClient(object): container_ring = pipeline_property('container_ring') account_ring = pipeline_property('account_ring') auto_create_account_prefix = pipeline_property( - 'auto_create_account_prefix', default='.') + 'auto_create_account_prefix', default=AUTO_CREATE_ACCOUNT_PREFIX) def make_request( self, method, path, headers, acceptable_statuses, body_file=None, diff --git a/swift/common/utils.py b/swift/common/utils.py index 68005f324b..38e2f4de20 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -45,6 +45,7 @@ from random import random, shuffle from contextlib import contextmanager, closing import ctypes import ctypes.util +from copy import deepcopy from optparse import OptionParser from tempfile import gettempdir, mkstemp, NamedTemporaryFile @@ -328,7 +329,7 @@ def get_swift_info(admin=False, disallowed_sections=None): :returns: dictionary of information about the swift cluster. """ disallowed_sections = disallowed_sections or [] - info = dict(_swift_info) + info = deepcopy(_swift_info) for section in disallowed_sections: key_to_pop = None sub_section_dict = info diff --git a/swift/common/wsgi.py b/swift/common/wsgi.py index 476894074b..d544d9d2c0 100644 --- a/swift/common/wsgi.py +++ b/swift/common/wsgi.py @@ -402,6 +402,9 @@ def loadapp(conf_file, global_conf=None, allow_modify_pipeline=True): func = getattr(app, 'modify_wsgi_pipeline', None) if func and allow_modify_pipeline: func(PipelineWrapper(ctx)) + # cache the freshly created app so we con't have to redo + # initialization checks and log startup messages again + ctx.app_context.create = lambda: app return ctx.create() diff --git a/swift/container/server.py b/swift/container/server.py index 831237cf88..44f484bacb 100644 --- a/swift/container/server.py +++ b/swift/container/server.py @@ -40,7 +40,8 @@ from swift.common.utils import get_logger, hash_path, public, \ override_bytes_from_content_type, get_log_line, \ config_fallocate_value, fs_has_free_space, list_from_csv, \ ShardRange -from swift.common.constraints import valid_timestamp, check_utf8, check_drive +from swift.common.constraints import valid_timestamp, check_utf8, \ + check_drive, AUTO_CREATE_ACCOUNT_PREFIX from swift.common import constraints from swift.common.bufferedhttp import http_connect from swift.common.exceptions import ConnectionTimeout @@ -142,8 +143,17 @@ class ContainerController(BaseStorageServer): self.replicator_rpc = ContainerReplicatorRpc( self.root, DATADIR, ContainerBroker, self.mount_check, logger=self.logger) - self.auto_create_account_prefix = \ - conf.get('auto_create_account_prefix') or '.' + if conf.get('auto_create_account_prefix'): + self.logger.warning('Option auto_create_account_prefix is ' + 'deprecated. Configure ' + 'auto_create_account_prefix under the ' + 'swift-constraints section of ' + 'swift.conf. This option will ' + 'be ignored in a future release.') + self.auto_create_account_prefix = \ + conf['auto_create_account_prefix'] + else: + self.auto_create_account_prefix = AUTO_CREATE_ACCOUNT_PREFIX if config_true_value(conf.get('allow_versions', 'f')): self.save_headers.append('x-versions-location') if 'allow_versions' in conf: diff --git a/swift/container/sharder.py b/swift/container/sharder.py index 98d6588559..0a1c4e9174 100644 --- a/swift/container/sharder.py +++ b/swift/container/sharder.py @@ -25,7 +25,7 @@ from six.moves.urllib.parse import quote from eventlet import Timeout from swift.common import internal_client -from swift.common.constraints import check_drive +from swift.common.constraints import check_drive, AUTO_CREATE_ACCOUNT_PREFIX from swift.common.direct_client import (direct_put_container, DirectClientException) from swift.common.exceptions import DeviceUnavailable @@ -334,8 +334,18 @@ class ContainerSharder(ContainerReplicator): def __init__(self, conf, logger=None): logger = logger or get_logger(conf, log_route='container-sharder') super(ContainerSharder, self).__init__(conf, logger=logger) - self.shards_account_prefix = ( - (conf.get('auto_create_account_prefix') or '.') + 'shards_') + if conf.get('auto_create_account_prefix'): + self.logger.warning('Option auto_create_account_prefix is ' + 'deprecated. Configure ' + 'auto_create_account_prefix under the ' + 'swift-constraints section of ' + 'swift.conf. This option will ' + 'be ignored in a future release.') + auto_create_account_prefix = \ + self.conf['auto_create_account_prefix'] + else: + auto_create_account_prefix = AUTO_CREATE_ACCOUNT_PREFIX + self.shards_account_prefix = (auto_create_account_prefix + 'shards_') def percent_value(key, default): try: diff --git a/swift/obj/expirer.py b/swift/obj/expirer.py index 28f1142180..1aef5fc2ff 100644 --- a/swift/obj/expirer.py +++ b/swift/obj/expirer.py @@ -25,6 +25,7 @@ import hashlib from eventlet import sleep, Timeout from eventlet.greenpool import GreenPool +from swift.common.constraints import AUTO_CREATE_ACCOUNT_PREFIX from swift.common.daemon import Daemon from swift.common.internal_client import InternalClient, UnexpectedResponse from swift.common.utils import get_logger, dump_recon_cache, split_path, \ @@ -112,8 +113,19 @@ class ObjectExpirer(Daemon): self.reclaim_age = int(conf.get('reclaim_age', 604800)) def read_conf_for_queue_access(self, swift): - self.expiring_objects_account = \ - (self.conf.get('auto_create_account_prefix') or '.') + \ + if self.conf.get('auto_create_account_prefix'): + self.logger.warning('Option auto_create_account_prefix is ' + 'deprecated. Configure ' + 'auto_create_account_prefix under the ' + 'swift-constraints section of ' + 'swift.conf. This option will ' + 'be ignored in a future release.') + auto_create_account_prefix = \ + self.conf['auto_create_account_prefix'] + else: + auto_create_account_prefix = AUTO_CREATE_ACCOUNT_PREFIX + + self.expiring_objects_account = auto_create_account_prefix + \ (self.conf.get('expiring_objects_account_name') or 'expiring_objects') diff --git a/swift/obj/server.py b/swift/obj/server.py index d1a1d85e93..b52343216d 100644 --- a/swift/obj/server.py +++ b/swift/obj/server.py @@ -39,7 +39,7 @@ from swift.common.utils import public, get_logger, \ normalize_timestamp from swift.common.bufferedhttp import http_connect from swift.common.constraints import check_object_creation, \ - valid_timestamp, check_utf8 + valid_timestamp, check_utf8, AUTO_CREATE_ACCOUNT_PREFIX from swift.common.exceptions import ConnectionTimeout, DiskFileQuarantined, \ DiskFileNotExist, DiskFileCollision, DiskFileNoSpace, DiskFileDeleted, \ DiskFileDeviceUnavailable, DiskFileExpired, ChunkReadTimeout, \ @@ -173,8 +173,18 @@ class ObjectController(BaseStorageServer): for header in extra_allowed_headers: if header not in RESERVED_DATAFILE_META: self.allowed_headers.add(header) - self.auto_create_account_prefix = \ - conf.get('auto_create_account_prefix') or '.' + if conf.get('auto_create_account_prefix'): + self.logger.warning('Option auto_create_account_prefix is ' + 'deprecated. Configure ' + 'auto_create_account_prefix under the ' + 'swift-constraints section of ' + 'swift.conf. This option will ' + 'be ignored in a future release.') + self.auto_create_account_prefix = \ + conf['auto_create_account_prefix'] + else: + self.auto_create_account_prefix = AUTO_CREATE_ACCOUNT_PREFIX + self.expiring_objects_account = self.auto_create_account_prefix + \ (conf.get('expiring_objects_account_name') or 'expiring_objects') self.expiring_objects_container_divisor = \ diff --git a/swift/proxy/controllers/base.py b/swift/proxy/controllers/base.py index e9ee894125..6b446754fb 100644 --- a/swift/proxy/controllers/base.py +++ b/swift/proxy/controllers/base.py @@ -347,7 +347,8 @@ def get_container_info(env, app, swift_source=None): # account is successful whether the account actually has .db files # on disk or not. is_autocreate_account = account.startswith( - getattr(app, 'auto_create_account_prefix', '.')) + getattr(app, 'auto_create_account_prefix', + constraints.AUTO_CREATE_ACCOUNT_PREFIX)) if not is_autocreate_account: account_info = get_account_info(env, app, swift_source) if not account_info or not is_success(account_info['status']): diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 6b3e3c9bc5..852f66dbc1 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -223,8 +223,18 @@ class Application(object): [os.path.join(swift_dir, 'mime.types')]) self.account_autocreate = \ config_true_value(conf.get('account_autocreate', 'no')) - self.auto_create_account_prefix = ( - conf.get('auto_create_account_prefix') or '.') + if conf.get('auto_create_account_prefix'): + self.logger.warning('Option auto_create_account_prefix is ' + 'deprecated. Configure ' + 'auto_create_account_prefix under the ' + 'swift-constraints section of ' + 'swift.conf. This option will ' + 'be ignored in a future release.') + self.auto_create_account_prefix = \ + conf['auto_create_account_prefix'] + else: + self.auto_create_account_prefix = \ + constraints.AUTO_CREATE_ACCOUNT_PREFIX self.expiring_objects_account = self.auto_create_account_prefix + \ (conf.get('expiring_objects_account_name') or 'expiring_objects') self.expiring_objects_container_divisor = \ @@ -295,7 +305,10 @@ class Application(object): self.expose_info = config_true_value( conf.get('expose_info', 'yes')) self.disallowed_sections = list_from_csv( - conf.get('disallowed_sections', 'swift.valid_api_versions')) + conf.get('disallowed_sections', ', '.join([ + 'swift.auto_create_account_prefix', + 'swift.valid_api_versions', + ]))) self.admin_key = conf.get('admin_key', None) register_swift_info( version=swift_version, diff --git a/test/probe/test_container_merge_policy_index.py b/test/probe/test_container_merge_policy_index.py index ccfea1bbf2..4b62804f13 100644 --- a/test/probe/test_container_merge_policy_index.py +++ b/test/probe/test_container_merge_policy_index.py @@ -23,6 +23,7 @@ from swift.common.internal_client import InternalClient from swift.common import utils, direct_client from swift.common.storage_policy import POLICIES from swift.common.http import HTTP_NOT_FOUND +from swift.container.reconciler import MISPLACED_OBJECTS_ACCOUNT from test.probe.brain import BrainSplitter from test.probe.common import (ReplProbeTest, ENABLED_POLICIES, POLICIES_BY_TYPE, REPL_POLICY) @@ -495,8 +496,8 @@ class TestContainerMergePolicyIndex(ReplProbeTest): server.once(number=self.config_number(node)) # verify entry in the queue for the "misplaced" new_policy - for container in int_client.iter_containers('.misplaced_objects'): - for obj in int_client.iter_objects('.misplaced_objects', + for container in int_client.iter_containers(MISPLACED_OBJECTS_ACCOUNT): + for obj in int_client.iter_objects(MISPLACED_OBJECTS_ACCOUNT, container['name']): expected = '%d:/%s/%s/%s' % (new_policy, self.account, self.container_name, @@ -519,8 +520,8 @@ class TestContainerMergePolicyIndex(ReplProbeTest): self.get_to_final_state() # verify entry in the queue - for container in int_client.iter_containers('.misplaced_objects'): - for obj in int_client.iter_objects('.misplaced_objects', + for container in int_client.iter_containers(MISPLACED_OBJECTS_ACCOUNT): + for obj in int_client.iter_objects(MISPLACED_OBJECTS_ACCOUNT, container['name']): expected = '%d:/%s/%s/%s' % (old_policy, self.account, self.container_name, @@ -540,8 +541,8 @@ class TestContainerMergePolicyIndex(ReplProbeTest): # make sure the queue is settled self.get_to_final_state() - for container in int_client.iter_containers('.misplaced_objects'): - for obj in int_client.iter_objects('.misplaced_objects', + for container in int_client.iter_containers(MISPLACED_OBJECTS_ACCOUNT): + for obj in int_client.iter_objects(MISPLACED_OBJECTS_ACCOUNT, container['name']): self.fail('Found unexpected object %r in the queue' % obj) diff --git a/test/unit/account/test_server.py b/test/unit/account/test_server.py index c1168a852f..3ba58cddf8 100644 --- a/test/unit/account/test_server.py +++ b/test/unit/account/test_server.py @@ -51,9 +51,10 @@ class TestAccountController(unittest.TestCase): self.testdir_base = mkdtemp() self.testdir = os.path.join(self.testdir_base, 'account_server') mkdirs(os.path.join(self.testdir, 'sda1')) + self.logger = debug_logger() self.controller = AccountController( {'devices': self.testdir, 'mount_check': 'false'}, - logger=debug_logger()) + logger=self.logger) self.ts = make_timestamp_iter() def tearDown(self): @@ -64,6 +65,22 @@ class TestAccountController(unittest.TestCase): if err.errno != errno.ENOENT: raise + def test_init(self): + conf = { + 'devices': self.testdir, + 'mount_check': 'false', + } + AccountController(conf, logger=self.logger) + self.assertEqual(self.logger.get_lines_for_level('warning'), []) + conf['auto_create_account_prefix'] = '-' + AccountController(conf, logger=self.logger) + self.assertEqual(self.logger.get_lines_for_level('warning'), [ + 'Option auto_create_account_prefix is deprecated. ' + 'Configure auto_create_account_prefix under the ' + 'swift-constraints section of swift.conf. This option ' + 'will be ignored in a future release.' + ]) + def test_OPTIONS(self): server_handler = AccountController( {'devices': self.testdir, 'mount_check': 'false'}) diff --git a/test/unit/common/test_constraints.py b/test/unit/common/test_constraints.py index 5090fddeb6..d0dde93e53 100644 --- a/test/unit/common/test_constraints.py +++ b/test/unit/common/test_constraints.py @@ -649,10 +649,15 @@ class TestConstraintsConfig(unittest.TestCase): f.flush() with mock.patch.object(utils, 'SWIFT_CONF_FILE', f.name): constraints.reload_constraints() - for key in constraints.DEFAULT_CONSTRAINTS: + for key, default in constraints.DEFAULT_CONSTRAINTS.items(): # module level attrs should all be 1 module_level_value = getattr(constraints, key.upper()) - self.assertEqual(module_level_value, 1) + if isinstance(default, int): + self.assertEqual(module_level_value, 1) + elif isinstance(default, str): + self.assertEqual(module_level_value, '1') + else: + self.assertEqual(module_level_value, ['1']) # all keys should be in OVERRIDE self.assertEqual(constraints.OVERRIDE_CONSTRAINTS[key], module_level_value) diff --git a/test/unit/common/test_internal_client.py b/test/unit/common/test_internal_client.py index e957ce894c..99ee0b319a 100644 --- a/test/unit/common/test_internal_client.py +++ b/test/unit/common/test_internal_client.py @@ -25,14 +25,13 @@ from textwrap import dedent import six from six.moves import range, zip_longest from six.moves.urllib.parse import quote, parse_qsl -from test.unit import FakeLogger from swift.common import exceptions, internal_client, swob from swift.common.header_key_dict import HeaderKeyDict from swift.common.storage_policy import StoragePolicy from swift.common.middleware.proxy_logging import ProxyLoggingMiddleware from test.unit import with_tempdir, write_fake_ring, patch_policies, \ - DebugLogger + debug_logger from test.unit.common.middleware.helpers import FakeSwift, LeakTrackingIter if six.PY3: @@ -256,8 +255,17 @@ class TestInternalClient(unittest.TestCase): write_fake_ring(container_ring_path) object_ring_path = os.path.join(tempdir, 'object.ring.gz') write_fake_ring(object_ring_path) + logger = debug_logger('test-ic') + self.assertEqual(logger.get_lines_for_level('warning'), []) with patch_policies([StoragePolicy(0, 'legacy', True)]): - client = internal_client.InternalClient(conf_path, 'test', 1) + with mock.patch('swift.proxy.server.get_logger', + lambda *a, **kw: logger): + client = internal_client.InternalClient(conf_path, 'test', 1) + self.assertEqual(logger.get_lines_for_level('warning'), [ + 'Option auto_create_account_prefix is deprecated. ' + 'Configure auto_create_account_prefix under the ' + 'swift-constraints section of swift.conf. This option will ' + 'be ignored in a future release.']) self.assertEqual(client.account_ring, client.app.app.app.account_ring) self.assertEqual(client.account_ring.serialized_path, @@ -454,7 +462,7 @@ class TestInternalClient(unittest.TestCase): def test_make_request_error_case(self): class InternalClient(internal_client.InternalClient): def __init__(self): - self.logger = DebugLogger() + self.logger = debug_logger('test-ic') # wrap the fake app with ProxyLoggingMiddleware self.app = ProxyLoggingMiddleware( self.fake_app, {}, self.logger) @@ -484,7 +492,7 @@ class TestInternalClient(unittest.TestCase): def test_make_request_acceptable_status_not_2xx(self): class InternalClient(internal_client.InternalClient): def __init__(self, resp_status): - self.logger = DebugLogger() + self.logger = debug_logger('test-ic') # wrap the fake app with ProxyLoggingMiddleware self.app = ProxyLoggingMiddleware( self.fake_app, {}, self.logger) @@ -1408,7 +1416,7 @@ class TestSimpleClient(unittest.TestCase): urlopen.return_value.getcode.return_value = 200 urlopen.return_value.info.return_value = {'content-length': '345'} sc = internal_client.SimpleClient(url='http://127.0.0.1') - logger = FakeLogger() + logger = debug_logger('test-ic') retval = sc.retry_request( method, headers={'content-length': '123'}, logger=logger) self.assertEqual(urlopen.call_count, 1) @@ -1417,10 +1425,11 @@ class TestSimpleClient(unittest.TestCase): data=None) self.assertEqual([{'content-length': '345'}, None], retval) self.assertEqual(method, request.return_value.get_method()) - self.assertEqual(logger.log_dict['debug'], [( - ('-> 2014-05-27T20:54:11 ' + method + - ' http://127.0.0.1%3Fformat%3Djson 200 ' - '123 345 1401224050.98 1401224051.98 1.0 -',), {})]) + self.assertEqual(logger.get_lines_for_level('debug'), [ + '-> 2014-05-27T20:54:11 ' + method + + ' http://127.0.0.1%3Fformat%3Djson 200 ' + '123 345 1401224050.98 1401224051.98 1.0 -' + ]) # Check if JSON is decoded urlopen.return_value.read.return_value = b'{}' diff --git a/test/unit/container/test_server.py b/test/unit/container/test_server.py index c45ec01c9b..a6d48c5082 100644 --- a/test/unit/container/test_server.py +++ b/test/unit/container/test_server.py @@ -116,9 +116,18 @@ class TestContainerController(unittest.TestCase): def test_creation(self): # later config should be extended to assert more config options - replicator = container_server.ContainerController( - {'node_timeout': '3.5'}) - self.assertEqual(replicator.node_timeout, 3.5) + app = container_server.ContainerController( + {'node_timeout': '3.5'}, logger=self.logger) + self.assertEqual(app.node_timeout, 3.5) + self.assertEqual(self.logger.get_lines_for_level('warning'), []) + app = container_server.ContainerController( + {'auto_create_account_prefix': '-'}, logger=self.logger) + self.assertEqual(self.logger.get_lines_for_level('warning'), [ + 'Option auto_create_account_prefix is deprecated. ' + 'Configure auto_create_account_prefix under the ' + 'swift-constraints section of swift.conf. This option ' + 'will be ignored in a future release.' + ]) def test_get_and_validate_policy_index(self): # no policy is OK diff --git a/test/unit/container/test_sharder.py b/test/unit/container/test_sharder.py index 1175c7620a..c814445dab 100644 --- a/test/unit/container/test_sharder.py +++ b/test/unit/container/test_sharder.py @@ -45,7 +45,7 @@ from swift.common.utils import ShardRange, Timestamp, hash_path, \ encode_timestamps, parse_db_filename, quorum_size, Everything from test import annotate_failure -from test.unit import FakeLogger, debug_logger, FakeRing, \ +from test.unit import debug_logger, FakeRing, \ make_timestamp_iter, unlink_files, mocked_http_conn, mock_timestamp_now, \ attach_fake_replication_rpc @@ -54,6 +54,7 @@ class BaseTestSharder(unittest.TestCase): def setUp(self): self.tempdir = mkdtemp() self.ts_iter = make_timestamp_iter() + self.logger = debug_logger('sharder-test') def tearDown(self): shutil.rmtree(self.tempdir, ignore_errors=True) @@ -74,7 +75,7 @@ class BaseTestSharder(unittest.TestCase): db_file = os.path.join(datadir, filename) broker = ContainerBroker( db_file, account=account, container=container, - logger=debug_logger()) + logger=self.logger) broker.initialize() return broker @@ -108,7 +109,9 @@ class BaseTestSharder(unittest.TestCase): class TestSharder(BaseTestSharder): def test_init(self): - def do_test(conf, expected): + def do_test(conf, expected, logger=self.logger): + if logger: + logger.clear() with mock.patch( 'swift.container.sharder.internal_client.InternalClient') \ as mock_ic: @@ -116,17 +119,15 @@ class TestSharder(BaseTestSharder): as mock_ring: mock_ring.return_value = mock.MagicMock() mock_ring.return_value.replica_count = 3 - sharder = ContainerSharder(conf) + sharder = ContainerSharder(conf, logger=logger) mock_ring.assert_called_once_with( '/etc/swift', ring_name='container') - self.assertEqual( - 'container-sharder', sharder.logger.logger.name) for k, v in expected.items(): self.assertTrue(hasattr(sharder, k), 'Missing attr %s' % k) self.assertEqual(v, getattr(sharder, k), 'Incorrect value: expected %s=%s but got %s' % (k, v, getattr(sharder, k))) - return mock_ic + return sharder, mock_ic expected = { 'mount_check': True, 'bind_ip': '0.0.0.0', 'port': 6201, @@ -150,7 +151,9 @@ class TestSharder(BaseTestSharder): 'shard_replication_quorum': 2, 'existing_shard_replication_quorum': 2 } - mock_ic = do_test({}, expected) + sharder, mock_ic = do_test({}, expected, logger=None) + self.assertEqual( + 'container-sharder', sharder.logger.logger.name) mock_ic.assert_called_once_with( '/etc/swift/internal-client.conf', 'Swift Container Sharder', 3, allow_modify_pipeline=False) @@ -200,16 +203,33 @@ class TestSharder(BaseTestSharder): 'shard_replication_quorum': 1, 'existing_shard_replication_quorum': 0 } - mock_ic = do_test(conf, expected) + sharder, mock_ic = do_test(conf, expected) mock_ic.assert_called_once_with( '/etc/swift/my-sharder-ic.conf', 'Swift Container Sharder', 2, allow_modify_pipeline=False) + self.assertEqual(self.logger.get_lines_for_level('warning'), [ + 'Option auto_create_account_prefix is deprecated. ' + 'Configure auto_create_account_prefix under the ' + 'swift-constraints section of swift.conf. This option ' + 'will be ignored in a future release.']) expected.update({'shard_replication_quorum': 3, 'existing_shard_replication_quorum': 3}) conf.update({'shard_replication_quorum': 4, 'existing_shard_replication_quorum': 4}) do_test(conf, expected) + warnings = self.logger.get_lines_for_level('warning') + self.assertEqual(warnings[:1], [ + 'Option auto_create_account_prefix is deprecated. ' + 'Configure auto_create_account_prefix under the ' + 'swift-constraints section of swift.conf. This option ' + 'will be ignored in a future release.']) + self.assertEqual(warnings[1:], [ + 'shard_replication_quorum of 4 exceeds replica count 3, ' + 'reducing to 3', + 'existing_shard_replication_quorum of 4 exceeds replica count 3, ' + 'reducing to 3', + ]) with self.assertRaises(ValueError) as cm: do_test({'shard_shrink_point': 101}, {}) @@ -487,7 +507,6 @@ class TestSharder(BaseTestSharder): mock.patch('swift.container.sharder.is_local_device', return_value=True): sharder.reported = time.time() - sharder.logger = debug_logger() brokers = [] device_ids = set(d['id'] for d in sharder.ring.devs) @@ -694,6 +713,7 @@ class TestSharder(BaseTestSharder): @contextmanager def _mock_sharder(self, conf=None, replicas=3): + self.logger.clear() conf = conf or {} conf['devices'] = self.tempdir with mock.patch( @@ -701,7 +721,7 @@ class TestSharder(BaseTestSharder): with mock.patch( 'swift.common.db_replicator.ring.Ring', lambda *args, **kwargs: FakeRing(replicas=replicas)): - sharder = ContainerSharder(conf, logger=FakeLogger()) + sharder = ContainerSharder(conf, logger=self.logger) sharder._local_device_ids = {0, 1, 2} sharder._replicate_object = mock.MagicMock( return_value=(True, [True] * sharder.ring.replica_count)) @@ -1080,7 +1100,6 @@ class TestSharder(BaseTestSharder): # run cleave again - should process the fourth range with self._mock_sharder(conf=conf) as sharder: - sharder.logger = debug_logger() self.assertFalse(sharder._cleave(broker)) expected = {'attempted': 1, 'success': 1, 'failure': 0, @@ -1845,7 +1864,6 @@ class TestSharder(BaseTestSharder): side_effect=[(True, [True, True, True]), # misplaced obj (False, [False, True, True])]) sharder._audit_container = mock.MagicMock() - sharder.logger = debug_logger() sharder._process_broker(broker, node, 99) self.assertEqual(SHARDED, broker.get_db_state()) @@ -2003,7 +2021,6 @@ class TestSharder(BaseTestSharder): sharder._replicate_object = mock.MagicMock( side_effect=[(False, [False, True, True])]) sharder._audit_container = mock.MagicMock() - sharder.logger = debug_logger() sharder._process_broker(broker, node, 99) self.assertEqual(SHARDING, broker.get_db_state()) self.assertEqual(ShardRange.SHARDING, @@ -3354,7 +3371,6 @@ class TestSharder(BaseTestSharder): self.assertTrue(broker.set_sharded_state()) with self._mock_sharder() as sharder: - sharder.logger = debug_logger() sharder._move_misplaced_objects(broker) sharder._replicate_object.assert_has_calls( @@ -3865,7 +3881,6 @@ class TestSharder(BaseTestSharder): with self._mock_sharder() as sharder: with mock_timestamp_now() as now: with mock.patch.object(sharder, '_audit_container'): - sharder.logger = debug_logger() sharder._process_broker(broker, node, 99) own_shard_range = broker.get_own_shard_range( no_default=True) @@ -3980,8 +3995,6 @@ class TestSharder(BaseTestSharder): own_sr.epoch = epoch broker.merge_shard_ranges([own_sr]) with self._mock_sharder() as sharder: - - sharder.logger = debug_logger() with mock_timestamp_now() as now: # we're not testing rest of the process here so prevent any # attempt to progress shard range states @@ -4156,7 +4169,6 @@ class TestSharder(BaseTestSharder): def call_audit_container(exc=None): with self._mock_sharder() as sharder: - sharder.logger = debug_logger() with mock.patch.object(sharder, '_audit_root_container') \ as mocked, mock.patch.object( sharder, 'int_client') as mock_swift: diff --git a/test/unit/obj/test_expirer.py b/test/unit/obj/test_expirer.py index 077b84d420..c61cd5d7bf 100644 --- a/test/unit/obj/test_expirer.py +++ b/test/unit/obj/test_expirer.py @@ -154,6 +154,20 @@ class TestObjectExpirer(TestCase): internal_client.sleep = self.old_sleep internal_client.loadapp = self.old_loadapp + def test_init(self): + x = expirer.ObjectExpirer({}, logger=self.logger) + self.assertEqual(self.logger.get_lines_for_level('warning'), []) + self.assertEqual(x.expiring_objects_account, '.expiring_objects') + x = expirer.ObjectExpirer({'auto_create_account_prefix': '-'}, + logger=self.logger) + self.assertEqual(self.logger.get_lines_for_level('warning'), [ + 'Option auto_create_account_prefix is deprecated. ' + 'Configure auto_create_account_prefix under the ' + 'swift-constraints section of swift.conf. This option ' + 'will be ignored in a future release.' + ]) + self.assertEqual(x.expiring_objects_account, '-expiring_objects') + def test_get_process_values_from_kwargs(self): x = expirer.ObjectExpirer({}) vals = { diff --git a/test/unit/obj/test_server.py b/test/unit/obj/test_server.py index 3e5a5f16f7..bf518c99ad 100644 --- a/test/unit/obj/test_server.py +++ b/test/unit/obj/test_server.py @@ -148,8 +148,9 @@ class TestObjectController(unittest.TestCase): mkdirs(os.path.join(self.testdir, 'sda1')) self.conf = {'devices': self.testdir, 'mount_check': 'false', 'container_update_timeout': 0.0} + self.logger = debug_logger() self.object_controller = object_server.ObjectController( - self.conf, logger=debug_logger()) + self.conf, logger=self.logger) self.object_controller.bytes_per_sync = 1 self._orig_tpool_exc = tpool.execute tpool.execute = lambda f, *args, **kwargs: f(*args, **kwargs) @@ -174,6 +175,27 @@ class TestObjectController(unittest.TestCase): self.policy = policy yield policy + def test_init(self): + conf = { + 'devices': self.testdir, + 'mount_check': 'false', + 'container_update_timeout': 0.0, + } + app = object_server.ObjectController(conf, logger=self.logger) + self.assertEqual(app.container_update_timeout, 0.0) + self.assertEqual(app.auto_create_account_prefix, '.') + self.assertEqual(self.logger.get_lines_for_level('warning'), []) + + conf['auto_create_account_prefix'] = '-' + app = object_server.ObjectController(conf, logger=self.logger) + self.assertEqual(app.auto_create_account_prefix, '-') + self.assertEqual(self.logger.get_lines_for_level('warning'), [ + 'Option auto_create_account_prefix is deprecated. ' + 'Configure auto_create_account_prefix under the ' + 'swift-constraints section of swift.conf. This option ' + 'will be ignored in a future release.' + ]) + def check_all_api_methods(self, obj_name='o', alt_res=None): path = '/sda1/p/a/c/%s' % obj_name body = b'SPECIAL_STRING' diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 3b509c380d..5f50419224 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -571,11 +571,31 @@ class TestController(unittest.TestCase): @patch_policies([StoragePolicy(0, 'zero', True, object_ring=FakeRing())]) class TestProxyServerConfiguration(unittest.TestCase): + + def setUp(self): + self.logger = debug_logger('test-proxy-config') + def _make_app(self, conf): + self.logger.clear() # helper function to instantiate a proxy server instance return proxy_server.Application(conf, FakeMemcache(), container_ring=FakeRing(), - account_ring=FakeRing()) + account_ring=FakeRing(), + logger=self.logger) + + def test_auto_create_account(self): + app = self._make_app({}) + self.assertEqual(app.auto_create_account_prefix, '.') + self.assertEqual(self.logger.get_lines_for_level('warning'), []) + + app = self._make_app({'auto_create_account_prefix': '-'}) + self.assertEqual(app.auto_create_account_prefix, '-') + self.assertEqual(self.logger.get_lines_for_level('warning'), [ + 'Option auto_create_account_prefix is deprecated. ' + 'Configure auto_create_account_prefix under the ' + 'swift-constraints section of swift.conf. This option ' + 'will be ignored in a future release.' + ]) def test_node_timeout(self): # later config should be extended to assert more config options @@ -1092,9 +1112,11 @@ class TestProxyServer(unittest.TestCase): self.assertTrue(app.expose_info) self.assertIsInstance(app.disallowed_sections, list) - self.assertEqual(1, len(app.disallowed_sections)) - self.assertEqual(['swift.valid_api_versions'], - app.disallowed_sections) + self.assertEqual(2, len(app.disallowed_sections)) + self.assertEqual([ + 'swift.auto_create_account_prefix', + 'swift.valid_api_versions', + ], sorted(app.disallowed_sections)) self.assertIsNone(app.admin_key) def test_get_info_controller(self): @@ -10780,11 +10802,13 @@ class TestSwiftInfo(unittest.TestCase): utils._swift_admin_info = {} def test_registered_defaults(self): - proxy_server.Application({}, FakeMemcache(), - account_ring=FakeRing(), - container_ring=FakeRing()) + app = proxy_server.Application({}, FakeMemcache(), + account_ring=FakeRing(), + container_ring=FakeRing()) + req = Request.blank('/info') + resp = req.get_response(app) + si = json.loads(resp.body)['swift'] - si = utils.get_swift_info()['swift'] self.assertIn('version', si) self.assertEqual(si['max_file_size'], constraints.MAX_FILE_SIZE) self.assertEqual(si['max_meta_name_length'], @@ -10808,12 +10832,16 @@ class TestSwiftInfo(unittest.TestCase): self.assertIn('strict_cors_mode', si) self.assertFalse(si['allow_account_management']) self.assertFalse(si['account_autocreate']) - # This setting is by default excluded by disallowed_sections - self.assertEqual(si['valid_api_versions'], - constraints.VALID_API_VERSIONS) # this next test is deliberately brittle in order to alert if # other items are added to swift info - self.assertEqual(len(si), 18) + self.assertEqual(len(si), 17) + + si = utils.get_swift_info()['swift'] + # Tehse settings is by default excluded by disallowed_sections + self.assertEqual(si['valid_api_versions'], + constraints.VALID_API_VERSIONS) + self.assertEqual(si['auto_create_account_prefix'], + constraints.AUTO_CREATE_ACCOUNT_PREFIX) self.assertIn('policies', si) sorted_pols = sorted(si['policies'], key=operator.itemgetter('name'))