diff --git a/doc/manpages/account-server.conf.5 b/doc/manpages/account-server.conf.5 index cec0752999..955eedfa76 100644 --- a/doc/manpages/account-server.conf.5 +++ b/doc/manpages/account-server.conf.5 @@ -176,8 +176,6 @@ Syslog log facility. The default is LOG_LOCAL0. Logging level. The default is INFO. .IP \fBlog_address\fR Logging address. The default is /dev/log. -.IP \fBvm_test_mode\fR -Indicates that you are using a VM environment. The default is no. .IP \fBper_diff\fR The default is 1000. .IP \fBmax_diffs\fR diff --git a/doc/manpages/container-server.conf.5 b/doc/manpages/container-server.conf.5 index 2cd1623dc1..65981308bd 100644 --- a/doc/manpages/container-server.conf.5 +++ b/doc/manpages/container-server.conf.5 @@ -182,8 +182,6 @@ Syslog log facility. The default is LOG_LOCAL0. Logging level. The default is INFO. .IP \fBlog_address\fR Logging address. The default is /dev/log. -.IP \fBvm_test_mode\fR -Indicates that you are using a VM environment. The default is no. .IP \fBer_diff\fR The default is 1000. .IP \fBmax_diffs\fR diff --git a/doc/manpages/object-server.conf.5 b/doc/manpages/object-server.conf.5 index e82a56ffb5..c790729c16 100644 --- a/doc/manpages/object-server.conf.5 +++ b/doc/manpages/object-server.conf.5 @@ -185,8 +185,6 @@ Syslog log facility. The default is LOG_LOCAL0. Logging level. The default is INFO. .IP \fBlog_address\fR Logging address. The default is /dev/log. -.IP \fBvm_test_mode\fR -Indicates that you are using a VM environment. The default is no. .IP \fBdaemonize\fR Whether or not to run replication as a daemon. The default is yes. .IP "\fBrun_pause [deprecated]\fR" diff --git a/doc/saio/swift/account-server/1.conf b/doc/saio/swift/account-server/1.conf index ce599a7c8d..91ded8905d 100644 --- a/doc/saio/swift/account-server/1.conf +++ b/doc/saio/swift/account-server/1.conf @@ -20,7 +20,7 @@ use = egg:swift#account use = egg:swift#recon [account-replicator] -vm_test_mode = yes +rsync_module = {replication_ip}::account{replication_port} [account-auditor] diff --git a/doc/saio/swift/account-server/2.conf b/doc/saio/swift/account-server/2.conf index 0593e9ad39..70f8fae896 100644 --- a/doc/saio/swift/account-server/2.conf +++ b/doc/saio/swift/account-server/2.conf @@ -20,7 +20,7 @@ use = egg:swift#account use = egg:swift#recon [account-replicator] -vm_test_mode = yes +rsync_module = {replication_ip}::account{replication_port} [account-auditor] diff --git a/doc/saio/swift/account-server/3.conf b/doc/saio/swift/account-server/3.conf index abc27c2580..6090883bd7 100644 --- a/doc/saio/swift/account-server/3.conf +++ b/doc/saio/swift/account-server/3.conf @@ -20,7 +20,7 @@ use = egg:swift#account use = egg:swift#recon [account-replicator] -vm_test_mode = yes +rsync_module = {replication_ip}::account{replication_port} [account-auditor] diff --git a/doc/saio/swift/account-server/4.conf b/doc/saio/swift/account-server/4.conf index cda9ac87b9..e435842d1a 100644 --- a/doc/saio/swift/account-server/4.conf +++ b/doc/saio/swift/account-server/4.conf @@ -20,7 +20,7 @@ use = egg:swift#account use = egg:swift#recon [account-replicator] -vm_test_mode = yes +rsync_module = {replication_ip}::account{replication_port} [account-auditor] diff --git a/doc/saio/swift/container-server/1.conf b/doc/saio/swift/container-server/1.conf index 176096dbe1..5bf3c0f28c 100644 --- a/doc/saio/swift/container-server/1.conf +++ b/doc/saio/swift/container-server/1.conf @@ -20,7 +20,7 @@ use = egg:swift#container use = egg:swift#recon [container-replicator] -vm_test_mode = yes +rsync_module = {replication_ip}::container{replication_port} [container-updater] diff --git a/doc/saio/swift/container-server/2.conf b/doc/saio/swift/container-server/2.conf index 7100710b3c..7046236fae 100644 --- a/doc/saio/swift/container-server/2.conf +++ b/doc/saio/swift/container-server/2.conf @@ -20,7 +20,7 @@ use = egg:swift#container use = egg:swift#recon [container-replicator] -vm_test_mode = yes +rsync_module = {replication_ip}::container{replication_port} [container-updater] diff --git a/doc/saio/swift/container-server/3.conf b/doc/saio/swift/container-server/3.conf index 06ec47414d..c7f726199c 100644 --- a/doc/saio/swift/container-server/3.conf +++ b/doc/saio/swift/container-server/3.conf @@ -20,7 +20,7 @@ use = egg:swift#container use = egg:swift#recon [container-replicator] -vm_test_mode = yes +rsync_module = {replication_ip}::container{replication_port} [container-updater] diff --git a/doc/saio/swift/container-server/4.conf b/doc/saio/swift/container-server/4.conf index 1acc3b5c54..ff9c0e806a 100644 --- a/doc/saio/swift/container-server/4.conf +++ b/doc/saio/swift/container-server/4.conf @@ -20,7 +20,7 @@ use = egg:swift#container use = egg:swift#recon [container-replicator] -vm_test_mode = yes +rsync_module = {replication_ip}::container{replication_port} [container-updater] diff --git a/doc/saio/swift/object-server/1.conf b/doc/saio/swift/object-server/1.conf index 178e3fcba3..c6d1b1464f 100644 --- a/doc/saio/swift/object-server/1.conf +++ b/doc/saio/swift/object-server/1.conf @@ -20,7 +20,7 @@ use = egg:swift#object use = egg:swift#recon [object-replicator] -vm_test_mode = yes +rsync_module = {replication_ip}::object{replication_port} [object-reconstructor] diff --git a/doc/saio/swift/object-server/2.conf b/doc/saio/swift/object-server/2.conf index 6b611ca25a..3d863cf59b 100644 --- a/doc/saio/swift/object-server/2.conf +++ b/doc/saio/swift/object-server/2.conf @@ -20,7 +20,7 @@ use = egg:swift#object use = egg:swift#recon [object-replicator] -vm_test_mode = yes +rsync_module = {replication_ip}::object{replication_port} [object-reconstructor] diff --git a/doc/saio/swift/object-server/3.conf b/doc/saio/swift/object-server/3.conf index 7352592319..d99995bb5e 100644 --- a/doc/saio/swift/object-server/3.conf +++ b/doc/saio/swift/object-server/3.conf @@ -20,7 +20,7 @@ use = egg:swift#object use = egg:swift#recon [object-replicator] -vm_test_mode = yes +rsync_module = {replication_ip}::object{replication_port} [object-reconstructor] diff --git a/doc/saio/swift/object-server/4.conf b/doc/saio/swift/object-server/4.conf index be1211047b..9f6fd1fc13 100644 --- a/doc/saio/swift/object-server/4.conf +++ b/doc/saio/swift/object-server/4.conf @@ -20,7 +20,7 @@ use = egg:swift#object use = egg:swift#recon [object-replicator] -vm_test_mode = yes +rsync_module = {replication_ip}::object{replication_port} [object-reconstructor] diff --git a/doc/source/deployment_guide.rst b/doc/source/deployment_guide.rst index da9d0b4fa2..1516d083ca 100644 --- a/doc/source/deployment_guide.rst +++ b/doc/source/deployment_guide.rst @@ -594,6 +594,17 @@ node_timeout DEFAULT or 10 Request timeout to external services. in the DEFAULT section, or 10 (though other sections use 3 as the final default). +rsync_module {replication_ip}::object + Format of the rsync module where the + replicator will send data. The + configuration value can include some + variables that will be extracted from + the ring. Variables must follow the + format {NAME} where NAME is one of: + ip, port, replication_ip, + replication_port, region, zone, device, + meta. See etc/rsyncd.conf-sample for + some examples. ================== ================= ======================================= [object-updater] @@ -723,6 +734,18 @@ conn_timeout 0.5 Connection timeout to external services reclaim_age 604800 Time elapsed in seconds before a container can be reclaimed +rsync_module {replication_ip}::container + Format of the rsync module where the + replicator will send data. The + configuration value can include some + variables that will be extracted from + the ring. Variables must follow the + format {NAME} where NAME is one of: + ip, port, replication_ip, + replication_port, region, zone, + device, meta. See + etc/rsyncd.conf-sample for some + examples. ================== ==================== ==================================== [container-updater] @@ -850,6 +873,18 @@ node_timeout 10 Request timeout to external services conn_timeout 0.5 Connection timeout to external services reclaim_age 604800 Time elapsed in seconds before an account can be reclaimed +rsync_module {replication_ip}::account + Format of the rsync module where the + replicator will send data. The + configuration value can include some + variables that will be extracted from + the ring. Variables must follow the + format {NAME} where NAME is one of: + ip, port, replication_ip, + replication_port, region, zone, + device, meta. See + etc/rsyncd.conf-sample for some + examples. ================== ================== ====================================== [account-auditor] diff --git a/doc/source/replication_network.rst b/doc/source/replication_network.rst index 138cb42186..609ebc9688 100644 --- a/doc/source/replication_network.rst +++ b/doc/source/replication_network.rst @@ -146,7 +146,7 @@ For SAIO replication delete all configuration options in section [<*>-replicator] -#. Add configuration files for object-server, in /etc/swift/objec-server/ +#. Add configuration files for object-server, in /etc/swift/object-server/ * 5.conf:: @@ -170,7 +170,7 @@ For SAIO replication use = egg:swift#recon [object-replicator] - vm_test_mode = yes + rsync_module = {replication_ip}::object{replication_port} * 6.conf:: @@ -194,7 +194,7 @@ For SAIO replication use = egg:swift#recon [object-replicator] - vm_test_mode = yes + rsync_module = {replication_ip}::object{replication_port} * 7.conf:: @@ -218,7 +218,7 @@ For SAIO replication use = egg:swift#recon [object-replicator] - vm_test_mode = yes + rsync_module = {replication_ip}::object{replication_port} * 8.conf:: @@ -242,7 +242,7 @@ For SAIO replication use = egg:swift#recon [object-replicator] - vm_test_mode = yes + rsync_module = {replication_ip}::object{replication_port} #. Add configuration files for container-server, in /etc/swift/container-server/ @@ -268,7 +268,7 @@ For SAIO replication use = egg:swift#recon [container-replicator] - vm_test_mode = yes + rsync_module = {replication_ip}::container{replication_port} * 6.conf:: @@ -292,7 +292,7 @@ For SAIO replication use = egg:swift#recon [container-replicator] - vm_test_mode = yes + rsync_module = {replication_ip}::container{replication_port} * 7.conf:: @@ -316,7 +316,7 @@ For SAIO replication use = egg:swift#recon [container-replicator] - vm_test_mode = yes + rsync_module = {replication_ip}::container{replication_port} * 8.conf:: @@ -340,7 +340,7 @@ For SAIO replication use = egg:swift#recon [container-replicator] - vm_test_mode = yes + rsync_module = {replication_ip}::container{replication_port} #. Add configuration files for account-server, in /etc/swift/account-server/ @@ -366,7 +366,7 @@ For SAIO replication use = egg:swift#recon [account-replicator] - vm_test_mode = yes + rsync_module = {replication_ip}::account{replication_port} * 6.conf:: @@ -390,7 +390,7 @@ For SAIO replication use = egg:swift#recon [account-replicator] - vm_test_mode = yes + rsync_module = {replication_ip}::account{replication_port} * 7.conf:: @@ -414,7 +414,7 @@ For SAIO replication use = egg:swift#recon [account-replicator] - vm_test_mode = yes + rsync_module = {replication_ip}::account{replication_port} * 8.conf:: @@ -438,7 +438,7 @@ For SAIO replication use = egg:swift#recon [account-replicator] - vm_test_mode = yes + rsync_module = {replication_ip}::account{replication_port} --------------------------------- diff --git a/etc/account-server.conf-sample b/etc/account-server.conf-sample index 8cfbfa0098..9eb61a9d96 100644 --- a/etc/account-server.conf-sample +++ b/etc/account-server.conf-sample @@ -90,7 +90,6 @@ use = egg:swift#recon # log_level = INFO # log_address = /dev/log # -# vm_test_mode = no # per_diff = 1000 # max_diffs = 100 # concurrency = 8 @@ -111,6 +110,10 @@ use = egg:swift#recon # a different region than the local one. # rsync_compress = no # +# Format of the rysnc module where the replicator will send data. See +# etc/rsyncd.conf-sample for some usage examples. +# rsync_module = {replication_ip}::account +# # recon_cache_path = /var/cache/swift [account-auditor] diff --git a/etc/container-server.conf-sample b/etc/container-server.conf-sample index 54daee1e8e..0eebf366e0 100644 --- a/etc/container-server.conf-sample +++ b/etc/container-server.conf-sample @@ -99,7 +99,6 @@ use = egg:swift#recon # log_level = INFO # log_address = /dev/log # -# vm_test_mode = no # per_diff = 1000 # max_diffs = 100 # concurrency = 8 @@ -120,6 +119,10 @@ use = egg:swift#recon # a different region than the local one. # rsync_compress = no # +# Format of the rysnc module where the replicator will send data. See +# etc/rsyncd.conf-sample for some usage examples. +# rsync_module = {replication_ip}::container +# # recon_cache_path = /var/cache/swift [container-updater] diff --git a/etc/object-server.conf-sample b/etc/object-server.conf-sample index 31bd160a3e..84328b0c3f 100644 --- a/etc/object-server.conf-sample +++ b/etc/object-server.conf-sample @@ -162,7 +162,6 @@ use = egg:swift#recon # log_level = INFO # log_address = /dev/log # -# vm_test_mode = no # daemonize = on # # Time in seconds to wait between replication passes @@ -195,6 +194,10 @@ use = egg:swift#recon # slow down the syncing process. # rsync_compress = no # +# Format of the rysnc module where the replicator will send data. See +# etc/rsyncd.conf-sample for some usage examples. +# rsync_module = {replication_ip}::object +# # node_timeout = # max duration of an http request; this is for REPLICATE finalization calls and # so should be longer than node_timeout diff --git a/etc/rsyncd.conf-sample b/etc/rsyncd.conf-sample index c3b9952b16..103a85a57b 100644 --- a/etc/rsyncd.conf-sample +++ b/etc/rsyncd.conf-sample @@ -20,3 +20,59 @@ max connections = 8 path = /srv/node read only = false lock file = /var/lock/object.lock + + +# If rsync_module includes the device, you can tune rsyncd to permit 4 +# connections per device instead of simply allowing 8 connections for all +# devices: +# rsync_module = {replication_ip}::object_{device} +# +# (if devices in your object ring are named sda, sdb and sdc) +# +#[object_sda] +#max connections = 4 +#path = /srv/node +#read only = false +#lock file = /var/lock/object_sda.lock +# +#[object_sdb] +#max connections = 4 +#path = /srv/node +#read only = false +#lock file = /var/lock/object_sdb.lock +# +#[object_sdc] +#max connections = 4 +#path = /srv/node +#read only = false +#lock file = /var/lock/object_sdc.lock + + +# To emulate the deprecated option vm_test_mode = yes, set: +# rsync_module = {replication_ip}::object{replication_port} +# +# So, on your SAIO, you have to set the following rsyncd configuration: +# +#[object6010] +#max connections = 25 +#path = /srv/1/node/ +#read only = false +#lock file = /var/lock/object6010.lock +# +#[object6020] +#max connections = 25 +#path = /srv/2/node/ +#read only = false +#lock file = /var/lock/object6020.lock +# +#[object6030] +#max connections = 25 +#path = /srv/3/node/ +#read only = false +#lock file = /var/lock/object6030.lock +# +#[object6040] +#max connections = 25 +#path = /srv/4/node/ +#read only = false +#lock file = /var/lock/object6040.lock diff --git a/swift/common/db_replicator.py b/swift/common/db_replicator.py index 7a6e8d549f..71d1634c42 100644 --- a/swift/common/db_replicator.py +++ b/swift/common/db_replicator.py @@ -31,7 +31,8 @@ import swift.common.db from swift.common.direct_client import quote from swift.common.utils import get_logger, whataremyips, storage_directory, \ renamer, mkdirs, lock_parent_directory, config_true_value, \ - unlink_older_than, dump_recon_cache, rsync_ip, ismount, json, Timestamp + unlink_older_than, dump_recon_cache, rsync_module_interpolation, ismount, \ + json, Timestamp from swift.common import ring from swift.common.ring.utils import is_local_device from swift.common.http import HTTP_NOT_FOUND, HTTP_INSUFFICIENT_STORAGE @@ -165,11 +166,20 @@ class Replicator(Daemon): self.max_diffs = int(conf.get('max_diffs') or 100) self.interval = int(conf.get('interval') or conf.get('run_pause') or 30) - self.vm_test_mode = config_true_value(conf.get('vm_test_mode', 'no')) self.node_timeout = int(conf.get('node_timeout', 10)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.rsync_compress = config_true_value( conf.get('rsync_compress', 'no')) + self.rsync_module = conf.get('rsync_module', '').rstrip('/') + if not self.rsync_module: + self.rsync_module = '{replication_ip}::%s' % self.server_type + if config_true_value(conf.get('vm_test_mode', 'no')): + self.logger.warn('Option %(type)s-replicator/vm_test_mode is ' + 'deprecated and will be removed in a future ' + 'version. Update your configuration to use ' + 'option %(type)s-replicator/rsync_module.' + % {'type': self.server_type}) + self.rsync_module += '{replication_port}' self.reclaim_age = float(conf.get('reclaim_age', 86400 * 7)) swift.common.db.DB_PREALLOCATION = \ config_true_value(conf.get('db_preallocation', 'f')) @@ -267,14 +277,9 @@ class Replicator(Daemon): :param different_region: if True, the destination node is in a different region """ - device_ip = rsync_ip(device['replication_ip']) - if self.vm_test_mode: - remote_file = '%s::%s%s/%s/tmp/%s' % ( - device_ip, self.server_type, device['replication_port'], - device['device'], local_id) - else: - remote_file = '%s::%s/%s/tmp/%s' % ( - device_ip, self.server_type, device['device'], local_id) + rsync_module = rsync_module_interpolation(self.rsync_module, device) + rsync_path = '%s/tmp/%s' % (device['device'], local_id) + remote_file = '%s/%s' % (rsync_module, rsync_path) mtime = os.path.getmtime(broker.db_file) if not self._rsync_file(broker.db_file, remote_file, different_region=different_region): diff --git a/swift/common/utils.py b/swift/common/utils.py index 0dbbda6c09..996486916d 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -2703,6 +2703,33 @@ def rsync_ip(ip): return '[%s]' % ip +def rsync_module_interpolation(template, device): + """ + Interpolate devices variables inside a rsync module template + + :param template: rsync module template as a string + :param device: a device from a ring + + :returns: a string with all variables replaced by device attributes + """ + replacements = { + 'ip': rsync_ip(device.get('ip', '')), + 'port': device.get('port', ''), + 'replication_ip': rsync_ip(device.get('replication_ip', '')), + 'replication_port': device.get('replication_port', ''), + 'region': device.get('region', ''), + 'zone': device.get('zone', ''), + 'device': device.get('device', ''), + 'meta': device.get('meta', ''), + } + try: + module = template.format(**replacements) + except KeyError as e: + raise ValueError('Cannot interpolate rsync_module, invalid variable: ' + '%s' % e) + return module + + def get_valid_utf8_str(str_or_unicode): """ Get valid parts of utf-8 str from str, unicode and even invalid utf-8 str diff --git a/swift/obj/replicator.py b/swift/obj/replicator.py index 842a2a859c..124df3e786 100644 --- a/swift/obj/replicator.py +++ b/swift/obj/replicator.py @@ -31,8 +31,8 @@ from eventlet.support.greenlets import GreenletExit from swift.common.ring.utils import is_local_device from swift.common.utils import whataremyips, unlink_older_than, \ compute_eta, get_logger, dump_recon_cache, ismount, \ - rsync_ip, mkdirs, config_true_value, list_from_csv, get_hub, \ - tpool_reraise, config_auto_int_value, storage_directory + rsync_module_interpolation, mkdirs, config_true_value, list_from_csv, \ + get_hub, tpool_reraise, config_auto_int_value, storage_directory from swift.common.bufferedhttp import http_connect from swift.common.daemon import Daemon from swift.common.http import HTTP_OK, HTTP_INSUFFICIENT_STORAGE @@ -62,7 +62,6 @@ class ObjectReplicator(Daemon): self.logger = logger or get_logger(conf, log_route='object-replicator') self.devices_dir = conf.get('devices', '/srv/node') self.mount_check = config_true_value(conf.get('mount_check', 'true')) - self.vm_test_mode = config_true_value(conf.get('vm_test_mode', 'no')) self.swift_dir = conf.get('swift_dir', '/etc/swift') self.bind_ip = conf.get('bind_ip', '0.0.0.0') self.servers_per_port = int(conf.get('servers_per_port', '0') or 0) @@ -81,6 +80,15 @@ class ObjectReplicator(Daemon): self.rsync_bwlimit = conf.get('rsync_bwlimit', '0') self.rsync_compress = config_true_value( conf.get('rsync_compress', 'no')) + self.rsync_module = conf.get('rsync_module', '').rstrip('/') + if not self.rsync_module: + self.rsync_module = '{replication_ip}::object' + if config_true_value(conf.get('vm_test_mode', 'no')): + self.logger.warn('Option object-replicator/vm_test_mode is ' + 'deprecated and will be removed in a future ' + 'version. Update your configuration to use ' + 'option object-replicator/rsync_module.') + self.rsync_module += '{replication_port}' self.http_timeout = int(conf.get('http_timeout', 60)) self.lockup_timeout = int(conf.get('lockup_timeout', 1800)) self.recon_cache_path = conf.get('recon_cache_path', @@ -223,11 +231,7 @@ class ObjectReplicator(Daemon): # Allow for compression, but only if the remote node is in # a different region than the local one. args.append('--compress') - node_ip = rsync_ip(node['replication_ip']) - if self.vm_test_mode: - rsync_module = '%s::object%s' % (node_ip, node['replication_port']) - else: - rsync_module = '%s::object' % node_ip + rsync_module = rsync_module_interpolation(self.rsync_module, node) had_any = False for suffix in suffixes: spath = join(job['path'], suffix) diff --git a/test/probe/common.py b/test/probe/common.py index 45a907444d..37bf885536 100644 --- a/test/probe/common.py +++ b/test/probe/common.py @@ -29,7 +29,8 @@ from six.moves.http_client import HTTPConnection from swiftclient import get_auth, head_account from swift.obj.diskfile import get_data_dir from swift.common.ring import Ring -from swift.common.utils import readconf, renamer +from swift.common.utils import readconf, renamer, \ + config_true_value, rsync_module_interpolation from swift.common.manager import Manager from swift.common.storage_policy import POLICIES, EC_POLICY, REPL_POLICY @@ -219,11 +220,12 @@ def get_ring(ring_name, required_replicas, required_devices, "unable to find ring device %s under %s's devices (%s)" % ( dev['device'], server, conf['devices'])) # verify server is exposing rsync device - if conf.get('vm_test_mode', False): - rsync_export = '%s%s' % (server, dev['replication_port']) - else: - rsync_export = server - cmd = "rsync rsync://localhost/%s" % rsync_export + rsync_export = conf.get('rsync_module', '').rstrip('/') + if not rsync_export: + rsync_export = '{replication_ip}::%s' % server + if config_true_value(conf.get('vm_test_mode', 'no')): + rsync_export += '{replication_port}' + cmd = "rsync %s" % rsync_module_interpolation(rsync_export, dev) p = Popen(cmd, shell=True, stdout=PIPE) stdout, _stderr = p.communicate() if p.returncode: diff --git a/test/unit/common/test_db_replicator.py b/test/unit/common/test_db_replicator.py index 91a5adfbfd..03c3864350 100644 --- a/test/unit/common/test_db_replicator.py +++ b/test/unit/common/test_db_replicator.py @@ -364,10 +364,6 @@ class TestDBReplicator(unittest.TestCase): 'replication_ip': '127.0.0.1', 'replication_port': '0', 'device': 'sda1'} - def mock_rsync_ip(ip): - self.assertEquals(fake_device['ip'], ip) - return 'rsync_ip(%s)' % ip - class MyTestReplicator(TestReplicator): def __init__(self, db_file, remote_file): super(MyTestReplicator, self).__init__({}) @@ -381,20 +377,11 @@ class TestDBReplicator(unittest.TestCase): self_._rsync_file_called = True return False - with patch('swift.common.db_replicator.rsync_ip', mock_rsync_ip): - broker = FakeBroker() - remote_file = 'rsync_ip(127.0.0.1)::container/sda1/tmp/abcd' - replicator = MyTestReplicator(broker.db_file, remote_file) - replicator._rsync_db(broker, fake_device, ReplHttp(), 'abcd') - self.assertTrue(replicator._rsync_file_called) - - with patch('swift.common.db_replicator.rsync_ip', mock_rsync_ip): - broker = FakeBroker() - remote_file = 'rsync_ip(127.0.0.1)::container0/sda1/tmp/abcd' - replicator = MyTestReplicator(broker.db_file, remote_file) - replicator.vm_test_mode = True - replicator._rsync_db(broker, fake_device, ReplHttp(), 'abcd') - self.assertTrue(replicator._rsync_file_called) + broker = FakeBroker() + remote_file = '127.0.0.1::container/sda1/tmp/abcd' + replicator = MyTestReplicator(broker.db_file, remote_file) + replicator._rsync_db(broker, fake_device, ReplHttp(), 'abcd') + self.assertTrue(replicator._rsync_file_called) def test_rsync_db_rsync_file_failure(self): class MyTestReplicator(TestReplicator): diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index 653d939cce..d1fdb1f411 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -2244,6 +2244,58 @@ cluster_dfw1 = http://dfw1.host/v1/ self.assertEqual( utils.rsync_ip('::ffff:192.0.2.128'), '[::ffff:192.0.2.128]') + def test_rsync_module_interpolation(self): + fake_device = {'ip': '127.0.0.1', 'port': 11, + 'replication_ip': '127.0.0.2', 'replication_port': 12, + 'region': '1', 'zone': '2', 'device': 'sda1', + 'meta': 'just_a_string'} + + self.assertEqual( + utils.rsync_module_interpolation('{ip}', fake_device), + '127.0.0.1') + self.assertEqual( + utils.rsync_module_interpolation('{port}', fake_device), + '11') + self.assertEqual( + utils.rsync_module_interpolation('{replication_ip}', fake_device), + '127.0.0.2') + self.assertEqual( + utils.rsync_module_interpolation('{replication_port}', + fake_device), + '12') + self.assertEqual( + utils.rsync_module_interpolation('{region}', fake_device), + '1') + self.assertEqual( + utils.rsync_module_interpolation('{zone}', fake_device), + '2') + self.assertEqual( + utils.rsync_module_interpolation('{device}', fake_device), + 'sda1') + self.assertEqual( + utils.rsync_module_interpolation('{meta}', fake_device), + 'just_a_string') + + self.assertEqual( + utils.rsync_module_interpolation('{replication_ip}::object', + fake_device), + '127.0.0.2::object') + self.assertEqual( + utils.rsync_module_interpolation('{ip}::container{port}', + fake_device), + '127.0.0.1::container11') + self.assertEqual( + utils.rsync_module_interpolation( + '{replication_ip}::object_{device}', fake_device), + '127.0.0.2::object_sda1') + self.assertEqual( + utils.rsync_module_interpolation( + '127.0.0.3::object_{replication_port}', fake_device), + '127.0.0.3::object_12') + + self.assertRaises(ValueError, utils.rsync_module_interpolation, + '{replication_ip}::object_{deivce}', fake_device) + def test_fallocate_reserve(self): class StatVFS(object):