Browse Source

Add option to make probetests more brittle

Currently probetests take advantage of a number of assumptions about the SUT.
Unfortunately after some time with a working SAIO, configuration drift may
result in a system that is no longer compatible with these assumptions.  To
help weary developers more quickly identify the changes they've made since
they last ran probetests successfully, some handy validators have been added
to test.probe.common

Additionally a new option 'validate_rsync' in test.conf, when enabled, will
run a series of up front validations during the setup of each probetest by
inspecting the ring, the mounted devices, and the rsync exports ("modules") in
order to ensure that when probetests fail the do so early and with specific
complaints.

To preserve existing failures, the option is disabled by default.

Change-Id: I2be11c7e67ccd0bc0589c360c170049b6288c152
changes/54/37854/1
Clay Gerrard 8 years ago
parent
commit
1f43ee050b
  1. 2
      test/probe/__init__.py
  2. 66
      test/probe/common.py
  3. 1
      test/sample.conf

2
test/probe/__init__.py

@ -1,3 +1,5 @@
from test import get_config
from swift.common.utils import config_true_value
config = get_config('probe_test')
CHECK_SERVER_TIMEOUT = int(config.get('check_server_timeout', 30))
VALIDATE_RSYNC = config_true_value(config.get('validate_rsync', False))

66
test/probe/common.py

@ -14,16 +14,19 @@
# limitations under the License.
from httplib import HTTPConnection
import os
from os import kill, path
from signal import SIGTERM
from subprocess import Popen, PIPE
import sys
from time import sleep, time
from swiftclient import get_auth, head_account
from swift.common.ring import Ring
from swift.common.utils import readconf
from test.probe import CHECK_SERVER_TIMEOUT
from test.probe import CHECK_SERVER_TIMEOUT, VALIDATE_RSYNC
def start_server(port, port2server, pids, check=True):
@ -130,12 +133,61 @@ def kill_nonprimary_server(primary_nodes, port2server, pids):
return port
def get_ring(server, force_validate=None):
ring = Ring('/etc/swift/%s.ring.gz' % server)
if not VALIDATE_RSYNC and not force_validate:
return ring
# easy sanity checks
assert 3 == ring.replica_count, '%s has %s replicas instead of 3' % (
ring.serialized_path, ring.replica_count)
assert 4 == len(ring.devs), '%s has %s devices instead of 4' % (
ring.serialized_path, len(ring.devs))
# map server to config by port
port_to_config = {}
for node_id in range(1,5):
conf = readconf('/etc/swift/%s-server/%d.conf' % (server, node_id),
section_name='%s-replicator' % server)
port_to_config[int(conf['bind_port'])] = conf
for dev in ring.devs:
# verify server is exposing mounted device
conf = port_to_config[dev['port']]
for device in os.listdir(conf['devices']):
if device == dev['device']:
full_path = path.realpath(path.join(conf['devices'], device))
assert path.ismount(full_path), 'device %s in %s was not ' \
'mounted (%s)' % (device, conf['devices'], full_path)
break
else:
assert False, "unable to find ring device %s " \
"under %s's devices (%s)" % (
dev['device'], server, conf['devices'])
# verify server is exposing rsync device
rsync_export = '%s%s' % (server, dev['replication_port'])
cmd = "rsync rsync://localhost/%s" % rsync_export
p = Popen(cmd, shell=True, stdout=PIPE)
stdout, _stderr = p.communicate()
if p.returncode:
raise AssertionError('unable to connect to rsync '
'export %s (%s)' % (rsync_export, cmd))
for line in stdout.splitlines():
if line.rsplit(None, 1)[-1] == dev['device']:
break
else:
assert False, "unable to find ring device %s under rsync's " \
"exported devices for %s (%s)" % (
dev['device'], rsync_export, cmd)
return ring
def reset_environment():
p = Popen("resetswift 2>&1", shell=True, stdout=PIPE)
stdout, _stderr = p.communicate()
print stdout
pids = {}
try:
account_ring = get_ring('account')
container_ring = get_ring('container')
object_ring = get_ring('object')
port2server = {}
config_dict = {}
for server, port in [('account', 6002), ('container', 6001),
@ -148,15 +200,15 @@ def reset_environment():
check_server(port, port2server, pids)
port2server[8080] = 'proxy'
url, token, account = start_server(8080, port2server, pids)
account_ring = Ring('/etc/swift/account.ring.gz')
container_ring = Ring('/etc/swift/container.ring.gz')
object_ring = Ring('/etc/swift/object.ring.gz')
for name in ('account', 'container', 'object'):
for server in (name, '%s-replicator' % name):
config_dict[server] = '/etc/swift/%s-server/%%d.conf' % name
except BaseException:
try:
raise
except AssertionError, e:
print >>sys.stderr, 'ERROR: %s' % e
os._exit(1)
finally:
try:
kill_servers(port2server, pids)
@ -202,3 +254,9 @@ def get_to_final_state():
'once']))
for process in processes:
process.wait()
if __name__ == "__main__":
for server in ('account', 'container', 'object'):
get_ring(server, force_validate=True)
print '%s OK' % server

1
test/sample.conf

@ -47,3 +47,4 @@ fake_syslog = False
[probe_test]
# check_server_timeout = 30
# validate_rsync = false
Loading…
Cancel
Save