Merge remote-tracking branch 'gerrit/master' into feature/deep

Mainly because I want the new probetests-during-check patch.

Change-Id: Iee0e6ed3bf1a8d8fe0b1c4851b92ffa130eab5cd

.zuul.yaml

@@ -93,6 +93,19 @@
     parent: swift-tox-func-ec
     nodeset: centos-7
 
+- job:
+    name: swift-probetests-centos-7
+    parent: unittests
+    nodeset: centos-7
+    voting: false
+    description: |
+      Setup a SAIO dev environment and run Swift's probe tests
+    pre-run:
+      - playbooks/saio_single_node_setup/install_dependencies.yaml
+      - playbooks/saio_single_node_setup/setup_saio.yaml
+      - playbooks/saio_single_node_setup/make_rings.yaml
+    run: playbooks/probetests/run.yaml
+
 - project:
     check:
       jobs:
@@ -101,6 +114,7 @@
         - swift-tox-func
         - swift-tox-func-encryption
         - swift-tox-func-ec
+        - swift-probetests-centos-7
     gate:
       jobs:
         - swift-tox-py27

playbooks/probetests/run.yaml

@@ -0,0 +1,26 @@
+
+
+# Copyright (c) 2018 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+- hosts: all
+
+  tasks:
+    - name: run probe tests
+      shell:
+        cmd: |
+          source ~/.bashrc
+          nosetests test/probe/
+        executable: /bin/bash
+        chdir: '{{ zuul.project.src_dir }}'

playbooks/saio_single_node_setup/install_dependencies.yaml

@@ -0,0 +1,24 @@
+# Copyright (c) 2018 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+- hosts: all
+  become: true
+  tasks:
+    - name: installing dependencies
+      yum: name={{ item }} state=present
+      with_items:
+        - python-eventlet
+        - python-pyeclib
+        - python-nose
+        - python-swiftclient

playbooks/saio_single_node_setup/make_rings.yaml

@@ -0,0 +1,29 @@
+
+# Copyright (c) 2018 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+- hosts: all
+  tasks:
+    - name: install swift
+      become: true
+      shell:
+        cmd: python setup.py develop
+        executable: /bin/bash
+        chdir: '{{ zuul.project.src_dir }}'
+
+    - name: make rings
+      shell:
+        cmd: remakerings
+        executable: /bin/bash
+        chdir: '/etc/swift'

playbooks/saio_single_node_setup/setup_saio.yaml

@@ -0,0 +1,174 @@
+# Copyright (c) 2018 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+- hosts: all
+  become: true
+  tasks:
+    - name: assure /srv directory exists
+      file: path=/srv state=directory
+
+    - name: create loopback device
+      command: truncate -s 1GB /srv/swift-disk creates=/srv/swift-disk
+
+    - name: create filesystem /srv/swift-disk
+      become: true
+      filesystem: fstype=xfs dev=/srv/swift-disk
+
+    - name: create mount path /mnt/sdb1
+      file: path=/mnt/sdb1 state=directory
+
+    - name: mount /mnt/sdb1
+      mount: name=/mnt/sdb1 src=/srv/swift-disk fstype=xfs opts="loop,noatime,nodiratime,nobarrier,logbufs=8" dump=0 passno=0 state=mounted
+
+    - name: create sub-partitions
+      file: >
+        path=/mnt/sdb1/{{ item }}
+        state=directory
+        owner={{ ansible_user_id }}
+        group={{ ansible_user_gid }}
+      with_items:
+        - 1
+        - 2
+        - 3
+        - 4
+
+    - name: create symlinks
+      become: true
+      file: >
+        src=/mnt/sdb1/{{ item }}
+        dest=/srv/{{ item }}
+        owner={{ ansible_user_id }}
+        group={{ ansible_user_gid }}
+        state=link
+      with_items:
+        - 1
+        - 2
+        - 3
+        - 4
+
+    - name: create node partition directories
+      file: >
+        path=/srv/{{ item[1] }}/node/sdb{{ item[0] + item[1] }}
+        owner={{ ansible_user_id }}
+        group={{ ansible_user_gid }}
+        state=directory
+      with_nested:
+        - [0, 4]
+        - [1, 2, 3, 4]
+
+    - name: create /var/run/swift
+      file: >
+        path=/var/run/swift
+        owner={{ ansible_user_id }}
+        group={{ ansible_user_gid }}
+        state=directory
+
+    - name: create /var/cache/swift
+      file: >
+        path=/var/cache/swift
+        owner={{ ansible_user_id }}
+        group={{ ansible_user_gid }}
+        state=directory
+
+    - name: create /var/cache/swift[n]
+      file: >
+        path=/var/cache/swift{{ item }}
+        owner={{ ansible_user_id }}
+        group={{ ansible_user_gid }}
+        state=directory
+      with_items:
+        - 2
+        - 3
+        - 4
+
+    - name: create rc.local from template
+      template: src=rc.local.j2 dest=/etc/rc.d/rc.local owner=root group=root mode=0755
+
+    - name: create /etc/rsyncd.conf
+      command: cp {{ zuul.project.src_dir }}/doc/saio/rsyncd.conf /etc/
+
+    - name: update rsyncd.conf with correct username
+      replace: dest=/etc/rsyncd.conf regexp=<your-user-name> replace={{ ansible_user_id }}
+
+    - name: enable rsync
+      lineinfile: dest=/etc/xinetd.d/rsync line="disable = no" create=yes
+
+    - name: set selinux to permissive
+      selinux: policy=targeted state=disabled
+
+    - name: restart rsync
+      service: name=rsyncd state=restarted enabled=yes
+
+    - name: start memcache
+      service: name=memcached state=started enabled=yes
+
+    - name: configure rsyslog
+      command: cp {{ zuul.project.src_dir }}/doc/saio/rsyslog.d/10-swift.conf /etc/rsyslog.d/
+
+    - name: modify /etc/rsyslog.conf
+      lineinfile: dest=/etc/rsyslog.conf
+                  line="$PrivDropToGroup adm"
+                  create=yes
+                  insertafter="^#### GLOBAL DIRECTIVES"
+
+    - name: assure /var/log/swift directory exists
+      file: path=/var/log/swift
+            state=directory
+            owner=root
+            group=adm
+            mode="g+w"
+
+    - name: restart rsyslog
+      service: name=rsyslog state=restarted enabled=yes
+
+    - name: clean up /etc/swift directory
+      file: path=/etc/swift state=absent
+
+    - name: create clean /etc/swift
+      command: cp -r {{ zuul.project.src_dir }}/doc/saio/swift /etc/swift
+
+    - name: copy the sample configuration files for running tests
+      command: cp -r {{ zuul.project.src_dir }}/test/sample.conf /etc/swift/test.conf
+
+    - name: set correct ownership of /etc/swift
+      file: path=/etc/swift owner={{ ansible_user_id }} group={{ ansible_user_gid }} recurse=yes
+
+    - name: find config files to modify user option
+      find: paths="/etc/swift" patterns="*.conf" recurse=yes
+      register: find_result
+
+    - name: replace user name
+      replace: dest={{ item.path }} regexp=<your-user-name> replace={{ ansible_user_id }}
+      with_items: "{{ find_result.files }}"
+
+    - name: copy the SAIO scripts for resetting the environment
+      command: cp -r {{ zuul.project.src_dir }}/doc/saio/bin /home/{{ ansible_ssh_user }}/bin creates=/home/{{ ansible_ssh_user }}/bin
+
+    - name: set the correct file mode for SAIO scripts
+      file: dest=/home/{{ ansible_ssh_user }}/bin mode=0777 recurse=yes
+
+    - name: add new env. variable for loopback device
+      lineinfile: dest=/home/{{ ansible_ssh_user }}/.bashrc line="export SAIO_BLOCK_DEVICE=/srv/swift-disk"
+
+    - name: remove line from resetswift
+      lineinfile: dest=/home/{{ ansible_ssh_user }}/bin/resetswift line="sudo find /var/log/swift -type f -exec rm -f {} \;" state=absent
+
+    - name: add new env. variable for running tests
+      lineinfile: dest=/home/{{ ansible_ssh_user }}/.bashrc line="export SWIFT_TEST_CONFIG_FILE=/etc/swift/test.conf"
+
+    - name: make sure PATH includes the bin directory
+      lineinfile: dest=/home/{{ ansible_ssh_user }}/.bashrc line="export PATH=${PATH}:/home/{{ ansible_ssh_user }}/bin"
+
+    - name: increase open files limit to run probe tests
+      lineinfile: dest=/home/{{ ansible_ssh_user }}/.bashrc line="ulimit -n 4096"

playbooks/saio_single_node_setup/templates/rc.local.j2

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+mkdir -p /var/cache/swift /var/cache/swift2 /var/cache/swift3 /var/cache/swift4
+chown {{ ansible_user_id }}:{{ ansible_user_gid }} /var/cache/swift*
+mkdir -p /var/run/swift
+chown {{ ansible_user_id }}:{{ ansible_user_gid }} /var/run/swift
+
+exit 0

swift/common/db_replicator.py

@@ -69,6 +69,17 @@ def quarantine_db(object_file, server_type):
         renamer(object_dir, quarantine_dir, fsync=False)
 
 
+def looks_like_partition(dir_name):
+    """
+    True if the directory name is a valid partition number, False otherwise.
+    """
+    try:
+        part = int(dir_name)
+        return part >= 0
+    except ValueError:
+        return False
+
+
 def roundrobin_datadirs(datadirs):
     """
     Generator to walk the data dirs in a round robin manner, evenly
@@ -81,7 +92,8 @@ def roundrobin_datadirs(datadirs):
     """
 
     def walk_datadir(datadir, node_id):
-        partitions = os.listdir(datadir)
+        partitions = [pd for pd in os.listdir(datadir)
+                      if looks_like_partition(pd)]
         random.shuffle(partitions)
         for partition in partitions:
             part_dir = os.path.join(datadir, partition)

swift/common/wsgi.py

@@ -119,6 +119,8 @@ class ConfigString(NamedConfigLoader):
         }
         self.parser = loadwsgi.NicerConfigParser("string", defaults=defaults)
         self.parser.optionxform = str  # Don't lower-case keys
+        # Defaults don't need interpolation (crazy PasteDeploy...)
+        self.parser.defaults = lambda: dict(self.parser._defaults, **defaults)
         self.parser.readfp(self.contents)
 
 

swift/obj/diskfile.py

@@ -608,10 +608,10 @@ class DiskFileRouter(object):
         self.policy_to_manager = {}
         for policy in POLICIES:
             manager_cls = self.policy_type_to_manager_cls[policy.policy_type]
-            self.policy_to_manager[policy] = manager_cls(*args, **kwargs)
+            self.policy_to_manager[int(policy)] = manager_cls(*args, **kwargs)
 
     def __getitem__(self, policy):
-        return self.policy_to_manager[policy]
+        return self.policy_to_manager[int(policy)]
 
 
 class BaseDiskFileManager(object):

swift/obj/replicator.py

@@ -169,7 +169,8 @@ class ObjectReplicator(Daemon):
         :returns: return code of rsync process. 0 is successful
         """
         start_time = time.time()
-        ret_val = None
+        proc = ret_val = None
+
         try:
             with Timeout(self.rsync_timeout):
                 proc = subprocess.Popen(args,
@@ -177,9 +178,24 @@ class ObjectReplicator(Daemon):
                                         stderr=subprocess.STDOUT)
                 results = proc.stdout.read()
                 ret_val = proc.wait()
+        except GreenletExit:
+            self.logger.error(_("Killing by lockup detector"))
+            if proc:
+                # Assume rsync is still responsive and give it a chance
+                # to shut down gracefully
+                proc.terminate()
+                # Final good-faith effort to clean up the process table.
+                # Note that this blocks, but worst-case we wait for the
+                # lockup detector to come around and kill us. This can
+                # happen if the process is stuck down in kernel-space
+                # waiting on I/O or something.
+                proc.wait()
+            raise
         except Timeout:
             self.logger.error(_("Killing long-running rsync: %s"), str(args))
-            proc.kill()
+            if proc:
+                proc.kill()
+                proc.wait()
             return 1  # failure response code
         total_time = time.time() - start_time
         for result in results.split('\n'):

swift/obj/server.py

@@ -15,6 +15,7 @@
 
 """ Object Server for Swift """
 
+import six
 import six.moves.cPickle as pickle
 import json
 import os
@@ -170,7 +171,9 @@ class ObjectController(BaseStorageServer):
         # disk_chunk_size parameter. However, it affects all created sockets
         # using this class so we have chosen to tie it to the
         # network_chunk_size parameter value instead.
-        socket._fileobject.default_bufsize = self.network_chunk_size
+        if six.PY2:
+            socket._fileobject.default_bufsize = self.network_chunk_size
+        # TODO: find a way to enable similar functionality in py3
 
         # Provide further setup specific to an object server implementation.
         self.setup(conf)

swift/proxy/controllers/obj.py

@@ -151,11 +151,11 @@ class ObjectControllerRouter(object):
     def __init__(self):
         self.policy_to_controller_cls = {}
         for policy in POLICIES:
-            self.policy_to_controller_cls[policy] = \
+            self.policy_to_controller_cls[int(policy)] = \
                 self.policy_type_to_controller_map[policy.policy_type]
 
     def __getitem__(self, policy):
-        return self.policy_to_controller_cls[policy]
+        return self.policy_to_controller_cls[int(policy)]
 
 
 class BaseObjectController(Controller):

swift/proxy/server.py

@@ -276,7 +276,10 @@ class Application(object):
         #
         # ** Because it affects the client as well, currently, we use the
         # client chunk size as the govenor and not the object chunk size.
-        socket._fileobject.default_bufsize = self.client_chunk_size
+        if sys.version_info < (3,):
+            socket._fileobject.default_bufsize = self.client_chunk_size
+        # TODO: find a way to enable similar functionality in py3
+
         self.expose_info = config_true_value(
             conf.get('expose_info', 'yes'))
         self.disallowed_sections = list_from_csv(

test/functional/swift_test_client.py

@@ -543,7 +543,7 @@ class Container(Base):
     def delete_files(self):
         for f in listing_items(self.files):
             file_item = self.file(f)
-            if not file_item.delete():
+            if not file_item.delete(tolerate_missing=True):
                 return False
 
         return listing_empty(self.files)
@@ -764,14 +764,19 @@ class File(Base):
                                 self.conn.make_path(self.path))
         return True
 
-    def delete(self, hdrs=None, parms=None, cfg=None):
+    def delete(self, hdrs=None, parms=None, cfg=None, tolerate_missing=False):
         if hdrs is None:
             hdrs = {}
         if parms is None:
             parms = {}
-        if self.conn.make_request('DELETE', self.path, hdrs=hdrs,
+        if tolerate_missing:
-                                  cfg=cfg, parms=parms) != 204:
+            allowed_statuses = (204, 404)
+        else:
+            allowed_statuses = (204,)
 
+        if self.conn.make_request(
+                'DELETE', self.path, hdrs=hdrs, cfg=cfg,
+                parms=parms) not in allowed_statuses:
             raise ResponseError(self.conn.response, 'DELETE',
                                 self.conn.make_path(self.path))
 

test/unit/common/test_db_replicator.py

@@ -1323,9 +1323,11 @@ class TestDBReplicator(unittest.TestCase):
                 return []
             path = path[len('/srv/node/sdx/containers'):]
             if path == '':
-                return ['123', '456', '789', '9999']
+                return ['123', '456', '789', '9999', "-5", "not-a-partition"]
                 # 456 will pretend to be a file
                 # 9999 will be an empty partition with no contents
+                # -5 and not-a-partition were created by something outside
+                #   Swift
             elif path == '/123':
                 return ['abc', 'def.db']  # def.db will pretend to be a file
             elif path == '/123/abc':
@@ -1349,6 +1351,10 @@ class TestDBReplicator(unittest.TestCase):
                         'weird2']  # weird2 will pretend to be a dir, if asked
             elif path == '9999':
                 return []
+            elif path == 'not-a-partition':
+                raise Exception("shouldn't look in not-a-partition")
+            elif path == '-5':
+                raise Exception("shouldn't look in -5")
             return []
 
         def _isdir(path):

test/unit/common/test_utils.py

@@ -624,16 +624,7 @@ class TestTimestamp(unittest.TestCase):
                             '%r is not greater than %r given %r' % (
                                 timestamp, int(other), value))
 
-    def test_greater_with_offset(self):
+    def _test_greater_with_offset(self, now, test_values):
-        now = time.time()
-        older = now - 1
-        test_values = (
-            0, '0', 0.0, '0.0', '0000.0000', '000.000_000',
-            1, '1', 1.1, '1.1', '1111.1111', '111.111_111',
-            1402443346.935174, '1402443346.93517', '1402443346.935169_ffff',
-            older, '%f' % older, '%f_0000ffff' % older,
-            now, '%f' % now, '%f_00000000' % now,
-        )
         for offset in range(1, 1000, 100):
             timestamp = utils.Timestamp(now, offset=offset)
             for value in test_values:
@@ -658,6 +649,43 @@ class TestTimestamp(unittest.TestCase):
                                 '%r is not greater than %r given %r' % (
                                     timestamp, int(other), value))
 
+    def test_greater_with_offset(self):
+        # Part 1: use the natural time of the Python. This is deliciously
+        # unpredictable, but completely legitimate and realistic. Finds bugs!
+        now = time.time()
+        older = now - 1
+        test_values = (
+            0, '0', 0.0, '0.0', '0000.0000', '000.000_000',
+            1, '1', 1.1, '1.1', '1111.1111', '111.111_111',
+            1402443346.935174, '1402443346.93517', '1402443346.935169_ffff',
+            older, now,
+        )
+        self._test_greater_with_offset(now, test_values)
+        # Part 2: Same as above, but with fixed time values that reproduce
+        # specific corner cases.
+        now = 1519830570.6949348
+        older = now - 1
+        test_values = (
+            0, '0', 0.0, '0.0', '0000.0000', '000.000_000',
+            1, '1', 1.1, '1.1', '1111.1111', '111.111_111',
+            1402443346.935174, '1402443346.93517', '1402443346.935169_ffff',
+            older, now,
+        )
+        self._test_greater_with_offset(now, test_values)
+        # Part 3: The '%f' problem. Timestamps cannot be converted to %f
+        # strings, then back to timestamps, then compared with originals.
+        # You can only "import" a floating point representation once.
+        now = 1519830570.6949348
+        now = float('%f' % now)
+        older = now - 1
+        test_values = (
+            0, '0', 0.0, '0.0', '0000.0000', '000.000_000',
+            1, '1', 1.1, '1.1', '1111.1111', '111.111_111',
+            older, '%f' % older, '%f_0000ffff' % older,
+            now, '%f' % now, '%s_00000000' % now,
+        )
+        self._test_greater_with_offset(now, test_values)
+
     def test_smaller_no_offset(self):
         now = time.time()
         newer = now + 1

test/unit/common/test_wsgi.py

@@ -203,7 +203,7 @@ class TestWSGI(unittest.TestCase):
         conf_file = os.path.join(tempdir, 'file.conf')
 
         def _write_and_load_conf_file(conf):
-            with open(conf_file, 'wb') as fd:
+            with open(conf_file, 'wt') as fd:
                 fd.write(dedent(conf))
             return wsgi.load_app_config(conf_file)
 
@@ -659,12 +659,12 @@ class TestWSGI(unittest.TestCase):
         oldenv = {}
         newenv = wsgi.make_pre_authed_env(oldenv)
         self.assertTrue('wsgi.input' in newenv)
-        self.assertEqual(newenv['wsgi.input'].read(), '')
+        self.assertEqual(newenv['wsgi.input'].read(), b'')
 
         oldenv = {'wsgi.input': BytesIO(b'original wsgi.input')}
         newenv = wsgi.make_pre_authed_env(oldenv)
         self.assertTrue('wsgi.input' in newenv)
-        self.assertEqual(newenv['wsgi.input'].read(), '')
+        self.assertEqual(newenv['wsgi.input'].read(), b'')
 
         oldenv = {'swift.source': 'UT'}
         newenv = wsgi.make_pre_authed_env(oldenv)
@@ -677,7 +677,7 @@ class TestWSGI(unittest.TestCase):
     def test_pre_auth_req(self):
         class FakeReq(object):
             @classmethod
-            def fake_blank(cls, path, environ=None, body='', headers=None):
+            def fake_blank(cls, path, environ=None, body=b'', headers=None):
                 if environ is None:
                     environ = {}
                 if headers is None:
@@ -687,7 +687,7 @@ class TestWSGI(unittest.TestCase):
         was_blank = Request.blank
         Request.blank = FakeReq.fake_blank
         wsgi.make_pre_authed_request({'HTTP_X_TRANS_ID': '1234'},
-                                     'PUT', '/', body='tester', headers={})
+                                     'PUT', '/', body=b'tester', headers={})
         wsgi.make_pre_authed_request({'HTTP_X_TRANS_ID': '1234'},
                                      'PUT', '/', headers={})
         Request.blank = was_blank
@@ -695,7 +695,7 @@ class TestWSGI(unittest.TestCase):
     def test_pre_auth_req_with_quoted_path(self):
         r = wsgi.make_pre_authed_request(
             {'HTTP_X_TRANS_ID': '1234'}, 'PUT', path=quote('/a space'),
-            body='tester', headers={})
+            body=b'tester', headers={})
         self.assertEqual(r.path, quote('/a space'))
 
     def test_pre_auth_req_drops_query(self):
@@ -711,8 +711,8 @@ class TestWSGI(unittest.TestCase):
 
     def test_pre_auth_req_with_body(self):
         r = wsgi.make_pre_authed_request(
-            {'QUERY_STRING': 'original'}, 'GET', 'path', 'the body')
+            {'QUERY_STRING': 'original'}, 'GET', 'path', b'the body')
-        self.assertEqual(r.body, 'the body')
+        self.assertEqual(r.body, b'the body')
 
     def test_pre_auth_creates_script_name(self):
         e = wsgi.make_pre_authed_env({})
@@ -730,9 +730,9 @@ class TestWSGI(unittest.TestCase):
 
     def test_pre_auth_req_swift_source(self):
         r = wsgi.make_pre_authed_request(
-            {'QUERY_STRING': 'original'}, 'GET', 'path', 'the body',
+            {'QUERY_STRING': 'original'}, 'GET', 'path', b'the body',
             swift_source='UT')
-        self.assertEqual(r.body, 'the body')
+        self.assertEqual(r.body, b'the body')
         self.assertEqual(r.environ['swift.source'], 'UT')
 
     def test_run_server_global_conf_callback(self):
@@ -1363,7 +1363,8 @@ class TestWSGIContext(unittest.TestCase):
         self.assertEqual('aaaaa', next(iterator))
         self.assertEqual('bbbbb', next(iterator))
         iterable.close()
-        self.assertRaises(StopIteration, iterator.next)
+        with self.assertRaises(StopIteration):
+            next(iterator)
 
     def test_update_content_length(self):
         statuses = ['200 Ok']

test/unit/obj/test_replicator.py

@@ -27,10 +27,11 @@ from collections import defaultdict
 from errno import ENOENT, ENOTEMPTY, ENOTDIR
 
 from eventlet.green import subprocess
-from eventlet import Timeout
+from eventlet import Timeout, sleep
 
 from test.unit import (debug_logger, patch_policies, make_timestamp_iter,
-                       mocked_http_conn, mock_check_drive, skip_if_no_xattrs)
+                       mocked_http_conn, mock_check_drive, skip_if_no_xattrs,
+                       SkipTest)
 from swift.common import utils
 from swift.common.utils import (hash_path, mkdirs, normalize_timestamp,
                                 storage_directory)
@@ -132,6 +133,32 @@ def _mock_process(ret):
     object_replicator.subprocess.Popen = orig_process
 
 
+class MockHungProcess(object):
+    def __init__(self, *args, **kwargs):
+        class MockStdout(object):
+            def read(self):
+                pass
+        self.stdout = MockStdout()
+        self._state = 'running'
+        self._calls = []
+
+    def wait(self):
+        self._calls.append(('wait', self._state))
+        if self._state == 'running':
+            # Sleep so we trip either the lockup detector or the rsync timeout
+            sleep(1)
+            raise BaseException('You need to mock out some timeouts')
+
+    def terminate(self):
+        self._calls.append(('terminate', self._state))
+        if self._state == 'running':
+            self._state = 'terminating'
+
+    def kill(self):
+        self._calls.append(('kill', self._state))
+        self._state = 'killed'
+
+
 def _create_test_rings(path, devs=None, next_part_power=None):
     testgz = os.path.join(path, 'object.ring.gz')
     intended_replica2part2dev_id = [
@@ -2009,6 +2036,68 @@ class TestObjectReplicator(unittest.TestCase):
         self.assertIn(
             "next_part_power set in policy 'one'. Skipping", warnings)
 
+    def test_replicate_lockup_detector(self):
+        raise SkipTest("this is not a reliable test and must be fixed")
+        cur_part = '0'
+        df = self.df_mgr.get_diskfile('sda', cur_part, 'a', 'c', 'o',
+                                      policy=POLICIES[0])
+        mkdirs(df._datadir)
+        f = open(os.path.join(df._datadir,
+                              normalize_timestamp(time.time()) + '.data'),
+                 'wb')
+        f.write('1234567890')
+        f.close()
+
+        mock_procs = []
+
+        def new_mock(*a, **kw):
+            proc = MockHungProcess()
+            mock_procs.append(proc)
+            return proc
+
+        with mock.patch('swift.obj.replicator.http_connect',
+                        mock_http_connect(200)), \
+                mock.patch.object(self.replicator, 'lockup_timeout', 0.01), \
+                mock.patch('eventlet.green.subprocess.Popen', new_mock):
+            self.replicator.replicate()
+        for proc in mock_procs:
+            self.assertEqual(proc._calls, [
+                ('wait', 'running'),
+                ('terminate', 'running'),
+                ('wait', 'terminating'),
+            ])
+        self.assertEqual(len(mock_procs), 1)
+
+    def test_replicate_rsync_timeout(self):
+        cur_part = '0'
+        df = self.df_mgr.get_diskfile('sda', cur_part, 'a', 'c', 'o',
+                                      policy=POLICIES[0])
+        mkdirs(df._datadir)
+        f = open(os.path.join(df._datadir,
+                              normalize_timestamp(time.time()) + '.data'),
+                 'wb')
+        f.write('1234567890')
+        f.close()
+
+        mock_procs = []
+
+        def new_mock(*a, **kw):
+            proc = MockHungProcess()
+            mock_procs.append(proc)
+            return proc
+
+        with mock.patch('swift.obj.replicator.http_connect',
+                        mock_http_connect(200)), \
+                mock.patch.object(self.replicator, 'rsync_timeout', 0.01), \
+                mock.patch('eventlet.green.subprocess.Popen', new_mock):
+            self.replicator.replicate()
+        for proc in mock_procs:
+            self.assertEqual(proc._calls, [
+                ('wait', 'running'),
+                ('kill', 'running'),
+                ('wait', 'killed'),
+            ])
+        self.assertEqual(len(mock_procs), 2)
 
 if __name__ == '__main__':
     unittest.main()

tools/test-setup.sh

@@ -25,6 +25,6 @@ function is_rhel7 {
 if is_rhel7; then
     # Install CentOS OpenStack repos so that we have access to some extra
     # packages.
-    sudo yum install -y centos-release-openstack-pike
+    sudo yum install -y centos-release-openstack-queens
     sudo yum install -y liberasurecode-devel
 fi

tox.ini

@@ -40,7 +40,8 @@ commands =
       test/unit/common/test_manager.py \
       test/unit/common/test_splice.py \
       test/unit/common/test_storage_policy.py \
-      test/unit/common/test_utils.py
+      test/unit/common/test_utils.py \
+      test/unit/common/test_wsgi.py
 
 [testenv:py35]
 commands = {[testenv:py34]commands}