Move the object client to tempest.lib

Move the object client to tempest.lib and cleanup the last bits
of code that were required to manage internal-only tempest service
clients.

Change-Id: Ia9aeab78e530c798dfa7b7d6f71e4b3ea3a84b7f

releasenotes/notes/make-object-storage-client-as-stable-interface-d1b07c7e8f17bef6.yaml

@@ -8,3 +8,4 @@ features:
 
       * account_client
       * container_client
+      * object_client

tempest/clients.py

@@ -17,7 +17,6 @@ from tempest import config
 from tempest.lib import auth
 from tempest.lib import exceptions as lib_exc
 from tempest.lib.services import clients
-from tempest.services import object_storage
 
 CONF = config.CONF
 
@@ -281,21 +280,11 @@ class Manager(clients.ServiceClients):
             self.snapshots_client_latest = self.snapshots_v3_client
 
     def _set_object_storage_clients(self):
-        # NOTE(andreaf) Load configuration from config. Once object storage
+        self.account_client = self.object_storage.AccountClient()
-        # is in lib, configuration will be pulled directly from the registry
+        self.bulk_client = self.object_storage.BulkMiddlewareClient()
-        # and this will not be required anymore.
+        self.capabilities_client = self.object_storage.CapabilitiesClient()
-        params = config.service_client_config('object-storage')
+        self.container_client = self.object_storage.ContainerClient()
-
+        self.object_client = self.object_storage.ObjectClient()
-        self.account_client = object_storage.AccountClient(self.auth_provider,
-                                                           **params)
-        self.bulk_client = object_storage.BulkMiddlewareClient(
-            self.auth_provider, **params)
-        self.capabilities_client = object_storage.CapabilitiesClient(
-            self.auth_provider, **params)
-        self.container_client = object_storage.ContainerClient(
-            self.auth_provider, **params)
-        self.object_client = object_storage.ObjectClient(self.auth_provider,
-                                                         **params)
 
 
 def get_auth_provider_class(credentials):

tempest/cmd/verify_tempest_config.py

@@ -76,7 +76,6 @@ from tempest.common import credentials_factory as credentials
 from tempest import config
 import tempest.lib.common.http
 from tempest.lib import exceptions as lib_exc
-from tempest.services import object_storage
 
 
 CONF = config.CONF
@@ -236,11 +235,10 @@ def verify_api_versions(os, service, update):
 
 
 def get_extension_client(os, service):
-    params = config.service_client_config('object-storage')
     extensions_client = {
         'nova': os.compute.ExtensionsClient(),
         'neutron': os.network.ExtensionsClient(),
-        'swift': object_storage.CapabilitiesClient(os.auth_provider, **params),
+        'swift': os.object_storage.CapabilitiesClient(),
         # NOTE: Cinder v3 API is current and v2 and v1 are deprecated.
         # V3 extension API is the same as v2, so we reuse the v2 client
         # for v3 API also.

tempest/config.py

@@ -1384,7 +1384,7 @@ def _register_tempest_service_clients():
         module = service_clients[service_client]
         configs = service_client.split('.')[0]
         service_client_data = dict(
-            name=service_client.replace('.', '_'),
+            name=service_client.replace('.', '_').replace('-', '_'),
             service_version=service_client,
             module_path=module.__name__,
             client_names=module.__all__,

tempest/lib/services/clients.py

@@ -31,6 +31,7 @@ from tempest.lib.services import compute
 from tempest.lib.services import identity
 from tempest.lib.services import image
 from tempest.lib.services import network
+from tempest.lib.services import object_storage
 from tempest.lib.services import volume
 
 warnings.simplefilter("once")
@@ -50,20 +51,13 @@ def tempest_modules():
         'image.v1': image.v1,
         'image.v2': image.v2,
         'network': network,
+        'object-storage': object_storage,
         'volume.v1': volume.v1,
         'volume.v2': volume.v2,
         'volume.v3': volume.v3
     }
 
 
-def _tempest_internal_modules():
-    # Set of unstable service clients available in Tempest
-    # NOTE(andreaf) This list will exists only as long the remain clients
-    # are migrated to tempest.lib, and it will then be deleted without
-    # deprecation or advance notice
-    return set(['object-storage'])
-
-
 def available_modules():
     """Set of service client modules available in Tempest and plugins
 
@@ -101,17 +95,6 @@ def available_modules():
                                                 plug_service_versions))
                 name_conflicts.append(exceptions.PluginRegistrationException(
                     name=plugin_name, detailed_error=detailed_error))
-            # NOTE(andreaf) Once all tempest clients are stable, the following
-            # if will have to be removed.
-            if not plug_service_versions.isdisjoint(
-                    _tempest_internal_modules()):
-                detailed_error = (
-                    'Plugin %s is trying to register a service %s already '
-                    'claimed by a Tempest one' % (plugin_name,
-                                                  _tempest_internal_modules() &
-                                                  plug_service_versions))
-                name_conflicts.append(exceptions.PluginRegistrationException(
-                    name=plugin_name, detailed_error=detailed_error))
         extra_service_versions |= plug_service_versions
     if name_conflicts:
         LOG.error(
@@ -369,7 +352,7 @@ class ServiceClients(object):
         client_parameters = client_parameters or {}
         self.parameters = {}
         # Parameters are provided for unversioned services
-        all_modules = available_modules() | _tempest_internal_modules()
+        all_modules = available_modules()
         unversioned_services = set(
             [x.split('.')[0] for x in all_modules])
         for service in unversioned_services:
@@ -456,9 +439,7 @@ class ServiceClients(object):
 
     @property
     def registered_services(self):
-        # NOTE(andreaf) Once all tempest modules are stable this needs to
+        return self._registered_services
-        # be updated to remove _tempest_internal_modules
-        return self._registered_services | _tempest_internal_modules()
 
     def _setup_parameters(self, parameters):
         """Setup default values for client parameters

tempest/lib/services/object_storage/__init__.py

@@ -0,0 +1,25 @@
+# Copyright (c) 2016 Hewlett-Packard Enterprise Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+from tempest.lib.services.object_storage.account_client import AccountClient
+from tempest.lib.services.object_storage.bulk_middleware_client import \
+    BulkMiddlewareClient
+from tempest.lib.services.object_storage.capabilities_client import \
+    CapabilitiesClient
+from tempest.lib.services.object_storage.container_client import \
+    ContainerClient
+from tempest.lib.services.object_storage.object_client import ObjectClient
+
+__all__ = ['AccountClient', 'BulkMiddlewareClient', 'CapabilitiesClient',
+           'ContainerClient', 'ObjectClient']

tempest/services/object_storage/__init__.py

@@ -1,25 +0,0 @@
-# Copyright (c) 2016 Hewlett-Packard Enterprise Development Company, L.P.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may not
-# use this file except in compliance with the License. You may obtain a copy of
-# the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations under
-# the License.
-
-from tempest.lib.services.object_storage.account_client import AccountClient
-from tempest.lib.services.object_storage.bulk_middleware_client import \
-    BulkMiddlewareClient
-from tempest.lib.services.object_storage.capabilities_client import \
-    CapabilitiesClient
-from tempest.lib.services.object_storage.container_client import \
-    ContainerClient
-from tempest.services.object_storage.object_client import ObjectClient
-
-__all__ = ['AccountClient', 'BulkMiddlewareClient', 'CapabilitiesClient',
-           'ContainerClient', 'ObjectClient']

tempest/tests/lib/services/object_storage/test_object_client.py

@@ -0,0 +1,108 @@
+# Copyright 2016 IBM Corp.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+import mock
+
+from tempest.lib import exceptions
+from tempest.lib.services.object_storage import object_client
+from tempest.tests import base
+from tempest.tests.lib import fake_auth_provider
+
+
+class TestObjectClient(base.TestCase):
+
+    def setUp(self):
+        super(TestObjectClient, self).setUp()
+        self.fake_auth = fake_auth_provider.FakeAuthProvider()
+        self.url = self.fake_auth.base_url(None)
+        self.object_client = object_client.ObjectClient(self.fake_auth,
+                                                        'swift', 'region1')
+
+    @mock.patch.object(object_client, '_create_connection')
+    def test_create_object_continue_no_data(self, mock_poc):
+        self._validate_create_object_continue(None, mock_poc)
+
+    @mock.patch.object(object_client, '_create_connection')
+    def test_create_object_continue_with_data(self, mock_poc):
+        self._validate_create_object_continue('hello', mock_poc)
+
+    @mock.patch.object(object_client, '_create_connection')
+    def test_create_continue_with_no_continue_received(self, mock_poc):
+        self._validate_create_object_continue('hello', mock_poc,
+                                              initial_status=201)
+
+    def _validate_create_object_continue(self, req_data,
+                                         mock_poc, initial_status=100):
+
+        expected_hdrs = {
+            'X-Auth-Token': self.fake_auth.get_token(),
+            'content-length': 0 if req_data is None else len(req_data),
+            'Expect': '100-continue'}
+
+        # Setup the Mocks prior to invoking the object creation
+        mock_resp_cls = mock.Mock()
+        mock_resp_cls._read_status.return_value = ("1", initial_status, "OK")
+
+        mock_poc.return_value.response_class.return_value = mock_resp_cls
+
+        # This is the final expected return value
+        mock_poc.return_value.getresponse.return_value.status = 201
+        mock_poc.return_value.getresponse.return_value.reason = 'OK'
+
+        # Call method to PUT object using expect:100-continue
+        cnt = "container1"
+        obj = "object1"
+        path = "/%s/%s" % (cnt, obj)
+
+        # If the expected initial status is not 100, then an exception
+        # should be thrown and the connection closed
+        if initial_status is 100:
+            status, reason = \
+                self.object_client.create_object_continue(cnt, obj, req_data)
+        else:
+            self.assertRaises(exceptions.UnexpectedResponseCode,
+                              self.object_client.create_object_continue, cnt,
+                              obj, req_data)
+            mock_poc.return_value.close.assert_called_once_with()
+
+        # Verify that putrequest is called 1 time with the appropriate values
+        mock_poc.return_value.putrequest.assert_called_once_with('PUT', path)
+
+        # Verify that headers were written, including "Expect:100-continue"
+        calls = []
+
+        for header, value in expected_hdrs.items():
+            calls.append(mock.call(header, value))
+
+        mock_poc.return_value.putheader.assert_has_calls(calls, False)
+        mock_poc.return_value.endheaders.assert_called_once_with()
+
+        # The following steps are only taken if the initial status is 100
+        if initial_status is 100:
+            # Verify that the method returned what it was supposed to
+            self.assertEqual(status, 201)
+
+            # Verify that _safe_read was called once to remove the CRLF
+            # after the 100 response
+            mock_rc = mock_poc.return_value.response_class.return_value
+            mock_rc._safe_read.assert_called_once_with(2)
+
+            # Verify the actual data was written via send
+            mock_poc.return_value.send.assert_called_once_with(req_data)
+
+            # Verify that the getresponse method was called to receive
+            # the final
+            mock_poc.return_value.getresponse.assert_called_once_with()

tempest/tests/lib/services/registry_fixture.py

@@ -38,7 +38,7 @@ class RegistryFixture(fixtures.Fixture):
         """Initialise the registry fixture"""
         self.services = set(['compute', 'identity.v2', 'identity.v3',
                              'image.v1', 'image.v2', 'network', 'volume.v1',
-                             'volume.v2', 'volume.v3'])
+                             'volume.v2', 'volume.v3', 'object-storage'])
 
     def _setUp(self):
         # Cleanup the registry
@@ -50,7 +50,7 @@ class RegistryFixture(fixtures.Fixture):
         for sc in self.services:
             sc_module = service_clients[sc]
             sc_unversioned = sc.split('.')[0]
-            sc_name = sc.replace('.', '_')
+            sc_name = sc.replace('.', '_').replace('-', '_')
             # Pass the bare minimum params to satisfy the clients interface
             service_client_data = dict(
                 name=sc_name, service_version=sc, service=sc_unversioned,

tempest/tests/lib/services/test_clients.py

@@ -189,9 +189,7 @@ class TestServiceClients(base.TestCase):
     def setUp(self):
         super(TestServiceClients, self).setUp()
         self.useFixture(fixtures.MockPatch(
-            'tempest.lib.services.clients.tempest_modules', return_value={}))
+            'tempest.lib.services.clients.tempest_modules',
-        self.useFixture(fixtures.MockPatch(
-            'tempest.lib.services.clients._tempest_internal_modules',
             return_value=set(['fake_service1'])))
 
     def test___init___creds_v2_uri(self):
@@ -416,6 +414,7 @@ class TestServiceClients(base.TestCase):
         _manager = self._get_manager()
         duplicate_service = 'fake_service1'
         expected_error = '.*' + duplicate_service
+        _manager._registered_services = [duplicate_service]
         with testtools.ExpectedException(
                 exceptions.ServiceClientRegistrationException, expected_error):
             _manager.register_service_client_module(