Refactor Freezer authentication with keystone

Supports v2 and v3 of keystone and using keystoneauth library for
authentication with sessions. Supports new OS environment variables

Change-Id: I5cf8a5460b81c071f5982c1a7b7750d1c861a86f
Implements: blueprint refactor-os-clients

freezer/main.py

@@ -27,7 +27,6 @@ from oslo_log import log
 from freezer.common import config as freezer_config
 from freezer.engine.tar import tar_engine
 from freezer import job
-from freezer.openstack import openstack
 from freezer.openstack import osclients
 from freezer.storage import local
 from freezer.storage import multiple
@@ -58,23 +57,21 @@ def freezer_main(backup_args):
     validator.validate(backup_args)
 
     work_dir = backup_args.work_dir
-    os_identity = backup_args.os_identity_api_version
     max_segment_size = backup_args.max_segment_size
-    if backup_args.storage == 'swift' or (backup_args.__dict__['backup_media']
-        in ['nova', 'cinder', 'cindernative']):
-        client_manager = get_client_manager(backup_args.__dict__)
-
+    if backup_args.storage == 'swift' or (
+                backup_args.backup_media in ['nova', 'cinder', 'cindernative']):
+        backup_args.client_manager = get_client_manager(backup_args.__dict__)
 
     if backup_args.storages:
         storage = multiple.MultipleStorage(
             work_dir,
-            [storage_from_dict(x, work_dir, max_segment_size, os_identity)
+            [storage_from_dict(x, work_dir, max_segment_size)
              for x in backup_args.storages])
     else:
         storage = storage_from_dict(backup_args.__dict__, work_dir,
-                                    max_segment_size, os_identity)
+                                    max_segment_size)
 
-    backup_args.__dict__['engine'] = tar_engine.TarBackupEngine(
+    backup_args.engine = tar_engine.TarBackupEngine(
         backup_args.compression,
         backup_args.dereference_symlink,
         backup_args.exclude,
@@ -146,26 +143,24 @@ def parse_osrc(file_name):
     with open(file_name, 'r') as osrc_file:
         return config.osrc_parse(osrc_file.read())
 
-def get_client_manager(backup_args, os_identity_api_version=None):
+
+def get_client_manager(backup_args):
     if "osrc" in backup_args:
-        options = openstack.OpenstackOptions.create_from_dict(
+        options = osclients.OpenstackOpts.create_from_dict(
             parse_osrc(backup_args['osrc']))
     else:
-        options = openstack.OpenstackOptions.create_from_env()
-    identity_api_version = (os_identity_api_version or
-                            options.identity_api_version)
-    client_manager = osclients.ClientManager(
-        options=options,
-        insecure=backup_args.get('insecure') or False,
-        swift_auth_version=identity_api_version,
-        dry_run=backup_args.get('dry_run') or False)
+        options = osclients.OpenstackOpts.create_from_env().get_opts_dicts()
 
-    backup_args['client_manager'] = client_manager
+    client_manager = osclients.OSClientManager(
+        auth_url=options.pop('auth_url', None),
+        auth_method=options.pop('auth_method', 'password'),
+        dry_run=backup_args.get('dry_run', None),
+        **options
+    )
     return client_manager
 
 
-def storage_from_dict(backup_args, work_dir, max_segment_size,
-                      os_identity_api_version=None):
+def storage_from_dict(backup_args, work_dir, max_segment_size):
     storage_name = backup_args['storage']
     container = backup_args['container']
     if storage_name == "swift":
@@ -188,7 +183,7 @@ def storage_from_dict(backup_args, work_dir, max_segment_size,
 
 
 def main():
-    """freezer-agent/freezerc binary main execution"""
+    """freezer-agent binary main execution"""
     backup_args = None
     try:
         freezer_config.config()
@@ -203,5 +198,4 @@ def main():
         return fail(1, err, quiet)
 
 if __name__ == '__main__':
-
     sys.exit(main())

freezer/openstack/backup.py

@@ -28,7 +28,7 @@ logging = log.getLogger(__name__)
 home = os.path.expanduser("~")
 
 
-class BackupOs:
+class BackupOs(object):
 
     def __init__(self, client_manager, container, storage):
         """
@@ -76,8 +76,8 @@ class BackupOs:
         headers = {"x-object-meta-name": instance._info['name'],
                    "x-object-meta-flavor-id": instance._info['flavor']['id']}
         self.storage.add_stream(stream, package, headers)
-        logging.info("[*] Deleting temporary image")
-        glance.images.delete(image)
+        logging.info("[*] Deleting temporary image {0}".format(image))
+        glance.images.delete(image.id)
 
     def backup_cinder_by_glance(self, volume_id):
         """

freezer/openstack/openstack.py

@@ -1,81 +0,0 @@
-"""
-(c) Copyright 2015,2016 Hewlett-Packard Development Company, L.P.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-import os
-
-
-class OpenstackOptions:
-    """
-    Stores credentials for OpenStack API.
-    Can be created using
-    >> create_from_env()
-    or
-    >> create_from_dict(dict)
-    """
-    def __init__(self, user_name, tenant_name, project_name, auth_url,
-                 password, identity_api_version, tenant_id=None,
-                 region_name=None, endpoint_type=None, cert=None,
-                 insecure=False, verify=True):
-        self.user_name = user_name
-        self.tenant_name = tenant_name
-        self.auth_url = auth_url
-        self.password = password
-        self.tenant_id = tenant_id
-        self.project_name = project_name
-        self.identity_api_version = identity_api_version
-        self.region_name = region_name
-        self.endpoint_type = endpoint_type
-        self.cert = cert
-        self.insecure = insecure
-        self.verify = verify
-        if not (self.password and self.user_name and self.auth_url and
-           (self.tenant_name or self.project_name)):
-            raise Exception("Please set up in your env:"
-                            "OS_USERNAME, OS_TENANT_NAME/OS_PROJECT_NAME,"
-                            " OS_AUTH_URL, OS_PASSWORD")
-
-    @property
-    def os_options(self):
-        """
-        :return: The OpenStack options which can have tenant_id,
-                 auth_token, service_type, endpoint_type, tenant_name,
-                 object_storage_url, region_name
-        """
-        return {'tenant_id': self.tenant_id,
-                'tenant_name': self.tenant_name,
-                'project_name': self.project_name,
-                'identity_api_version': self.identity_api_version,
-                'region_name': self.region_name,
-                'endpoint_type': self.endpoint_type}
-
-    @staticmethod
-    def create_from_env():
-        return OpenstackOptions.create_from_dict(os.environ)
-
-    @staticmethod
-    def create_from_dict(src_dict):
-        return OpenstackOptions(
-            user_name=src_dict.get('OS_USERNAME', None),
-            tenant_name=src_dict.get('OS_TENANT_NAME', None),
-            project_name=src_dict.get('OS_PROJECT_NAME', None),
-            auth_url=src_dict.get('OS_AUTH_URL', None),
-            identity_api_version=src_dict.get('OS_IDENTITY_API_VERSION',
-                                              '2.0'),
-            password=src_dict.get('OS_PASSWORD', None),
-            tenant_id=src_dict.get('OS_TENANT_ID', None),
-            region_name=src_dict.get('OS_REGION_NAME', None),
-            endpoint_type=src_dict.get('OS_ENDPOINT_TYPE', None),
-            cert=src_dict.get('OS_CERT', None)
-        )

freezer/openstack/osclients.py

@@ -1,4 +1,5 @@
 # (c) Copyright 2014,2015 Hewlett-Packard Development Company, L.P.
+# (c) Copyright 2016 Hewlett-Packard Enterprise Development Company, L.P.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,14 +13,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
+import swiftclient
 import time
 
-from cinderclient import client as cclient
-from glanceclient import client as gclient
-from novaclient import client as nclient
+from cinderclient.client import Client as cinder_client
+from glanceclient.client import Client as glance_client
+from keystoneauth1 import loading
+from keystoneauth1 import session
+from novaclient.client import Client as nova_client
 from oslo_config import cfg
 from oslo_log import log
-import swiftclient
 
 from freezer.utils import utils
 
@@ -27,158 +31,156 @@ CONF = cfg.CONF
 logging = log.getLogger(__name__)
 
 
-class ClientManager:
-    
-    """
-    :type swift: swiftclient.Connection
-    :type glance: glanceclient.v1.client.Client
-    :type nova: novaclient.v2.client.Client
-    :type cinder: cinderclient.v1.client.Client
-    """
-    def __init__(self, options, insecure=True,
-                 swift_auth_version=2, dry_run=False):
-        """
-        Creates manager of connections to swift, nova, glance and cinder
-        :param options: OpenstackOptions
-        :type options: freezer.openstack.openstack.OpenstackOptions
-        :param insecure:
-        :param swift_auth_version:
-        :param dry_run:
-        :return:
-        """
-        self.options = options
-        self.insecure = insecure
-        self.swift_auth_version = swift_auth_version
-        self.dry_run = dry_run
-        self.cinder = None
+class OSClientManager(object):
+
+    def __init__(self, auth_url, auth_method='password', **kwargs):
         self.swift = None
         self.glance = None
         self.nova = None
+        self.cinder = None
+        self.dry_run = kwargs.pop('dry_run', None)
+        loader = loading.get_plugin_loader(auth_method)
+        # copy the args for swift authentication !
+        self.swift_args = kwargs.copy()
+        self.swift_args['auth_url'] = auth_url
+        # client specific arguments !
+        self.client_kwargs = {}
+        # session specific arguments
+        session_kwargs = {}
+        if 'verify' in kwargs.keys():
+            session_kwargs['verify'] = kwargs.pop('verify')
+        if 'cacert' in kwargs.keys():
+            session_kwargs['cert'] = kwargs.pop('cacert')
+        # client specific args
+        if 'insecure' in kwargs.keys():
+            self.client_kwargs['insecure'] = kwargs.pop('insecure')
+            session_kwargs['verify'] = False
+        if 'region_name' in kwargs.keys():
+            self.client_kwargs['region_name'] = kwargs.pop('region_name')
+        if 'endpoint_type' in kwargs.keys():
+            self.client_kwargs['endpoint_type'] = kwargs.pop('endpoint_type')
+        if 'identity_api_version' in kwargs.keys():
+            kwargs.pop('identity_api_version')
+        if 'auth_version' in kwargs.keys():
+            kwargs.pop('auth_version')
+        if 'interface' in kwargs.keys():
+            self.client_kwargs['interface'] = kwargs.pop('interface')
 
-    def get_cinder(self):
-        """
-        :rtype cinderclient.v1.client.Client
-        :return:
-        """
-        if not self.cinder:
-            self.create_cinder()
-        return self.cinder
+        self.compute_version = kwargs.pop('compute_api_version', 2)
+        self.image_version = kwargs.pop('image_api_version', 2)
+        self.volume_version = kwargs.pop('volume_api_version', 2)
+        self.auth = loader.load_from_options(auth_url=auth_url, **kwargs)
 
-    def get_swift(self):
-        """
-        :rtype swiftclient.Connection
-        :return: instance of swift client
-        """
-        if not self.swift:
-            self.create_swift()
-        return self.swift
+        self.sess = session.Session(auth=self.auth, **session_kwargs)
 
-    def get_glance(self):
+    def create_nova(self):
         """
-        :rtype glanceclient.v1.client.Client
-        :return:
+        Use pre-initialized session to create an instance of nova client.
+        :return: novaclient instance
         """
-        if not self.glance:
-            self.create_glance()
-        return self.glance
-
-    def get_nova(self):
-        """
-        :rtype
-        :return:
-        """
-        if not self.nova:
-            self.create_nova()
+        self.nova = nova_client(self.compute_version, session=self.sess,
+                                **self.client_kwargs)
         return self.nova
 
+    def create_glance(self):
+        """
+        Use pre-initialized session to create an instance of glance client.
+        :return: glanceclient instance
+        """
+        if 'endpoint_type' in self.client_kwargs.keys():
+            self.client_kwargs.pop('endpoint_type')
+        if 'insecure' in self.client_kwargs.keys():
+            self.client_kwargs.pop('insecure')
+
+        self.glance = glance_client(self.image_version, session=self.sess,
+                                    **self.client_kwargs)
+        return self.glance
+
     def create_cinder(self):
         """
-        Creates client for cinder and caches it
-        :rtype cinderclient.v1.client.Client
-        :return: instance of cinder client
+        Use pre-initialized session to create an instance of cinder client.
+        :return: cinderclient instance
         """
-        options = self.options
-        logging.info("[*] Creation of cinder client")
-        self.cinder = cclient.Client(
-            version="2",
-            username=options.user_name,
-            api_key=options.password,
-            project_id=options.tenant_name,
-            auth_url=options.auth_url,
-            region_name=options.region_name,
-            insecure=self.insecure,
-            endpoint_type=options.endpoint_type or 'publicURL',
-            service_type="volume",
-            cacert=options.cert)
+        self.cinder = cinder_client(self.volume_version, session=self.sess,
+                                    **self.client_kwargs)
         return self.cinder
 
     def create_swift(self):
         """
-        Creates client for swift and caches it
-        :rtype swiftclient.Connection
-        :return: instance of swift client
+        Swift client needs to be treated differently so we need to copy the
+        arguments and provide it to swiftclient the correct way !
+        :return: swiftclient instance
         """
-        options = self.options
-        logging.info("[*] Creation of swift client")
+        os_options = {}
+        auth_version = None
+        if 'region_name' in self.swift_args.keys():
+            os_options['region_name'] = self.swift_args.get('region_name')
+        if 'endpoint_type' in self.swift_args.keys():
+            os_options['endpoint_type'] = self.swift_args.pop('endpoint_type')
+        if 'tenant_id' in self.swift_args.keys():
+            os_options['tenant_id'] = self.swift_args.pop('tenant_id')
+        if 'identity_api_version' in self.swift_args.keys():
+            os_options['identity_api_version'] = \
+                self.swift_args.pop('identity_api_version')
+            auth_version = os_options['identity_api_version']
 
+        if 'token' in self.swift_args.keys():
+            os_options['auth_token'] = self.swift_args.pop('token')
+        if 'auth_version' in self.swift_args.keys():
+            auth_version = self.swift_args.get('auth_version')
+
+        tenant_name = self.swift_args.get('project_name') or self.swift_args.\
+            get('tenant_name')
         self.swift = swiftclient.client.Connection(
-            authurl=options.auth_url,
-            user=options.user_name, key=options.password,
-            tenant_name=options.tenant_name,
-            os_options=options.os_options,
-            auth_version=self.swift_auth_version,
-            insecure=self.insecure, retries=6,
-            cacert=options.cert)
+            authurl=self.swift_args.get('auth_url'),
+            user=self.swift_args.get('username'),
+            key=self.swift_args.get('password'),
+            tenant_name=tenant_name,
+            insecure=self.swift_args.get('insecure', False),
+            cacert=self.swift_args.get('cacert', None),
+            os_options=os_options,
+            auth_version=auth_version
+        )
 
         if self.dry_run:
             self.swift = DryRunSwiftclientConnectionWrapper(self.swift)
         return self.swift
 
-    def create_glance(self):
+    def get_nova(self):
         """
-        Creates client for glance and caches it
-        :rtype glanceclient.v1.client.Client
-        :return: instance of glance client
+        Get novaclient instance
+        :return: novaclient instance
         """
+        if not self.nova:
+            self.nova = self.create_nova()
+        return self.nova
 
-        from glanceclient.shell import OpenStackImagesShell
-
-        options = self.options
-
-        logging.info("[*] Creation of glance client")
-
-        self.glance = OpenStackImagesShell()._get_versioned_client('1',
-            utils.Bunch(os_username=options.user_name,
-                        os_password=options.password,
-                        os_tenant_name=options.tenant_name,
-                        os_project_name=options.project_name,
-                        os_auth_url=options.auth_url,
-                        os_region_name=options.region_name,
-                        endpoint_type=options.endpoint_type,
-                        force_auth=False,
-                        cacert=options.cert))
-
+    def get_glance(self):
+        """
+        Get glanceclient instance
+        :return: glanceclient instance
+        """
+        if not self.glance:
+            self.glance = self.create_glance()
         return self.glance
 
-    def create_nova(self):
+    def get_cinder(self):
         """
-        Creates client for nova and caches it
-        :return:
+        Get cinderclient instance
+        :return: cinderclient instance
         """
-        options = self.options
-        logging.info("[*] Creation of nova client")
+        if not self.cinder:
+            self.cinder = self.create_cinder()
+        return self.cinder
 
-        self.nova = nclient.Client(
-            version='2',
-            username=options.user_name,
-            api_key=options.password,
-            project_id=options.tenant_name,
-            auth_url=options.auth_url,
-            region_name=options.region_name,
-            insecure=self.insecure,
-            cacert=options.cert)
-
-        return self.nova
+    def get_swift(self):
+        """
+        Get swiftclient instance
+        :return: swiftclient instance
+        """
+        if not self.swift:
+            self.swift = self.create_swift()
+        return self.swift
 
     def provide_snapshot(self, volume, snapshot_name):
         """
@@ -275,7 +277,188 @@ class ClientManager:
         return utils.ReSizeStream(stream, image.size, 1000000)
 
 
-class DryRunSwiftclientConnectionWrapper:
+class OpenstackOpts(object):
+    """
+    Gathering and maintaining the right Openstack credentials that will be used
+    to authenticate against keystone. Now we support keystone v2 and v3.
+    We need to provide a correct url that ends with either v2.0 or v3 or provide
+    auth_version or identity_api_version
+    """
+    def __init__(self, auth_url, auth_method='password', auth_version=None,
+                 username=None, password=None, region_name=None, cacert=None,
+                 identity_api_version=None, project_id=None, project_name=None,
+                 tenant_id=None, tenant_name=None, token=None, insecure=False,
+                 endpoint_type='internalURL', interface=None,
+                 compute_api_version=2, image_api_version=2,
+                 volume_api_version=2, user_domain_name=None, domain_id=None,
+                 user_domain_id=None, project_domain_id=None, domain_name=None,
+                 project_domain_name=None):
+        """
+        Authentication Options to build a valid opts dict to be used to
+        authenticate against keystone. You must provide auth_url with a vaild
+        Openstack version at the end v2.0 or v3 or provide auth_version.
+        :param auth_url: string Keystone API URL
+        :param auth_method: string defaults to password or token (not tested)
+        :param auth_version: string Keystone API version. 2.0 or 3
+        :param username: string A valid Username
+        :param password: string A valid Password
+        :param region_name: string Region name or None
+        :param cacert: string Path to CA certificate
+        :param identity_api_version: string Keystone API version to use
+        :param project_id: UUID string Project ID
+        :param project_name: string Project Name
+        :param tenant_id: string Project/ Tenant ID. Use with keystone v2.0 only
+        :param tenant_name: string Project/ Tenant Name. keystone v2.0 only
+        :param token: string Valid token. Only if auth_method is token
+        :param insecure: boolean Use insecure connections
+        :param endpoint_type: string publicURL, adminURL, internalURL
+        :param interface: string internal, ...
+        :param compute_api_version: int NOVA API version to use default 2
+        :param image_api_version: int Glance API version, default 2
+        :param volume_api_version: int Cinder API version, default 2
+        :param user_domain_name: string User Domain Name. only with keystone v3
+        :param domain_id: string Domain ID. Only with keystone v3
+        :param user_domain_id: string User Domain ID. only with keystone v3
+        :param project_domain_id: string Project Domain ID. keystone v3 only
+        :param domain_name: string Domain Name. only with keystone v3
+        :param project_domain_name: string Project Domain Name. keystone v3 only
+        :return: None
+        """
+        self.auth_url = auth_url
+        self.auth_method = auth_method
+        self.auth_version = auth_version
+        self.username = username
+        self.password = password
+        self.region_name = region_name
+        self.cacert = cacert
+        self.identity_api_version = identity_api_version
+        self.tenant_id = tenant_id or project_id
+        self.project_id = project_id or tenant_id
+        self.project_name = project_name or tenant_name
+        self.tenant_name = tenant_name or project_name
+        self.token = token
+        self.insecure = insecure
+        self.endpoint_type = endpoint_type
+        self.interface = interface
+        self.compute_api_version = compute_api_version
+        self.image_api_version = image_api_version
+        self.volume_api_version = volume_api_version
+        self.user_domain_id = user_domain_id
+        self.user_domain_name = user_domain_name
+        self.project_domain_id = project_domain_id
+        self.project_domain_name = project_domain_name
+        self.domain_id = domain_id
+        self.domain_name = domain_name
+        if auth_url is None:
+            raise Exception('auth_url required to authenticate. Make sure to '
+                            'export OS_AUTH_URL=http://keystone_url:5000/v3')
+        if auth_version is None and identity_api_version is None:
+            version = auth_url.rstrip('/').rsplit('/')[-1]
+            if version == 'v3':
+                self.auth_version = self.identity_api_version = str('3')
+            elif version == 'v2.0':
+                self.auth_version = self.identity_api_version = str('2.0')
+            else:
+                raise Exception('Keystone Auth version {0} is not supported!. '
+                                'Generated from auth_url: {1}'.format(version,
+                                                                      auth_url))
+        logging.info('Authenticating with Keystone version: {0}, auth_url: {1},'
+                     ' username: {2}, project: {3}'.format(self.auth_version,
+                                                           self.auth_url,
+                                                           self.username,
+                                                           self.project_name))
+
+    def get_opts_dicts(self):
+        """
+        Return opentack auth arguments as dict
+        detects the auth version from url if not provided
+        handles certificate issues
+        """
+        opts = self.__dict__
+        if self.auth_method == 'password':
+            opts.pop('token', None)
+        elif self.auth_method == 'token':
+            opts.pop('username', None)
+            opts.pop('password', None)
+
+        if not self.cacert:
+            opts['verify'] = False
+            opts['insecure'] = True
+        self.auth_version = str(self.auth_version)
+        self.identity_api_version = str(self.identity_api_version)
+
+        if self.auth_version == '3' or self.identity_api_version == '3':
+            opts['auth_version'] = opts['identity_api_version'] = '3'
+            opts.pop('tenant_id', None)
+            opts.pop('tenant_name', None)
+
+        elif self.auth_version in ['2.0', '2'] or self.identity_api_version in \
+                ['2.0', '2']:
+            opts['auth_version'] = opts['identity_api_version'] = '2.0'
+            # these parameters won't work with keystone v2.0
+            opts.pop('project_id', None)
+            opts.pop('project_name', None)
+            opts.pop('project_domain_id', None)
+            opts.pop('project_domain_name', None)
+            opts.pop('user_domain_id', None)
+            opts.pop('user_domain_name', None)
+            opts.pop('domain_id', None)
+            opts.pop('domain_name', None)
+        else:
+            raise Exception('Keystone Auth version {0} is not supported!. '
+                            'Generated from auth_url: {1}'.
+                            format(self.auth_version, self.auth_url))
+        for i in opts.copy().keys():
+            if opts.get(i) is None:
+                opts.pop(i)
+        return opts
+
+    @staticmethod
+    def create_from_env():
+        """
+        Parse environment variables and load Openstack related options.
+        :return:
+        """
+        return OpenstackOpts.create_from_dict(os.environ)
+
+    @staticmethod
+    def create_from_dict(src_dict):
+        """
+        Load Openstack arguments from dict and return OpenstackOpts object with
+        the correct parameters to authenticate.
+        :param src_dict: dict
+        :return: OpenstackOpts object with the passed arguments in place
+        """
+        return OpenstackOpts(
+            auth_url=src_dict.get('OS_AUTH_URL'),
+            auth_method=src_dict.get('OS_AUTH_METHOD', 'password'),
+            auth_version=src_dict.get('OS_AUTH_VERSION', None),
+            username=src_dict.get('OS_USERNAME', None),
+            password=src_dict.get('OS_PASSWORD', None),
+            tenant_id=src_dict.get('OS_TENANT_ID', None),
+            tenant_name=src_dict.get('OS_TENANT_NAME', None),
+            project_id=src_dict.get('OS_PROJECT_ID', None),
+            project_name=src_dict.get('OS_PROJECT_NAME', None),
+            region_name=src_dict.get('OS_REGION_NAME', None),
+            endpoint_type=src_dict.get('OS_ENDPOINT_TYPE', 'publicURL'),
+            cacert=src_dict.get('OS_CACERT', None),
+            identity_api_version=src_dict.get('OS_IDENTITY_API_VERSION', None),
+            insecure=src_dict.get('OS_INSECURE', False),
+            token=src_dict.get('OS_TOKEN', None),
+            interface=src_dict.get('OS_INTERFACE', None),
+            user_domain_name=src_dict.get('OS_USER_DOMAIN_NAME', None),
+            user_domain_id=src_dict.get('OS_USER_DOMAIN_ID', None),
+            project_domain_id=src_dict.get('OS_PROJECT_DOMAIN_ID', None),
+            project_domain_name=src_dict.get('OS_PROJECT_DOMAIN_NAME', None),
+            domain_id=src_dict.get('OS_DOMAIN_ID'),
+            domain_name=src_dict.get('OS_DOMAIN_NAME'),
+            compute_api_version=src_dict.get('OS_COMPUTE_API_VERSION', 2),
+            volume_api_version=src_dict.get('OS_VOLUME_API_VERSION', 2),
+            image_api_version=src_dict.get('OS_IMAGE_API_VERSION', 2)
+                             )
+
+
+class DryRunSwiftclientConnectionWrapper(object):
     def __init__(self, sw_connector):
         self.sw_connector = sw_connector
         self.get_object = sw_connector.get_object

freezer/tests/commons.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 
 # (c) Copyright 2014,2015 Hewlett-Packard Development Company, L.P.
+# (c) Copyright 2016 Hewlett-Packard Enterprise Development Company, L.P
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,6 +17,7 @@
 
 
 from mock import MagicMock
+from mock import Mock
 
 import swiftclient
 import multiprocessing
@@ -27,13 +29,14 @@ import re
 from glanceclient.common.utils import IterableWithLength
 from freezer.storage import swift
 from freezer.utils import utils
-from freezer.openstack import openstack
+from freezer.openstack.osclients import OpenstackOpts
+from freezer.openstack.osclients import OSClientManager
 from freezer.engine.tar import tar_engine
 
 os.environ['OS_REGION_NAME'] = 'testregion'
 os.environ['OS_TENANT_ID'] = '0123456789'
 os.environ['OS_PASSWORD'] = 'testpassword'
-os.environ['OS_AUTH_URL'] = 'testauthurl'
+os.environ['OS_AUTH_URL'] = 'http://testauthurl/v2.0'
 os.environ['OS_USERNAME'] = 'testusername'
 os.environ['OS_TENANT_NAME'] = 'testtenantename'
 
@@ -316,10 +319,14 @@ class BackupOpt1:
         self.cindernative_vol_id = ''
         self.nova_inst_id = ''
         self.lvm_snapperm = 'ro'
-        self.options = openstack.OpenstackOptions.create_from_dict(os.environ)
-        from freezer.openstack.osclients import ClientManager
-        from mock import Mock
-        self.client_manager = ClientManager(None, False, 2, False)
+
+        self.compression = 'gzip'
+        self.storage = MagicMock()
+        self.engine = MagicMock()
+        opts = OpenstackOpts.create_from_env().get_opts_dicts()
+        self.client_manager = OSClientManager(opts.pop('auth_url'),
+                                              opts.pop('auth_method'),
+                                              **opts)
         self.client_manager.get_swift = Mock(
             return_value=FakeSwiftClient().client.Connection())
         self.client_manager.create_swift = self.client_manager.get_swift
@@ -327,8 +334,6 @@ class BackupOpt1:
                                           self.container,
                                           self.work_dir,
                                           self.max_segment_size)
-        self.compression = 'gzip'
-
         self.engine = tar_engine.TarBackupEngine(
             self.compression, self.dereference_symlink,
             self.exclude, self.storage, False)
@@ -337,6 +342,7 @@ class BackupOpt1:
         nova_client = MagicMock()
 
         self.client_manager.get_nova = Mock(return_value=nova_client)
+
         self.command = None
 
 
@@ -387,7 +393,7 @@ class Os:
     @classmethod
     def exists(cls, directory=True):
         return True
-    
+
     @classmethod
     def notexists(cls, directory=True):
         return False

freezer/utils/streaming.py

@@ -1,6 +1,6 @@
 """
 (c) Copyright 2014,2015 Hewlett-Packard Development Company, L.P.
-
+(c) Copyright 2016 Hewlett-Packard Enterprise Development Company, L.P.
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
@@ -103,4 +103,4 @@ class QueuedThread(threading.Thread):
             super(QueuedThread, self).run()
         except Exception as e:
             self.rich_queue.force_stop()
-            raise e
+            raise e

requirements.txt

@@ -14,6 +14,7 @@ oslo.utils>=3.5.0 # Apache-2.0
 oslo.i18n>=2.1.0 # Apache-2.0
 oslo.log>=1.14.0 # Apache-2.0
 oslo.config>=3.9.0 # Apache-2.0
+keystoneauth1>=2.1.0  # Apache-2.0
 
 PyMySQL>=0.6.2 # MIT License
 pymongo!=3.1,>=3.0.2 # Apache-2.0

tests/unit/openstack/test_osclients.py

@@ -1,4 +1,5 @@
 # (c) Copyright 2014,2015 Hewlett-Packard Development Company, L.P.
+# (c) Copyright 2016 Hewlett-Packard Enterprise Development Company, L.P.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,61 +13,46 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 import unittest
 import mock
 
-from freezer.openstack import openstack
 from freezer.openstack import osclients
-from freezer.utils import utils
 
 
 class TestOsClients(unittest.TestCase):
-
-    fake_options = openstack.OpenstackOptions(
-        user_name="user", tenant_name="tenant", project_name="project",
-        auth_url="url", password="password", identity_api_version="3",
-        insecure=False, cert='cert', verify=True)
+    def setUp(self):
+        self.opts = osclients.OpenstackOpts(
+            username="user", tenant_name="tenant", project_name="project",
+            auth_url="url/v3", password="password", identity_api_version="3",
+            insecure=False, cacert='cert', user_domain_name='Default',
+            project_domain_name='Default').get_opts_dicts()
+        self.client_manager = osclients.OSClientManager(auth_method=self.opts.pop('auth_method'),
+                                                        auth_url=self.opts.pop('auth_url'),
+                                                        **self.opts)
 
     def test_init(self):
-        osclients.ClientManager(self.fake_options, None, None, None)
+        self.client_manager.get_cinder()
 
     def test_create_cinder(self):
-        client = osclients.ClientManager(self.fake_options, None, None, None)
-        client.create_cinder()
+        self.client_manager.create_cinder()
 
     def test_create_swift(self):
-        client = osclients.ClientManager(self.fake_options, None, None, None)
-        client.create_swift()
+        self.client_manager.create_swift()
 
     def test_create_nova(self):
-        client = osclients.ClientManager(self.fake_options, None, None, None)
-        client.create_nova()
-
-    def test_create_swift_public(self):
-        options = openstack.OpenstackOptions(
-            user_name="user", tenant_name="tenant", project_name="project",
-            auth_url="url", password="password", identity_api_version="3",
-            endpoint_type="adminURL", insecure=False, cert='cert',
-            verify=True)
-        client = osclients.ClientManager(options, None, None, None)
-        client.create_swift()
+        self.client_manager.create_nova()
 
     def test_dry_run(self):
         osclients.DryRunSwiftclientConnectionWrapper(mock.Mock())
 
     def test_get_cinder(self):
-        client = osclients.ClientManager(self.fake_options, None, None, None)
-        client.get_cinder()
+        self.client_manager.get_cinder()
 
     def test_get_swift(self):
-        client = osclients.ClientManager(self.fake_options, None, None, None)
-        client.get_swift()
+        self.client_manager.get_swift()
 
     def get_glance(self):
-        client = osclients.ClientManager(self.fake_options, None, None, None)
-        client.get_glance()
+        self.client_manager.get_glance()
 
     def get_nova(self):
-        client = osclients.ClientManager(self.fake_options, None, None, None)
-        client.get_nova()
+        self.client_manager.get_nova()

tests/unit/openstack/test_restore.py

@@ -1,7 +1,7 @@
 """Freezer restore.py related tests
 
 (c) Copyright 2014,2015 Hewlett-Packard Development Company, L.P.
-
+(c) Copyright 2016 Hewlett-Packard Enterprise Development Company, L.P
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

tests/unit/storages/test_swift.py

@@ -1,4 +1,5 @@
 # (c) Copyright 2014,2015 Hewlett-Packard Development Company, L.P.
+# (c) Copyright 2016 Hewlett-Packard Enterprise Development Company, L.P
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,7 +16,6 @@
 
 import unittest
 
-from freezer.openstack import openstack
 from freezer.openstack import osclients
 from freezer.storage import swift
 from freezer.storage import base
@@ -24,11 +24,12 @@ from freezer.storage import base
 class TestSwiftStorage(unittest.TestCase):
 
     def setUp(self):
-
+        opts = osclients.OpenstackOpts.create_from_env().get_opts_dicts()
         self.storage = swift.SwiftStorage(
-            osclients.ClientManager(
-                openstack.OpenstackOptions.create_from_env()
-            ),
+            osclients.OSClientManager(opts.pop('auth_url'),
+                                      opts.pop('auth_method', 'password'),
+                                      **opts
+                                      ),
             "freezer_ops-aw1ops1-gerrit0001.aw1.hpcloud.net",
             "/tmp/",
             100, skip_prepare=True

tests/unit/utils/test_utils.py

@@ -1,4 +1,5 @@
 # (c) Copyright 2014,2015 Hewlett-Packard Development Company, L.P.
+# (c) Copyright 2016 Hewlett-Packard Enterprise Development Company, L.P
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,7 +17,7 @@
 import datetime
 import unittest
 
-from freezer.openstack import openstack
+from freezer.openstack.osclients import OpenstackOpts
 from freezer.tests.commons import *
 from freezer.utils import utils
 
@@ -65,14 +66,19 @@ class TestUtils(unittest.TestCase):
         self.assertRaises(ValueError, utils.human2bytes, '12 foo')
 
     def test_OpenstackOptions_creation_success(self):
+        class FreezerOpts:
+            def __init__(self, opts):
+                self.__dict__.update(opts)
         env_dict = dict(OS_USERNAME='testusername',
                         OS_TENANT_NAME='testtenantename',
                         OS_AUTH_URL='testauthurl',
                         OS_PASSWORD='testpassword',
                         OS_REGION_NAME='testregion',
-                        OS_TENANT_ID='0123456789')
-        options = openstack.OpenstackOptions.create_from_dict(env_dict)
-        assert options.user_name == env_dict['OS_USERNAME']
+                        OS_TENANT_ID='0123456789',
+                        OS_AUTH_VERSION='2.0')
+        options = OpenstackOpts.create_from_dict(env_dict).get_opts_dicts()
+        options = FreezerOpts(options)
+        assert options.username == env_dict['OS_USERNAME']
         assert options.tenant_name == env_dict['OS_TENANT_NAME']
         assert options.auth_url == env_dict['OS_AUTH_URL']
         assert options.password == env_dict['OS_PASSWORD']
@@ -82,14 +88,14 @@ class TestUtils(unittest.TestCase):
         env_dict = dict(OS_USERNAME='testusername',
                         OS_TENANT_NAME='testtenantename',
                         OS_AUTH_URL='testauthurl',
-                        OS_PASSWORD='testpassword')
-        options = openstack.OpenstackOptions.create_from_dict(env_dict)
-        assert options.user_name == env_dict['OS_USERNAME']
+                        OS_PASSWORD='testpassword',
+                        OS_AUTH_VERSION='2.0')
+        options = OpenstackOpts.create_from_dict(env_dict).get_opts_dicts()
+        options = FreezerOpts(options)
+        assert options.username == env_dict['OS_USERNAME']
         assert options.tenant_name == env_dict['OS_TENANT_NAME']
         assert options.auth_url == env_dict['OS_AUTH_URL']
         assert options.password == env_dict['OS_PASSWORD']
-        assert options.region_name is None
-        assert options.tenant_id is None
 
     def test_date_to_timestamp(self):
         # ensure that timestamp is check with appropriate timezone offset
@@ -136,8 +142,8 @@ class TestDateTime:
         d = utils.DateTime(1425750464)
         assert 1425750464 == d.timestamp
         #ensure that time is check with appropriate timezone offset
-        t = time.strftime("%Y-%m-%d %H:%M:%S", 
-              time.localtime((time.mktime(time.strptime("2015-03-07 17:47:44", 
+        t = time.strftime("%Y-%m-%d %H:%M:%S",
+              time.localtime((time.mktime(time.strptime("2015-03-07 17:47:44",
                                                     "%Y-%m-%d %H:%M:%S")))-time.timezone))
         assert t == '{}'.format(d)