Add support for vendor hooks

Add possibility to pass a hook in the vendor config, clouds-public.* or 
upon building a connection. This should be a string parameter - function
name to be executed. This gives possibility to register new services of 
the cloud automatically or alter behavior of the present services.
It would have not been necessary, if public clouds followed 
upstream-first aproach.

While we are here fix warnings on not closed files in the test_json

Change-Id: Ifd6c0847102af4f46e361dcb1a665829c77553b9

openstack/__init__.py

@@ -57,4 +57,6 @@ def connect(
         load_yaml_config=load_yaml_config,
         load_envvars=load_envvars,
         options=options, **kwargs)
-    return openstack.connection.Connection(config=cloud_region)
+    return openstack.connection.Connection(
+        config=cloud_region,
+        vendor_hook=kwargs.get('vendor_hook'))

openstack/config/loader.py

@@ -481,7 +481,8 @@ class OpenStackConfig(object):
                 "'profile' keyword.".format(self.config_filename))
 
         vendor_filename, vendor_file = self._load_vendor_file()
-        if vendor_file and profile_name in vendor_file['public-clouds']:
+        if (vendor_file and 'public-clouds' in vendor_file
+                and profile_name in vendor_file['public-clouds']):
             _auth_update(cloud, vendor_file['public-clouds'][profile_name])
         else:
             profile_data = vendors.get_profile(profile_name)

openstack/config/schema.json

@@ -104,6 +104,11 @@
       "description": "Volume API Version",
       "default": "2",
       "type": "string"
+    },
+    "vendor_hook": {
+      "name": "Hook for vendor customization",
+      "description": "A possibility for a vendor to alter connection object",
+      "type": "string"
     }
   },
   "required": [

openstack/config/vendor-schema.json

@@ -217,6 +217,11 @@
           "name": "Baremetal API Version",
           "description": "Baremetal API Version",
           "type": "string"
+        },
+        "vendor_hook": {
+          "name": "Hook for vendor customization",
+          "description": "A possibility for a vendor to alter connection object",
+          "type": "string"
         }
       }
     }

openstack/config/vendors/otc.json

@@ -2,12 +2,14 @@
   "name": "otc",
   "profile": {
     "auth": {
-      "auth_url": "https://iam.%(region_name)s.otc.t-systems.com/v3"
+      "auth_url": "https://iam.{region_name}.otc.t-systems.com/v3"
     },
     "regions": [
       "eu-de"
     ],
     "identity_api_version": "3",
-    "image_format": "vhd"
+    "interface": "public",
+    "image_format": "vhd",
+    "vendor_hook": "otcextensions.sdk:load"
   }
 }

openstack/connection.py

@@ -345,6 +345,32 @@ class Connection(six.with_metaclass(_meta.ConnectionMeta,
         _orchestration.OrchestrationCloudMixin.__init__(self)
         _security_group.SecurityGroupCloudMixin.__init__(self)
 
+        # Allow vendors to provide hooks. They will normally only receive a
+        # connection object and a responsible to register additional services
+        vendor_hook = kwargs.get('vendor_hook')
+        if not vendor_hook and 'vendor_hook' in self.config.config:
+            # Get the one from profile
+            vendor_hook = self.config.config.get('vendor_hook')
+        if vendor_hook:
+            try:
+                # NOTE(gtema): no class name in the hook, plain module:function
+                # Split string hook into module and function
+                try:
+                    (package_name, function) = vendor_hook.rsplit(':')
+
+                    if package_name and function:
+                        import pkg_resources
+                        ep = pkg_resources.EntryPoint(
+                            'vendor_hook', package_name, attrs=(function,))
+                        hook = ep.resolve()
+                        hook(self)
+                except ValueError:
+                    self.log.warning('Hook should be in the entrypoint '
+                                     'module:attribute format')
+            except (ImportError, TypeError) as e:
+                self.log.warning('Configured hook %s cannot be executed: %s',
+                                 vendor_hook, e)
+
     @property
     def session(self):
         if not self._session:

openstack/tests/unit/config/test_json.py

@@ -34,14 +34,16 @@ class TestConfig(base.TestCase):
         _schema_path = os.path.join(
             os.path.dirname(os.path.realpath(defaults.__file__)),
             'schema.json')
-        schema = json.load(open(_schema_path, 'r'))
+        with open(_schema_path, 'r') as f:
+            schema = json.load(f)
         self.validator = jsonschema.Draft4Validator(schema)
         self.addOnException(self.json_diagnostics)
 
         self.filename = os.path.join(
             os.path.dirname(os.path.realpath(defaults.__file__)),
             'defaults.json')
-        self.json_data = json.load(open(self.filename, 'r'))
+        with open(self.filename, 'r') as f:
+            self.json_data = json.load(f)
 
         self.assertTrue(self.validator.is_valid(self.json_data))
 
@@ -49,14 +51,17 @@ class TestConfig(base.TestCase):
         _schema_path = os.path.join(
             os.path.dirname(os.path.realpath(defaults.__file__)),
             'vendor-schema.json')
-        schema = json.load(open(_schema_path, 'r'))
-        self.validator = jsonschema.Draft4Validator(schema)
+        with open(_schema_path, 'r') as f:
+            schema = json.load(f)
+            self.validator = jsonschema.Draft4Validator(schema)
+
         self.addOnException(self.json_diagnostics)
 
         _vendors_path = os.path.join(
             os.path.dirname(os.path.realpath(defaults.__file__)),
             'vendors')
         for self.filename in glob.glob(os.path.join(_vendors_path, '*.json')):
-            self.json_data = json.load(open(self.filename, 'r'))
+            with open(self.filename, 'r') as f:
+                self.json_data = json.load(f)
 
-            self.assertTrue(self.validator.is_valid(self.json_data))
+                self.assertTrue(self.validator.is_valid(self.json_data))

openstack/tests/unit/test_connection.py

@@ -59,10 +59,37 @@ clouds:
       password: {password}
       project_name: {project}
     cacert: {cacert}
+  profiled-cloud:
+    profile: dummy
+    auth:
+      username: {username}
+      password: {password}
+      project_name: {project}
+    cacert: {cacert}
 """.format(auth_url=CONFIG_AUTH_URL, username=CONFIG_USERNAME,
            password=CONFIG_PASSWORD, project=CONFIG_PROJECT,
            cacert=CONFIG_CACERT)
 
+VENDOR_CONFIG = """
+{{
+  "name": "dummy",
+  "profile": {{
+    "auth": {{
+      "auth_url": "{auth_url}"
+    }},
+    "vendor_hook": "openstack.tests.unit.test_connection:vendor_hook"
+  }}
+}}
+""".format(auth_url=CONFIG_AUTH_URL)
+
+PUBLIC_CLOUDS_YAML = """
+public-clouds:
+  dummy:
+    auth:
+      auth_url: {auth_url}
+    vendor_hook: openstack.tests.unit.test_connection:vendor_hook
+""".format(auth_url=CONFIG_AUTH_URL)
+
 
 class TestConnection(base.TestCase):
 
@@ -334,3 +361,65 @@ class TestNewService(base.TestCase):
 
         # ensure dns service responds as we expect from replacement
         self.assertFalse(conn.dns.dummy())
+
+
+def vendor_hook(conn):
+    setattr(conn, 'test', 'test_val')
+
+
+class TestVendorProfile(base.TestCase):
+
+    def setUp(self):
+        super(TestVendorProfile, self).setUp()
+        # Create a temporary directory where our test config will live
+        # and insert it into the search path via OS_CLIENT_CONFIG_FILE.
+        config_dir = self.useFixture(fixtures.TempDir()).path
+        config_path = os.path.join(config_dir, "clouds.yaml")
+        public_clouds = os.path.join(config_dir, "clouds-public.yaml")
+
+        with open(config_path, "w") as conf:
+            conf.write(CLOUD_CONFIG)
+
+        with open(public_clouds, "w") as conf:
+            conf.write(PUBLIC_CLOUDS_YAML)
+
+        self.useFixture(fixtures.EnvironmentVariable(
+            "OS_CLIENT_CONFIG_FILE", config_path))
+        self.use_keystone_v2()
+
+        self.config = openstack.config.loader.OpenStackConfig(
+            vendor_files=[public_clouds])
+
+    def test_conn_from_profile(self):
+
+        self.cloud = self.config.get_one(cloud='profiled-cloud')
+
+        conn = connection.Connection(config=self.cloud)
+
+        self.assertIsNotNone(conn)
+
+    def test_hook_from_profile(self):
+
+        self.cloud = self.config.get_one(cloud='profiled-cloud')
+
+        conn = connection.Connection(config=self.cloud)
+
+        self.assertEqual('test_val', conn.test)
+
+    def test_hook_from_connection_param(self):
+
+        conn = connection.Connection(
+            cloud='sample-cloud',
+            vendor_hook='openstack.tests.unit.test_connection:vendor_hook'
+        )
+
+        self.assertEqual('test_val', conn.test)
+
+    def test_hook_from_connection_ignore_missing(self):
+
+        conn = connection.Connection(
+            cloud='sample-cloud',
+            vendor_hook='openstack.tests.unit.test_connection:missing'
+        )
+
+        self.assertIsNotNone(conn)

releasenotes/notes/add_vendor_hook-e87b6afb7f215a30.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    Add possibility to automatically invoke vendor hooks. This can be done
+    either through extending profile (vendor_hook), or passing `vendor_hook`
+    parameter to the connection. The format of the vendor_hook is the same as
+    in the setuptools (module.name:function_name). The hook will get connection
+    as the only parameter.