Add "idp-metadata-url" option

Add an option to allow retrieving the IDP metadata from a URL and
keeping the metadata auto-updated on charm hook execution.

Change-Id: I65b20e52835497a3fe57571794f332b2b4327fba
Signed-off-by: Yanos Angelopoulos <yanos@admin.grnet.gr>
Co-authored-by: Stamatis Katsaounis <skatsaounis@admin.grnet.gr>
Co-authored-by: Dmitrii Shcherbakov <dmitrii.shcherbakov@canonical.com>

src/config.yaml

@@ -69,3 +69,10 @@ options:
       IDP discovery service URL. If set to "" (default) no discovery
       service will be used. If used, the resource "idp-metadata" must
       be an XML file containing descriptors for multiple IDPs
+  idp-metadata-url:
+    type: string
+    default:
+    description: |
+      An optional URL to retrieve IDP metadata from. If set, takes priority
+      over the "idp-metadata" resource. Auto-updates of metadata occur during
+      any hook execution, including update-status.

src/lib/charm/openstack/keystone_saml_mellon.py

@@ -24,11 +24,14 @@ import charms_openstack.adapters
 
 import os
 import subprocess
+import urllib.request
+import urllib.error
 
 from lxml import etree
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import serialization
 
+
 CONFIGS = (IDP_METADATA, SP_METADATA, SP_PRIVATE_KEY,
            SP_LOCATION_CONFIG,) = [
                os.path.join('/etc/apache2/mellon',
@@ -130,29 +133,63 @@ class KeystoneSAMLMellonConfigurationAdapter(
 
     IDP_METADATA_INVALID = ('idp-metadata resource is not a well-formed'
                             ' xml file')
+    IDP_METADATA_URL_ERROR = ('an error has occurred during idp-metadata-url'
+                              ' config option processing')
+    IDP_METADATA_NOT_PROVIDED = ('idp-metadata resource has not been provided')
 
     @property
     def idp_metadata(self):
-        idp_metadata_path = hookenv.resource_get('idp-metadata')
+        idp_metadata_content = None
-        if os.path.exists(idp_metadata_path) and not self._idp_metadata:
+        if self.idp_metadata_url is None:
-            with open(idp_metadata_path) as f:
+            # Get metadata from resource
-                content = f.read()
+            idp_metadata_path = hookenv.resource_get('idp-metadata')
-                try:
+            if (idp_metadata_path and
-                    etree.fromstring(content.encode())
+                    os.path.exists(idp_metadata_path) and not
-                    self._idp_metadata = content
+                    self._idp_metadata):
-                    self._validation_errors['idp-metadata'] = None
+                with open(idp_metadata_path, 'r', encoding='utf-8') as f:
-                except etree.XMLSyntaxError:
+                    idp_metadata_content = f.read()
-                    self._idp_metadata = ''
+            else:
-                    self._validation_errors['idp-metadata'] = (
+                self._validation_errors['idp-metadata'] =\
-                        self.IDP_METADATA_INVALID)
+                    self.IDP_METADATA_NOT_PROVIDED
+        else:
+            try:
+                response = urllib.request.urlopen(self.idp_metadata_url)
+            except urllib.error.URLError as e:
+                self._validation_errors['idp-metadata'] = '{}: {}'.format(
+                    self.IDP_METADATA_URL_ERROR,
+                    e.reason
+                )
+                return self._idp_metadata
+            encoded_content = response.read()
+            idp_metadata_content = encoded_content.decode("utf-8")
+
+        # Metadata has been provided either via a resource or downloaded from
+        # the specified URL.
+        if idp_metadata_content is not None:
+            try:
+                etree.fromstring(idp_metadata_content.encode())
+                self._idp_metadata = idp_metadata_content
+                self._validation_errors['idp-metadata'] = None
+            except etree.XMLSyntaxError:
+                self._idp_metadata = None
+                self._validation_errors['idp-metadata'] = (
+                    self.IDP_METADATA_INVALID)
+
         return self._idp_metadata
 
     SP_SIGNING_KEYINFO_INVALID = ('sp-signing-keyinfo resource is not a'
                                   ' well-formed xml file')
+    SP_SIGNING_KEYINFO_NOT_PROVIDED = ('sp-signing-keyinfo resource has not'
+                                       ' been provided')
 
     @property
     def sp_signing_keyinfo(self):
         info_path = hookenv.resource_get('sp-signing-keyinfo')
+        if not info_path:
+            self._sp_signing_keyinfo = None
+            self._validation_errors['sp-signing-keyinfo'] = (
+                self.SP_SIGNING_KEYINFO_NOT_PROVIDED)
+            return self._sp_signing_keyinfo
         if os.path.exists(info_path) and not self._sp_signing_keyinfo:
             self._sp_signing_keyinfo = None
             with open(info_path) as f:
@@ -167,12 +204,19 @@ class KeystoneSAMLMellonConfigurationAdapter(
                         self.SP_SIGNING_KEYINFO_INVALID)
         return self._sp_signing_keyinfo
 
-    SP_PRIVATE_KEY_INVALID = ('resource is not a well-formed'
+    SP_PRIVATE_KEY_INVALID = ('sp-private-key resource is not a well-formed'
                               ' RFC 5958 (PKCS#8) key')
+    SP_PRIVATE_KEY_NOT_PROVIDED = ('sp-private-key resource has not'
+                                   ' been provided')
 
     @property
     def sp_private_key(self):
         pk_path = hookenv.resource_get('sp-private-key')
+        if not pk_path:
+            self._sp_private_key = None
+            self._validation_errors['sp-private-key'] = (
+                self.SP_PRIVATE_KEY_NOT_PROVIDED)
+            return self._sp_private_key
         if os.path.exists(pk_path) and not self._sp_private_key:
             with open(pk_path) as f:
                 content = f.read()
@@ -245,8 +289,8 @@ class KeystoneSAMLMellonCharm(charms_openstack.charm.OpenStackCharm):
             'protocol-name': self.options.protocol_name,
             'user-facing-name': self.options.user_facing_name,
             'idp-metadata': self.options.idp_metadata,
-            'sp-private-key': self.options.sp_private_key,
             'sp-signing-keyinfo': self.options.sp_signing_keyinfo,
+            'sp-private-key': self.options.sp_private_key,
             'nameid-formats': self.options.nameid_formats,
         }
 

src/tests/tests.yaml

@@ -32,7 +32,7 @@ target_deploy_status:
     workload-status-message: Vault needs to be initialized
   keystone-saml-mellon:
     workload-status: blocked
-    workload-status-message: "Configuration is incomplete. idp-metadata: idp-metadata resource is not a well-formed xml file"
+    workload-status-message: "Configuration is incomplete. idp-metadata: idp-metadata resource has not been provided,sp-signing-keyinfo: sp-signing-keyinfo resource has not been provided,sp-private-key: sp-private-key resource has not been provided"
 
 tests_options:
   force_deploy:

unit_tests/test_lib_charm_openstack_keystone_saml_mellon.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import mock
+import urllib.error
 
 import charms_openstack.test_utils as test_utils
 
@@ -53,7 +54,8 @@ class Helper(test_utils.PatchHelper):
             "protocol-name": self.protocol_name,
             "user-facing-name": self.user_facing_name,
             "nameid-formats": self.nameid_formats,
-            "subject-confirmation-data-address-check": False
+            "subject-confirmation-data-address-check": False,
+            "idp-metadata-url": None
         }
         self.resources = {
             "idp-metadata": "/path/to/idp-metadata.xml",
@@ -194,16 +196,91 @@ class TestKeystoneSAMLMellonConfigurationAdapter(Helper):
             "</EntityDescriptor>")
         self.file.read.return_value = self.idp_metadata_xml
         self.assertEqual(ksmca.idp_metadata, self.idp_metadata_xml)
-        self.open.assert_called_with(self.resources["idp-metadata"])
+        self.open.assert_called_with(self.resources["idp-metadata"],
+                                     'r', encoding='utf-8')
 
-        # Inalid XML
+        # Invalid XML
         ksmca._idp_metadata = None
         self.file.read.return_value = "INVALID XML"
-        self.assertEqual(ksmca.idp_metadata, "")
+        self.assertEqual(ksmca.idp_metadata, None)
         self.assertEqual(
             ksmca._validation_errors,
             {"idp-metadata": ksmca.IDP_METADATA_INVALID})
 
+    def test_no_resources(self):
+        resources = {
+            "idp-metadata": False,
+            "sp-private-key": False,
+            "sp-signing-keyinfo": False
+        }
+        self.patch_object(keystone_saml_mellon.hookenv, 'resource_get',
+                          side_effect=FakeResourceGet(resources))
+        ksmca = keystone_saml_mellon.KeystoneSAMLMellonConfigurationAdapter()
+
+        self.assertEqual(ksmca.idp_metadata, None)
+        self.assertEqual(
+            ksmca._validation_errors,
+            {"idp-metadata": ksmca.IDP_METADATA_NOT_PROVIDED})
+
+        self.assertEqual(ksmca.sp_signing_keyinfo, None)
+        self.assertEqual(
+            ksmca._validation_errors,
+            {"idp-metadata": ksmca.IDP_METADATA_NOT_PROVIDED,
+             "sp-signing-keyinfo": ksmca.SP_SIGNING_KEYINFO_NOT_PROVIDED})
+
+        self.assertEqual(ksmca.sp_private_key, None)
+        self.assertEqual(
+            ksmca._validation_errors,
+            {"idp-metadata": ksmca.IDP_METADATA_NOT_PROVIDED,
+             "sp-signing-keyinfo": ksmca.SP_SIGNING_KEYINFO_NOT_PROVIDED,
+             "sp-private-key": ksmca.SP_PRIVATE_KEY_NOT_PROVIDED})
+
+    def test_idp_metadata_url(self):
+        self.test_config.update(
+            {'idp-metadata-url': 'https://samltest.id/saml/idp'}
+        )
+        self.patch_object(keystone_saml_mellon.hookenv, 'config',
+                          side_effect=FakeConfig(self.test_config))
+        resources = {
+            "idp-metadata": False,
+            "sp-private-key": False,
+            "sp-signing-keyinfo": False
+        }
+        self.patch_object(keystone_saml_mellon.hookenv, 'resource_get',
+                          side_effect=FakeResourceGet(resources))
+        response = mock.MagicMock()
+        idp_meta_xml = (
+            "<?xml version='1.0' encoding='UTF-8'?>"
+            "<EntityDescriptor  entityID='https://samltest.id/saml/idp'> "
+            "</EntityDescriptor>"
+        )
+        response.read.return_value = idp_meta_xml.encode('utf-8')
+        self.patch_object(keystone_saml_mellon.urllib.request, 'urlopen',
+                          return_value=response)
+
+        ksmca = keystone_saml_mellon.KeystoneSAMLMellonConfigurationAdapter()
+        # Valid XML
+        self.assertEqual(ksmca.idp_metadata, idp_meta_xml)
+
+        # Invalid XML
+        response = mock.MagicMock()
+        response.read.return_value = "foobar42".encode('utf-8')
+        self.patch_object(keystone_saml_mellon.urllib.request, 'urlopen',
+                          return_value=response)
+        ksmca._idp_metadata = None
+        self.assertEqual(ksmca.idp_metadata, None)
+        self.assertEqual(
+            ksmca._validation_errors,
+            {"idp-metadata": ksmca.IDP_METADATA_INVALID})
+
+        # Invalid URL
+        self.patch_object(keystone_saml_mellon.urllib.request, 'urlopen',
+                          side_effect=urllib.error.URLError('invalid URL'))
+        self.assertEqual(ksmca.idp_metadata, None)
+        self.assertEqual(
+            ksmca._validation_errors,
+            {"idp-metadata": ksmca.IDP_METADATA_URL_ERROR + ': invalid URL'})
+
     def test_sp_signing_keyinfo(self):
         self.os.path.exists.return_value = True
         ksmca = keystone_saml_mellon.KeystoneSAMLMellonConfigurationAdapter()