Add support for identity resource library

* Add new library identity_resource
* Implement provider for identity_resource
* Make keystone client helpers to return dictionaries
instead of keystone resource objects
* Update identity_service library based on above changes.

Change-Id: Ib8278947a5bf3ccd4770e15bc49f54c9bff17b70

charms/keystone-k8s/lib/charms/keystone_k8s/v0/identity_resource.py

@@ -0,0 +1,348 @@
+"""IdentityResourceProvides and Requires module.
+
+
+This library contains the Requires and Provides classes for handling
+the identity_ops interface.
+
+Import `IdentityResourceRequires` in your charm, with the charm object and the
+relation name:
+    - self
+    - "identity_ops"
+
+Also provide additional parameters to the charm object:
+    - request
+
+Three events are also available to respond to:
+    - provider_ready
+    - provider_goneaway
+    - response_avaialable
+
+A basic example showing the usage of this relation follows:
+
+```
+from charms.keystone_k8s.v0.identity_resource import IdentityResourceRequires
+
+class IdentityResourceClientCharm(CharmBase):
+    def __init__(self, *args):
+        super().__init__(*args)
+        # IdentityResource Requires
+        self.identity_resource = IdentityResourceRequires(
+            self, "identity_ops",
+        )
+        self.framework.observe(
+            self.identity_resource.on.provider_ready, self._on_identity_resource_ready)
+        self.framework.observe(
+            self.identity_resource.on.provider_goneaway, self._on_identity_resource_goneaway)
+        self.framework.observe(
+            self.identity_resource.on.response_available, self._on_identity_resource_response)
+
+    def _on_identity_resource_ready(self, event):
+        '''React to the IdentityResource provider_ready event.
+
+        This event happens when n IdentityResource relation is added to the
+        model. Ready to send any ops to keystone.
+        '''
+        # Ready to send any ops.
+        pass
+
+    def _on_identity_resource_response(self, event):
+        '''React to the IdentityResource response_available event.
+
+        The IdentityResource interface will provide the response for the ops sent.
+        '''
+        # Read the response for the ops sent.
+        pass
+
+    def _on_identity_resource_goneaway(self, event):
+        '''React to the IdentityResource goneaway event.
+
+        This event happens when an IdentityResource relation is removed.
+        '''
+        # IdentityResource Relation has goneaway. No ops can be sent.
+        pass
+```
+"""
+
+import json
+import logging
+
+from ops.framework import (
+    EventBase,
+    EventSource,
+    Object,
+    ObjectEvents,
+    StoredState,
+)
+from ops.model import (
+    Relation,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# The unique Charmhub library identifier, never change it
+LIBID = "b419d4d8249e423487daafc3665ed06f"
+
+# Increment this major API version when introducing breaking changes
+LIBAPI = 0
+
+# Increment this PATCH version before using `charmcraft publish-lib` or reset
+# to 0 if you are raising the major API version
+LIBPATCH = 1
+
+
+REQUEST_NOT_SENT = 1
+REQUEST_SENT = 2
+REQUEST_PROCESSED = 3
+
+
+class IdentityOpsProviderReadyEvent(EventBase):
+    """Has IdentityOpsProviderReady Event."""
+
+    pass
+
+
+class IdentityOpsResponseEvent(EventBase):
+    """Has IdentityOpsResponse Event."""
+
+    pass
+
+
+class IdentityOpsProviderGoneAwayEvent(EventBase):
+    """Has IdentityOpsProviderGoneAway Event."""
+
+    pass
+
+
+class IdentityResourceResponseEvents(ObjectEvents):
+    """Events class for `on`."""
+
+    provider_ready = EventSource(IdentityOpsProviderReadyEvent)
+    response_available = EventSource(IdentityOpsResponseEvent)
+    provider_goneaway = EventSource(IdentityOpsProviderGoneAwayEvent)
+
+
+class IdentityResourceRequires(Object):
+    """IdentityResourceRequires class."""
+
+    on = IdentityResourceResponseEvents()
+    _stored = StoredState()
+
+    def __init__(self, charm, relation_name):
+        super().__init__(charm, relation_name)
+        self.charm = charm
+        self.relation_name = relation_name
+        self._stored.set_default(provider_ready=False, requests=[])
+        self.framework.observe(
+            self.charm.on[relation_name].relation_joined,
+            self._on_identity_resource_relation_joined,
+        )
+        self.framework.observe(
+            self.charm.on[relation_name].relation_changed,
+            self._on_identity_resource_relation_changed,
+        )
+        self.framework.observe(
+            self.charm.on[relation_name].relation_broken,
+            self._on_identity_resource_relation_broken,
+        )
+
+    def _on_identity_resource_relation_joined(self, event):
+        """Handle IdentityResource joined."""
+        self._stored.provider_ready = True
+        self.on.provider_ready.emit()
+
+    def _on_identity_resource_relation_changed(self, event):
+        """Handle IdentityResource changed."""
+        id_ = self.response.get("id")
+        self.save_request_in_store(id_, None, None, REQUEST_PROCESSED)
+        self.on.response_available.emit()
+
+    def _on_identity_resource_relation_broken(self, event):
+        """Handle IdentityResource broken."""
+        self._stored.provider_ready = False
+        self.on.provider_goneaway.emit()
+
+    @property
+    def _identity_resource_rel(self) -> Relation:
+        """The IdentityResource relation."""
+        return self.framework.model.get_relation(self.relation_name)
+
+    @property
+    def response(self) -> dict:
+        """Response object from keystone."""
+        response = self.get_remote_app_data("response")
+        if not response:
+            return {}
+
+        try:
+            return json.loads(response)
+        except Exception as e:
+            logger.debug(str(e))
+
+        return {}
+
+    def save_request_in_store(self, id: str, tag: str, ops: list, state: int):
+        """Save request in the store."""
+        if id is None:
+            return
+
+        for request in self._stored.requests:
+            if request.get("id") == id:
+                if tag:
+                    request["tag"] = tag
+                if ops:
+                    request["ops"] = ops
+                request["state"] = state
+                return
+
+        # New request
+        self._stored.requests.append(
+            {"id": id, "tag": tag, "ops": ops, "state": state}
+        )
+
+    def get_request_from_store(self, id: str) -> dict:
+        """Get request from the stote."""
+        for request in self._stored.requests:
+            if request.get("id") == id:
+                return request
+
+        return {}
+
+    def is_request_processed(self, id: str) -> bool:
+        """Check if request is processed."""
+        for request in self._stored.requests:
+            if (
+                request.get("id") == id
+                and request.get("state") == REQUEST_PROCESSED
+            ):
+                return True
+
+        return False
+
+    def get_remote_app_data(self, key: str) -> str:
+        """Return the value for the given key from remote app data."""
+        data = self._identity_resource_rel.data[
+            self._identity_resource_rel.app
+        ]
+        return data.get(key)
+
+    def ready(self) -> bool:
+        """Interface is ready or not.
+
+        Interface is considered ready if the op request is processed
+        and response is sent. In case of non leader unit, just consider
+        the interface is ready.
+        """
+        if not self.model.unit.is_leader():
+            logger.debug("Not a leader unit, set the interface to ready")
+            return True
+
+        try:
+            app_data = self._identity_resource_rel.data[self.charm.app]
+            if "request" not in app_data:
+                return False
+
+            request = json.loads(app_data["request"])
+            request_id = request.get("id")
+            response_id = self.response.get("id")
+            if request_id == response_id:
+                return True
+        except Exception as e:
+            logger.debug(str(e))
+
+        return False
+
+    def request_ops(self, request: dict) -> None:
+        """Request keystone ops."""
+        if not self.model.unit.is_leader():
+            logger.debug("Not a leader unit, not sending request")
+            return
+
+        id_ = request.get("id")
+        tag = request.get("tag")
+        ops = request.get("ops")
+        req = self.get_request_from_store(id_)
+        if req and req.get("state") == REQUEST_PROCESSED:
+            logger.debug("Request {id_} already processed")
+            return
+
+        if not self._stored.provider_ready:
+            self.save_request_in_store(id_, tag, ops, REQUEST_NOT_SENT)
+            logger.debug("Keystone not yet ready to take requests")
+            return
+
+        logger.debug("Requesting ops to keystone")
+        app_data = self._identity_resource_rel.data[self.charm.app]
+        app_data["request"] = json.dumps(request)
+        self.save_request_in_store(id_, tag, ops, REQUEST_SENT)
+
+
+class IdentityOpsRequestEvent(EventBase):
+    """Has IdentityOpsRequest Event."""
+
+    def __init__(self, handle, relation_id, relation_name, request):
+        """Initialise event."""
+        super().__init__(handle)
+        self.relation_id = relation_id
+        self.relation_name = relation_name
+        self.request = request
+
+    def snapshot(self):
+        """Snapshot the event."""
+        return {
+            "relation_id": self.relation_id,
+            "relation_name": self.relation_name,
+            "request": self.request,
+        }
+
+    def restore(self, snapshot):
+        """Restore the event."""
+        super().restore(snapshot)
+        self.relation_id = snapshot["relation_id"]
+        self.relation_name = snapshot["relation_name"]
+        self.request = snapshot["request"]
+
+
+class IdentityResourceProviderEvents(ObjectEvents):
+    """Events class for `on`."""
+
+    process_op = EventSource(IdentityOpsRequestEvent)
+
+
+class IdentityResourceProvides(Object):
+    """IdentityResourceProvides class."""
+
+    on = IdentityResourceProviderEvents()
+
+    def __init__(self, charm, relation_name):
+        super().__init__(charm, relation_name)
+        self.charm = charm
+        self.relation_name = relation_name
+        self.framework.observe(
+            self.charm.on[relation_name].relation_changed,
+            self._on_identity_resource_relation_changed,
+        )
+
+    def _on_identity_resource_relation_changed(self, event):
+        """Handle IdentityResource changed."""
+        request = event.relation.data[event.relation.app].get("request", {})
+        self.on.process_op.emit(
+            event.relation.id, event.relation.name, request
+        )
+
+    def set_ops_response(
+        self, relation_id: str, relation_name: str, ops_response: dict
+    ):
+        """Set response to ops request."""
+        if not self.model.unit.is_leader():
+            logger.debug("Not a leader unit, not sending response")
+            return
+
+        logger.debug("Update response from keystone")
+        _identity_resource_rel = self.charm.model.get_relation(relation_name, relation_id)
+        if not _identity_resource_rel:
+            # Relation has disappeared so skip send of data
+            return
+
+        app_data = _identity_resource_rel.data[self.charm.app]
+        app_data["response"] = json.dumps(ops_response)

charms/keystone-k8s/lib/charms/keystone_k8s/v1/identity_service.py

@@ -100,7 +100,7 @@ LIBAPI = 1
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
-LIBPATCH = 1
+LIBPATCH = 2
 
 
 logger = logging.getLogger(__name__)
@@ -477,12 +477,12 @@ class IdentityServiceProvides(Object):
                                          service_host: str,
                                          service_port: str,
                                          service_protocol: str,
-                                         admin_domain: str,
-                                         admin_project: str,
-                                         admin_user: str,
-                                         service_domain: str,
-                                         service_project: str,
-                                         service_user: str,
+                                         admin_domain: dict,
+                                         admin_project: dict,
+                                         admin_user: dict,
+                                         service_domain: dict,
+                                         service_project: dict,
+                                         service_user: dict,
                                          internal_auth_url: str,
                                          admin_auth_url: str,
                                          public_auth_url: str,
@@ -507,17 +507,17 @@ class IdentityServiceProvides(Object):
         app_data["service-host"] = service_host
         app_data["service-port"] = str(service_port)
         app_data["service-protocol"] = service_protocol
-        app_data["admin-domain-name"] = admin_domain.name
-        app_data["admin-domain-id"] = admin_domain.id
-        app_data["admin-project-name"] = admin_project.name
-        app_data["admin-project-id"] = admin_project.id
-        app_data["admin-user-name"] = admin_user.name
-        app_data["admin-user-id"] = admin_user.id
-        app_data["service-domain-name"] = service_domain.name
-        app_data["service-domain-id"] = service_domain.id
-        app_data["service-project-name"] = service_project.name
-        app_data["service-project-id"] = service_project.id
-        app_data["service-user-id"] = service_user.id
+        app_data["admin-domain-name"] = admin_domain.get("name")
+        app_data["admin-domain-id"] = admin_domain.get("id")
+        app_data["admin-project-name"] = admin_project.get("name")
+        app_data["admin-project-id"] = admin_project.get("id")
+        app_data["admin-user-name"] = admin_user.get("name")
+        app_data["admin-user-id"] = admin_user.get("id")
+        app_data["service-domain-name"] = service_domain.get("name")
+        app_data["service-domain-id"] = service_domain.get("id")
+        app_data["service-project-name"] = service_project.get("name")
+        app_data["service-project-id"] = service_project.get("id")
+        app_data["service-user-id"] = service_user.get("id")
         app_data["internal-auth-url"] = internal_auth_url
         app_data["admin-auth-url"] = admin_auth_url
         app_data["public-auth-url"] = public_auth_url

charms/keystone-k8s/metadata.yaml

@@ -27,6 +27,8 @@ provides:
     interface: keystone
   identity-credentials:
     interface: keystone-credentials
+  identity-ops:
+    interface: keystone-resources
 
 requires:
   database:

charms/keystone-k8s/src/charm.py

@@ -33,6 +33,7 @@ from typing import (
 )
 
 import charms.keystone_k8s.v0.identity_credentials as sunbeam_cc_svc
+import charms.keystone_k8s.v0.identity_resource as sunbeam_ops_svc
 import charms.keystone_k8s.v1.identity_service as sunbeam_id_svc
 import ops.charm
 import ops.pebble
@@ -201,6 +202,40 @@ class IdentityCredentialsProvidesHandler(sunbeam_rhandlers.RelationHandler):
         return True
 
 
+class IdentityResourceProvidesHandler(sunbeam_rhandlers.RelationHandler):
+    """Handler for identity resource relation."""
+
+    def __init__(
+        self,
+        charm: ops.charm.CharmBase,
+        relation_name: str,
+        callback_f: Callable,
+    ):
+        super().__init__(charm, relation_name, callback_f)
+
+    def setup_event_handler(self):
+        """Configure event handlers for an Identity resource relation."""
+        logger.debug("Setting up Identity Resource event handler")
+        ops_svc = sunbeam_ops_svc.IdentityResourceProvides(
+            self.charm,
+            self.relation_name,
+        )
+        self.framework.observe(
+            ops_svc.on.process_op,
+            self._on_process_op,
+        )
+        return ops_svc
+
+    def _on_process_op(self, event) -> None:
+        """Handles keystone ops events."""
+        self.callback_f(event)
+
+    @property
+    def ready(self) -> bool:
+        """Check if handler is ready."""
+        return True
+
+
 class WSGIKeystonePebbleHandler(sunbeam_chandlers.WSGIPebbleHandler):
     """Keystone Pebble Handler."""
 
@@ -242,6 +277,7 @@ class KeystoneOperatorCharm(sunbeam_charm.OSBaseOperatorAPICharm):
     ]
     IDSVC_RELATION_NAME = "identity-service"
     IDCREDS_RELATION_NAME = "identity-credentials"
+    IDOPS_RELATION_NAME = "identity-ops"
 
     def __init__(self, framework):
         super().__init__(framework)
@@ -370,26 +406,26 @@ export OS_AUTH_VERSION=3
         except SecretNotFoundError:
             logger.warning("Secret for {username} not found")
 
-        service_domain = self.keystone_manager.get_domain(
+        service_domain = self.keystone_manager.ksclient.show_domain(
             name="service_domain"
         )
-        service_project = self.keystone_manager.get_project(
-            name=self.service_project, domain=service_domain
+        service_project = self.keystone_manager.ksclient.show_project(
+            name=self.service_project, domain=service_domain.get("name")
         )
         self.keystone_manager.create_service_account(
             username=username,
             password=user_password,
-            project=service_project,
-            domain=service_domain,
+            project=service_project.get("name"),
+            domain=service_domain.get("name"),
         )
 
         event.set_results(
             {
                 "username": username,
                 "password": user_password,
-                "user-domain-name": service_domain.name,
-                "project-name": service_project.name,
-                "project-domain-name": service_domain.name,
+                "user-domain-name": service_domain.get("name"),
+                "project-name": service_project.get("name"),
+                "project-domain-name": service_domain.get("name"),
                 "region": self.model.config["region"],
                 "internal-endpoint": self.internal_endpoint,
                 "public-endpoint": self.public_endpoint,
@@ -412,7 +448,9 @@ export OS_AUTH_VERSION=3
             credentials_id = self._retrieve_or_set_secret(username)
             credentials = self.model.get_secret(id=credentials_id)
             password = pwgen.pwgen(12)
-            self.keystone_manager.update_user(name=username, password=password)
+            self.keystone_manager.ksclient.update_user(
+                user=username, password=password
+            )
             credentials.set_content(
                 {"username": username, "password": password}
             )
@@ -632,7 +670,7 @@ export OS_AUTH_VERSION=3
                     event.secret.label
                 ):
                     logger.info(f"Deleting user {user} from keystone")
-                    self.keystone_manager.delete_user(user)
+                    self.keystone_manager.ksclient.delete_user(user)
                     deleted_users.append(user)
             service_users_to_delete = [
                 x for x in service_users_to_delete if x not in deleted_users
@@ -676,6 +714,14 @@ export OS_AUTH_VERSION=3
             )
             handlers.append(self.cc_svc)
 
+        if self.can_add_handler(self.IDOPS_RELATION_NAME, handlers):
+            self.ops_svc = IdentityResourceProvidesHandler(
+                self,
+                self.IDOPS_RELATION_NAME,
+                self.handle_ops_from_event,
+            )
+            handlers.append(self.ops_svc)
+
         return super().get_relation_handlers(handlers)
 
     @property
@@ -721,11 +767,8 @@ export OS_AUTH_VERSION=3
             )
             return False
 
-    def check_outstanding_requests(self) -> bool:
-        """Process any outstanding client requests."""
-        logger.debug("Checking for outstanding client requests")
-        if not self.can_service_requests():
-            return
+    def check_outstanding_identity_service_requests(self) -> None:
+        """Check requests from identity service relation."""
         for relation in self.framework.model.relations[
             self.IDSVC_RELATION_NAME
         ]:
@@ -755,6 +798,9 @@ export OS_AUTH_VERSION=3
                         "Cannot process client request, 'service-endpoints' "
                         "not supplied"
                     )
+
+    def check_outstanding_identity_credentials_requests(self) -> None:
+        """Check requests from identity credentials relation."""
         for relation in self.framework.model.relations[
             self.IDCREDS_RELATION_NAME
         ]:
@@ -781,6 +827,38 @@ export OS_AUTH_VERSION=3
                         "supplied"
                     )
 
+    def check_outstanding_identity_ops_requests(self) -> None:
+        """Check requests from identity ops relation."""
+        for relation in self.framework.model.relations[
+            self.IDOPS_RELATION_NAME
+        ]:
+            app_data = relation.data[relation.app]
+            request = {}
+            response = {}
+            if app_data.get("request"):
+                request = json.loads(app_data.get("request"))
+            if relation.data[self.app].get("response"):
+                response = json.loads(relation.data[self.app].get("response"))
+
+            request_id = request.get("id")
+            if request_id != response.get("id"):
+                logger.debug(
+                    "Processing identity ops request from"
+                    f"{relation.app.name} {relation.name}/{relation.id}"
+                    f" for request id {request_id}"
+                )
+                self.handle_op_request(relation.id, relation.name, request)
+
+    def check_outstanding_requests(self) -> bool:
+        """Process any outstanding client requests."""
+        logger.debug("Checking for outstanding client requests")
+        if not self.can_service_requests():
+            return
+
+        self.check_outstanding_identity_service_requests()
+        self.check_outstanding_identity_credentials_requests()
+        self.check_outstanding_identity_ops_requests()
+
     def register_service_from_event(self, event):
         """Process service request event.
 
@@ -810,20 +888,23 @@ export OS_AUTH_VERSION=3
         binding = self.framework.model.get_binding(relation)
         ingress_address = str(binding.network.ingress_address)
 
-        service_domain = self.keystone_manager.get_domain(
+        service_domain = self.keystone_manager.ksclient.show_domain(
             name="service_domain"
         )
-        service_project = self.keystone_manager.get_project(
-            name=self.service_project, domain=service_domain
+        admin_domain = self.keystone_manager.ksclient.show_domain(
+            name="admin_domain"
         )
-        admin_domain = self.keystone_manager.get_domain(name="admin_domain")
-        admin_project = self.keystone_manager.get_project(
-            name="admin", domain=admin_domain
+        service_project = self.keystone_manager.ksclient.show_project(
+            name=self.service_project, domain=service_domain.get("name")
         )
-        admin_user = self.keystone_manager.get_user(
+        admin_project = self.keystone_manager.ksclient.show_project(
+            name="admin", domain=admin_domain.get("name")
+        )
+        admin_user = self.keystone_manager.ksclient.show_user(
             name=self.model.config["admin-user"],
-            project=admin_project,
-            domain=admin_domain,
+            domain=admin_domain.get("name"),
+            project=admin_project.get("name"),
+            project_domain=admin_domain.get("name"),
         )
 
         for ep_data in service_endpoints:
@@ -853,18 +934,18 @@ export OS_AUTH_VERSION=3
             service_user = self.keystone_manager.create_service_account(
                 username=service_username,
                 password=service_password,
-                project=service_project,
-                domain=service_domain,
+                project=service_project.get("name"),
+                domain=service_domain.get("name"),
             )
 
-            service = self.keystone_manager.create_service(
+            service = self.keystone_manager.ksclient.create_service(
                 name=ep_data["service_name"],
                 service_type=ep_data["type"],
                 description=ep_data["description"],
                 may_exist=True,
             )
             for interface in ["admin", "internal", "public"]:
-                self.keystone_manager.create_endpoint(
+                self.keystone_manager.ksclient.create_endpoint(
                     service=service,
                     interface=interface,
                     url=ep_data[f"{interface}_url"],
@@ -930,17 +1011,17 @@ export OS_AUTH_VERSION=3
         except SecretNotFoundError:
             logger.warning(f"Secret for {username} not found")
 
-        service_domain = self.keystone_manager.get_domain(
+        service_domain = self.keystone_manager.ksclient.show_domain(
             name="service_domain"
         )
-        service_project = self.keystone_manager.get_project(
-            name=self.service_project, domain=service_domain
+        service_project = self.keystone_manager.ksclient.show_project(
+            name=self.service_project, domain=service_domain.get("name")
         )
         self.keystone_manager.create_service_account(
             username=username,
             password=user_password,
-            project=service_project,
-            domain=service_domain,
+            project=service_project.get("name"),
+            domain=service_domain.get("name"),
         )
 
         self.cc_svc.interface.set_identity_credentials(
@@ -954,12 +1035,12 @@ export OS_AUTH_VERSION=3
             internal_port=self.default_public_ingress_port,
             internal_protocol="http",
             credentials=credentials_id,
-            project_name=service_project.name,
-            project_id=service_project.id,
-            user_domain_name=service_domain.name,
-            user_domain_id=service_domain.id,
-            project_domain_name=service_domain.name,
-            project_domain_id=service_domain.id,
+            project_name=service_project.get("name"),
+            project_id=service_project.get("id"),
+            user_domain_name=service_domain.get("name"),
+            user_domain_id=service_domain.get("id"),
+            project_domain_name=service_domain.get("name"),
+            project_domain_id=service_domain.get("id"),
             region=self.model.config["region"],  # XXX(wolsen) region matters?
             admin_role=self.admin_role,
         )
@@ -1265,6 +1346,49 @@ export OS_AUTH_VERSION=3
             self.keystone_manager.update_service_catalog_for_keystone()
         self.configure_charm(event)
 
+    def handle_ops_from_event(self, event):
+        """Process ops request event."""
+        logger.debug("Handle ops from event")
+        if not self.can_service_requests():
+            logger.debug(
+                f"handle_ops_from_event: Service not ready, request {event.request} not processed"
+            )
+            return
+
+        request = json.loads(event.request)
+        self.handle_op_request(
+            event.relation_id, event.relation_name, request=request
+        )
+
+    def handle_op_request(
+        self, relation_id: str, relation_name: str, request: dict
+    ):
+        """Process op request."""
+        response = {}
+        response["id"] = request.get("id")
+        response["tag"] = request.get("tag")
+        response["ops"] = [
+            {"name": op.get("name"), "return-code": -2, "value": None}
+            for op in request.get("ops", [])
+        ]
+
+        for idx, op in enumerate(request.get("ops", [])):
+            try:
+                func_name = op.get("name")
+                func = getattr(self.keystone_manager.ksclient, func_name)
+                params = op.get("params", {})
+                result = func(**params)
+                response["ops"][idx]["return-code"] = 0
+                response["ops"][idx]["value"] = result
+            except Exception as e:
+                response["ops"][idx]["return-code"] = -1
+                response["ops"][idx]["value"] = str(e)
+
+        logger.debug(f"handle_op_request: Sending response {response}")
+        self.ops_svc.interface.set_ops_response(
+            relation_id, relation_name, ops_response=response
+        )
+
 
 if __name__ == "__main__":
     main(KeystoneOperatorCharm)

charms/keystone-k8s/src/utils/manager.py

@@ -15,7 +15,10 @@
 """Manager for interacting with keystone."""
 
 import logging
-import typing
+from typing import (
+    Mapping,
+    Optional,
+)
 
 import ops.pebble
 import ops_sunbeam.guard as sunbeam_guard
@@ -28,27 +31,6 @@ from keystoneauth1.identity import (
 from keystoneclient.v3 import (
     client,
 )
-from keystoneclient.v3.domains import (
-    Domain,
-)
-from keystoneclient.v3.endpoints import (
-    Endpoint,
-)
-from keystoneclient.v3.projects import (
-    Project,
-)
-from keystoneclient.v3.regions import (
-    Region,
-)
-from keystoneclient.v3.roles import (
-    Role,
-)
-from keystoneclient.v3.services import (
-    Service,
-)
-from keystoneclient.v3.users import (
-    User,
-)
 from ops import (
     framework,
 )
@@ -56,15 +38,14 @@ from ops.model import (
     MaintenanceStatus,
 )
 
+from utils.client import (
+    KeystoneClient,
+    KeystoneExceptionError,
+)
+
 logger = logging.getLogger(__name__)
 
 
-class KeystoneExceptionError(Exception):
-    """Error interacting with Keystone."""
-
-    pass
-
-
 class KeystoneManager(framework.Object):
     """Class for managing interactions with keystone api."""
 
@@ -74,6 +55,7 @@ class KeystoneManager(framework.Object):
         self.charm = charm
         self.container_name = container_name
         self._api = None
+        self._ksclient = None
 
     def run_cmd(self, cmd, exception_on_error=True, **kwargs):
         """Run command in container."""
@@ -108,6 +90,14 @@ class KeystoneManager(framework.Object):
         )
         return self._api
 
+    @property
+    def ksclient(self) -> KeystoneClient:
+        """Keystone client."""
+        if self._ksclient:
+            return self._ksclient
+
+        return KeystoneClient(self.api)
+
     @property
     def admin_endpoint(self):
         """Admin endpoint for this keystone."""
@@ -202,15 +192,13 @@ class KeystoneManager(framework.Object):
                 ]
             )
 
-    def read_keys(self, key_repository: str) -> typing.Mapping[str, str]:
+    def read_keys(self, key_repository: str) -> Mapping[str, str]:
         """Pull the fernet keys from the on-disk repository."""
         container = self.charm.unit.get_container(self.container_name)
         files = container.list_files(key_repository)
         return {file.name: container.pull(file.path).read() for file in files}
 
-    def write_keys(
-        self, key_repository: str, keys: typing.Mapping[str, str]
-    ) -> None:
+    def write_keys(self, key_repository: str, keys: Mapping[str, str]) -> None:
         """Update the local fernet key repository with the provided keys."""
         container = self.charm.unit.get_container(self.container_name)
 
@@ -375,108 +363,107 @@ class KeystoneManager(framework.Object):
     def _setup_admin_accounts(self):
         """Setup admin accounts."""
         # Get the default domain id
-        default_domain = self.get_domain("default")
+        default_domain = self.ksclient.get_domain_object("default")
         logger.debug(f"Default domain id: {default_domain.id}")
         self.charm._state.default_domain_id = default_domain.id  # noqa
 
         # Get the admin domain id
-        admin_domain = self.create_domain(name="admin_domain", may_exist=True)
-        logger.debug(f"Admin domain id: {admin_domain.id}")
-        self.charm._state.admin_domain_id = admin_domain.id  # noqa
-        self.charm._state.admin_domain_name = admin_domain.name  # noqa
+        admin_domain = self.ksclient.create_domain(name="admin_domain")
+        admin_domain_id = admin_domain.get("id")
+        logger.debug(f"Admin domain id: {admin_domain_id}")
+        self.charm._state.admin_domain_id = admin_domain_id  # noqa
+        self.charm._state.admin_domain_name = admin_domain.get("name")  # noqa
 
         # Ensure that we have the necessary projects: admin and service
-        admin_project = self.create_project(
-            name="admin", domain=admin_domain, may_exist=True
+        admin_project = self.ksclient.create_project(
+            name="admin", domain=self.charm.admin_domain_name
         )
 
         logger.debug("Ensuring admin user exists")
-        admin_user = self.create_user(
+        self.ksclient.create_user(
             name=self.charm.admin_user,
             password=self.charm.admin_password,
-            domain=admin_domain,
-            may_exist=True,
+            domain=self.charm.admin_domain_name,
         )
 
         logger.debug("Ensuring roles exist for admin")
         # I seem to recall all kinds of grief between Member and member and
         # _member_ and inconsistencies in what other projects expect.
-        member_role = self.create_role(name="member", may_exist=True)
-        admin_role = self.create_role(
-            name=self.charm.admin_role, may_exist=True
-        )
+        member_role = self.ksclient.create_role(name="member")
+        self.ksclient.create_role(name=self.charm.admin_role)
 
         logger.debug("Granting roles to admin user")
         # Make the admin a member of the admin project
-        self.grant_role(
-            role=member_role,
-            user=admin_user,
-            project=admin_project,
-            may_exist=True,
+        self.ksclient.grant_role(
+            role=member_role.get("name"),
+            user=self.charm.admin_user,
+            project=admin_project.get("name"),
+            project_domain=self.charm.admin_domain_name,
+            user_domain=self.charm.admin_domain_name,
         )
         # Make the admin an admin of the admin project
-        self.grant_role(
-            role=admin_role,
-            user=admin_user,
-            project=admin_project,
-            may_exist=True,
+        self.ksclient.grant_role(
+            role=self.charm.admin_role,
+            user=self.charm.admin_user,
+            project=admin_project.get("name"),
+            project_domain=self.charm.admin_domain_name,
+            user_domain=self.charm.admin_domain_name,
         )
         # Make the admin a domain-level admin
-        self.grant_role(
-            role=admin_role,
-            user=admin_user,
-            domain=admin_domain,
-            may_exist=True,
+        self.ksclient.grant_role(
+            role=self.charm.admin_role,
+            user=self.charm.admin_user,
+            domain=self.charm.admin_domain_name,
+            user_domain=self.charm.admin_domain_name,
         )
 
     def _setup_service_accounts(self):
         """Create service accounts."""
         # Get the service domain id
-        service_domain = self.create_domain(
+        service_domain = self.ksclient.create_domain(
             name="service_domain", may_exist=True
         )
-        logger.debug(f"Service domain id: {service_domain.id}.")
+        service_domain_id = service_domain.get("id")
+        logger.debug(f"Service domain id: {service_domain_id}.")
 
-        service_project = self.create_project(
+        service_project = self.ksclient.create_project(
             name=self.charm.service_project,
-            domain=service_domain,
-            may_exist=True,
+            domain=service_domain.get("name"),
         )
-        logger.debug(f"Service project id: {service_project.id}.")
-        self.charm._state.service_project_id = service_project.id  # noqa
+        service_project_id = service_project.get("id")
+        logger.debug(f"Service project id: {service_project_id}.")
+        self.charm._state.service_project_id = service_project_id  # noqa
 
     def create_service_account(
         self,
         username: str,
         password: str,
-        project: "Project" = None,
-        domain: "Domain" = None,
-    ) -> "User":
+        project: Optional[str] = None,
+        domain: Optional[str] = None,
+    ) -> dict:
         """Helper function to create service account."""
         if not domain:
-            domain = self.get_domain(name="service_domain")
+            domain = "service_domain"
         if not project:
-            project = self.get_project(
-                name=self.charm.service_project, domain=domain
-            )
-        admin_role = self.get_role(name=self.charm.admin_role)
-        service_user = self.create_user(
+            project = self.charm_service_project
+
+        service_user = self.ksclient.create_user(
             name=username,
             password=password,
-            domain=domain.id,
-            may_exist=True,
+            domain=domain,
         )
-        self.grant_role(
-            role=admin_role,
-            user=service_user,
-            project=project,
-            may_exist=True,
+        self.ksclient.grant_role(
+            role=self.charm.admin_role,
+            project=self.charm.service_project,
+            user=service_user.get("name"),
+            project_domain="service_domain",
+            user_domain="service_domain",
         )
         return service_user
 
     def update_service_catalog_for_keystone(self):
         """Create identity service in catalogue."""
-        service = self.create_service(
+        service = self.ksclient.create_service(
             name="keystone",
             service_type="identity",
             description="Keystone Identity Service",
@@ -494,314 +481,10 @@ class KeystoneManager(framework.Object):
                 continue
 
             for interface, url in endpoints.items():
-                self.create_endpoint(
+                self.ksclient.create_endpoint(
                     service=service,
                     interface=interface,
                     url=url,
                     region=region,
                     may_exist=True,
                 )
-
-    def get_domain(self, name: str) -> "Domain":
-        """Get domain by name.
-
-        Returns the domain specified by the name, or None if a matching
-        domain could not be found.
-
-        :param name: the name of the domain
-        :type name: str
-        :rtype: 'Domain' or None
-        """
-        for domain in self.api.domains.list():
-            if domain.name.lower() == name.lower():
-                return domain
-
-        return None
-
-    def create_domain(
-        self,
-        name: str,
-        description: str = "Created by Juju",
-        may_exist: bool = False,
-    ) -> "Domain":
-        """Create a domain."""
-        if may_exist:
-            domain = self.get_domain(name)
-            if domain:
-                logger.debug(
-                    f"Domain {name} already exists with domain "
-                    f"id {domain.id}."
-                )
-                return domain
-
-        domain = self.api.domains.create(name=name, description=description)
-        logger.debug(f"Created domain {name} with id {domain.id}")
-        return domain
-
-    def create_project(
-        self,
-        name: str,
-        domain: str,
-        description: str = "Created by Juju",
-        may_exist: bool = False,
-    ) -> "Project":
-        """Create a project."""
-        if may_exist:
-            for project in self.api.projects.list(domain=domain):
-                if project.name.lower() == name.lower():
-                    logger.debug(
-                        f"Project {name} already exists with project "
-                        f"id {project.id}."
-                    )
-                    return project
-
-        project = self.api.projects.create(
-            name=name, description=description, domain=domain
-        )
-        logger.debug(f"Created project {name} with id {project.id}")
-        return project
-
-    def get_project(
-        self, name: str, domain: typing.Union[str, "Domain"] = None
-    ):
-        """Get a project from name."""
-        projects = self.api.projects.list(domain=domain)
-        for project in projects:
-            if project.name.lower() == name.lower():
-                return project
-        return None
-
-    def create_user(
-        self,
-        name: str,
-        password: str,
-        email: str = None,
-        project: "Project" = None,
-        domain: "Domain" = None,
-        may_exist: bool = False,
-    ) -> "User":
-        """Create a user."""
-        if may_exist:
-            user = self.get_user(name, project=project, domain=domain)
-            if user:
-                logger.debug(
-                    f"User {name} already exists with user " f"id {user.id}."
-                )
-                return user
-
-        user = self.api.users.create(
-            name=name,
-            default_project=project,
-            domain=domain,
-            password=password,
-            email=email,
-        )
-        logger.debug(f"Created user {user.name} with id {user.id}.")
-        return user
-
-    def get_user(
-        self,
-        name: str,
-        project: "Project" = None,
-        domain: typing.Union[str, "Domain"] = None,
-    ) -> "User":
-        """Get a user from name."""
-        users = self.api.users.list(default_project=project, domain=domain)
-        for user in users:
-            if user.name.lower() == name.lower():
-                return user
-
-        return None
-
-    def update_user(
-        self,
-        name: str,
-        password: str,
-        email: str = None,
-        project: "Project" = None,
-        domain: "Domain" = None,
-    ) -> "User":
-        """Update password for user."""
-        user = self.get_user(name=name, domain=domain, project=project)
-        user = self.api.users.update(
-            user,
-            name=name,
-            default_project=project,
-            domain=domain,
-            password=password,
-            email=email,
-        )
-        logger.debug(f"Updated user {user.name}.")
-        return user
-
-    def delete_user(
-        self,
-        name: str,
-    ) -> None:
-        """Delete a user from name."""
-        user = self.get_user(name)
-        self.api.users.delete(user)
-
-    def create_role(
-        self,
-        name: str,
-        domain: typing.Union["Domain", str] = None,
-        may_exist: bool = False,
-    ) -> "Role":
-        """Create a role."""
-        if may_exist:
-            role = self.get_role(name=name, domain=domain)
-            if role:
-                logger.debug(
-                    f"Role {name} already exists with role " f"id {role.id}"
-                )
-                return role
-
-        role = self.api.roles.create(name=name, domain=domain)
-        logger.debug(f"Created role {name} with id {role.id}.")
-        return role
-
-    def get_role(self, name: str, domain: "Domain" = None) -> "Role":
-        """Get role for user."""
-        for role in self.api.roles.list(domain=domain):
-            if role.name == name:
-                return role
-
-        return None
-
-    def get_roles(
-        self, user: "User", project: "Project" = None, domain: "Project" = None
-    ) -> typing.List["Role"]:
-        """Get Roles for user."""
-        if project and domain:
-            raise ValueError("Project and domain are mutually exclusive")
-        if not project and not domain:
-            raise ValueError("Project or domain must be specified")
-
-        if project:
-            roles = self.api.roles.list(user=user, project=project)
-        else:
-            roles = self.api.roles.list(user=user, domain=domain)
-
-        return roles
-
-    def grant_role(
-        self,
-        role: typing.Union["Role", str],
-        user: "User",
-        project: typing.Union["Project", str] = None,
-        domain: typing.Union["Domain", str] = None,
-        may_exist: bool = False,
-    ) -> "Role":
-        """Grant role to user."""
-        if project and domain:
-            raise ValueError("Project and domain are mutually exclusive")
-        if not project and not domain:
-            raise ValueError("Project or domain must be specified")
-
-        if domain:
-            ctxt_str = f"domain {domain.name}"
-        else:
-            ctxt_str = f"project {project.name}"
-
-        if may_exist:
-            roles = self.get_roles(user=user, project=project, domain=domain)
-            for r in roles:
-                if role.id == r.id:
-                    logger.debug(
-                        f"User {user.name} already has role "
-                        f"{role.name} for {ctxt_str}"
-                    )
-                    return r
-
-        role = self.api.roles.grant(
-            role=role, user=user, project=project, domain=domain
-        )
-        logger.debug(f"Granted user {user} role {role} for " f"{ctxt_str}.")
-        return role
-
-    def create_region(
-        self, name: str, description: str = None, may_exist: bool = False
-    ) -> "Region":
-        """Create Region in keystone."""
-        if may_exist:
-            for region in self.api.regions.list():
-                if region.id == name:
-                    logger.debug(f"Region {name} already exists.")
-                    return region
-
-        region = self.api.regions.create(id=name, description=description)
-        logger.debug(f"Created region {name}.")
-        return region
-
-    def create_service(
-        self,
-        name: str,
-        service_type: str,
-        description: str,
-        owner: str = None,
-        may_exist: bool = False,
-    ) -> "Service":
-        """Create service in Keystone."""
-        if may_exist:
-            services = self.api.services.list(name=name, type=service_type)
-            # TODO(wolsen) can we have more than one service with the same
-            #  service name? I don't think so, so we'll just handle the first
-            #  one for now.
-            logger.debug(f"FOUND: {services}")
-            for service in services:
-                logger.debug(
-                    f"Service {name} already exists with "
-                    f"service id {service.id}."
-                )
-                return service
-
-        service = self.api.services.create(
-            name=name, type=service_type, description=description
-        )
-        logger.debug(f"Created service {service.name} with id {service.id}")
-        return service
-
-    def create_endpoint(
-        self,
-        service: "Service",
-        url: str,
-        interface: str,
-        region: str,
-        may_exist: bool = False,
-    ) -> "Endpoint":
-        """Create endpoint in keystone."""
-        ep_string = (
-            f"{interface} endpoint for service {service} in "
-            f"region {region}"
-        )
-        if may_exist:
-            endpoints = self.api.endpoints.list(
-                service=service, interface=interface, region=region
-            )
-            if endpoints:
-                # NOTE(wolsen) if we have endpoints found, there should be only
-                # one endpoint; but assert it to make sure
-                assert len(endpoints) == 1
-                endpoint = endpoints[0]
-                if endpoint.url != url:
-                    logger.debug(
-                        f"{ep_string} ({endpoint.url}) does "
-                        f"not match requested url ({url}). Updating."
-                    )
-                    endpoint = self.api.endpoints.update(
-                        endpoint=endpoint, url=url
-                    )
-                    logger.debug(f"Endpoint updated to use {url}")
-                else:
-                    logger.debug(
-                        f"Endpoint {ep_string} already exists with "
-                        f"id {endpoint.id}"
-                    )
-                return endpoint
-
-        endpoint = self.api.endpoints.create(
-            service=service, url=url, interface=interface, region=region
-        )
-        logger.debug(f"Created endpoint {ep_string} with id {endpoint.id}")
-        return endpoint

charms/keystone-k8s/tests/unit/test_keystone_charm.py

@@ -98,10 +98,7 @@ class TestKeystoneOperatorCharm(test_utils.CharmTestCase):
         """Create keystone manager mock."""
 
         def _create_mock(p_name, p_id):
-            _mock = mock.MagicMock()
-            type(_mock).name = mock.PropertyMock(return_value=p_name)
-            type(_mock).id = mock.PropertyMock(return_value=p_id)
-            return _mock
+            return {"id": p_id, "name": p_name}
 
         def _get_domain_side_effect(name: str):
             if name == "admin_domain":
@@ -120,12 +117,11 @@ class TestKeystoneOperatorCharm(test_utils.CharmTestCase):
         admin_role_mock = _create_mock("arole_name", "arole_id")
 
         km_mock = mock.MagicMock()
-        km_mock.get_domain.side_effect = _get_domain_side_effect
-        km_mock.get_project.return_value = admin_project_mock
-        km_mock.get_user.return_value = admin_user_mock
-        km_mock.create_domain.return_value = service_domain_mock
-        km_mock.create_user.return_value = service_user_mock
-        km_mock.create_role.return_value = admin_role_mock
+        km_mock.ksclient.show_domain.side_effect = _get_domain_side_effect
+        km_mock.ksclient.show_project.return_value = admin_project_mock
+        km_mock.ksclient.show_user.return_value = admin_user_mock
+        km_mock.ksclient.create_user.return_value = service_user_mock
+        km_mock.ksclient.create_role.return_value = admin_role_mock
         km_mock.create_service_account.return_value = service_user_mock
         km_mock.read_keys.return_value = {
             "0": "Qf4vHdf6XC2dGKpEwtGapq7oDOqUWepcH2tKgQ0qOKc=",
@@ -172,7 +168,6 @@ class TestKeystoneOperatorCharm(test_utils.CharmTestCase):
     # This function need to be moved to operator
     def get_secret_by_label(self, label: str) -> str:
         """Get secret by label from harness class."""
-        print(self.harness._backend._secrets)
         for secret in self.harness._backend._secrets:
             if secret.label == label:
                 return secret.id