Add TLS support for tempest-k8s

Updated charmcraft.yaml to include receive-ca-cert relation, and inject
the cacert to tempest container, and expose the path to the cacert to
pebble.

Change-Id: I3d7f3775d700643f688fb3b6140058dfb257f2bc

charms/tempest-k8s/.sunbeam-build.yaml

@@ -2,5 +2,8 @@ external-libraries:
   - charms.observability_libs.v1.kubernetes_service_patch
   - charms.grafana_k8s.v0.grafana_dashboard
   - charms.loki_k8s.v1.loki_push_api
+  - charms.certificate_transfer_interface.v0.certificate_transfer
 internal-libraries:
   - charms.keystone_k8s.v0.identity_resource
+templates:
+  - ca-bundle.pem.j2

charms/tempest-k8s/charmcraft.yaml

@@ -62,6 +62,8 @@ requires:
     interface: keystone-resources
   logging:
     interface: loki_push_api
+  receive-ca-cert:
+    interface: certificate_transfer
     optional: true
 
 provides:

charms/tempest-k8s/src/charm.py

@@ -55,12 +55,10 @@ from utils.alert_rules import (
     ensure_alert_rules_disabled,
     update_alert_rules_files,
 )
-from utils.cleanup import (
-    CleanUpError,
-    run_extensive_cleanup,
-)
 from utils.constants import (
     CONTAINER,
+    OS_CACERT_PATH,
+    RECEIVE_CA_CERT_RELATION_NAME,
     TEMPEST_ACCOUNTS_COUNT,
     TEMPEST_ADHOC_OUTPUT,
     TEMPEST_CONCURRENCY,
@@ -141,6 +139,12 @@ class TempestOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S):
                 "tempest",
                 0o750,
             ),
+            sunbeam_core.ContainerConfigFile(
+                OS_CACERT_PATH,
+                "root",
+                "tempest",
+                0o640,
+            ),
         ]
 
     def get_schedule(self) -> Schedule:
@@ -224,6 +228,12 @@ class TempestOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S):
             if (value := os.environ.get(proxy_var))
         }
 
+    def _get_os_cacert_environment(self) -> Dict[str, str]:
+        """Return the path to the OS cacert file if receive-ca-cert relation exist."""
+        if not list(self.model.relations[RECEIVE_CA_CERT_RELATION_NAME]):
+            return {}
+        return {"OS_CACERT": OS_CACERT_PATH}
+
     def _get_environment_for_tempest(
         self, variant: TempestEnvVariant
     ) -> Dict[str, str]:
@@ -259,6 +269,7 @@ class TempestOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S):
             "TEMPEST_OUTPUT": variant.output_path(),
         }
         tempest_env.update(self._get_proxy_environment())
+        tempest_env.update(self._get_os_cacert_environment())
         return tempest_env
 
     def _get_cleanup_env(self) -> Dict[str, str]:
@@ -278,6 +289,7 @@ class TempestOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S):
             "OS_PROJECT_DOMAIN_ID": credential.get("domain-id"),
         }
         cleanup_env.update(self._get_proxy_environment())
+        cleanup_env.update(self._get_os_cacert_environment())
         return cleanup_env
 
     def get_unit_data(self, key: str) -> Optional[str]:
@@ -309,10 +321,13 @@ class TempestOperatorCharm(sunbeam_charm.OSBaseOperatorCharmK8S):
         env = self._get_environment_for_tempest(TempestEnvVariant.PERIODIC)
         pebble = self.pebble_handler()
 
+        # Push auxiliary files first before doing anything
+        pebble.push_auxiliary_files()
+
         try:
             # do an extensive clean-up before tempest init to remove stalled resources
-            run_extensive_cleanup(self._get_cleanup_env())
-        except CleanUpError:
+            pebble.run_extensive_cleanup(self._get_cleanup_env())
+        except ops.pebble.ExecError:
             logger.debug("Clean-up failed and tempest init not run.")
             self.set_tempest_ready(False)
             return

charms/tempest-k8s/src/handlers.py

@@ -40,10 +40,6 @@ import ops_sunbeam.relation_handlers as sunbeam_rhandlers
 from utils.alert_rules import (
     ALERT_RULES_PATH,
 )
-from utils.cleanup import (
-    CleanUpError,
-    run_extensive_cleanup,
-)
 from utils.constants import (
     OPENSTACK_DOMAIN,
     OPENSTACK_PROJECT,
@@ -160,14 +156,13 @@ class TempestPebbleHandler(sunbeam_chandlers.ServicePebbleHandler):
         return [x.name for x in files]
 
     @assert_ready
-    def init_tempest(self, env: Dict[str, str]):
-        """Init the openstack environment for tempest.
+    def push_auxiliary_files(self) -> None:
+        """Push auxiliary files to the container.
 
-        Raise a RuntimeError if something goes wrong.
+        The auxiliary files are:
+        * the cleanup script
+        * the exclude list for tempest
         """
-        # push auxiliary files to the container
-        # * the cleanup script
-        # * the exclude list for tempest
         aux_files = [
             "src/utils/cleanup.py",
             "src/utils/tempest_exclude_list.txt",
@@ -182,6 +177,12 @@ class TempestPebbleHandler(sunbeam_chandlers.ServicePebbleHandler):
                     make_dirs=True,
                 )
 
+    @assert_ready
+    def init_tempest(self, env: Dict[str, str]):
+        """Init the openstack environment for tempest.
+
+        Raise a RuntimeError if something goes wrong.
+        """
         # Pebble runs cron, which runs tempest periodically
         # when periodic checks are enabled.
         # This ensures that tempest gets the env, inherited from cron.
@@ -284,6 +285,21 @@ class TempestPebbleHandler(sunbeam_chandlers.ServicePebbleHandler):
 
         return summary
 
+    @assert_ready
+    def run_extensive_cleanup(self, env: Dict[str, str]) -> None:
+        """Wrapper for running extensive cleanup."""
+        try:
+            self.execute(
+                ["python3", "cleanup.py", "extensive"],
+                user="tempest",
+                group="tempest",
+                working_dir=TEMPEST_HOME,
+                exception_on_error=True,
+                environment=env,
+            )
+        except ops.pebble.ExecError:
+            logger.warning("Clean-up failed")
+
 
 class TempestUserIdentityRelationHandler(sunbeam_rhandlers.RelationHandler):
     """Relation handler for identity ops."""
@@ -619,22 +635,12 @@ class TempestUserIdentityRelationHandler(sunbeam_rhandlers.RelationHandler):
         # and the environment should be inited again if rejoined.
         self.charm.set_tempest_ready(False)
 
-        credential = self.get_user_credential()
-        if credential and credential.get("auth-url"):
-            env = {
-                "OS_AUTH_URL": credential.get("auth-url"),
-                "OS_USERNAME": credential.get("username"),
-                "OS_PASSWORD": credential.get("password"),
-                "OS_PROJECT_NAME": credential.get("project-name"),
-                "OS_DOMAIN_ID": credential.get("domain-id"),
-                "OS_USER_DOMAIN_ID": credential.get("domain-id"),
-                "OS_PROJECT_DOMAIN_ID": credential.get("domain-id"),
-            }
-            try:
-                # do an extensive clean-up upon identity relation removal
-                run_extensive_cleanup(env)
-            except CleanUpError as e:
-                logger.warning("Clean-up failed: %s", str(e))
+        # Do an extensive clean-up upon identity relation removal if credential
+        # exists.
+        env = self.charm._get_cleanup_env()
+        if env and env.get("OS_AUTH_URL"):
+            pebble = self.charm.pebble_handler()
+            pebble.run_extensive_cleanup(env)
 
         # Delete the stored keystone credentials,
         # because they are no longer valid.

charms/tempest-k8s/src/templates/tempest-run-wrapper.j2

@@ -23,7 +23,7 @@ echo ":: discover-tempest-config" >> "$TMP_FILE"
 if discover-tempest-config --test-accounts "$TEMPEST_TEST_ACCOUNTS" --out "$TEMPEST_CONF" >> "$TMP_FILE" 2>&1; then
     echo ":: tempest run" >> "$TMP_FILE"
     tempest run --exclude-list "$TEMPEST_EXCLUDE_LIST" --workspace "$TEMPEST_WORKSPACE" -w "$TEMPEST_CONCURRENCY" "$@" >> "$TMP_FILE" 2>&1
-    python3 "$TEMPEST_HOME/cleanup.py"
+    python3 "$TEMPEST_HOME/cleanup.py" quick
 else
     echo ":: skipping tempest run because discover-tempest-config had errors" >> "$TMP_FILE"
 fi

charms/tempest-k8s/src/utils/cleanup.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Utils for cleaning up tempest-related resources."""
+import argparse
 import os
 from collections.abc import (
     Callable,
@@ -47,6 +48,7 @@ def _connect_to_os(env: dict) -> Connection:
         password=env["OS_PASSWORD"],
         user_domain_id=env["OS_USER_DOMAIN_ID"],
         project_domain_id=env["OS_USER_DOMAIN_ID"],
+        cacert=env["OS_CACERT"],
     )
 
 
@@ -298,6 +300,25 @@ def run_extensive_cleanup(env: dict) -> None:
         raise CleanUpError("\n".join(failure_message))
 
 
+def parse_command_line() -> argparse.Namespace:
+    """Parse command line interface."""
+    parser = argparse.ArgumentParser(
+        description="Clean up OpenStack resources created by tempest or discover-tempest-conf."
+    )
+    subparsers = parser.add_subparsers(dest="command")
+    subparsers.add_parser(
+        "quick",
+        description="Run a quick cleanup, suitable in between tempest test runs.",
+        help="run a quick cleanup, suitable in between tempest test runs",
+    )
+    subparsers.add_parser(
+        "extensive",
+        description="Run an extensive cleanup, suitable for a clean environment before creating initial tempest resources.",
+        help="run an extensive cleanup, suitable for a clean environment before creating initial tempest resources",
+    )
+    return parser.parse_args()
+
+
 def main() -> None:
     """Entrypoint for executing the script directly.
 
@@ -305,6 +326,7 @@ def main() -> None:
     Quick cleanup will be performed.
     """
     env = {
+        "OS_CACERT": os.getenv("OS_CACERT", ""),
         "OS_AUTH_URL": os.getenv("OS_AUTH_URL", ""),
         "OS_USERNAME": os.getenv("OS_USERNAME", ""),
         "OS_PASSWORD": os.getenv("OS_PASSWORD", ""),
@@ -315,7 +337,12 @@ def main() -> None:
         "TEMPEST_TEST_ACCOUNTS": os.getenv("TEMPEST_TEST_ACCOUNTS", ""),
     }
 
-    run_quick_cleanup(env)
+    args = parse_command_line()
+
+    if args.command == "quick":
+        run_quick_cleanup(env)
+    else:
+        run_extensive_cleanup(env)
 
 
 if __name__ == "__main__":

charms/tempest-k8s/src/utils/constants.py

@@ -32,6 +32,13 @@ def get_tempest_concurrency() -> str:
     return str(min(4, cpu_count()))
 
 
+# It's the target location for the OS cacert template file.  See
+# https://opendev.org/openstack/sunbeam-charms/src/branch/main/templates/ca-bundle.pem.j2
+OS_CACERT_PATH = "/usr/local/share/ca-certificates/ca-bundle.pem"
+# The relation name of 'receive-ca-cert'. See
+# https://opendev.org/openstack/sunbeam-charms/src/branch/main/ops-sunbeam/ops_sunbeam/charm.py#L194
+RECEIVE_CA_CERT_RELATION_NAME = "receive-ca-cert"
+
 TEMPEST_CONCURRENCY = get_tempest_concurrency()
 
 # It's desirable to have more accounts than the concurrency,

charms/tempest-k8s/tests/unit/test_cleanup.py

@@ -63,6 +63,7 @@ class TestCleanup(unittest.TestCase):
     def test_connect_to_os(self):
         """Test establishing OS connection."""
         env = {
+            "OS_CACERT": "/usr/local/share/ca-certificates/ca-bundle.pem",
             "OS_AUTH_URL": "http://10.6.0.20/openstack-keystone",
             "OS_USERNAME": "test_user",
             "OS_PASSWORD": "userpass",
@@ -80,6 +81,7 @@ class TestCleanup(unittest.TestCase):
             password=env["OS_PASSWORD"],
             user_domain_id=env["OS_USER_DOMAIN_ID"],
             project_domain_id=env["OS_USER_DOMAIN_ID"],
+            cacert=env["OS_CACERT"],
         )
 
     def test_get_exclude_resources(self):

tests/tempest/smoke.yaml.j2

@@ -210,3 +210,5 @@ relations:
 
 - - tempest:identity-ops
   - keystone:identity-ops
+- - tempest:receive-ca-cert
+  - keystone:send-ca-cert