Sync global tox, requirement files

Following changes are done as part of this patch:
* Sync global tox, requirements.txt, test-requirements
from release-tools project
* Resolve flake errors
* Fix templates to handle optional ingress relations
* Remove run_tests script
* Restructure unit tests directory

charms/keystone-k8s/.stestr.conf

@@ -1,3 +1,3 @@
 [DEFAULT]
-test_path=./unit_tests
+test_path=./tests/unit
-top_dir=./
+top_dir=./tests

charms/keystone-k8s/CONTRIBUTING.md

@@ -39,9 +39,9 @@ TODO
 ## Testing
 
 The Python operator framework includes a very nice harness for testing
-operator behaviour without full deployment. Just `run_tests`:
+operator behaviour without full deployment. Run tests using command:
 
-    ./run_tests
+    tox -e py3
 
 ## Deployment
 

charms/keystone-k8s/lib/charms/sunbeam_keystone_operator/v0/cloud_credentials.py

@@ -75,6 +75,17 @@ class IdentityServiceClientCharm(CharmBase):
 ```
 """
 
+import logging
+
+from ops.framework import (
+    StoredState,
+    EventBase,
+    ObjectEvents,
+    EventSource,
+    Object,
+)
+from ops.model import Relation
+
 # The unique Charmhub library identifier, never change it
 LIBID = "deadbeef"
 
@@ -85,22 +96,6 @@ LIBAPI = 0
 # to 0 if you are raising the major API version
 LIBPATCH = 1
 
-import json
-import logging
-import requests
-
-from ops.framework import (
-    StoredState,
-    EventBase,
-    ObjectEvents,
-    EventSource,
-    Object,
-)
-
-from ops.model import Relation
-
-from typing import List
-
 logger = logging.getLogger(__name__)
 
 

charms/keystone-k8s/lib/charms/sunbeam_keystone_operator/v0/identity_service.py

@@ -75,6 +75,20 @@ class IdentityServiceClientCharm(CharmBase):
 ```
 """
 
+import json
+import logging
+
+from ops.framework import (
+    StoredState,
+    EventBase,
+    ObjectEvents,
+    EventSource,
+    Object,
+)
+from ops.model import Relation
+
+logger = logging.getLogger(__name__)
+
 # The unique Charmhub library identifier, never change it
 LIBID = "6a7cb19b98314ecf916e3fcb02708608"
 
@@ -85,21 +99,6 @@ LIBAPI = 0
 # to 0 if you are raising the major API version
 LIBPATCH = 1
 
-import json
-import logging
-import requests
-
-from ops.framework import (
-    StoredState,
-    EventBase,
-    ObjectEvents,
-    EventSource,
-    Object,
-)
-
-from ops.model import Relation
-
-from typing import List
 
 logger = logging.getLogger(__name__)
 
@@ -309,25 +308,21 @@ class IdentityServiceRequires(Object):
         """Return the service_user_id."""
         return self.get_remote_app_data('service-user-id')
 
-
     @property
     def internal_auth_url(self) -> str:
         """Return the internal_auth_url."""
         return self.get_remote_app_data('internal-auth-url')
 
-
     @property
     def admin_auth_url(self) -> str:
         """Return the admin_auth_url."""
         return self.get_remote_app_data('admin-auth-url')
 
-
     @property
     def public_auth_url(self) -> str:
         """Return the public_auth_url."""
         return self.get_remote_app_data('public-auth-url')
 
-
     def register_services(self, service_endpoints: dict,
                           region: str) -> None:
         """Request access to the IdentityService server."""
@@ -356,7 +351,6 @@ class ReadyIdentityServiceClientsEvent(EventBase):
         self.region = region
         self.client_app_name = client_app_name
 
-
     def snapshot(self):
         return {
             "relation_id": self.relation_id,
@@ -420,7 +414,8 @@ class IdentityServiceProvides(Object):
 
         values = [
             event.relation.data[event.relation.app].get(k)
-            for k in REQUIRED_KEYS ]
+            for k in REQUIRED_KEYS
+        ]
         # Validate data on the relation
         if all(values):
             service_eps = json.loads(

charms/keystone-k8s/requirements.txt

@@ -1,7 +1,31 @@
-ops
+# This file is managed centrally by release-tools and should not be modified
+# within individual charm repos.  See the 'global' dir contents for available
+# choices of *requirements.txt files for OpenStack Charms:
+#     https://github.com/openstack-charmers/release-tools
+#
+
+# NOTE: newer versions of cryptography require a Rust compiler to build,
+# see
+# * https://github.com/openstack-charmers/zaza/issues/421
+# * https://mail.python.org/pipermail/cryptography-dev/2021-January/001003.html
+#
+cryptography<3.4
 jinja2
-git+https://github.com/openstack-charmers/advanced-sunbeam-openstack#egg=ops_sunbeam
 lightkube
 lightkube-models
-python-keystoneclient
+ops
-serialized-data-interface
+git+https://github.com/openstack-charmers/advanced-sunbeam-openstack#egg=ops_sunbeam
+
+python-keystoneclient  # keystone-k8s
+
+# Used for Traefik
+# Note: Remove when traefik-k8s-operator v1 library is released
+serialized_data_interface
+
+# Get resources from github until cacerts issue is charmbuild image is fixed.
+git+https://github.com/openstack/charm-ops-interface-tls-certificates#egg=interface_tls_certificates
+
+# Note: Required for cinder-k8s, cinder-ceph-k8s, glance-k8s, nova-k8s
+git+https://github.com/openstack/charm-ops-interface-ceph-client#egg=interface_ceph_client
+# Charmhelpers is only present as interface_ceph_client uses it.
+git+https://github.com/juju/charm-helpers.git#egg=charmhelpers

charms/keystone-k8s/run_tests

@@ -1,17 +0,0 @@
-#!/bin/sh -e
-# Copyright 2021 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-if [ -z "$VIRTUAL_ENV" -a -d venv/ ]; then
-    . venv/bin/activate
-fi
-
-if [ -z "$PYTHONPATH" ]; then
-    export PYTHONPATH="lib:src"
-else
-    export PYTHONPATH="lib:src:$PYTHONPATH"
-fi
-
-flake8
-coverage run --source=src -m unittest -v "$@"
-coverage report -m

charms/keystone-k8s/src/charm.py

@@ -432,7 +432,7 @@ class KeystoneOperatorCharm(sunbeam_charm.OSBaseOperatorAPICharm):
 
     @property
     def internal_endpoint(self):
-        if self.ingress_internal:
+        if self.ingress_internal and self.ingress_internal.url:
             return self.ingress_internal.url
 
         internal_hostname = self.model.config.get('os-internal-hostname')
@@ -445,7 +445,7 @@ class KeystoneOperatorCharm(sunbeam_charm.OSBaseOperatorAPICharm):
 
     @property
     def public_endpoint(self):
-        if self.ingress_public:
+        if self.ingress_public and self.ingress_public.url:
             return self.ingress_public.url
 
         address = self.public_ingress_address

charms/keystone-k8s/src/templates/keystone.conf.j2

@@ -8,7 +8,9 @@ use_syslog = True
 log_config_append = /etc/keystone/logging.conf
 debug = {{ options.debug }}
 
+{% if ks_config.public_endpoint -%}
 public_endpoint = {{ ks_config.public_endpoint }}
+{% endif -%}
 
 [identity]
 driver = {{ ks_config.identity_backend }}

charms/keystone-k8s/src/templates/wsgi-keystone.conf.j2

@@ -4,7 +4,7 @@ Listen 0.0.0.0:35357
 <VirtualHost *:5000>
     WSGIDaemonProcess keystone-public processes=4 threads=1 user=keystone group=keystone display-name=%{GROUP} python-path=/usr/lib/python3/site-packages
     WSGIProcessGroup keystone-public
-    {% if ingress_internal.ingress_path -%}
+    {% if ingress_internal and ingress_internal.ingress_path -%}
     WSGIScriptAlias {{ ingress_internal.ingress_path }} /usr/bin/keystone-wsgi-public
     {% endif -%}
     WSGIScriptAlias / /usr/bin/keystone-wsgi-public

charms/keystone-k8s/src/utils/manager.py

@@ -347,7 +347,8 @@ class KeystoneManager(framework.Object):
         logger.debug(f'Created project {name} with id {project.id}')
         return project
 
-    def get_project(self, name: str, domain: typing.Union[str, 'Domain']=None):
+    def get_project(self, name: str,
+                    domain: typing.Union[str, 'Domain'] = None):
         """
 
         """

charms/keystone-k8s/test-requirements.txt

@@ -1,12 +1,10 @@
-# This file is managed centrally.  If you find the need to modify this as a
+# This file is managed centrally by release-tools and should not be modified
-# one-off, please don't.  Intead, consult #openstack-charms and ask about
+# within individual charm repos.  See the 'global' dir contents for available
-# requirements management in charms via bot-control.  Thank you.
+# choices of *requirements.txt files for OpenStack Charms:
-coverage>=3.6
+#     https://github.com/openstack-charmers/release-tools
-cryptography<3.4
+#
-mock>=1.2
+
-flake8>=2.2.4,<=2.4.1
+coverage
-pyflakes==2.1.1
+mock
-stestr>=2.2.0
+flake8
-requests>=2.18.4
+stestr
-psutil
-pytz    # workaround for 14.04 pip/tox

charms/keystone-k8s/tests/unit/test_keystone_charm.py

@@ -16,10 +16,6 @@
 
 import mock
 import json
-import sys
-
-sys.path.append('lib')  # noqa
-sys.path.append('src')  # noqa
 
 import charm
 import ops_sunbeam.test_utils as test_utils

charms/keystone-k8s/tox.ini

@@ -1,78 +1,76 @@
-# Operator charm: tox.ini
+# Source charm: ./tox.ini
+# This file is managed centrally by release-tools and should not be modified
+# within individual charm repos.  See the 'global' dir contents for available
+# choices of tox.ini for OpenStack Charms:
+#     https://github.com/openstack-charmers/release-tools
 
 [tox]
-envlist = pep8,py3
 skipsdist = True
-# NOTE: Avoid build/test env pollution by not enabling sitepackages.
+envlist = pep8,py3
 sitepackages = False
-# NOTE: Avoid false positives by not skipping missing interpreters.
 skip_missing_interpreters = False
-# NOTES:
-# * We avoid the new dependency resolver by pinning pip < 20.3, see
-#   https://github.com/pypa/pip/issues/9187
-# * Pinning dependencies requires tox >= 3.2.0, see
-#   https://tox.readthedocs.io/en/latest/config.html#conf-requires
-# * It is also necessary to pin virtualenv as a newer virtualenv would still
-#   lead to fetching the latest pip in the func* tox targets, see
-#   https://stackoverflow.com/a/38133283
-requires = pip < 20.3
-           virtualenv < 20.0
-# NOTE: https://wiki.canonical.com/engineering/OpenStack/InstallLatestToxOnOsci
 minversion = 3.18.0
 
+[vars]
+src_path = {toxinidir}/src/
+tst_path = {toxinidir}/tests/
+lib_path = {toxinidir}/lib/
+
 [testenv]
-setenv = VIRTUAL_ENV={envdir}
+basepython = python3
-         PYTHONHASHSEED=0
+setenv =
-         CHARM_DIR={envdir}
+  PYTHONPATH = {toxinidir}:{[vars]lib_path}:{[vars]src_path}
+passenv =
+  PYTHONPATH
 install_command =
   pip install {opts} {packages}
 commands = stestr run --slowest {posargs}
-whitelist_externals =
+allowlist_externals =
   git
-    add-to-archive.py
-    bash
   charmcraft
-passenv = HOME TERM CS_* OS_* TEST_*
+  fetch-libs.sh
-deps = -r{toxinidir}/test-requirements.txt
+deps =
+  -r{toxinidir}/test-requirements.txt
+
+[testenv:build]
+basepython = python3
+deps =
+commands =
+  charmcraft clean
+  charmcraft -v pack
 
 [testenv:fetch]
 basepython = python3
 deps =
 commands =
-  ./fetch-libs.sh
+  {toxinidir}/fetch-libs.sh
-
-[testenv:py3.8]
-basepython = python3.8
-deps = -r{toxinidir}/requirements.txt
-       -r{toxinidir}/test-requirements.txt
-
-[testenv:py3.9]
-basepython = python3.9
-deps = -r{toxinidir}/requirements.txt
-       -r{toxinidir}/test-requirements.txt
-
-[testenv:py3.10]
-basepython = python3.10
-deps = -r{toxinidir}/requirements.txt
-       -r{toxinidir}/test-requirements.txt
 
 [testenv:py3]
 basepython = python3
-deps = -r{toxinidir}/requirements.txt
+deps =
-       -r{toxinidir}/test-requirements.txt
+  {[testenv]deps}
+  -r{toxinidir}/requirements.txt
+
+[testenv:py3.8]
+basepython = python3.8
+deps = {[testenv:py3]deps}
+
+[testenv:py3.9]
+basepython = python3.9
+deps = {[testenv:py3]deps}
+
+[testenv:py3.10]
+basepython = python3.10
+deps = {[testenv:py3]deps}
 
 [testenv:pep8]
 basepython = python3
-deps = -r{toxinidir}/requirements.txt
+deps = {[testenv]deps}
-       -r{toxinidir}/test-requirements.txt
+commands = flake8 {posargs} {[vars]src_path} {[vars]tst_path}
-commands = flake8 {posargs} src unit_tests tests
 
 [testenv:cover]
-# Technique based heavily upon
-# https://github.com/openstack/nova/blob/master/tox.ini
 basepython = python3
-deps = -r{toxinidir}/requirements.txt
+deps = {[testenv:py3]deps}
-       -r{toxinidir}/test-requirements.txt
 setenv =
     {[testenv]setenv}
     PYTHON=coverage run
@@ -92,46 +90,8 @@ source =
     .
 omit =
     .tox/*
-    */charmhelpers/*
-    src/templates/*
-    unit_tests/*
     tests/*
-
+    src/templates/*
-[testenv:venv]
-basepython = python3
-commands = {posargs}
-
-[testenv:build]
-basepython = python3
-deps = -r{toxinidir}/build-requirements.txt
-commands =
-    charmcraft build
-
-[testenv:func-noop]
-basepython = python3
-commands =
-    functest-run-suite --help
-
-[testenv:func]
-basepython = python3
-commands =
-    functest-run-suite --keep-model
-
-[testenv:func-smoke]
-basepython = python3
-commands =
-    functest-run-suite --keep-model --smoke
-
-[testenv:func-dev]
-basepython = python3
-commands =
-    functest-run-suite --keep-model --dev
-
-[testenv:func-target]
-basepython = python3
-commands =
-    functest-run-suite --keep-model --bundle {posargs}
 
 [flake8]
-# Ignore E902 because the unit_tests directory is missing in the built charm.
+ignore=E226,W504
-ignore = E402,E226,E902