Sync global tox, requirement files

Following changes are done as part of this patch:
* Sync global tox, requirements.txt, test-requirements
from release-tools project
* Resolve flake errors
* Fix templates to handle optional ingress relations
* Remove run_tests script
* Restructure unit tests directory

charms/nova-k8s/.stestr.conf

@@ -1,3 +1,3 @@
 [DEFAULT]
-test_path=./unit_tests
+test_path=./tests/unit
-top_dir=./
+top_dir=./tests

charms/nova-k8s/1

@@ -1,51 +0,0 @@
-Listen {{ wsgi_config.public_port }}
-Listen {{ wsgi_nova_metadata.public_port }}
-<VirtualHost *:{{ wsgi_config.public_port }}>
-    WSGIDaemonProcess nova-api processes=4 threads=1 user={{ wsgi_config.user }} group={{ wsgi_config.group }} \
-                      display-name=%{GROUP}
-    WSGIProcessGroup nova-api
-    {% if ingress_internal.ingress_path -%}
-    WSGIScriptAlias {{ ingress_internal.ingress_path }} {{ wsgi_config.wsgi_public_script }}
-    {% endif -%}
-    WSGIScriptAlias / {{ wsgi_config.wsgi_public_script }}
-    WSGIApplicationGroup %{GLOBAL}
-    WSGIPassAuthorization On
-    <IfVersion >= 2.4>
-      ErrorLogFormat "%{cu}t %M"
-    </IfVersion>
-    ErrorLog {{ wsgi_config.error_log }}
-    CustomLog {{ wsgi_config.custom_log }} combined
-
-    <Directory /usr/bin>
-        <IfVersion >= 2.4>
-            Require all granted
-        </IfVersion>
-        <IfVersion < 2.4>
-            Order allow,deny
-            Allow from all
-        </IfVersion>
-    </Directory>
-</VirtualHost>
-<VirtualHost *:{{ wsgi_nova_metadata.public_port }}>
-    WSGIDaemonProcess nova-metadata processes=4 threads=1 user={{ wsgi_nova_metadata.user }} group={{ wsgi_nova_metadata.group }} \
-                      display-name=%{GROUP}
-    WSGIProcessGroup nova-metadata
-    WSGIScriptAlias / {{ wsgi_nova_metadata.wsgi_public_script }}
-    WSGIApplicationGroup %{GLOBAL}
-    WSGIPassAuthorization On
-    <IfVersion >= 2.4>
-      ErrorLogFormat "%{cu}t %M"
-    </IfVersion>
-    ErrorLog {{ wsgi_nova_metadata.error_log }}
-    CustomLog {{ wsgi_nova_metadata.custom_log }} combined
-
-    <Directory /usr/bin>
-        <IfVersion >= 2.4>
-            Require all granted
-        </IfVersion>
-        <IfVersion < 2.4>
-            Order allow,deny
-            Allow from all
-        </IfVersion>
-    </Directory>
-</VirtualHost>

charms/nova-k8s/CONTRIBUTING.md

@@ -37,9 +37,9 @@ TODO
 ## Testing
 
 The Python operator framework includes a very nice harness for testing
-operator behaviour without full deployment. Just `run_tests`:
+operator behaviour without full deployment. Run tests using command:
 
-    ./run_tests
+    tox -e py3
 
 ## Deployment
 

charms/nova-k8s/lib/charms/sunbeam_keystone_operator/v0/identity_service.py

@@ -75,6 +75,20 @@ class IdentityServiceClientCharm(CharmBase):
 ```
 """
 
+import json
+import logging
+
+from ops.framework import (
+    StoredState,
+    EventBase,
+    ObjectEvents,
+    EventSource,
+    Object,
+)
+from ops.model import Relation
+
+logger = logging.getLogger(__name__)
+
 # The unique Charmhub library identifier, never change it
 LIBID = "6a7cb19b98314ecf916e3fcb02708608"
 
@@ -85,21 +99,6 @@ LIBAPI = 0
 # to 0 if you are raising the major API version
 LIBPATCH = 1
 
-import json
-import logging
-import requests
-
-from ops.framework import (
-    StoredState,
-    EventBase,
-    ObjectEvents,
-    EventSource,
-    Object,
-)
-
-from ops.model import Relation
-
-from typing import List
 
 logger = logging.getLogger(__name__)
 
@@ -309,25 +308,21 @@ class IdentityServiceRequires(Object):
         """Return the service_user_id."""
         return self.get_remote_app_data('service-user-id')
 
-
     @property
     def internal_auth_url(self) -> str:
         """Return the internal_auth_url."""
         return self.get_remote_app_data('internal-auth-url')
 
-
     @property
     def admin_auth_url(self) -> str:
         """Return the admin_auth_url."""
         return self.get_remote_app_data('admin-auth-url')
 
-
     @property
     def public_auth_url(self) -> str:
         """Return the public_auth_url."""
         return self.get_remote_app_data('public-auth-url')
 
-
     def register_services(self, service_endpoints: dict,
                           region: str) -> None:
         """Request access to the IdentityService server."""
@@ -355,7 +350,6 @@ class ReadyIdentityServiceClientsEvent(EventBase):
         self.service_endpoints = service_endpoints
         self.region = region
         self.client_app_name = client_app_name
-            
 
     def snapshot(self):
         return {
@@ -417,10 +411,11 @@ class IdentityServiceProvides(Object):
         REQUIRED_KEYS = [
             'service-endpoints',
             'region']
-        
+
         values = [
             event.relation.data[event.relation.app].get(k)
-            for k in REQUIRED_KEYS ]
+            for k in REQUIRED_KEYS
+        ]
         # Validate data on the relation
         if all(values):
             service_eps = json.loads(

charms/nova-k8s/requirements.txt

@@ -1,19 +1,31 @@
-ops
+# This file is managed centrally by release-tools and should not be modified
-jinja2
+# within individual charm repos.  See the 'global' dir contents for available
-# Get resources from github until cacerts issue is charmbuild image is fixed.
+# choices of *requirements.txt files for OpenStack Charms:
-# git+https://opendev.org/openstack/charm-ops-openstack#egg=ops_openstack
+#     https://github.com/openstack-charmers/release-tools
-# git+https://opendev.org/openstack/charm-ops-interface-tls-certificates#egg=interface_tls_certificates
+#
-# git+https://github.com/openstack/charm-ops-openstack#egg=ops_openstack
-git+https://github.com/openstack/charm-ops-interface-tls-certificates#egg=interface_tls_certificates
 
-git+https://github.com/openstack-charmers/advanced-sunbeam-openstack#egg=ops_sunbeam
+# NOTE: newer versions of cryptography require a Rust compiler to build,
+# see
+# * https://github.com/openstack-charmers/zaza/issues/421
+# * https://mail.python.org/pipermail/cryptography-dev/2021-January/001003.html
+#
+cryptography<3.4
+jinja2
 lightkube
 lightkube-models
-cryptography < 3.4
+ops
+git+https://github.com/openstack-charmers/advanced-sunbeam-openstack#egg=ops_sunbeam
+
+python-keystoneclient  # keystone-k8s
+
+# Used for Traefik
+# Note: Remove when traefik-k8s-operator v1 library is released
+serialized_data_interface
+
+# Get resources from github until cacerts issue is charmbuild image is fixed.
+git+https://github.com/openstack/charm-ops-interface-tls-certificates#egg=interface_tls_certificates
+
+# Note: Required for cinder-k8s, cinder-ceph-k8s, glance-k8s, nova-k8s
 git+https://github.com/openstack/charm-ops-interface-ceph-client#egg=interface_ceph_client
 # Charmhelpers is only present as interface_ceph_client uses it.
 git+https://github.com/juju/charm-helpers.git#egg=charmhelpers
-
-# Used for traefik
-serialized_data_interface
-

charms/nova-k8s/run_tests

@@ -1,17 +0,0 @@
-#!/bin/sh -e
-# Copyright 2021 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-if [ -z "$VIRTUAL_ENV" -a -d venv/ ]; then
-    . venv/bin/activate
-fi
-
-if [ -z "$PYTHONPATH" ]; then
-    export PYTHONPATH="lib:src"
-else
-    export PYTHONPATH="lib:src:$PYTHONPATH"
-fi
-
-flake8
-coverage run --branch --source=src -m unittest -v "$@"
-coverage report -m

charms/nova-k8s/src/charm.py

@@ -177,7 +177,7 @@ class NovaOperatorCharm(sunbeam_charm.OSBaseOperatorAPICharm):
     @property
     def service_conf(self) -> str:
         """Service default configuration file."""
-        return f"/etc/nova/nova.conf"
+        return "/etc/nova/nova.conf"
 
     @property
     def service_user(self) -> str:

charms/nova-k8s/src/templates/wsgi-nova-api.conf.j2

@@ -4,7 +4,7 @@ Listen {{ wsgi_nova_metadata.public_port }}
     WSGIDaemonProcess nova-api processes=4 threads=1 user={{ wsgi_config.user }} group={{ wsgi_config.group }} \
                       display-name=%{GROUP}
     WSGIProcessGroup nova-api
-    {% if ingress_internal.ingress_path -%}
+    {% if ingress_internal and ingress_internal.ingress_path -%}
     WSGIScriptAlias {{ ingress_internal.ingress_path }} {{ wsgi_config.wsgi_public_script }}
     {% endif -%}
     WSGIScriptAlias / {{ wsgi_config.wsgi_public_script }}

charms/nova-k8s/test-requirements.txt

@@ -1,16 +1,10 @@
-# This file is managed centrally.  If you find the need to modify this as a
+# This file is managed centrally by release-tools and should not be modified
-# one-off, please don't.  Intead, consult #openstack-charms and ask about
+# within individual charm repos.  See the 'global' dir contents for available
-# requirements management in charms via bot-control.  Thank you.
+# choices of *requirements.txt files for OpenStack Charms:
-coverage>=3.6
+#     https://github.com/openstack-charmers/release-tools
-mock>=1.2
+#
-flake8>=2.2.4,<=2.4.1
+
-pyflakes==2.1.1
+coverage
-stestr>=2.2.0
+mock
-requests>=2.18.4
+flake8
-psutil
+stestr
-# oslo.i18n dropped py35 support
-oslo.i18n<4.0.0
-git+https://github.com/openstack-charmers/zaza.git#egg=zaza
-git+https://github.com/openstack-charmers/zaza-openstack-tests.git#egg=zaza.openstack
-pytz    # workaround for 14.04 pip/tox
-pyudev  # for ceph-* charm unit tests (not mocked?)

charms/nova-k8s/tests/unit/test_nova_charm.py

@@ -15,10 +15,6 @@
 # limitations under the License.
 
 import mock
-import sys
-
-sys.path.append('lib')  # noqa
-sys.path.append('src')  # noqa
 
 import charm
 import ops_sunbeam.test_utils as test_utils

charms/nova-k8s/tox.ini

@@ -1,78 +1,76 @@
-# Operator charm (with zaza): tox.ini
+# Source charm: ./tox.ini
+# This file is managed centrally by release-tools and should not be modified
+# within individual charm repos.  See the 'global' dir contents for available
+# choices of tox.ini for OpenStack Charms:
+#     https://github.com/openstack-charmers/release-tools
 
 [tox]
-envlist = pep8,py3
 skipsdist = True
-# NOTE: Avoid build/test env pollution by not enabling sitepackages.
+envlist = pep8,py3
 sitepackages = False
-# NOTE: Avoid false positives by not skipping missing interpreters.
 skip_missing_interpreters = False
-# NOTES:
-# * We avoid the new dependency resolver by pinning pip < 20.3, see
-#   https://github.com/pypa/pip/issues/9187
-# * Pinning dependencies requires tox >= 3.2.0, see
-#   https://tox.readthedocs.io/en/latest/config.html#conf-requires
-# * It is also necessary to pin virtualenv as a newer virtualenv would still
-#   lead to fetching the latest pip in the func* tox targets, see
-#   https://stackoverflow.com/a/38133283
-requires = pip < 20.3
-           virtualenv < 20.0
-# NOTE: https://wiki.canonical.com/engineering/OpenStack/InstallLatestToxOnOsci
 minversion = 3.18.0
 
+[vars]
+src_path = {toxinidir}/src/
+tst_path = {toxinidir}/tests/
+lib_path = {toxinidir}/lib/
+
 [testenv]
-setenv = VIRTUAL_ENV={envdir}
+basepython = python3
-         PYTHONHASHSEED=0
+setenv =
-         CHARM_DIR={envdir}
+  PYTHONPATH = {toxinidir}:{[vars]lib_path}:{[vars]src_path}
+passenv =
+  PYTHONPATH
 install_command =
   pip install {opts} {packages}
 commands = stestr run --slowest {posargs}
-whitelist_externals =
+allowlist_externals =
-    git
+  git
-    add-to-archive.py
+  charmcraft
-    bash
+  fetch-libs.sh
-    charmcraft
+deps =
-passenv = HOME TERM CS_* OS_* TEST_*
+  -r{toxinidir}/test-requirements.txt
-deps = -r{toxinidir}/test-requirements.txt
+
+[testenv:build]
+basepython = python3
+deps =
+commands =
+  charmcraft clean
+  charmcraft -v pack
 
 [testenv:fetch]
 basepython = python3
 deps =
 commands =
-  ./fetch-libs.sh
+  {toxinidir}/fetch-libs.sh
-
-[testenv:py3.8]
-basepython = python3.8
-deps = -r{toxinidir}/requirements.txt
-       -r{toxinidir}/test-requirements.txt
-
-[testenv:py3.9]
-basepython = python3.9
-deps = -r{toxinidir}/requirements.txt
-       -r{toxinidir}/test-requirements.txt
-
-[testenv:py3.10]
-basepython = python3.10
-deps = -r{toxinidir}/requirements.txt
-       -r{toxinidir}/test-requirements.txt
 
 [testenv:py3]
 basepython = python3
-deps = -r{toxinidir}/requirements.txt
+deps =
-       -r{toxinidir}/test-requirements.txt
+  {[testenv]deps}
+  -r{toxinidir}/requirements.txt
+
+[testenv:py3.8]
+basepython = python3.8
+deps = {[testenv:py3]deps}
+
+[testenv:py3.9]
+basepython = python3.9
+deps = {[testenv:py3]deps}
+
+[testenv:py3.10]
+basepython = python3.10
+deps = {[testenv:py3]deps}
 
 [testenv:pep8]
 basepython = python3
-deps = -r{toxinidir}/requirements.txt
+deps = {[testenv]deps}
-       -r{toxinidir}/test-requirements.txt
+commands = flake8 {posargs} {[vars]src_path} {[vars]tst_path}
-commands = flake8 {posargs} src unit_tests tests
 
 [testenv:cover]
-# Technique based heavily upon
-# https://github.com/openstack/nova/blob/master/tox.ini
 basepython = python3
-deps = -r{toxinidir}/requirements.txt
+deps = {[testenv:py3]deps}
-       -r{toxinidir}/test-requirements.txt
 setenv =
     {[testenv]setenv}
     PYTHON=coverage run
@@ -92,45 +90,8 @@ source =
     .
 omit =
     .tox/*
-    */charmhelpers/*
+    tests/*
-    unit_tests/*
     src/templates/*
 
-[testenv:venv]
-basepython = python3
-commands = {posargs}
-
-[testenv:build]
-basepython = python3
-deps = -r{toxinidir}/build-requirements.txt
-commands =
-    charmcraft build
-
-[testenv:func-noop]
-basepython = python3
-commands =
-    functest-run-suite --help
-
-[testenv:func]
-basepython = python3
-commands =
-    functest-run-suite --keep-model
-
-[testenv:func-smoke]
-basepython = python3
-commands =
-    functest-run-suite --keep-model --smoke
-
-[testenv:func-dev]
-basepython = python3
-commands =
-    functest-run-suite --keep-model --dev
-
-[testenv:func-target]
-basepython = python3
-commands =
-    functest-run-suite --keep-model --bundle {posargs}
-
 [flake8]
-# Ignore E902 because the unit_tests directory is missing in the built charm.
+ignore=E226,W504
-ignore = E402,E226,E902