Keystone Fernet Token implementation

This patchset adds more Fernet token implementation:

1. Adds a cron job to rotate / sync keys to other units.
2. Adds additional tests around gating on config.
3. Adds rotation / syncing with more robust key handling.

Change-Id: Ied021ad83c241f241dbb5f9acdede9045e43a8a3

config.yaml

@@ -98,7 +98,7 @@ options:
     description: Admin role to be associated with admin and service users.
   token-provider:
     type: string
-    default: 'uuid'
+    default:
     description: |
       Transitional configuration option to enable migration to Fernet tokens
       prior to upgrade to OpenStack Rocky.
@@ -112,6 +112,21 @@ options:
     type: int
     default: 3600
     description: Amount of time (in seconds) a token should remain valid.
+  fernet-max-active-keys:
+    type: int
+    default: 3
+    description: |
+      This is the maximum number of active keys when `token-provider` is set to
+      "fernet".  If has a minimum of 3, which includes the spare and staging
+      keys.  When set to 3, the rotation time for the keys is the same as the
+      token expiration time.  When set to a higher value, the rotation time is
+      less than the `token-expiration` time as calculated by:
+      .
+      rotation-time = token-expiration / (fernet-max-active-keys - 2)
+      .
+      Please see the charm documentation for further details about how to use
+      the Fernet token parameters to achieve a key strategy appropriate for the
+      system in question.
   service-tenant:
     type: string
     default: "services"

hooks/keystone_context.py

@@ -26,14 +26,21 @@ from charmhelpers.contrib.hahelpers.cluster import (
 )
 
 from charmhelpers.core.hookenv import (
+    charm_dir,
     config,
     log,
     leader_get,
+    local_unit,
     related_units,
     relation_ids,
     relation_get,
 )
 
+from charmhelpers.contrib.openstack.utils import (
+    CompareOpenStackReleases,
+    os_release,
+)
+
 
 class ApacheSSLContext(context.ApacheSSLContext):
     interfaces = ['https']
@@ -175,6 +182,7 @@ class KeystoneContext(context.OSContextGenerator):
         ctxt['identity_backend'] = config('identity-backend')
         ctxt['assignment_backend'] = config('assignment-backend')
         ctxt['token_provider'] = config('token-provider')
+        ctxt['fernet_max_active_keys'] = config('fernet-max-active-keys')
         if config('identity-backend') == 'ldap':
             ctxt['ldap_server'] = config('ldap-server')
             ctxt['ldap_user'] = config('ldap-user')
@@ -242,11 +250,41 @@ class TokenFlushContext(context.OSContextGenerator):
 
     def __call__(self):
         ctxt = {
-            'token_flush': is_elected_leader(DC_RESOURCE_NAME)
+            'token_flush': (not fernet_enabled() and
+                            is_elected_leader(DC_RESOURCE_NAME))
         }
         return ctxt
 
 
+class FernetCronContext(context.OSContextGenerator):
+
+    def __call__(self):
+        token_expiration = int(config('token-expiration'))
+        ctxt = {
+            'enabled': (fernet_enabled() and
+                        is_elected_leader(DC_RESOURCE_NAME)),
+            'unit_name': local_unit(),
+            'charm_dir': charm_dir(),
+            'minute': ('*/5' if token_expiration > 300 else '*')
+        }
+        return ctxt
+
+
+def fernet_enabled():
+    """Helper function for determinining whether Fernet tokens are enabled.
+
+    :returns: True if the fernet keys should be configured.
+    :rtype: bool
+    """
+    cmp_release = CompareOpenStackReleases(os_release('keystone'))
+    if cmp_release < 'ocata':
+        return False
+    elif 'ocata' >= cmp_release < 'rocky':
+        return config('token-provider') == 'fernet'
+    else:
+        return True
+
+
 class KeystoneFIDServiceProviderContext(context.OSContextGenerator):
     interfaces = ['keystone-fid-service-provider']
 

hooks/keystone_hooks.py

@@ -66,6 +66,8 @@ from charmhelpers.contrib.openstack.utils import (
     enable_memcache,
 )
 
+from keystone_context import fernet_enabled
+
 from keystone_utils import (
     add_service_to_keystone,
     add_credentials_to_keystone,
@@ -101,10 +103,9 @@ from keystone_utils import (
     ADMIN_PROJECT,
     create_or_show_domain,
     restart_keystone,
-    fernet_enabled,
-    fernet_leader_set,
-    fernet_setup,
-    fernet_write_keys,
+    key_leader_set,
+    key_setup,
+    key_write,
 )
 
 from charmhelpers.contrib.hahelpers.cluster import (
@@ -227,8 +228,8 @@ def config_changed_postupgrade():
         apt_install(filter_installed_packages(determine_packages()))
 
     if is_leader() and fernet_enabled():
-        fernet_setup()
-        fernet_leader_set()
+        key_setup()
+        key_leader_set()
 
     configure_https()
     open_port(config('service-port'))
@@ -502,7 +503,7 @@ def leader_settings_changed():
         CONFIGS.write(POLICY_JSON)
 
     if fernet_enabled():
-        fernet_write_keys()
+        key_write()
 
     update_all_identity_relation_units()
 

hooks/keystone_utils.py

@@ -84,6 +84,8 @@ from charmhelpers.core.hookenv import (
     related_units,
     DEBUG,
     INFO,
+    ERROR,
+    WARNING,
 )
 
 from charmhelpers.fetch import (
@@ -191,6 +193,10 @@ ADMIN_PROJECT = 'admin'
 DEFAULT_DOMAIN = 'default'
 SERVICE_DOMAIN = 'service_domain'
 TOKEN_FLUSH_CRON_FILE = '/etc/cron.d/keystone-token-flush'
+KEY_SETUP_FILE = '/etc/keystone/key-setup'
+CREDENTIAL_KEY_REPOSITORY = '/etc/keystone/credential-keys/'
+FERNET_KEY_REPOSITORY = '/etc/keystone/fernet-keys/'
+FERNET_KEY_ROTATE_SYNC_CRON_FILE = '/etc/cron.d/keystone-fernet-rotate-sync'
 WSGI_KEYSTONE_API_CONF = '/etc/apache2/sites-enabled/wsgi-openstack-api.conf'
 UNUSED_APACHE_SITE_FILES = ['/etc/apache2/sites-enabled/keystone.conf',
                             '/etc/apache2/sites-enabled/wsgi-keystone.conf']
@@ -254,6 +260,11 @@ BASE_RESOURCE_MAP = OrderedDict([
                      context.SyslogContext()],
         'services': [],
     }),
+    (FERNET_KEY_ROTATE_SYNC_CRON_FILE, {
+        'contexts': [keystone_context.FernetCronContext(),
+                     context.SyslogContext()],
+        'services': [],
+    }),
 ])
 
 valid_services = {
@@ -1904,54 +1915,161 @@ def restart_keystone():
             service_restart(keystone_service())
 
 
-def fernet_enabled():
-    """Helper function for determinining whether Fernet tokens are enabled"""
-    cmp_release = CompareOpenStackReleases(os_release('keystone'))
-    if cmp_release < 'ocata':
-        return False
-    elif 'ocata' >= cmp_release < 'rocky':
-        return config('token-provider') == 'fernet'
-    else:
-        return True
+def key_setup():
+    """Initialize Fernet and Credential encryption key repositories
 
+    To setup the key repositories, calls (as user "keystone"):
 
-def fernet_setup():
-    """Initialize Fernet key repositories"""
+        keystone-manage fernet_setup
+        keystone-manage credential_setup
+
+    In addition we migrate any credentials currently stored in database using
+    the null key to be encrypted by the new credential key:
+
+        keystone-manage credential_migrate
+
+    Note that we only want to do this once, so we store success in the leader
+    settings (which we should be).
+
+    :raises: `:class:subprocess.CallProcessError` if either of the commands
+        fails.
+    """
+    if os.path.exists(KEY_SETUP_FILE) or not is_leader():
+        return
     base_cmd = ['sudo', '-u', 'keystone', 'keystone-manage']
-    subprocess.check_output(base_cmd + ['fernet_setup'])
-    subprocess.check_output(base_cmd + ['credential_setup'])
+    try:
+        log("Setting up key repositories for Fernet tokens and Credential "
+            "encryption", level=DEBUG)
+        subprocess.check_call(base_cmd + ['fernet_setup'])
+        subprocess.check_call(base_cmd + ['credential_setup'])
+        subprocess.check_call(base_cmd + ['credential_migrate'])
+        # touch the file to create
+        open(KEY_SETUP_FILE, "w").close()
+    except subprocess.CalledProcessError as e:
+        log("Key repository setup failed, will retry in config-changed hook: "
+            "{}".format(e), level=ERROR)
 
 
 def fernet_rotate():
-    """Rotate Fernet keys"""
+    """Rotate Fernet keys
+
+    To rotate the Fernet tokens, and create a new staging key, it calls (as the
+    "keystone" user):
+
+        keystone-manage fernet_rotate
+
+    Note that we do not rotate the Credential encryption keys.
+
+    Note that this does NOT synchronise the keys between the units.  This is
+    performed in `:function:`hooks.keystone_utils.fernet_leader_set`
+
+    :raises: `:class:subprocess.CallProcessError` if the command fails.
+    """
+    log("Rotating Fernet tokens", level=DEBUG)
     cmd = ['sudo', '-u', 'keystone', 'keystone-manage', 'fernet_rotate']
-    subprocess.check_output(cmd)
+    subprocess.check_call(cmd)
 
 
-def fernet_leader_set():
-    """Read current key set and update leader storage if necessary"""
-    key_repository = '/etc/keystone/fernet-keys/'
-    disk_keys = []
-    for key in os.listdir(key_repository):
-        with open(os.path.join(key_repository, str(key)), 'r') as f:
-            disk_keys.append(f.read())
-    leader_set({'fernet_keys': json.dumps(disk_keys)})
+def key_leader_set():
+    """Read current key sets and update leader storage
+
+    The keys are read from the `FERNET_KEY_REPOSITORY` and
+    `CREDENTIAL_KEY_REPOSITORY` directories.  Note that this function will fail
+    if it is called on the unit that is not the leader.
+
+    :raises: :class:`subprocess.CalledProcessError` if the leader_set fails.
+    """
+    disk_keys = {}
+    for key_repository in [FERNET_KEY_REPOSITORY, CREDENTIAL_KEY_REPOSITORY]:
+        disk_keys[key_repository] = {}
+        for key_number in os.listdir(key_repository):
+            with open(os.path.join(key_repository, key_number),
+                      'r') as f:
+                disk_keys[key_repository][key_number] = f.read()
+    leader_set({'key_repository': json.dumps(disk_keys)})
 
 
-def fernet_write_keys():
-    """Get keys from leader storage and write out to disk"""
-    key_repository = '/etc/keystone/fernet-keys/'
-    leader_keys = leader_get('fernet_keys')
+def key_write():
+    """Get keys from leader storage and write out to disk
+
+    The keys are written to the `FERNET_KEY_REPOSITORY` and
+    `CREDENTIAL_KEY_REPOSITORY` directories.  Note that the keys are first
+    written to a tmp file and then moved to the key to avoid any races.  Any
+    'excess' keys are deleted, which may occur if the "number of keys" has been
+    reduced on the leader.
+    """
+    leader_keys = leader_get('key_repository')
     if not leader_keys:
-        log('"fernet_keys" not in leader settings yet...', level=DEBUG)
+        log('"key_repository" not in leader settings yet...', level=DEBUG)
         return
-    mkdir(key_repository, owner=KEYSTONE_USER, group=KEYSTONE_USER,
-          perms=0o700)
-    for idx, key in enumerate(json.loads(leader_keys)):
-        tmp_filename = os.path.join(key_repository, "."+str(idx))
-        key_filename = os.path.join(key_repository, str(idx))
-        # write to tmp file first, move the key into place in an atomic
-        # operation avoiding any races with consumers of the key files
-        write_file(tmp_filename, key, owner=KEYSTONE_USER, group=KEYSTONE_USER,
-                   perms=0o600)
-        os.rename(tmp_filename, key_filename)
+    leader_keys = json.loads(leader_keys)
+    for key_repository in [FERNET_KEY_REPOSITORY, CREDENTIAL_KEY_REPOSITORY]:
+        mkdir(key_repository,
+              owner=KEYSTONE_USER,
+              group=KEYSTONE_USER,
+              perms=0o700)
+        for key_number, key in leader_keys[key_repository].items():
+            tmp_filename = os.path.join(key_repository,
+                                        ".{}".format(key_number))
+            key_filename = os.path.join(key_repository, key_number)
+            # write to tmp file first, move the key into place in an atomic
+            # operation avoiding any races with consumers of the key files
+            write_file(tmp_filename,
+                       key,
+                       owner=KEYSTONE_USER,
+                       group=KEYSTONE_USER,
+                       perms=0o600)
+            os.rename(tmp_filename, key_filename)
+        # now delete any keys that shouldn't be there
+        for key_number in os.listdir(key_repository):
+            if key_number not in leader_keys[key_repository].keys():
+                os.remove(os.path.join(key_repository, key_number))
+        # also say that keys have been setup for this system.
+        open(KEY_SETUP_FILE, "w").close()
+
+
+def fernet_keys_rotate_and_sync(log_func=log):
+    """Rotate and sync the keys if the unit is the leader and the primary key
+    has expired.
+
+    The modification time of the staging key (key with index '0') is used,
+    along with the config setting "token_expiration" to determine whether to
+    rotate the keys, along with the function `fernet_enabled()` to test
+    whether to do it at all.
+
+    Note that the reason for using modification time and not change time is
+    that the former can be set by the operator as part of restoring the key
+    from backup.
+
+    The rotation time = token-expiration / (max-active-keys - 2)
+
+    where max-active-keys has a minumum of 3.
+
+    :param log_func: Function to use for logging
+    :type log_func: func
+    """
+    if not keystone_context.fernet_enabled() or not is_leader():
+        return
+    # now see if the keys need to be rotated
+    try:
+        last_rotation = os.stat(
+            os.path.join(FERNET_KEY_REPOSITORY, '0')).st_mtime
+    except OSError:
+        log_func("Fernet key rotation requested but key repository not "
+                 "initialized yet", level=WARNING)
+        return
+    max_keys = max(config('fernet-max-active-keys'), 3)
+    expiration = config('token-expiration')
+    rotation_time = expiration // (max_keys - 2)
+    now = time.time()
+    if last_rotation + rotation_time > now:
+        # Nothing to do as not reached rotation time
+        log_func("No rotation until at least {}"
+                 .format(time.ctime(last_rotation + rotation_time)),
+                 level=DEBUG)
+        return
+    # now rotate the keys and sync them
+    fernet_rotate()
+    key_leader_set()
+    log_func("Rotated and started sync (via leader settings) of fernet keys",
+             level=INFO)

scripts/fernet_rotate_and_sync.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+# Copyright 2018 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import print_function
+
+import os
+import sys
+import time
+
+dir_path = os.path.dirname(os.path.realpath(__file__))
+hooks_path = os.path.abspath(os.path.join(dir_path, "..", "hooks"))
+
+if hooks_path not in sys.path:
+    sys.path.append(hooks_path)
+
+# now we can import charm related items
+import charmhelpers.core.hookenv
+
+import keystone_utils
+
+
+def cli_log(msg, level=charmhelpers.core.hookenv.INFO):
+    """Helper function to write log message to stdout/stderr for CLI usage."""
+    if level == charmhelpers.core.hookenv.DEBUG:
+        return charmhelpers.core.hookenv.log(msg, level=level)
+    elif level in [charmhelpers.core.hookenv.ERROR,
+                   charmhelpers.core.hookenv.WARNING]:
+        output = sys.stderr
+    else:
+        output = sys.stdout
+
+    print('{}: {}'.format(time.ctime(), msg), file=output)
+
+
+# the rotate_and_sync_keys() function checks for leadership AND whether to
+# rotate the keys or not.
+if __name__ == "__main__":
+    keystone_utils.fernet_keys_rotate_and_sync(log_func=cli_log)

templates/keystone-fernet-rotate-sync

@@ -0,0 +1,9 @@
+# call the rotate and sync function at 5 min intervals.  The actual function
+# works out when to do the rotate and sync of the keys.
+{% if enabled -%}
+{% if use_syslog -%}
+{{ minute }} * * * * root /usr/bin/juju-run {{ unit_name }} {{ charm_dir }}/scripts/fernet_rotate_and_sync.py 2>&1 | logger -t keystone-fernet-rotate-sync
+{% else -%}
+{{ minute }} * * * * root /usr/bin/juju-run {{ unit_name }} {{ charm_dir }}/scripts/fernet_rotate_and_sync.py >> /var/log/keystone/keystone-fernet-rotate-sync.log 2>&1
+{% endif -%}
+{% endif -%}

templates/ocata/keystone.conf

@@ -52,6 +52,11 @@ provider = uuid
 {% endif -%}
 expiration = {{ token_expiration }}
 
+{% if token_provider == 'fernet' -%}
+[fernet_tokens]
+max_active_keys = {{ fernet_max_active_keys }}
+{% endif -%}
+
 {% include "parts/section-signing" %}
 
 {% include "section-oslo-cache" %}

templates/rocky/keystone.conf

@@ -44,6 +44,9 @@ driver = sql
 [token]
 expiration = {{ token_expiration }}
 
+[fernet_tokens]
+max_active_keys = {{ fernet_max_active_keys }}
+
 {% include "parts/section-signing" %}
 
 {% include "section-oslo-cache" %}

unit_tests/__init__.py

@@ -16,3 +16,4 @@ import sys
 
 sys.path.append('actions/')
 sys.path.append('hooks/')
+sys.path.append('scripts/')

unit_tests/test_keystone_contexts.py

@@ -29,6 +29,7 @@ TO_PATCH = [
     'config',
     'determine_apache_port',
     'determine_api_port',
+    'os_release',
 ]
 
 
@@ -36,6 +37,7 @@ class TestKeystoneContexts(CharmTestCase):
 
     def setUp(self):
         super(TestKeystoneContexts, self).setUp(context, TO_PATCH)
+        self.config.side_effect = self.test_config.get
 
     @patch('charmhelpers.contrib.hahelpers.cluster.relation_ids')
     @patch('charmhelpers.contrib.openstack.ip.unit_get')
@@ -152,15 +154,80 @@ class TestKeystoneContexts(CharmTestCase):
                          ctxt())
 
     @patch.object(context, 'is_elected_leader')
-    def test_token_flush_context(self, mock_is_elected_leader):
+    @patch.object(context, 'fernet_enabled')
+    def test_token_flush_context(
+            self, mock_fernet_enabled, mock_is_elected_leader):
         ctxt = context.TokenFlushContext()
 
+        mock_fernet_enabled.return_value = False
         mock_is_elected_leader.return_value = False
         self.assertEqual({'token_flush': False}, ctxt())
 
         mock_is_elected_leader.return_value = True
         self.assertEqual({'token_flush': True}, ctxt())
 
+        mock_fernet_enabled.return_value = True
+        self.assertEqual({'token_flush': False}, ctxt())
+
+    @patch.object(context, 'charm_dir')
+    @patch.object(context, 'local_unit')
+    @patch.object(context, 'is_elected_leader')
+    @patch.object(context, 'fernet_enabled')
+    def test_fernet_cron_context(
+            self, mock_fernet_enabled, mock_is_elected_leader, mock_local_unit,
+            mock_charm_dir):
+        ctxt = context.FernetCronContext()
+
+        mock_charm_dir.return_value = "my-dir"
+        mock_local_unit.return_value = "the-local-unit"
+
+        expected = {
+            'enabled': False,
+            'unit_name': 'the-local-unit',
+            'charm_dir': 'my-dir',
+            'minute': '*/5',
+        }
+
+        mock_fernet_enabled.return_value = False
+        mock_is_elected_leader.return_value = False
+        self.assertEqual(expected, ctxt())
+
+        mock_is_elected_leader.return_value = True
+        self.assertEqual(expected, ctxt())
+
+        mock_fernet_enabled.return_value = True
+        expected['enabled'] = True
+        self.assertEqual(expected, ctxt())
+
+    def test_fernet_enabled_no_config(self):
+        self.os_release.return_value = 'ocata'
+        self.test_config.set('token-provider', 'uuid')
+        result = context.fernet_enabled()
+        self.assertFalse(result)
+
+    def test_fernet_enabled_yes_config(self):
+        self.os_release.return_value = 'ocata'
+        self.test_config.set('token-provider', 'fernet')
+        result = context.fernet_enabled()
+        self.assertTrue(result)
+
+    def test_fernet_enabled_no_release_override_config(self):
+        self.os_release.return_value = 'mitaka'
+        self.test_config.set('token-provider', 'fernet')
+        result = context.fernet_enabled()
+        self.assertFalse(result)
+
+    def test_fernet_enabled_yes_release(self):
+        self.os_release.return_value = 'rocky'
+        result = context.fernet_enabled()
+        self.assertTrue(result)
+
+    def test_fernet_enabled_yes_release_override_config(self):
+        self.os_release.return_value = 'rocky'
+        self.test_config.set('token-provider', 'uuid')
+        result = context.fernet_enabled()
+        self.assertTrue(result)
+
     @patch.object(context, 'relation_ids')
     @patch.object(context, 'related_units')
     @patch.object(context, 'relation_get')

unit_tests/test_keystone_hooks.py

@@ -90,9 +90,9 @@ TO_PATCH = [
     'create_or_show_domain',
     'get_api_version',
     'fernet_enabled',
-    'fernet_leader_set',
-    'fernet_setup',
-    'fernet_write_keys',
+    'key_leader_set',
+    'key_setup',
+    'key_write',
     # other
     'check_call',
     'execd_preinstall',

unit_tests/test_keystone_utils.py

@@ -16,6 +16,7 @@ import json
 import os
 import subprocess
 import sys
+import time
 
 from mock import MagicMock, call, mock_open, patch
 from test_utils import CharmTestCase
@@ -1142,43 +1143,23 @@ class TestKeystoneUtils(CharmTestCase):
         with self.assertRaises(ValueError):
             utils.get_api_version()
 
-    def test_fernet_enabled_no_config(self):
-        self.os_release.return_value = 'ocata'
-        self.test_config.set('token-provider', 'uuid')
-        result = utils.fernet_enabled()
-        self.assertFalse(result)
-
-    def test_fernet_enabled_yes_config(self):
-        self.os_release.return_value = 'ocata'
-        self.test_config.set('token-provider', 'fernet')
-        result = utils.fernet_enabled()
-        self.assertTrue(result)
-
-    def test_fernet_enabled_no_release_override_config(self):
-        self.os_release.return_value = 'mitaka'
-        self.test_config.set('token-provider', 'fernet')
-        result = utils.fernet_enabled()
-        self.assertFalse(result)
-
-    def test_fernet_enabled_yes_release(self):
-        self.os_release.return_value = 'rocky'
-        result = utils.fernet_enabled()
-        self.assertTrue(result)
-
-    def test_fernet_enabled_yes_release_override_config(self):
-        self.os_release.return_value = 'rocky'
-        self.test_config.set('token-provider', 'uuid')
-        result = utils.fernet_enabled()
-        self.assertTrue(result)
-
-    def test_fernet_setup(self):
+    @patch.object(utils, 'is_leader')
+    @patch('os.path.exists')
+    def test_key_setup(self, mock_path_exists, mock_is_leader):
         base_cmd = ['sudo', '-u', 'keystone', 'keystone-manage']
-        utils.fernet_setup()
+        mock_is_leader.return_value = True
+        mock_path_exists.return_value = False
+        with patch.object(builtins, 'open', mock_open()) as m:
+            utils.key_setup()
+            m.assert_called_once_with(utils.KEY_SETUP_FILE, "w")
         self.subprocess.check_output.has_calls(
             [
                 base_cmd + ['fernet_setup'],
                 base_cmd + ['credential_setup'],
+                base_cmd + ['credential_migrate'],
             ])
+        mock_path_exists.assert_called_once_with(utils.KEY_SETUP_FILE)
+        mock_is_leader.assert_called_once_with()
 
     def test_fernet_rotate(self):
         cmd = ['sudo', '-u', 'keystone', 'keystone-manage', 'fernet_rotate']
@@ -1187,34 +1168,109 @@ class TestKeystoneUtils(CharmTestCase):
 
     @patch.object(utils, 'leader_set')
     @patch('os.listdir')
-    def test_fernet_leader_set(self, listdir, leader_set):
-        listdir.return_value = [0, 1]
+    def test_key_leader_set(self, listdir, leader_set):
+        listdir.return_value = ['0', '1']
+        self.time.time.return_value = "the-time"
         with patch.object(builtins, 'open', mock_open(
                 read_data="some_data")):
-            utils.fernet_leader_set()
-        listdir.assert_called_with('/etc/keystone/fernet-keys/')
+            utils.key_leader_set()
+        listdir.has_calls([
+            call(utils.FERNET_KEY_REPOSITORY),
+            call(utils.CREDENTIAL_KEY_REPOSITORY)])
         leader_set.assert_called_with(
-            {'fernet_keys': json.dumps(['some_data', 'some_data'])})
+            {'key_repository': json.dumps(
+                {utils.FERNET_KEY_REPOSITORY:
+                    {'0': 'some_data', '1': 'some_data'},
+                 utils.CREDENTIAL_KEY_REPOSITORY:
+                    {'0': 'some_data', '1': 'some_data'}})
+             })
 
     @patch('os.rename')
     @patch.object(utils, 'leader_get')
-    def test_fernet_write_keys(self, leader_get, rename):
-        key_repository = '/etc/keystone/fernet-keys/'
-        leader_get.return_value = json.dumps(['key0', 'key1'])
-        utils.fernet_write_keys()
-        self.mkdir.assert_called_with(key_repository, owner='keystone',
-                                      group='keystone', perms=0o700)
+    @patch('os.listdir')
+    @patch('os.remove')
+    def test_key_write(self, remove, listdir, leader_get, rename):
+        leader_get.return_value = json.dumps(
+            {utils.FERNET_KEY_REPOSITORY:
+                {'0': 'key0', '1': 'key1'},
+             utils.CREDENTIAL_KEY_REPOSITORY:
+                {'0': 'key0', '1': 'key1'}})
+        listdir.return_value = ['0', '1', '2']
+        with patch.object(builtins, 'open', mock_open()) as m:
+            utils.key_write()
+            m.assert_called_with(utils.KEY_SETUP_FILE, "w")
+        self.mkdir.has_calls([call(utils.CREDENTIAL_KEY_REPOSITORY,
+                                   owner='keystone', group='keystone',
+                                   perms=0o700),
+                              call(utils.FERNET_KEY_REPOSITORY,
+                                   owner='keystone', group='keystone',
+                                   perms=0o700)])
+        # note 'any_order=True' as we are dealing with dictionaries in Py27
         self.write_file.assert_has_calls(
             [
-                call(os.path.join(key_repository, '.0'), u'key0',
+                call(os.path.join(utils.CREDENTIAL_KEY_REPOSITORY, '.0'),
+                     u'key0', owner='keystone', group='keystone', perms=0o600),
+                call(os.path.join(utils.CREDENTIAL_KEY_REPOSITORY, '.1'),
+                     u'key1', owner='keystone', group='keystone', perms=0o600),
+                call(os.path.join(utils.FERNET_KEY_REPOSITORY, '.0'), u'key0',
                      owner='keystone', group='keystone', perms=0o600),
-                call(os.path.join(key_repository, '.1'), u'key1',
+                call(os.path.join(utils.FERNET_KEY_REPOSITORY, '.1'), u'key1',
                      owner='keystone', group='keystone', perms=0o600),
-            ])
+            ], any_order=True)
         rename.assert_has_calls(
             [
-                call(os.path.join(key_repository, '.0'),
-                     os.path.join(key_repository, '0')),
-                call(os.path.join(key_repository, '.1'),
-                     os.path.join(key_repository, '1')),
-            ])
+                call(os.path.join(utils.CREDENTIAL_KEY_REPOSITORY, '.0'),
+                     os.path.join(utils.CREDENTIAL_KEY_REPOSITORY, '0')),
+                call(os.path.join(utils.CREDENTIAL_KEY_REPOSITORY, '.1'),
+                     os.path.join(utils.CREDENTIAL_KEY_REPOSITORY, '1')),
+                call(os.path.join(utils.FERNET_KEY_REPOSITORY, '.0'),
+                     os.path.join(utils.FERNET_KEY_REPOSITORY, '0')),
+                call(os.path.join(utils.FERNET_KEY_REPOSITORY, '.1'),
+                     os.path.join(utils.FERNET_KEY_REPOSITORY, '1')),
+            ], any_order=True)
+
+    @patch.object(utils, 'keystone_context')
+    @patch.object(utils, 'fernet_rotate')
+    @patch.object(utils, 'key_leader_set')
+    @patch.object(utils, 'os')
+    @patch.object(utils, 'is_leader')
+    def test_fernet_keys_rotate_and_sync(self, mock_is_leader, mock_os,
+                                         mock_key_leader_set,
+                                         mock_fernet_rotate,
+                                         mock_keystone_context):
+        self.test_config.set('fernet-max-active-keys', 3)
+        self.test_config.set('token-expiration', 60)
+        self.time.time.return_value = 0
+
+        # if not leader shouldn't do anything
+        mock_is_leader.return_value = False
+        utils.fernet_keys_rotate_and_sync()
+        mock_os.stat.assert_not_called()
+        # shouldn't do anything as the token provider is wrong
+        mock_keystone_context.fernet_enabled.return_value = False
+        mock_is_leader.return_value = True
+        utils.fernet_keys_rotate_and_sync()
+        mock_os.stat.assert_not_called()
+        # fail gracefully if key repository is not initialized
+        mock_keystone_context.fernet_enabled.return_value = True
+        mock_os.stat.side_effect = Exception()
+        with self.assertRaises(Exception):
+            utils.fernet_keys_rotate_and_sync()
+        self.time.time.assert_not_called()
+        mock_os.stat.side_effect = None
+        # now set up the times, so that it still shouldn't be called.
+        self.time.time.return_value = 30
+        self.time.ctime = time.ctime
+        _stat = MagicMock()
+        _stat.st_mtime = 10
+        mock_os.stat.return_value = _stat
+        utils.fernet_keys_rotate_and_sync(log_func=self.log)
+        self.log.assert_called_once_with(
+            'No rotation until at least Thu Jan  1 00:01:10 1970',
+            level='DEBUG')
+        mock_key_leader_set.assert_not_called()
+        # finally, set it up so that the rotation and sync occur
+        self.time.time.return_value = 71
+        utils.fernet_keys_rotate_and_sync()
+        mock_fernet_rotate.assert_called_once_with()
+        mock_key_leader_set.assert_called_once_with()

unit_tests/test_scripts_fernet_rotate_and_sync.py

@@ -0,0 +1,42 @@
+# Copyright 2018 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+from mock import patch
+
+from test_utils import CharmTestCase
+
+import fernet_rotate_and_sync as script
+
+
+class FernetRotateAndSync(CharmTestCase):
+
+    def setUp(self):
+        super(FernetRotateAndSync, self).setUp(
+            script, [])
+
+    @patch('charmhelpers.core.hookenv.log')
+    @patch('time.ctime')
+    @patch('__builtin__.print')
+    def test_cli_log(self, mock_print, mock_ctime, mock_ch_log):
+        mock_ctime.return_value = 'FAKE_TIMESTAMP'
+        script.cli_log('message', level='DEBUG')
+        mock_ch_log.assert_called_with('message', level='DEBUG')
+        script.cli_log('message', level='WARNING')
+        mock_print.assert_called_with('FAKE_TIMESTAMP: message',
+                                      file=sys.stderr)
+        script.cli_log('message', level='INFO')
+        mock_print.assert_called_with('FAKE_TIMESTAMP: message',
+                                      file=sys.stdout)