Fernet key synchronization

This update contains the following changes for Distributed
Cloud Fernet Key Synching & Management:

1.Distribute the fernet keys when the subcloud is managed
2.Setup a periodic task to rotate and re-distribute keys
3.Support fernet repo audit

Story: 2002842
Task: 22786

Depends-On: https://review.openstack.org/#/c/613620/

Change-Id: I203c937e9c2334da3f4766c0a49f32f71f7fd39e
Signed-off-by: Tao Liu <tao.liu@windriver.com>
This commit is contained in:
Tao Liu 2018-10-26 10:56:57 -05:00
parent c59024ac56
commit 3dceb62d37
6 changed files with 339 additions and 3 deletions

View File

@ -197,6 +197,12 @@ snmp_server_opts = [
help='interval of periodic updates in seconds')
]
fernet_opts = [
cfg.IntOpt('key_rotation_interval',
default=168,
help='Hours between running fernet key rotation tasks.')
]
scheduler_opt_group = cfg.OptGroup('scheduler',
title='Scheduler options for periodic job')
# The group stores DC Orchestrator global limit for all the projects.
@ -212,6 +218,9 @@ cache_opt_group = cfg.OptGroup(name='cache',
snmp_opt_group = cfg.OptGroup(name='snmp',
title='SNMP Options')
fernet_opt_group = cfg.OptGroup(name='fernet',
title='Fernet Options')
def list_opts():
yield default_quota_group.name, nova_quotas
@ -221,6 +230,7 @@ def list_opts():
yield scheduler_opt_group.name, scheduler_opts
yield pecan_group.name, pecan_opts
yield snmp_opt_group.name, snmp_server_opts
yield fernet_opt_group.name, fernet_opts
yield None, global_opts
yield None, common_opts

View File

@ -90,6 +90,7 @@ RESOURCE_TYPE_SYSINV_REMOTE_LOGGING = "remotelogging"
RESOURCE_TYPE_SYSINV_SNMP_COMM = "icommunity"
RESOURCE_TYPE_SYSINV_SNMP_TRAPDEST = "itrapdest"
RESOURCE_TYPE_SYSINV_USER = "iuser"
RESOURCE_TYPE_SYSINV_FERNET_REPO = "fernet_repo"
# Compute Resources
RESOURCE_TYPE_COMPUTE_FLAVOR = "flavor"
@ -178,3 +179,5 @@ ACTION_EXTRASPECS_DELETE = "extra_specs_delete"
ALARM_OK_STATUS = "OK"
ALARM_DEGRADED_STATUS = "degraded"
ALARM_CRITICAL_STATUS = "critical"
SECONDS_IN_HOUR = 3600

View File

@ -708,3 +708,50 @@ class SysinvClient(base.DriverBase):
raise exceptions.SyncRequestFailedRetry()
return iuser
def create_fernet_repo(self, key_list):
"""Add the fernet keys for this region
:param: key list payload
:return: Nothing
"""
# Example key_list:
# [{"id": 0, "key": "GgDAOfmyr19u0hXdm5r_zMgaMLjglVFpp5qn_N4GBJQ="},
# {"id": 1, "key": "7WfL_z54p67gWAkOmQhLA9P0ZygsbbJcKgff0uh28O8="},
# {"id": 2, "key": ""5gsUQeOZ2FzZP58DN32u8pRKRgAludrjmrZFJSOHOw0="}]
LOG.info("create_fernet_repo driver region={} "
"fernet_repo_list={}".format(self.region_name, key_list))
try:
self.client.fernet.create(key_list)
except Exception as e:
LOG.error("create_fernet_repo exception={}".format(e))
raise exceptions.SyncRequestFailedRetry()
def update_fernet_repo(self, key_list):
"""Update the fernet keys for this region
:param: key list payload
:return: Nothing
"""
LOG.info("update_fernet_repo driver region={} "
"fernet_repo_list={}".format(self.region_name, key_list))
try:
self.client.fernet.put(key_list)
except Exception as e:
LOG.error("update_fernet_repo exception={}".format(e))
raise exceptions.SyncRequestFailedRetry()
def get_fernet_keys(self):
"""Retrieve the fernet keys for this region
:return: a list of fernet keys
"""
try:
keys = self.client.fernet.list()
except Exception as e:
LOG.error("get_fernet_keys exception={}".format(e))
raise exceptions.SyncRequestFailedRetry()
return keys

View File

@ -0,0 +1,125 @@
# Copyright 2018 Wind River
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import subprocess
from oslo_config import cfg
from oslo_log import log as logging
from oslo_serialization import jsonutils
from dcorch.common import consts
from dcorch.common import context
from dcorch.common import exceptions
from dcorch.common.i18n import _
from dcorch.common import manager
from dcorch.common import utils
from dcorch.drivers.openstack import sdk_platform as sdk
from dcorch.objects import subcloud as subcloud_obj
FERNET_REPO_MASTER_ID = "keys"
KEY_ROTATE_CMD = "/usr/bin/keystone-fernet-keys-rotate-active"
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
class FernetKeyManager(manager.Manager):
"""Manages tasks related to fernet key management"""
def __init__(self, gsm, *args, **kwargs):
LOG.debug(_('FernetKeyManager initialization...'))
super(FernetKeyManager, self).__init__(service_name="fernet_manager",
*args, **kwargs)
self.gsm = gsm
self.context = context.get_admin_context()
self.endpoint_type = consts.ENDPOINT_TYPE_PLATFORM
self.resource_type = consts.RESOURCE_TYPE_SYSINV_FERNET_REPO
@classmethod
def to_resource_info(cls, key_list):
return dict((getattr(key, 'id'), getattr(key, 'key'))
for key in key_list)
@classmethod
def from_resource_info(cls, keys):
key_list = [dict(id=k, key=v) for k, v in keys.items()]
return key_list
@classmethod
def get_resource_hash(cls, resource_info):
return hash(tuple(sorted(hash(x) for x in resource_info.items())))
def _schedule_work(self, operation_type, subcloud=None):
keys = self._get_master_keys()
if not keys:
LOG.info(_("No fernet keys returned from %s") % consts.CLOUD_0)
return
try:
resource_info = FernetKeyManager.to_resource_info(keys)
utils.enqueue_work(self.context,
self.endpoint_type,
self.resource_type,
FERNET_REPO_MASTER_ID,
operation_type,
resource_info=jsonutils.dumps(resource_info),
subcloud=subcloud)
# wake up sync thread
if self.gsm:
self.gsm.sync_request(self.context, self.endpoint_type)
except Exception as e:
LOG.error(_("Exception in schedule_work: %s") % e.message)
@staticmethod
def _get_master_keys():
"""get the keys from the local fernet key repo"""
keys = []
try:
os_client = sdk.OpenStackDriver(consts.CLOUD_0)
keys = os_client.sysinv_client.get_fernet_keys()
except (exceptions.ConnectionRefused, exceptions.NotAuthorized,
exceptions.TimeOut):
LOG.info(_("Retrieving the fernet keys from %s timeout") %
consts.CLOUD_0)
except Exception as e:
LOG.info(_("Fail to retrieve the master fernet keys: %s") %
e.message)
return keys
def rotate_fernet_keys(self):
"""Rotate fernet keys."""
with open(os.devnull, "w") as fnull:
try:
subprocess.check_call(KEY_ROTATE_CMD,
stdout=fnull,
stderr=fnull)
except subprocess.CalledProcessError:
msg = _("Failed to rotate the keys")
LOG.exception(msg)
raise exceptions.InternalError(msg)
self._schedule_work(consts.OPERATION_TYPE_PUT)
def distribute_keys(self, ctxt, subcloud_name):
subclouds = subcloud_obj.SubcloudList.get_all(ctxt)
for sc in subclouds:
if sc.region_name == subcloud_name:
subcloud = sc
self._schedule_work(consts.OPERATION_TYPE_CREATE, subcloud)
break

View File

@ -25,6 +25,7 @@ from dcorch.common import exceptions
from dcorch.common.i18n import _
from dcorch.common import messaging as rpc_messaging
from dcorch.engine.alarm_aggregate_manager import AlarmAggregateManager
from dcorch.engine.fernet_key_manager import FernetKeyManager
from dcorch.engine.generic_sync_manager import GenericSyncManager
from dcorch.engine.quota_manager import QuotaManager
from dcorch.engine import scheduler
@ -77,6 +78,7 @@ class EngineService(service.Service):
self.qm = None
self.gsm = None
self.aam = None
self.fkm = None
def init_tgm(self):
self.TG = scheduler.ThreadGroupManager()
@ -92,12 +94,16 @@ class EngineService(service.Service):
def init_aam(self):
self.aam = AlarmAggregateManager()
def init_fkm(self):
self.fkm = FernetKeyManager(self.gsm)
def start(self):
self.engine_id = uuidutils.generate_uuid()
self.init_tgm()
self.init_qm()
self.init_gsm()
self.init_aam()
self.init_fkm()
target = oslo_messaging.Target(version=self.rpc_api_version,
server=self.host,
topic=self.topic)
@ -118,6 +124,11 @@ class EngineService(service.Service):
self.TG.add_timer(self.periodic_interval,
self.periodic_sync_audit,
initial_delay=self.periodic_interval / 2)
self.TG.add_timer(CONF.fernet.key_rotation_interval *
consts.SECONDS_IN_HOUR,
self.periodic_key_rotation,
initial_delay=(CONF.fernet.key_rotation_interval
* consts.SECONDS_IN_HOUR))
def service_registry_report(self):
ctx = context.get_admin_context()
@ -182,6 +193,7 @@ class EngineService(service.Service):
# keep equivalent functionality for now
if (management_state == dcm_consts.MANAGEMENT_MANAGED) and \
(availability_status == dcm_consts.AVAILABILITY_ONLINE):
self.fkm.distribute_keys(ctxt, subcloud_name)
self.aam.enable_snmp(ctxt, subcloud_name)
self.gsm.enable_subcloud(ctxt, subcloud_name)
else:
@ -233,3 +245,8 @@ class EngineService(service.Service):
# Terminate the engine process
LOG.info("All threads were gone, terminating engine")
super(EngineService, self).stop()
def periodic_key_rotation(self):
"""Periodic key rotation."""
LOG.info("Periodic key rotation started at: %s", time.strftime("%c"))
return self.fkm.rotate_fernet_keys()

View File

@ -22,7 +22,8 @@ from oslo_serialization import jsonutils
from dcorch.common import consts
from dcorch.common import exceptions
from dcorch.drivers.openstack import sdk_platform as sdk
from dcorch.engine.fernet_key_manager import FERNET_REPO_MASTER_ID
from dcorch.engine.fernet_key_manager import FernetKeyManager
from dcorch.engine.sync_thread import AUDIT_RESOURCE_MISSING
from dcorch.engine.sync_thread import SyncThread
@ -37,13 +38,15 @@ class SysinvSyncThread(SyncThread):
consts.RESOURCE_TYPE_SYSINV_PTP,
consts.RESOURCE_TYPE_SYSINV_REMOTE_LOGGING,
consts.RESOURCE_TYPE_SYSINV_USER,
consts.RESOURCE_TYPE_SYSINV_FERNET_REPO
]
SYSINV_ADD_DELETE_RESOURCES = [consts.RESOURCE_TYPE_SYSINV_SNMP_COMM,
consts.RESOURCE_TYPE_SYSINV_SNMP_TRAPDEST]
SYSINV_CREATE_RESOURCES = [consts.RESOURCE_TYPE_SYSINV_FIREWALL_RULES,
consts.RESOURCE_TYPE_SYSINV_CERTIFICATE]
consts.RESOURCE_TYPE_SYSINV_CERTIFICATE,
consts.RESOURCE_TYPE_SYSINV_FERNET_REPO]
FIREWALL_SIG_NULL = 'NoCustomFirewallRules'
CERTIFICATE_SIG_NULL = 'NoCertificate'
@ -68,6 +71,8 @@ class SysinvSyncThread(SyncThread):
consts.RESOURCE_TYPE_SYSINV_CERTIFICATE:
self.sync_certificate,
consts.RESOURCE_TYPE_SYSINV_USER: self.sync_user,
consts.RESOURCE_TYPE_SYSINV_FERNET_REPO:
self.sync_fernet_resources
}
self.region_name = self.subcloud_engine.subcloud.region_name
self.log_extra = {"instance": "{}/{}: ".format(
@ -83,6 +88,7 @@ class SysinvSyncThread(SyncThread):
consts.RESOURCE_TYPE_SYSINV_SNMP_COMM,
consts.RESOURCE_TYPE_SYSINV_SNMP_TRAPDEST,
consts.RESOURCE_TYPE_SYSINV_USER,
consts.RESOURCE_TYPE_SYSINV_FERNET_REPO,
]
# initialize the master clients
@ -764,6 +770,90 @@ class SysinvSyncThread(SyncThread):
.format(rsrc.id, subcloud_rsrc_id, passwd_hash),
extra=self.log_extra)
def sync_fernet_resources(self, request, rsrc):
switcher = {
consts.OPERATION_TYPE_PUT: self.update_fernet_repo,
consts.OPERATION_TYPE_PATCH: self.update_fernet_repo,
consts.OPERATION_TYPE_CREATE: self.create_fernet_repo,
}
func = switcher[request.orch_job.operation_type]
try:
func(request, rsrc)
except (keystone_exceptions.connection.ConnectTimeout,
keystone_exceptions.ConnectFailure) as e:
LOG.info("sync_fernet_resources: subcloud {} is not reachable [{}]"
.format(self.subcloud_engine.subcloud.region_name,
str(e)), extra=self.log_extra)
raise exceptions.SyncRequestTimeout
except Exception as e:
LOG.exception(e)
raise exceptions.SyncRequestFailedRetry
def create_fernet_repo(self, request, rsrc):
LOG.info("create_fernet_repo region {} resource_info={}".format(
self.subcloud_engine.subcloud.region_name,
request.orch_job.resource_info),
extra=self.log_extra)
resource_info = jsonutils.loads(request.orch_job.resource_info)
s_os_client = sdk.OpenStackDriver(self.region_name)
try:
s_os_client.sysinv_client.create_fernet_repo(
FernetKeyManager.from_resource_info(resource_info))
# Ensure subcloud resource is persisted to the DB for later
subcloud_rsrc_id = self.persist_db_subcloud_resource(
rsrc.id, rsrc.master_id)
except (exceptions.ConnectionRefused, exceptions.NotAuthorized,
exceptions.TimeOut):
LOG.info("create_fernet_repo Timeout,{}:{}".format(
rsrc.id, subcloud_rsrc_id))
s_os_client.delete_region_clients(self.region_name,
clear_token=True)
raise exceptions.SyncRequestTimeout
except (AttributeError, TypeError) as e:
LOG.info("create_fernet_repo error {}".format(e),
extra=self.log_extra)
s_os_client.delete_region_clients(self.region_name,
clear_token=True)
raise exceptions.SyncRequestFailedRetry
LOG.info("fernet_repo {} {} {} created".format(rsrc.id,
subcloud_rsrc_id, resource_info),
extra=self.log_extra)
def update_fernet_repo(self, request, rsrc):
LOG.info("update_fernet_repo region {} resource_info={}".format(
self.subcloud_engine.subcloud.region_name,
request.orch_job.resource_info),
extra=self.log_extra)
resource_info = jsonutils.loads(request.orch_job.resource_info)
s_os_client = sdk.OpenStackDriver(self.region_name)
try:
s_os_client.sysinv_client.update_fernet_repo(
FernetKeyManager.from_resource_info(resource_info))
# Ensure subcloud resource is persisted to the DB for later
subcloud_rsrc_id = self.persist_db_subcloud_resource(
rsrc.id, rsrc.master_id)
except (exceptions.ConnectionRefused, exceptions.NotAuthorized,
exceptions.TimeOut):
LOG.info("update_fernet_repo Timeout,{}:{}".format(
rsrc.id, subcloud_rsrc_id))
s_os_client.delete_region_clients(self.region_name,
clear_token=True)
raise exceptions.SyncRequestTimeout
except (AttributeError, TypeError) as e:
LOG.info("update_fernet_repo error {}".format(e),
extra=self.log_extra)
s_os_client.delete_region_clients(self.region_name,
clear_token=True)
raise exceptions.SyncRequestFailedRetry
LOG.info("fernet_repo {} {} {} update".format(rsrc.id,
subcloud_rsrc_id, resource_info),
extra=self.log_extra)
# SysInv Audit Related
def get_master_resources(self, resource_type):
os_client = sdk.OpenStackDriver(consts.CLOUD_0)
@ -785,6 +875,8 @@ class SysinvSyncThread(SyncThread):
return self.get_certificates_resources(os_client)
elif resource_type == consts.RESOURCE_TYPE_SYSINV_USER:
return [self.get_user_resource(os_client)]
elif resource_type == consts.RESOURCE_TYPE_SYSINV_FERNET_REPO:
return [self.get_fernet_resources(os_client)]
else:
LOG.error("Wrong resource type {}".format(resource_type),
extra=self.log_extra)
@ -810,6 +902,8 @@ class SysinvSyncThread(SyncThread):
return self.get_certificates_resources(os_client)
elif resource_type == consts.RESOURCE_TYPE_SYSINV_USER:
return [self.get_user_resource(os_client)]
elif resource_type == consts.RESOURCE_TYPE_SYSINV_FERNET_REPO:
return [self.get_fernet_resources(os_client)]
else:
LOG.error("Wrong resource type {}".format(resource_type),
extra=self.log_extra)
@ -1004,6 +1098,27 @@ class SysinvSyncThread(SyncThread):
LOG.exception(e)
return None
def get_fernet_resources(self, os_client):
try:
keys = os_client.sysinv_client.get_fernet_keys()
return FernetKeyManager.to_resource_info(keys)
except (keystone_exceptions.connection.ConnectTimeout,
keystone_exceptions.ConnectFailure) as e:
LOG.info("get_fernet_resource: subcloud {} is not reachable [{}]"
.format(self.subcloud_engine.subcloud.region_name,
str(e)), extra=self.log_extra)
os_client.delete_region_clients(self.region_name)
# None will force skip of audit
return None
except (AttributeError, TypeError) as e:
LOG.info("get_fernet_resource error {}".format(e),
extra=self.log_extra)
os_client.delete_region_clients(self.region_name, clear_token=True)
return None
except Exception as e:
LOG.exception(e)
return None
def get_resource_id(self, resource_type, resource):
if resource_type == consts.RESOURCE_TYPE_SYSINV_SNMP_COMM:
LOG.debug("get_resource_id for community {}".format(resource))
@ -1047,6 +1162,10 @@ class SysinvSyncThread(SyncThread):
else:
LOG.error("no get_resource_id for certificate")
return self.CERTIFICATE_SIG_NULL
elif resource_type == consts.RESOURCE_TYPE_SYSINV_FERNET_REPO:
LOG.info("get_resource_id {} resource={}".format(
resource_type, resource))
return FERNET_REPO_MASTER_ID
else:
if hasattr(resource, 'uuid'):
LOG.info("get_resource_id {} uuid={}".format(
@ -1155,6 +1274,15 @@ class SysinvSyncThread(SyncThread):
same_user = False
return same_user
def same_fernet_key(self, i1, i2):
LOG.info("same_fernet_repo i1={}, i2={}".format(i1, i2),
extra=self.log_extra)
same_fernet = True
if (FernetKeyManager.get_resource_hash(i1) !=
FernetKeyManager.get_resource_hash(i2)):
same_fernet = False
return same_fernet
def same_resource(self, resource_type, m_resource, sc_resource):
if resource_type == consts.RESOURCE_TYPE_SYSINV_DNS:
return self.same_dns(m_resource, sc_resource)
@ -1172,8 +1300,10 @@ class SysinvSyncThread(SyncThread):
return self.same_firewallrules(m_resource, sc_resource)
elif resource_type == consts.RESOURCE_TYPE_SYSINV_CERTIFICATE:
return self.same_certificate(m_resource, sc_resource)
if resource_type == consts.RESOURCE_TYPE_SYSINV_USER:
elif resource_type == consts.RESOURCE_TYPE_SYSINV_USER:
return self.same_user(m_resource, sc_resource)
elif resource_type == consts.RESOURCE_TYPE_SYSINV_FERNET_REPO:
return self.same_fernet_key(m_resource, sc_resource)
else:
LOG.warn("same_resource() unexpected resource_type {}".format(
resource_type),
@ -1279,6 +1409,10 @@ class SysinvSyncThread(SyncThread):
resource_type, dumps),
extra=self.log_extra)
return dumps
elif resource_type == consts.RESOURCE_TYPE_SYSINV_FERNET_REPO:
LOG.info("get_resource_info resource_type={} resource={}".format(
resource_type, resource), extra=self.log_extra)
return jsonutils.dumps(resource)
else:
LOG.warn("get_resource_info unsupported resource {}".format(
resource_type),