diff --git a/charmhelpers/contrib/network/ip.py b/charmhelpers/contrib/network/ip.py index cf9926b9..f3b4864f 100644 --- a/charmhelpers/contrib/network/ip.py +++ b/charmhelpers/contrib/network/ip.py @@ -16,6 +16,7 @@ import glob import re import subprocess import socket +import ssl from functools import partial @@ -527,19 +528,56 @@ def get_hostname(address, fqdn=True): return result.split('.')[0] -def port_has_listener(address, port): +class SSLPortCheckInfo(object): + + def __init__(self, key, cert, ca_cert, check_hostname=False): + self.key = key + self.cert = cert + self.ca_cert = ca_cert + # NOTE: by default we do not check hostname since the port check is + # typically performed using 0.0.0.0 which will not match the + # certificate. Hence the default for this is False. + self.check_hostname = check_hostname + + @property + def ssl_context(self): + context = ssl.create_default_context() + context.check_hostname = self.check_hostname + context.load_cert_chain(self.cert, self.key) + context.load_verify_locations(self.ca_cert) + return context + + +def port_has_listener(address, port, sslinfo=None): """ Returns True if the address:port is open and being listened to, - else False. + else False. By default uses netcat to check ports but if sslinfo is + provided will use an SSL connection instead. @param address: an IP address or hostname @param port: integer port + @param sslinfo: optional SSLPortCheckInfo object. + If provided, the check is performed using an ssl + connection. Note calls 'zc' via a subprocess shell """ - cmd = ['nc', '-z', address, str(port)] - result = subprocess.call(cmd) - return not (bool(result)) + if not sslinfo: + cmd = ['nc', '-z', address, str(port)] + result = subprocess.call(cmd) + return not (bool(result)) + + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) as sock: + ssock = sslinfo.ssl_context.wrap_socket(sock, + server_hostname=address) + ssock.connect((address, port)) + # this bit is crucial to ensure tls close_notify is sent + ssock.unwrap() + + return True + except ConnectionRefusedError: + return False def assert_charm_supports_ipv6(): diff --git a/charmhelpers/contrib/openstack/context.py b/charmhelpers/contrib/openstack/context.py index 1e667fb0..cd70b55c 100644 --- a/charmhelpers/contrib/openstack/context.py +++ b/charmhelpers/contrib/openstack/context.py @@ -202,6 +202,21 @@ class OSContextGenerator(object): return self.related +class KeystoneAuditMiddleware(OSContextGenerator): + def __init__(self, service: str) -> None: + self.service_name = service + + def __call__(self): + """Return context dictionary containing configuration status of + audit-middleware and the charm service name. + """ + ctxt = { + 'audit_middleware': config('audit-middleware') or False, + 'service_name': self.service_name + } + return ctxt + + class SharedDBContext(OSContextGenerator): interfaces = ['shared-db'] diff --git a/charmhelpers/contrib/openstack/templates/section-audit-middleware-notifications b/charmhelpers/contrib/openstack/templates/section-audit-middleware-notifications new file mode 100644 index 00000000..1f88014f --- /dev/null +++ b/charmhelpers/contrib/openstack/templates/section-audit-middleware-notifications @@ -0,0 +1,4 @@ +{% if audit_middleware -%} +[audit_middleware_notifications] +driver = log +{% endif -%} \ No newline at end of file diff --git a/charmhelpers/contrib/openstack/templates/section-filter-audit b/charmhelpers/contrib/openstack/templates/section-filter-audit new file mode 100644 index 00000000..11512aee --- /dev/null +++ b/charmhelpers/contrib/openstack/templates/section-filter-audit @@ -0,0 +1,6 @@ +{% if audit_middleware and service_name -%} +[filter:audit] +paste.filter_factory = keystonemiddleware.audit:filter_factory +audit_map_file = /etc/{{ service_name }}/api_audit_map.conf +service_name = {{ service_name }} +{% endif -%} \ No newline at end of file diff --git a/charmhelpers/contrib/openstack/utils.py b/charmhelpers/contrib/openstack/utils.py index da711c65..82c28d8e 100644 --- a/charmhelpers/contrib/openstack/utils.py +++ b/charmhelpers/contrib/openstack/utils.py @@ -1207,12 +1207,14 @@ def _ows_check_services_running(services, ports): return ows_check_services_running(services, ports) -def ows_check_services_running(services, ports): +def ows_check_services_running(services, ports, ssl_check_info=None): """Check that the services that should be running are actually running and that any ports specified are being listened to. @param services: list of strings OR dictionary specifying services/ports @param ports: list of ports + @param ssl_check_info: SSLPortCheckInfo object. If provided, port checks + will be done using an SSL connection. @returns state, message: strings or None, None """ messages = [] @@ -1228,7 +1230,7 @@ def ows_check_services_running(services, ports): # also verify that the ports that should be open are open # NB, that ServiceManager objects only OPTIONALLY have ports map_not_open, ports_open = ( - _check_listening_on_services_ports(services)) + _check_listening_on_services_ports(services, ssl_check_info)) if not all(ports_open): # find which service has missing ports. They are in service # order which makes it a bit easier. @@ -1243,7 +1245,8 @@ def ows_check_services_running(services, ports): if ports is not None: # and we can also check ports which we don't know the service for - ports_open, ports_open_bools = _check_listening_on_ports_list(ports) + ports_open, ports_open_bools = \ + _check_listening_on_ports_list(ports, ssl_check_info) if not all(ports_open_bools): messages.append( "Ports which should be open, but are not: {}" @@ -1302,7 +1305,8 @@ def _check_running_services(services): return list(zip(services, services_running)), services_running -def _check_listening_on_services_ports(services, test=False): +def _check_listening_on_services_ports(services, test=False, + ssl_check_info=None): """Check that the unit is actually listening (has the port open) on the ports that the service specifies are open. If test is True then the function returns the services with ports that are open rather than @@ -1312,11 +1316,14 @@ def _check_listening_on_services_ports(services, test=False): @param services: OrderedDict(service: [port, ...], ...) @param test: default=False, if False, test for closed, otherwise open. + @param ssl_check_info: SSLPortCheckInfo object. If provided, port checks + will be done using an SSL connection. @returns OrderedDict(service: [port-not-open, ...]...), [boolean] """ test = not (not (test)) # ensure test is True or False all_ports = list(itertools.chain(*services.values())) - ports_states = [port_has_listener('0.0.0.0', p) for p in all_ports] + ports_states = [port_has_listener('0.0.0.0', p, ssl_check_info) + for p in all_ports] map_ports = OrderedDict() matched_ports = [p for p, opened in zip(all_ports, ports_states) if opened == test] # essentially opened xor test @@ -1327,16 +1334,19 @@ def _check_listening_on_services_ports(services, test=False): return map_ports, ports_states -def _check_listening_on_ports_list(ports): +def _check_listening_on_ports_list(ports, ssl_check_info=None): """Check that the ports list given are being listened to Returns a list of ports being listened to and a list of the booleans. + @param ssl_check_info: SSLPortCheckInfo object. If provided, port checks + will be done using an SSL connection. @param ports: LIST of port numbers. @returns [(port_num, boolean), ...], [boolean] """ - ports_open = [port_has_listener('0.0.0.0', p) for p in ports] + ports_open = [port_has_listener('0.0.0.0', p, ssl_check_info) + for p in ports] return zip(ports, ports_open), ports_open diff --git a/charmhelpers/contrib/storage/linux/lvm.py b/charmhelpers/contrib/storage/linux/lvm.py index d0a57211..0d294c79 100644 --- a/charmhelpers/contrib/storage/linux/lvm.py +++ b/charmhelpers/contrib/storage/linux/lvm.py @@ -17,8 +17,6 @@ from subprocess import ( CalledProcessError, check_call, check_output, - Popen, - PIPE, ) @@ -58,9 +56,7 @@ def remove_lvm_physical_volume(block_device): :param block_device: str: Full path of block device to scrub. ''' - p = Popen(['pvremove', '-ff', block_device], - stdin=PIPE) - p.communicate(input='y\n') + check_call(['pvremove', '-ff', '--yes', block_device]) def list_lvm_volume_group(block_device): diff --git a/charmhelpers/core/host.py b/charmhelpers/core/host.py index 70dde6a5..def403c5 100644 --- a/charmhelpers/core/host.py +++ b/charmhelpers/core/host.py @@ -256,8 +256,11 @@ def service_resume(service_name, init_dir="/etc/init", upstart_file = os.path.join(init_dir, "{}.conf".format(service_name)) sysv_file = os.path.join(initd_dir, service_name) if init_is_systemd(service_name=service_name): - service('unmask', service_name) - service('enable', service_name) + if service('is-enabled', service_name): + log('service {} already enabled'.format(service_name), level=DEBUG) + else: + service('unmask', service_name) + service('enable', service_name) elif os.path.exists(upstart_file): override_path = os.path.join( init_dir, '{}.override'.format(service_name)) diff --git a/charmhelpers/osplatform.py b/charmhelpers/osplatform.py index 1ace468f..5d121866 100644 --- a/charmhelpers/osplatform.py +++ b/charmhelpers/osplatform.py @@ -9,19 +9,13 @@ def get_platform(): will be returned (which is the name of the module). This string is used to decide which platform module should be imported. """ - # linux_distribution is deprecated and will be removed in Python 3.7 - # Warnings *not* disabled, as we certainly need to fix this. - if hasattr(platform, 'linux_distribution'): - tuple_platform = platform.linux_distribution() - current_platform = tuple_platform[0] - else: - current_platform = _get_platform_from_fs() + current_platform = _get_current_platform() if "Ubuntu" in current_platform: return "ubuntu" elif "CentOS" in current_platform: return "centos" - elif "debian" in current_platform: + elif "debian" in current_platform or "Debian" in current_platform: # Stock Python does not detect Ubuntu and instead returns debian. # Or at least it does in some build environments like Travis CI return "ubuntu" @@ -36,6 +30,24 @@ def get_platform(): .format(current_platform)) +def _get_current_platform(): + """Return the current platform information for the OS. + + Attempts to lookup linux distribution information from the platform + module for releases of python < 3.7. For newer versions of python, + the platform is determined from the /etc/os-release file. + """ + # linux_distribution is deprecated and will be removed in Python 3.7 + # Warnings *not* disabled, as we certainly need to fix this. + if hasattr(platform, 'linux_distribution'): + tuple_platform = platform.linux_distribution() + current_platform = tuple_platform[0] + else: + current_platform = _get_platform_from_fs() + + return current_platform + + def _get_platform_from_fs(): """Get Platform from /etc/os-release.""" with open(os.path.join(os.sep, 'etc', 'os-release')) as fin: diff --git a/config.yaml b/config.yaml index 55b4dec8..89f37e5f 100644 --- a/config.yaml +++ b/config.yaml @@ -12,6 +12,11 @@ options: default: False description: | Setting this to True will allow supporting services to log to syslog. + audit-middleware: + type: boolean + default: False + description: | + Enable Keystone auditing middleware for logging API calls. openstack-origin: type: string default: bobcat diff --git a/hooks/cinder_utils.py b/hooks/cinder_utils.py index fa1d519f..14d226e2 100644 --- a/hooks/cinder_utils.py +++ b/hooks/cinder_utils.py @@ -167,6 +167,7 @@ CINDER_CONF_DIR = "/etc/cinder" CINDER_CONF = '%s/cinder.conf' % CINDER_CONF_DIR CINDER_API_CONF = '%s/api-paste.ini' % CINDER_CONF_DIR CINDER_POLICY_JSON = '%s/policy.json' % CINDER_CONF_DIR +CINDER_AUDIT_MAP = '%s/api_audit_map.conf' % CINDER_CONF_DIR CEPH_CONF = '/etc/ceph/ceph.conf' HAPROXY_CONF = '/etc/haproxy/haproxy.cfg' @@ -237,18 +238,24 @@ BASE_RESOURCE_MAP = OrderedDict([ cinder_contexts.VolumeUsageAuditContext(), context.MemcacheContext(), cinder_contexts.SectionalConfigContext(), - cinder_contexts.LVMContext()], + cinder_contexts.LVMContext(), + context.KeystoneAuditMiddleware(service='cinder')], 'services': ['cinder-api', 'cinder-volume', 'cinder-scheduler', 'haproxy'] }), (CINDER_API_CONF, { - 'contexts': [context.IdentityServiceContext()], + 'contexts': [context.IdentityServiceContext(), + context.KeystoneAuditMiddleware(service='cinder')], 'services': ['cinder-api'], }), (CINDER_POLICY_JSON, { 'contexts': [], 'services': ['cinder-api'] }), + (CINDER_AUDIT_MAP, { + 'contexts': [context.KeystoneAuditMiddleware(service='cinder')], + 'services': ['cinder-api'] + }), (ceph_config_file(), { 'contexts': [context.CephContext()], 'services': ['cinder-volume'] @@ -773,7 +780,7 @@ def check_local_db_actions_complete(): # Echo notification data = {CINDER_DB_INIT_ECHO_RKEY: init_id} # BUG: #1928383 - clear CINDER_DB_INIT_RKEY if not the leader - if not(is_elected_leader(CLUSTER_RES)): + if not is_elected_leader(CLUSTER_RES): data[CINDER_DB_INIT_RKEY] = None relation_set(**data) diff --git a/templates/yoga/api-paste.ini b/templates/yoga/api-paste.ini new file mode 100644 index 00000000..5a632263 --- /dev/null +++ b/templates/yoga/api-paste.ini @@ -0,0 +1,101 @@ +############# +# OpenStack # +############# + +[composite:osapi_volume] +use = call:cinder.api:root_app_factory +/: apiversions +/healthcheck: healthcheck +/v1: openstack_volume_api_v1 +/v2: openstack_volume_api_v2 +/v3: openstack_volume_api_v3 + +[composite:openstack_volume_api_v1] +use = call:cinder.api.middleware.auth:pipeline_factory +noauth = cors http_proxy_to_wsgi request_id faultwrap sizelimit osprofiler noauth apiv1 +{% if audit_middleware and service_name -%} +keystone = cors http_proxy_to_wsgi request_id faultwrap sizelimit osprofiler authtoken keystonecontext audit apiv1 +keystone_nolimit = cors http_proxy_to_wsgi request_id faultwrap sizelimit osprofiler authtoken keystonecontext audit apiv1 +{% else -%} +keystone = cors http_proxy_to_wsgi request_id faultwrap sizelimit osprofiler authtoken keystonecontext apiv1 +keystone_nolimit = cors http_proxy_to_wsgi request_id faultwrap sizelimit osprofiler authtoken keystonecontext apiv1 +{% endif %} + +[composite:openstack_volume_api_v2] +use = call:cinder.api.middleware.auth:pipeline_factory +noauth = cors http_proxy_to_wsgi request_id faultwrap sizelimit osprofiler noauth apiv2 +{% if audit_middleware and service_name -%} +keystone = cors http_proxy_to_wsgi request_id faultwrap sizelimit osprofiler authtoken keystonecontext audit apiv2 +keystone_nolimit = cors http_proxy_to_wsgi request_id faultwrap sizelimit osprofiler authtoken keystonecontext audit apiv2 +{% else -%} +keystone = cors http_proxy_to_wsgi request_id faultwrap sizelimit osprofiler authtoken keystonecontext apiv2 +keystone_nolimit = cors http_proxy_to_wsgi request_id faultwrap sizelimit osprofiler authtoken keystonecontext apiv2 +{% endif %} + +[composite:openstack_volume_api_v3] +use = call:cinder.api.middleware.auth:pipeline_factory +noauth = cors http_proxy_to_wsgi request_id faultwrap sizelimit osprofiler noauth apiv3 +{% if audit_middleware and service_name -%} +keystone = cors http_proxy_to_wsgi request_id faultwrap sizelimit osprofiler authtoken keystonecontext audit apiv3 +keystone_nolimit = cors http_proxy_to_wsgi request_id faultwrap sizelimit osprofiler authtoken keystonecontext audit apiv3 +{% else -%} +keystone = cors http_proxy_to_wsgi request_id faultwrap sizelimit osprofiler authtoken keystonecontext apiv3 +keystone_nolimit = cors http_proxy_to_wsgi request_id faultwrap sizelimit osprofiler authtoken keystonecontext apiv3 +{% endif %} + +[filter:request_id] +paste.filter_factory = oslo_middleware.request_id:RequestId.factory + +[filter:http_proxy_to_wsgi] +paste.filter_factory = oslo_middleware.http_proxy_to_wsgi:HTTPProxyToWSGI.factory + +[filter:cors] +paste.filter_factory = oslo_middleware.cors:filter_factory +oslo_config_project = cinder + +[filter:faultwrap] +paste.filter_factory = cinder.api.middleware.fault:FaultWrapper.factory + +[filter:osprofiler] +paste.filter_factory = osprofiler.web:WsgiMiddleware.factory + +[filter:noauth] +paste.filter_factory = cinder.api.middleware.auth:NoAuthMiddleware.factory + +[filter:sizelimit] +paste.filter_factory = oslo_middleware.sizelimit:RequestBodySizeLimiter.factory + +[app:apiv1] +paste.app_factory = cinder.api.v1.router:APIRouter.factory + +[app:apiv2] +paste.app_factory = cinder.api.v2.router:APIRouter.factory + +[app:apiv3] +paste.app_factory = cinder.api.v3.router:APIRouter.factory + +[pipeline:apiversions] +pipeline = cors http_proxy_to_wsgi faultwrap osvolumeversionapp + +[app:osvolumeversionapp] +paste.app_factory = cinder.api.versions:Versions.factory + +[pipeline:healthcheck] +pipeline = request_id healthcheckapp + +[app:healthcheckapp] +paste.app_factory = oslo_middleware:Healthcheck.app_factory +backends = disable_by_file +disable_by_file_path = /etc/cinder/healthcheck_disable + +########## +# Shared # +########## + +[filter:keystonecontext] +paste.filter_factory = cinder.api.middleware.auth:CinderKeystoneContext.factory + +[filter:authtoken] +paste.filter_factory = keystonemiddleware.auth_token:filter_factory + +{% include "section-filter-audit" %} \ No newline at end of file diff --git a/templates/yoga/api_audit_map.conf b/templates/yoga/api_audit_map.conf new file mode 100644 index 00000000..17889f8c --- /dev/null +++ b/templates/yoga/api_audit_map.conf @@ -0,0 +1,28 @@ +[DEFAULT] +# default target endpoint type +# should match the endpoint type defined in service catalog +target_endpoint_type = None + +# map urls ending with specific text to a unique action +[custom_actions] +associate = update/associate +disassociate = update/disassociate +disassociate_all = update/disassociate_all +associations = read/list/associations + +# possible end path of api requests +[path_keywords] +defaults = None +detail = None +limits = None +os-quota-specs = project +qos-specs = qos-spec +snapshots = snapshot +types = type +volumes = volume + +# map endpoint type defined in service catalog to CADF typeURI +[service_endpoints] +volume = service/storage/block +volumev2 = service/storage/block +volumev3 = service/storage/block \ No newline at end of file diff --git a/templates/yoga/cinder.conf b/templates/yoga/cinder.conf new file mode 100644 index 00000000..42f7421f --- /dev/null +++ b/templates/yoga/cinder.conf @@ -0,0 +1,89 @@ +############################################################################### +# [ WARNING ] +# cinder configuration file maintained by Juju +# local changes may be overwritten. +############################################################################### +[DEFAULT] +rootwrap_config = /etc/cinder/rootwrap.conf +api_paste_confg = /etc/cinder/api-paste.ini +iscsi_helper = tgtadm +verbose = {{ verbose }} +debug = {{ debug }} +use_syslog = {{ use_syslog }} +auth_strategy = keystone +state_path = /var/lib/cinder +osapi_volume_workers = {{ workers }} + +{% if transport_url %} +transport_url = {{ transport_url }} +{% endif %} + +{% if use_internal_endpoints -%} +swift_catalog_info = object-store:swift:internalURL +keystone_catalog_info = identity:Identity Service:internalURL +glance_catalog_info = image:glance:internalURL +nova_catalog_info = compute:Compute Service:internalURL +{% endif %} + +osapi_volume_listen = {{ bind_host }} +{% if osapi_volume_listen_port -%} +osapi_volume_listen_port = {{ osapi_volume_listen_port }} +{% endif -%} + +{% if glance_api_servers -%} +glance_api_servers = {{ glance_api_servers }} +{% endif -%} + +{% if glance_api_version -%} +glance_api_version = {{ glance_api_version }} +{% endif -%} + +{% if region -%} +os_region_name = {{ region }} +{% endif -%} + +{% if user_config_flags -%} +{% for key, value in user_config_flags.items() -%} +{{ key }} = {{ value }} +{% endfor -%} +{% endif -%} + +volume_usage_audit_period = {{ volume_usage_audit_period }} + +{% if auth_host -%} +cinder_internal_tenant_project_id = {{ admin_tenant_id }} +{% if admin_user_id -%} +cinder_internal_tenant_user_id = {{ admin_user_id }} +{% else -%} +cinder_internal_tenant_user_id = {{ admin_user }} +{% endif -%} +{% endif -%} + +{% include "parts/backends" %} +{% include "section-keystone-authtoken-mitaka" %} + +{% if keystone_authtoken -%} +{% include "section-service-user" %} +{% endif -%} + +{% include "parts/section-database" %} + +{% include "section-oslo-messaging-rabbit-ocata" %} + +{% include "section-oslo-notifications" %} + +{% include "section-audit-middleware-notifications" %} + +[oslo_concurrency] +lock_path = /var/lock/cinder + +[keymgr] +# XXX: hack to work around http://pad.lv/1516085 +# will be superseded by SRU to cinder package +encryption_auth_url = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/v3 + +{% include "section-oslo-middleware" %} + +[nova] +{% include "parts/service-auth" %} +