Browse Source

Sync libraries & common files prior to freeze

* charm-helpers sync for classic charms
* charms.ceph sync for ceph charms
* rebuild for reactive charms
* sync tox.ini files as needed
* sync requirements.txt files to sync to standard

Change-Id: I7c3e0a7e77e16b9bf318ca22c39f8220c5144ac7
changes/40/754540/2
Alex Kavanagh 12 months ago
parent
commit
8f45645505
  1. 1
      actions/openstack_upgrade.py
  2. 1
      actions/security_checklist.py
  3. 76
      hooks/charmhelpers/contrib/network/ovs/__init__.py
  4. 60
      hooks/charmhelpers/contrib/network/ovs/ovsdb.py
  5. 96
      hooks/charmhelpers/contrib/openstack/context.py
  6. 4
      hooks/charmhelpers/contrib/openstack/templates/ceph.conf
  7. 28
      hooks/charmhelpers/contrib/openstack/templates/section-ceph-bluestore-compression
  8. 1415
      hooks/charmhelpers/contrib/storage/linux/ceph.py
  9. 12
      hooks/neutron_api_context.py
  10. 6
      hooks/neutron_api_utils.py
  11. 1
      requirements.txt
  12. 3
      test-requirements.txt
  13. 2
      tox.ini
  14. 6
      unit_tests/test_utils.py

1
actions/openstack_upgrade.py

@ -45,5 +45,6 @@ def openstack_upgrade():
CONFIGS)):
config_changed()
if __name__ == '__main__':
openstack_upgrade()

1
actions/security_checklist.py

@ -63,5 +63,6 @@ def main():
config['neutron_config'] = dict(conf)
return audits.action_parse_results(audits.run(config))
if __name__ == "__main__":
sys.exit(main())

76
hooks/charmhelpers/contrib/network/ovs/__init__.py

@ -13,6 +13,7 @@
# limitations under the License.
''' Helpers for interacting with OpenvSwitch '''
import collections
import hashlib
import os
import re
@ -20,9 +21,9 @@ import six
import subprocess
from charmhelpers import deprecate
from charmhelpers.contrib.network.ovs import ovsdb as ch_ovsdb
from charmhelpers.fetch import apt_install
from charmhelpers.core.hookenv import (
log, WARNING, INFO, DEBUG
)
@ -592,3 +593,76 @@ def ovs_appctl(target, args):
cmd = ['ovs-appctl', '-t', target]
cmd.extend(args)
return subprocess.check_output(cmd, universal_newlines=True)
def uuid_for_port(port_name):
"""Get UUID of named port.
:param port_name: Name of port.
:type port_name: str
:returns: Port UUID.
:rtype: Optional[uuid.UUID]
"""
for port in ch_ovsdb.SimpleOVSDB(
'ovs-vsctl').port.find('name={}'.format(port_name)):
return port['_uuid']
def bridge_for_port(port_uuid):
"""Find which bridge a port is on.
:param port_uuid: UUID of port.
:type port_uuid: uuid.UUID
:returns: Name of bridge or None.
:rtype: Optional[str]
"""
for bridge in ch_ovsdb.SimpleOVSDB(
'ovs-vsctl').bridge:
# If there is a single port on a bridge the ports property will not be
# a list. ref: juju/charm-helpers#510
if (isinstance(bridge['ports'], list) and
port_uuid in bridge['ports'] or
port_uuid == bridge['ports']):
return bridge['name']
PatchPort = collections.namedtuple('PatchPort', ('bridge', 'port'))
Patch = collections.namedtuple('Patch', ('this_end', 'other_end'))
def patch_ports_on_bridge(bridge):
"""Find patch ports on a bridge.
:param bridge: Name of bridge
:type bridge: str
:returns: Iterator with bridge and port name for both ends of a patch.
:rtype: Iterator[Patch[PatchPort[str,str],PatchPort[str,str]]]
:raises: ValueError
"""
# On any given vSwitch there will be a small number of patch ports, so we
# start by iterating over ports with type `patch` then look up which bridge
# they belong to and act on any ports that match the criteria.
for interface in ch_ovsdb.SimpleOVSDB(
'ovs-vsctl').interface.find('type=patch'):
for port in ch_ovsdb.SimpleOVSDB(
'ovs-vsctl').port.find('name={}'.format(interface['name'])):
if bridge_for_port(port['_uuid']) == bridge:
this_end = PatchPort(bridge, port['name'])
other_end = PatchPort(bridge_for_port(
uuid_for_port(
interface['options']['peer'])),
interface['options']['peer'])
yield(Patch(this_end, other_end))
# We expect one result and it is ok if it turns out to be a port
# for a different bridge. However we need a break here to satisfy
# the for/else check which is in place to detect interface refering
# to non-existent port.
break
else:
raise ValueError('Port for interface named "{}" does unexpectedly '
'not exist.'.format(interface['name']))
else:
# Allow our caller to handle no patch ports found gracefully, in
# reference to PEP479 just doing a return will provide a emtpy iterator
# and not None.
return

60
hooks/charmhelpers/contrib/network/ovs/ovsdb.py

@ -36,6 +36,11 @@ class SimpleOVSDB(object):
for br in ovsdb.bridge:
if br['name'] == 'br-test':
ovsdb.bridge.set(br['uuid'], 'external_ids:charm', 'managed')
WARNING: If a list type field only have one item `ovs-vsctl` will present
it as a single item. Since we do not know the schema we have no way of
knowing what fields should be de-serialized as lists so the caller has
to be careful of checking the type of values returned from this library.
"""
# For validation we keep a complete map of currently known good tool and
@ -157,23 +162,59 @@ class SimpleOVSDB(object):
self._tool = tool
self._table = table
def _find_tbl(self, condition=None):
"""Run and parse output of OVSDB `find` command.
def _deserialize_ovsdb(self, data):
"""Deserialize OVSDB RFC7047 section 5.1 data.
:param condition: An optional RFC 7047 5.1 match condition
:type condition: Optional[str]
:returns: Dictionary with data
:rtype: Dict[str, any]
:param data: Multidimensional list where first row contains RFC7047
type information
:type data: List[str,any]
:returns: Deserialized data.
:rtype: any
"""
# When using json formatted output to OVS commands Internal OVSDB
# notation may occur that require further deserializing.
# Reference: https://tools.ietf.org/html/rfc7047#section-5.1
ovs_type_cb_map = {
'uuid': uuid.UUID,
# FIXME sets also appear to sometimes contain type/value tuples
# NOTE: OVSDB sets have overloaded type
# see special handling below
'set': list,
'map': dict,
}
assert len(data) > 1, ('Invalid data provided, expecting list '
'with at least two elements.')
if data[0] == 'set':
# special handling for set
#
# it is either a list of strings or a list of typed lists.
# taste first element to see which it is
for el in data[1]:
# NOTE: We lock this handling down to the `uuid` type as
# that is the only one we have a practical example of.
# We could potentially just handle this generally based on
# the types listed in `ovs_type_cb_map` but let's open for
# that as soon as we have a concrete example to validate on
if isinstance(
el, list) and len(el) and el[0] == 'uuid':
decoded_set = []
for el in data[1]:
decoded_set.append(self._deserialize_ovsdb(el))
return(decoded_set)
# fall back to normal processing below
break
# Use map to deserialize data with fallback to `str`
f = ovs_type_cb_map.get(data[0], str)
return f(data[1])
def _find_tbl(self, condition=None):
"""Run and parse output of OVSDB `find` command.
:param condition: An optional RFC 7047 5.1 match condition
:type condition: Optional[str]
:returns: Dictionary with data
:rtype: Dict[str, any]
"""
cmd = [self._tool, '-f', 'json', 'find', self._table]
if condition:
cmd.append(condition)
@ -182,9 +223,8 @@ class SimpleOVSDB(object):
for row in data['data']:
values = []
for col in row:
if isinstance(col, list):
f = ovs_type_cb_map.get(col[0], str)
values.append(f(col[1]))
if isinstance(col, list) and len(col) > 1:
values.append(self._deserialize_ovsdb(col))
else:
values.append(col)
yield dict(zip(data['headings'], values))

96
hooks/charmhelpers/contrib/openstack/context.py

@ -29,6 +29,8 @@ from subprocess import check_call, CalledProcessError
import six
import charmhelpers.contrib.storage.linux.ceph as ch_ceph
from charmhelpers.contrib.openstack.audits.openstack_security_guide import (
_config_ini as config_ini
)
@ -56,6 +58,7 @@ from charmhelpers.core.hookenv import (
status_set,
network_get_primary_address,
WARNING,
service_name,
)
from charmhelpers.core.sysctl import create as sysctl_create
@ -808,6 +811,12 @@ class CephContext(OSContextGenerator):
ctxt['mon_hosts'] = ' '.join(sorted(mon_hosts))
if config('pool-type') and config('pool-type') == 'erasure-coded':
base_pool_name = config('rbd-pool') or config('rbd-pool-name')
if not base_pool_name:
base_pool_name = service_name()
ctxt['rbd_default_data_pool'] = base_pool_name
if not os.path.isdir('/etc/ceph'):
os.mkdir('/etc/ceph')
@ -3175,3 +3184,90 @@ class SRIOVContext(OSContextGenerator):
:rtype: Dict[str,int]
"""
return self._map
class CephBlueStoreCompressionContext(OSContextGenerator):
"""Ceph BlueStore compression options."""
# Tuple with Tuples that map configuration option name to CephBrokerRq op
# property name
options = (
('bluestore-compression-algorithm',
'compression-algorithm'),
('bluestore-compression-mode',
'compression-mode'),
('bluestore-compression-required-ratio',
'compression-required-ratio'),
('bluestore-compression-min-blob-size',
'compression-min-blob-size'),
('bluestore-compression-min-blob-size-hdd',
'compression-min-blob-size-hdd'),
('bluestore-compression-min-blob-size-ssd',
'compression-min-blob-size-ssd'),
('bluestore-compression-max-blob-size',
'compression-max-blob-size'),
('bluestore-compression-max-blob-size-hdd',
'compression-max-blob-size-hdd'),
('bluestore-compression-max-blob-size-ssd',
'compression-max-blob-size-ssd'),
)
def __init__(self):
"""Initialize context by loading values from charm config.
We keep two maps, one suitable for use with CephBrokerRq's and one
suitable for template generation.
"""
charm_config = config()
# CephBrokerRq op map
self.op = {}
# Context exposed for template generation
self.ctxt = {}
for config_key, op_key in self.options:
value = charm_config.get(config_key)
self.ctxt.update({config_key.replace('-', '_'): value})
self.op.update({op_key: value})
def __call__(self):
"""Get context.
:returns: Context
:rtype: Dict[str,any]
"""
return self.ctxt
def get_op(self):
"""Get values for use in CephBrokerRq op.
:returns: Context values with CephBrokerRq op property name as key.
:rtype: Dict[str,any]
"""
return self.op
def get_kwargs(self):
"""Get values for use as keyword arguments.
:returns: Context values with key suitable for use as kwargs to
CephBrokerRq add_op_create_*_pool methods.
:rtype: Dict[str,any]
"""
return {
k.replace('-', '_'): v
for k, v in self.op.items()
}
def validate(self):
"""Validate options.
:raises: AssertionError
"""
# We slip in a dummy name on class instantiation to allow validation of
# the other options. It will not affect further use.
#
# NOTE: once we retire Python 3.5 we can fold this into a in-line
# dictionary comprehension in the call to the initializer.
dummy_op = {'name': 'dummy-name'}
dummy_op.update(self.op)
pool = ch_ceph.BasePool('dummy-service', op=dummy_op)
pool.validate()

4
hooks/charmhelpers/contrib/openstack/templates/ceph.conf

@ -22,3 +22,7 @@ rbd default features = {{ rbd_features }}
{{ key }} = {{ value }}
{% endfor -%}
{%- endif %}
{% if rbd_default_data_pool -%}
rbd default data pool = {{ rbd_default_data_pool }}
{% endif %}

28
hooks/charmhelpers/contrib/openstack/templates/section-ceph-bluestore-compression

@ -0,0 +1,28 @@
{# section header omitted as options can belong to multiple sections #}
{% if bluestore_compression_algorithm -%}
bluestore compression algorithm = {{ bluestore_compression_algorithm }}
{% endif -%}
{% if bluestore_compression_mode -%}
bluestore compression mode = {{ bluestore_compression_mode }}
{% endif -%}
{% if bluestore_compression_required_ratio -%}
bluestore compression required ratio = {{ bluestore_compression_required_ratio }}
{% endif -%}
{% if bluestore_compression_min_blob_size -%}
bluestore compression min blob size = {{ bluestore_compression_min_blob_size }}
{% endif -%}
{% if bluestore_compression_min_blob_size_hdd -%}
bluestore compression min blob size hdd = {{ bluestore_compression_min_blob_size_hdd }}
{% endif -%}
{% if bluestore_compression_min_blob_size_ssd -%}
bluestore compression min blob size ssd = {{ bluestore_compression_min_blob_size_ssd }}
{% endif -%}
{% if bluestore_compression_max_blob_size -%}
bluestore compression max blob size = {{ bluestore_compression_max_blob_size }}
{% endif -%}
{% if bluestore_compression_max_blob_size_hdd -%}
bluestore compression max blob size hdd = {{ bluestore_compression_max_blob_size_hdd }}
{% endif -%}
{% if bluestore_compression_max_blob_size_ssd -%}
bluestore compression max blob size ssd = {{ bluestore_compression_max_blob_size_ssd }}
{% endif -%}

1415
hooks/charmhelpers/contrib/storage/linux/ceph.py
File diff suppressed because it is too large
View File

12
hooks/neutron_api_context.py

@ -876,12 +876,12 @@ class NeutronApiApiPasteContext(context.OSContextGenerator):
t, n, c = [m.get(v) for v in ['type', 'name', 'config']]
# note that dict has to be non-empty
if not types_valid(t, n, c):
raise ValueError('Extra middleware key type(s) are'
' invalid: {}'.format(repr(m)))
raise ValueError('Extra middleware key type(s) are'
' invalid: {}'.format(repr(m)))
if not mtype_valid(t):
raise ValueError('Extra middleware type key is not'
' a valid PasteDeploy middleware '
'type {}'.format(repr(t)))
raise ValueError('Extra middleware type key is not'
' a valid PasteDeploy middleware '
'type {}'.format(repr(t)))
if not c:
raise ValueError('Extra middleware config dictionary'
' is empty')
@ -893,7 +893,7 @@ class NeutronApiApiPasteContext(context.OSContextGenerator):
if rdata_middleware:
try:
middleware = ast.literal_eval(rdata_middleware)
except:
except Exception:
import traceback
log(traceback.format_exc())
raise ValueError('Invalid extra middleware data'

6
hooks/neutron_api_utils.py

@ -877,8 +877,10 @@ def router_feature_present(feature):
return True
return False
l3ha_router_present = partial(router_feature_present, feature='ha')
dvr_router_present = partial(router_feature_present, feature='distributed')
@ -892,7 +894,7 @@ def neutron_ready():
neutron_client.list_routers()
log('neutron client ready')
return True
except:
except Exception:
log('neutron query failed, neutron not ready ')
return False
@ -953,7 +955,7 @@ def check_optional_relations(configs):
if relation_ids('ha'):
try:
get_hacluster_config()
except:
except Exception:
return ('blocked',
'hacluster missing configuration: '
'vip, vip_iface, vip_cidr')

1
requirements.txt

@ -7,6 +7,7 @@
# requirements. They are intertwined. Also, Zaza itself should specify
# all of its own requirements and if it doesn't, fix it there.
#
setuptools<50.0.0 # https://github.com/pypa/setuptools/commit/04e3df22df840c6bb244e9b27bc56750c44b7c85
pbr>=1.8.0,<1.9.0
simplejson>=2.2.0
netifaces>=0.10.4

3
test-requirements.txt

@ -7,10 +7,11 @@
# requirements. They are intertwined. Also, Zaza itself should specify
# all of its own requirements and if it doesn't, fix it there.
#
setuptools<50.0.0 # https://github.com/pypa/setuptools/commit/04e3df22df840c6bb244e9b27bc56750c44b7c85
charm-tools>=2.4.4
requests>=2.18.4
mock>=1.2
flake8>=2.2.4,<=2.4.1
flake8>=2.2.4
stestr>=2.2.0
coverage>=4.5.2
pyudev # for ceph-* charm unit tests (need to fix the ceph-* charm unit tests/mocking)

2
tox.ini

@ -116,5 +116,5 @@ commands =
functest-run-suite --keep-model --bundle {posargs}
[flake8]
ignore = E402,E226
ignore = E402,E226,W503,W504
exclude = */charmhelpers

6
unit_tests/test_utils.py

@ -101,9 +101,9 @@ class TestConfig(object):
return self.config
def set(self, attr, value):
if attr not in self.config:
raise KeyError
self.config[attr] = value
if attr not in self.config:
raise KeyError
self.config[attr] = value
class TestRelation(object):

Loading…
Cancel
Save