From aa5007332aa286461b5dfab160773126d5f7ce49 Mon Sep 17 00:00:00 2001 From: David Ames Date: Tue, 8 May 2018 12:00:33 -0700 Subject: [PATCH] Enable Bionic as a gate test Change bionic test from dev to gate for 18.05. Change-Id: Id71a674e9c12efc1c37bf8b616d5ed84d2d2be7c --- .../contrib/network/ovs/__init__.py | 19 ++++++ hooks/charmhelpers/contrib/network/ufw.py | 23 +++++++ .../templates/section-oslo-notifications | 3 + hooks/charmhelpers/contrib/openstack/utils.py | 2 +- .../contrib/openstack/vaultlocker.py | 62 ++++++++++++++----- hooks/charmhelpers/core/hookenv.py | 8 ++- hooks/charmhelpers/core/sysctl.py | 18 +++--- tests/charmhelpers/contrib/openstack/utils.py | 2 +- tests/charmhelpers/core/hookenv.py | 8 ++- tests/charmhelpers/core/sysctl.py | 18 +++--- ...bionic-queens => gate-basic-bionic-queens} | 0 tox.ini | 2 +- 12 files changed, 127 insertions(+), 38 deletions(-) rename tests/{dev-basic-bionic-queens => gate-basic-bionic-queens} (100%) diff --git a/hooks/charmhelpers/contrib/network/ovs/__init__.py b/hooks/charmhelpers/contrib/network/ovs/__init__.py index 02d60dfb..9b3583f3 100644 --- a/hooks/charmhelpers/contrib/network/ovs/__init__.py +++ b/hooks/charmhelpers/contrib/network/ovs/__init__.py @@ -175,3 +175,22 @@ def full_restart(): service('start', 'openvswitch-force-reload-kmod') else: service('force-reload-kmod', 'openvswitch-switch') + + +def enable_ipfix(bridge, target): + '''Enable IPfix on bridge to target. + :param bridge: Bridge to monitor + :param target: IPfix remote endpoint + ''' + cmd = ['ovs-vsctl', 'set', 'Bridge', bridge, 'ipfix=@i', '--', + '--id=@i', 'create', 'IPFIX', 'targets="{}"'.format(target)] + log('Enabling IPfix on {}.'.format(bridge)) + subprocess.check_call(cmd) + + +def disable_ipfix(bridge): + '''Diable IPfix on target bridge. + :param bridge: Bridge to modify + ''' + cmd = ['ovs-vsctl', 'clear', 'Bridge', bridge, 'ipfix'] + subprocess.check_call(cmd) diff --git a/hooks/charmhelpers/contrib/network/ufw.py b/hooks/charmhelpers/contrib/network/ufw.py index 5cff71bc..5db622fb 100644 --- a/hooks/charmhelpers/contrib/network/ufw.py +++ b/hooks/charmhelpers/contrib/network/ufw.py @@ -151,6 +151,29 @@ def enable(soft_fail=False): return True +def reload(): + """ + Reload ufw + + :returns: True if ufw is successfully enabled + """ + output = subprocess.check_output(['ufw', 'reload'], + universal_newlines=True, + env={'LANG': 'en_US', + 'PATH': os.environ['PATH']}) + + m = re.findall('^Firewall reloaded\n', + output, re.M) + hookenv.log(output, level='DEBUG') + + if len(m) == 0: + hookenv.log("ufw couldn't be reloaded", level='WARN') + return False + else: + hookenv.log("ufw reloaded", level='INFO') + return True + + def disable(): """ Disable ufw diff --git a/hooks/charmhelpers/contrib/openstack/templates/section-oslo-notifications b/hooks/charmhelpers/contrib/openstack/templates/section-oslo-notifications index 5dccd4bb..021a3c25 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/section-oslo-notifications +++ b/hooks/charmhelpers/contrib/openstack/templates/section-oslo-notifications @@ -5,4 +5,7 @@ transport_url = {{ transport_url }} {% if notification_topics -%} topics = {{ notification_topics }} {% endif -%} +{% if notification_format -%} +notification_format = {{ notification_format }} +{% endif -%} {% endif -%} diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index e7194264..6184abd0 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -306,7 +306,7 @@ def get_os_codename_install_source(src): if src.startswith('cloud:'): ca_rel = src.split(':')[1] - ca_rel = ca_rel.split('%s-' % ubuntu_rel)[1].split('/')[0] + ca_rel = ca_rel.split('-')[1].split('/')[0] return ca_rel # Best guess match based on deb string provided diff --git a/hooks/charmhelpers/contrib/openstack/vaultlocker.py b/hooks/charmhelpers/contrib/openstack/vaultlocker.py index 28c98d0c..a8e4bf88 100644 --- a/hooks/charmhelpers/contrib/openstack/vaultlocker.py +++ b/hooks/charmhelpers/contrib/openstack/vaultlocker.py @@ -21,6 +21,7 @@ import charmhelpers.contrib.openstack.context as context import charmhelpers.core.hookenv as hookenv import charmhelpers.core.host as host import charmhelpers.core.templating as templating +import charmhelpers.core.unitdata as unitdata VAULTLOCKER_BACKEND = 'charm-vaultlocker' @@ -36,30 +37,39 @@ class VaultKVContext(context.OSContextGenerator): ) def __call__(self): + db = unitdata.kv() + last_token = db.get('last-token') + secret_id = db.get('secret-id') for relation_id in hookenv.relation_ids(self.interfaces[0]): for unit in hookenv.related_units(relation_id): - vault_url = hookenv.relation_get( - 'vault_url', - unit=unit, - rid=relation_id - ) - role_id = hookenv.relation_get( - '{}_role_id'.format(hookenv.local_unit()), - unit=unit, - rid=relation_id - ) + data = hookenv.relation_get(unit=unit, + rid=relation_id) + vault_url = data.get('vault_url') + role_id = data.get('{}_role_id'.format(hookenv.local_unit())) + token = data.get('{}_token'.format(hookenv.local_unit())) + + if all([vault_url, role_id, token]): + token = json.loads(token) + vault_url = json.loads(vault_url) + + # Tokens may change when secret_id's are being + # reissued - if so use token to get new secret_id + if token != last_token: + secret_id = retrieve_secret_id( + url=vault_url, + token=token + ) + db.set('secret-id', secret_id) + db.set('last-token', token) + db.flush() - if vault_url and role_id: ctxt = { - 'vault_url': json.loads(vault_url), + 'vault_url': vault_url, 'role_id': json.loads(role_id), + 'secret_id': secret_id, 'secret_backend': self.secret_backend, } - vault_ca = hookenv.relation_get( - 'vault_ca', - unit=unit, - rid=relation_id - ) + vault_ca = data.get('vault_ca') if vault_ca: ctxt['vault_ca'] = json.loads(vault_ca) self.complete = True @@ -96,3 +106,21 @@ def vault_relation_complete(backend=None): vault_kv = VaultKVContext(secret_backend=backend or VAULTLOCKER_BACKEND) vault_kv() return vault_kv.complete + + +# TODO: contrib a high level unwrap method to hvac that works +def retrieve_secret_id(url, token): + """Retrieve a response-wrapped secret_id from Vault + + :param url: URL to Vault Server + :ptype url: str + :param token: One shot Token to use + :ptype token: str + :returns: secret_id to use for Vault Access + :rtype: str""" + import hvac + client = hvac.Client(url=url, token=token) + response = client._post('/v1/sys/wrapping/unwrap') + if response.status_code == 200: + data = response.json() + return data['data']['secret_id'] diff --git a/hooks/charmhelpers/core/hookenv.py b/hooks/charmhelpers/core/hookenv.py index e9df1509..627d8f79 100644 --- a/hooks/charmhelpers/core/hookenv.py +++ b/hooks/charmhelpers/core/hookenv.py @@ -290,7 +290,7 @@ class Config(dict): self.implicit_save = True self._prev_dict = None self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME) - if os.path.exists(self.path): + if os.path.exists(self.path) and os.stat(self.path).st_size: self.load_previous() atexit(self._implicit_save) @@ -310,7 +310,11 @@ class Config(dict): """ self.path = path or self.path with open(self.path) as f: - self._prev_dict = json.load(f) + try: + self._prev_dict = json.load(f) + except ValueError as e: + log('Unable to parse previous config data - {}'.format(str(e)), + level=ERROR) for k, v in copy.deepcopy(self._prev_dict).items(): if k not in self: self[k] = v diff --git a/hooks/charmhelpers/core/sysctl.py b/hooks/charmhelpers/core/sysctl.py index 6e413e31..1f188d8c 100644 --- a/hooks/charmhelpers/core/sysctl.py +++ b/hooks/charmhelpers/core/sysctl.py @@ -31,18 +31,22 @@ __author__ = 'Jorge Niedbalski R. ' def create(sysctl_dict, sysctl_file): """Creates a sysctl.conf file from a YAML associative array - :param sysctl_dict: a YAML-formatted string of sysctl options eg "{ 'kernel.max_pid': 1337 }" + :param sysctl_dict: a dict or YAML-formatted string of sysctl + options eg "{ 'kernel.max_pid': 1337 }" :type sysctl_dict: str :param sysctl_file: path to the sysctl file to be saved :type sysctl_file: str or unicode :returns: None """ - try: - sysctl_dict_parsed = yaml.safe_load(sysctl_dict) - except yaml.YAMLError: - log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict), - level=ERROR) - return + if type(sysctl_dict) is not dict: + try: + sysctl_dict_parsed = yaml.safe_load(sysctl_dict) + except yaml.YAMLError: + log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict), + level=ERROR) + return + else: + sysctl_dict_parsed = sysctl_dict with open(sysctl_file, "w") as fd: for key, value in sysctl_dict_parsed.items(): diff --git a/tests/charmhelpers/contrib/openstack/utils.py b/tests/charmhelpers/contrib/openstack/utils.py index e7194264..6184abd0 100644 --- a/tests/charmhelpers/contrib/openstack/utils.py +++ b/tests/charmhelpers/contrib/openstack/utils.py @@ -306,7 +306,7 @@ def get_os_codename_install_source(src): if src.startswith('cloud:'): ca_rel = src.split(':')[1] - ca_rel = ca_rel.split('%s-' % ubuntu_rel)[1].split('/')[0] + ca_rel = ca_rel.split('-')[1].split('/')[0] return ca_rel # Best guess match based on deb string provided diff --git a/tests/charmhelpers/core/hookenv.py b/tests/charmhelpers/core/hookenv.py index e9df1509..627d8f79 100644 --- a/tests/charmhelpers/core/hookenv.py +++ b/tests/charmhelpers/core/hookenv.py @@ -290,7 +290,7 @@ class Config(dict): self.implicit_save = True self._prev_dict = None self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME) - if os.path.exists(self.path): + if os.path.exists(self.path) and os.stat(self.path).st_size: self.load_previous() atexit(self._implicit_save) @@ -310,7 +310,11 @@ class Config(dict): """ self.path = path or self.path with open(self.path) as f: - self._prev_dict = json.load(f) + try: + self._prev_dict = json.load(f) + except ValueError as e: + log('Unable to parse previous config data - {}'.format(str(e)), + level=ERROR) for k, v in copy.deepcopy(self._prev_dict).items(): if k not in self: self[k] = v diff --git a/tests/charmhelpers/core/sysctl.py b/tests/charmhelpers/core/sysctl.py index 6e413e31..1f188d8c 100644 --- a/tests/charmhelpers/core/sysctl.py +++ b/tests/charmhelpers/core/sysctl.py @@ -31,18 +31,22 @@ __author__ = 'Jorge Niedbalski R. ' def create(sysctl_dict, sysctl_file): """Creates a sysctl.conf file from a YAML associative array - :param sysctl_dict: a YAML-formatted string of sysctl options eg "{ 'kernel.max_pid': 1337 }" + :param sysctl_dict: a dict or YAML-formatted string of sysctl + options eg "{ 'kernel.max_pid': 1337 }" :type sysctl_dict: str :param sysctl_file: path to the sysctl file to be saved :type sysctl_file: str or unicode :returns: None """ - try: - sysctl_dict_parsed = yaml.safe_load(sysctl_dict) - except yaml.YAMLError: - log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict), - level=ERROR) - return + if type(sysctl_dict) is not dict: + try: + sysctl_dict_parsed = yaml.safe_load(sysctl_dict) + except yaml.YAMLError: + log("Error parsing YAML sysctl_dict: {}".format(sysctl_dict), + level=ERROR) + return + else: + sysctl_dict_parsed = sysctl_dict with open(sysctl_file, "w") as fd: for key, value in sysctl_dict_parsed.items(): diff --git a/tests/dev-basic-bionic-queens b/tests/gate-basic-bionic-queens similarity index 100% rename from tests/dev-basic-bionic-queens rename to tests/gate-basic-bionic-queens diff --git a/tox.ini b/tox.ini index 777b800c..c5765bae 100644 --- a/tox.ini +++ b/tox.ini @@ -66,7 +66,7 @@ basepython = python2.7 deps = -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt commands = - bundletester -vl DEBUG -r json -o func-results.json gate-basic-xenial-pike --no-destroy + bundletester -vl DEBUG -r json -o func-results.json gate-basic-bionic-queens --no-destroy [testenv:func27-dfs] # Charm Functional Test