diff --git a/charm-helpers-hooks.yaml b/charm-helpers-hooks.yaml index 0911b52b..b263043a 100644 --- a/charm-helpers-hooks.yaml +++ b/charm-helpers-hooks.yaml @@ -1,4 +1,4 @@ -branch: lp:~james-page/charm-helpers/multiple-https-networks +branch: lp:charm-helpers destination: hooks/charmhelpers include: - core diff --git a/hooks/charmhelpers/contrib/hahelpers/apache.py b/hooks/charmhelpers/contrib/hahelpers/apache.py index 6595ddb8..6616ffff 100644 --- a/hooks/charmhelpers/contrib/hahelpers/apache.py +++ b/hooks/charmhelpers/contrib/hahelpers/apache.py @@ -20,7 +20,7 @@ from charmhelpers.core.hookenv import ( ) -def get_cert(cn): +def get_cert(cn=None): # TODO: deal with multiple https endpoints via charm config cert = config_get('ssl_cert') key = config_get('ssl_key') @@ -28,13 +28,19 @@ def get_cert(cn): log("Inspecting identity-service relations for SSL certificate.", level=INFO) cert = key = None + if cn: + ssl_cert_attr = 'ssl_cert_{}'.format(cn) + ssl_key_attr = 'ssl_key_{}'.format(cn) + else: + ssl_cert_attr = 'ssl_cert' + ssl_key_attr = 'ssl_key' for r_id in relation_ids('identity-service'): for unit in relation_list(r_id): if not cert: - cert = relation_get('ssl_cert_{}'.format(cn), + cert = relation_get(ssl_cert_attr, rid=r_id, unit=unit) if not key: - key = relation_get('ssl_key_{}'.format(cn), + key = relation_get(ssl_key_attr, rid=r_id, unit=unit) return (cert, key) diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index 9179eeb1..10d3b506 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -1,3 +1,6 @@ +from bzrlib.branch import Branch +import os +import re from charmhelpers.contrib.amulet.deployment import ( AmuletDeployment ) @@ -16,11 +19,41 @@ class OpenStackAmuletDeployment(AmuletDeployment): self.openstack = openstack self.source = source + def _is_dev_branch(self): + """Determine if branch being tested is a dev (i.e. next) branch.""" + branch = Branch.open(os.getcwd()) + parent = branch.get_parent() + pattern = re.compile("^.*/next/$") + if (pattern.match(parent)): + return True + else: + return False + + def _determine_branch_locations(self, other_services): + """Determine the branch locations for the other services. + + If the branch being tested is a dev branch, then determine the + development branch locations for the other services. Otherwise, + the default charm store branches will be used.""" + name = 0 + if self._is_dev_branch(): + updated_services = [] + for svc in other_services: + if svc[name] in ['mysql', 'mongodb', 'rabbitmq-server']: + location = 'lp:charms/{}'.format(svc[name]) + else: + temp = 'lp:~openstack-charmers/charms/trusty/{}/next' + location = temp.format(svc[name]) + updated_services.append(svc + (location,)) + other_services = updated_services + return other_services + def _add_services(self, this_service, other_services): - """Add services to the deployment and set openstack-origin.""" + """Add services to the deployment and set openstack-origin/source.""" + name = 0 + other_services = self._determine_branch_locations(other_services) super(OpenStackAmuletDeployment, self)._add_services(this_service, other_services) - name = 0 services = other_services services.append(this_service) use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph'] diff --git a/hooks/charmhelpers/contrib/openstack/amulet/utils.py b/hooks/charmhelpers/contrib/openstack/amulet/utils.py index bd327bdc..0f312b99 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/utils.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/utils.py @@ -187,15 +187,16 @@ class OpenStackAmuletUtils(AmuletUtils): f = opener.open("http://download.cirros-cloud.net/version/released") version = f.read().strip() - cirros_img = "tests/cirros-{}-x86_64-disk.img".format(version) + cirros_img = "cirros-{}-x86_64-disk.img".format(version) + local_path = os.path.join('tests', cirros_img) - if not os.path.exists(cirros_img): + if not os.path.exists(local_path): cirros_url = "http://{}/{}/{}".format("download.cirros-cloud.net", version, cirros_img) - opener.retrieve(cirros_url, cirros_img) + opener.retrieve(cirros_url, local_path) f.close() - with open(cirros_img) as f: + with open(local_path) as f: image = glance.images.create(name=image_name, is_public=True, disk_format='qcow2', container_format='bare', data=f) diff --git a/hooks/charmhelpers/contrib/openstack/context.py b/hooks/charmhelpers/contrib/openstack/context.py index bd280df5..2190e6bd 100644 --- a/hooks/charmhelpers/contrib/openstack/context.py +++ b/hooks/charmhelpers/contrib/openstack/context.py @@ -427,6 +427,11 @@ class HAProxyContext(OSContextGenerator): 'units': cluster_hosts, } + if config('haproxy-server-timeout'): + ctxt['haproxy-server-timeout'] = config('haproxy-server-timeout') + if config('haproxy-client-timeout'): + ctxt['haproxy-client-timeout'] = config('haproxy-client-timeout') + if config('prefer-ipv6'): ctxt['local_host'] = 'ip6-localhost' ctxt['haproxy_host'] = '::' @@ -500,9 +505,15 @@ class ApacheSSLContext(OSContextGenerator): ssl_dir = os.path.join('/etc/apache2/ssl/', self.service_namespace) mkdir(path=ssl_dir) cert, key = get_cert(cn) - write_file(path=os.path.join(ssl_dir, 'cert_{}'.format(cn)), + if cn: + cert_filename = 'cert_{}'.format(cn) + key_filename = 'key_{}'.format(cn) + else: + cert_filename = 'cert' + key_filename = 'key' + write_file(path=os.path.join(ssl_dir, cert_filename), content=b64decode(cert)) - write_file(path=os.path.join(ssl_dir, 'key_{}'.format(cn)), + write_file(path=os.path.join(ssl_dir, key_filename), content=b64decode(key)) def configure_ca(self): diff --git a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg index ce0e2738..888ee060 100644 --- a/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg +++ b/hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg @@ -14,8 +14,17 @@ defaults retries 3 timeout queue 1000 timeout connect 1000 +{% if haproxy-client-timeout -%} + timeout client {{ haproxy-client-timeout }} +{% else -%} timeout client 30000 +{% endif -%} + +{% if haproxy-server-timeout -%} + timeout server {{ haproxy-server-timeout }} +{% else -%} timeout server 30000 +{% endif -%} listen stats {{ stat_port }} mode http diff --git a/hooks/charmhelpers/core/host.py b/hooks/charmhelpers/core/host.py index 3ac70143..d7ce1e4c 100644 --- a/hooks/charmhelpers/core/host.py +++ b/hooks/charmhelpers/core/host.py @@ -68,8 +68,8 @@ def service_available(service_name): """Determine whether a system service is available""" try: subprocess.check_output(['service', service_name, 'status'], stderr=subprocess.STDOUT) - except subprocess.CalledProcessError: - return False + except subprocess.CalledProcessError as e: + return 'unrecognized service' not in e.output else: return True @@ -229,12 +229,12 @@ def check_hash(path, checksum, hash_type='md5'): """ Validate a file using a cryptographic checksum. - :param str checksum: Value of the checksum used to validate the file. - :param str hash_type: Hash algorithm used to generate :param:`checksum`. - Can be any hash alrgorithm supported by :mod:`hashlib`, - such as md5, sha1, sha256, sha512, etc. + :param str hash_type: Hash algorithm used to generate `checksum`. + Can be any hash alrgorithm supported by :mod:`hashlib`, + such as md5, sha1, sha256, sha512, etc. :raises ChecksumError: If the file fails the checksum + """ actual_checksum = file_hash(path, hash_type) if checksum != actual_checksum: diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index 20a20ac6..32a673d6 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -208,7 +208,8 @@ def add_source(source, key=None): """Add a package source to this system. @param source: a URL or sources.list entry, as supported by - add-apt-repository(1). Examples: + add-apt-repository(1). Examples:: + ppa:charmers/example deb https://stub:key@private.example.com/ubuntu trusty main diff --git a/hooks/charmhelpers/fetch/archiveurl.py b/hooks/charmhelpers/fetch/archiveurl.py index d1dcbc33..8c045650 100644 --- a/hooks/charmhelpers/fetch/archiveurl.py +++ b/hooks/charmhelpers/fetch/archiveurl.py @@ -74,18 +74,19 @@ class ArchiveUrlFetchHandler(BaseFetchHandler): """ Download and install an archive file, with optional checksum validation. - The checksum can also be given on the :param:`source` URL's fragment. + The checksum can also be given on the `source` URL's fragment. For example:: handler.install('http://example.com/file.tgz#sha1=deadbeef') :param str source: URL pointing to an archive file. - :param str dest: Local destination path to install to. If not given, - installs to `$CHARM_DIR/archives/archive_file_name`. + :param str dest: Local destination path to install to. If not given, + installs to `$CHARM_DIR/archives/archive_file_name`. :param str checksum: If given, validate the archive file after download. - :param str hash_type: Algorithm used to generate :param:`checksum`. - Can be any hash alrgorithm supported by :mod:`hashlib`, - such as md5, sha1, sha256, sha512, etc. + :param str hash_type: Algorithm used to generate `checksum`. + Can be any hash alrgorithm supported by :mod:`hashlib`, + such as md5, sha1, sha256, sha512, etc. + """ url_parts = self.parse_url(source) dest_dir = os.path.join(os.environ.get('CHARM_DIR'), 'fetched') diff --git a/tests/charmhelpers/contrib/amulet/deployment.py b/tests/charmhelpers/contrib/amulet/deployment.py index 8c0af487..e0e850dd 100644 --- a/tests/charmhelpers/contrib/amulet/deployment.py +++ b/tests/charmhelpers/contrib/amulet/deployment.py @@ -24,10 +24,10 @@ class AmuletDeployment(object): """Add services. Add services to the deployment where this_service is the local charm - that we're focused on testing and other_services are the other - charms that come from the charm store. + that we're testing and other_services are the other services that + are being used in the amulet tests. """ - name, units = range(2) + name, units, location = range(3) if this_service[name] != os.path.basename(os.getcwd()): s = this_service[name] @@ -37,12 +37,13 @@ class AmuletDeployment(object): self.d.add(this_service[name], units=this_service[units]) for svc in other_services: - if self.series: - self.d.add(svc[name], - charm='cs:{}/{}'.format(self.series, svc[name]), - units=svc[units]) + if len(svc) > 2: + branch_location = svc[location] + elif self.series: + branch_location = 'cs:{}/{}'.format(self.series, svc[name]), else: - self.d.add(svc[name], units=svc[units]) + branch_location = None + self.d.add(svc[name], charm=branch_location, units=svc[units]) def _add_relations(self, relations): """Add all of the relations for the services.""" @@ -57,7 +58,7 @@ class AmuletDeployment(object): def _deploy(self): """Deploy environment and wait for all hooks to finish executing.""" try: - self.d.setup() + self.d.setup(timeout=900) self.d.sentry.wait(timeout=900) except amulet.helpers.TimeoutError: amulet.raise_status(amulet.FAIL, msg="Deployment timed out") diff --git a/tests/charmhelpers/contrib/openstack/amulet/deployment.py b/tests/charmhelpers/contrib/openstack/amulet/deployment.py index 9179eeb1..10d3b506 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/tests/charmhelpers/contrib/openstack/amulet/deployment.py @@ -1,3 +1,6 @@ +from bzrlib.branch import Branch +import os +import re from charmhelpers.contrib.amulet.deployment import ( AmuletDeployment ) @@ -16,11 +19,41 @@ class OpenStackAmuletDeployment(AmuletDeployment): self.openstack = openstack self.source = source + def _is_dev_branch(self): + """Determine if branch being tested is a dev (i.e. next) branch.""" + branch = Branch.open(os.getcwd()) + parent = branch.get_parent() + pattern = re.compile("^.*/next/$") + if (pattern.match(parent)): + return True + else: + return False + + def _determine_branch_locations(self, other_services): + """Determine the branch locations for the other services. + + If the branch being tested is a dev branch, then determine the + development branch locations for the other services. Otherwise, + the default charm store branches will be used.""" + name = 0 + if self._is_dev_branch(): + updated_services = [] + for svc in other_services: + if svc[name] in ['mysql', 'mongodb', 'rabbitmq-server']: + location = 'lp:charms/{}'.format(svc[name]) + else: + temp = 'lp:~openstack-charmers/charms/trusty/{}/next' + location = temp.format(svc[name]) + updated_services.append(svc + (location,)) + other_services = updated_services + return other_services + def _add_services(self, this_service, other_services): - """Add services to the deployment and set openstack-origin.""" + """Add services to the deployment and set openstack-origin/source.""" + name = 0 + other_services = self._determine_branch_locations(other_services) super(OpenStackAmuletDeployment, self)._add_services(this_service, other_services) - name = 0 services = other_services services.append(this_service) use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph'] diff --git a/tests/charmhelpers/contrib/openstack/amulet/utils.py b/tests/charmhelpers/contrib/openstack/amulet/utils.py index bd327bdc..0f312b99 100644 --- a/tests/charmhelpers/contrib/openstack/amulet/utils.py +++ b/tests/charmhelpers/contrib/openstack/amulet/utils.py @@ -187,15 +187,16 @@ class OpenStackAmuletUtils(AmuletUtils): f = opener.open("http://download.cirros-cloud.net/version/released") version = f.read().strip() - cirros_img = "tests/cirros-{}-x86_64-disk.img".format(version) + cirros_img = "cirros-{}-x86_64-disk.img".format(version) + local_path = os.path.join('tests', cirros_img) - if not os.path.exists(cirros_img): + if not os.path.exists(local_path): cirros_url = "http://{}/{}/{}".format("download.cirros-cloud.net", version, cirros_img) - opener.retrieve(cirros_url, cirros_img) + opener.retrieve(cirros_url, local_path) f.close() - with open(cirros_img) as f: + with open(local_path) as f: image = glance.images.create(name=image_name, is_public=True, disk_format='qcow2', container_format='bare', data=f)