Browse Source

Add support for cephx pool grouping and permissions

Sync charmhelpers and add configuration option to allow access
to ceph pools to be limited based on grouping.

Glance only requires rwx access to pools containing images.

Change-Id: I72611b38887a686f6acaeffd70bc4705a425a07b
Partial-Bug: 1424771
changes/86/433586/2
James Page 2 years ago
parent
commit
29da04b58b
35 changed files with 4562 additions and 80 deletions
  1. 1
    0
      charm-helpers-tests.yaml
  2. 5
    1
      charmhelpers/contrib/network/ip.py
  3. 89
    18
      charmhelpers/contrib/openstack/amulet/utils.py
  4. 50
    1
      charmhelpers/contrib/openstack/context.py
  5. 100
    0
      charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf
  6. 43
    6
      charmhelpers/contrib/openstack/utils.py
  7. 14
    2
      charmhelpers/contrib/storage/linux/ceph.py
  8. 45
    0
      charmhelpers/core/hookenv.py
  9. 197
    30
      charmhelpers/core/host.py
  10. 6
    0
      charmhelpers/osplatform.py
  11. 5
    0
      config.yaml
  12. 8
    1
      hooks/glance_relations.py
  13. 89
    18
      tests/charmhelpers/contrib/openstack/amulet/utils.py
  14. 13
    0
      tests/charmhelpers/core/__init__.py
  15. 55
    0
      tests/charmhelpers/core/decorators.py
  16. 43
    0
      tests/charmhelpers/core/files.py
  17. 132
    0
      tests/charmhelpers/core/fstab.py
  18. 1068
    0
      tests/charmhelpers/core/hookenv.py
  19. 918
    0
      tests/charmhelpers/core/host.py
  20. 0
    0
      tests/charmhelpers/core/host_factory/__init__.py
  21. 56
    0
      tests/charmhelpers/core/host_factory/centos.py
  22. 56
    0
      tests/charmhelpers/core/host_factory/ubuntu.py
  23. 69
    0
      tests/charmhelpers/core/hugepage.py
  24. 72
    0
      tests/charmhelpers/core/kernel.py
  25. 0
    0
      tests/charmhelpers/core/kernel_factory/__init__.py
  26. 17
    0
      tests/charmhelpers/core/kernel_factory/centos.py
  27. 13
    0
      tests/charmhelpers/core/kernel_factory/ubuntu.py
  28. 16
    0
      tests/charmhelpers/core/services/__init__.py
  29. 351
    0
      tests/charmhelpers/core/services/base.py
  30. 290
    0
      tests/charmhelpers/core/services/helpers.py
  31. 70
    0
      tests/charmhelpers/core/strutils.py
  32. 54
    0
      tests/charmhelpers/core/sysctl.py
  33. 84
    0
      tests/charmhelpers/core/templating.py
  34. 518
    0
      tests/charmhelpers/core/unitdata.py
  35. 15
    3
      unit_tests/test_glance_relations.py

+ 1
- 0
charm-helpers-tests.yaml View File

@@ -1,5 +1,6 @@
1 1
 branch: lp:charm-helpers
2 2
 destination: tests/charmhelpers
3 3
 include:
4
+    - core
4 5
     - contrib.amulet
5 6
     - contrib.openstack.amulet

+ 5
- 1
charmhelpers/contrib/network/ip.py View File

@@ -424,7 +424,11 @@ def ns_query(address):
424 424
     else:
425 425
         return None
426 426
 
427
-    answers = dns.resolver.query(address, rtype)
427
+    try:
428
+        answers = dns.resolver.query(address, rtype)
429
+    except dns.resolver.NXDOMAIN:
430
+        return None
431
+
428 432
     if answers:
429 433
         return str(answers[0])
430 434
     return None

+ 89
- 18
charmhelpers/contrib/openstack/amulet/utils.py View File

@@ -20,6 +20,7 @@ import re
20 20
 import six
21 21
 import time
22 22
 import urllib
23
+import urlparse
23 24
 
24 25
 import cinderclient.v1.client as cinder_client
25 26
 import glanceclient.v1.client as glance_client
@@ -37,6 +38,7 @@ import swiftclient
37 38
 from charmhelpers.contrib.amulet.utils import (
38 39
     AmuletUtils
39 40
 )
41
+from charmhelpers.core.decorators import retry_on_exception
40 42
 
41 43
 DEBUG = logging.DEBUG
42 44
 ERROR = logging.ERROR
@@ -303,6 +305,46 @@ class OpenStackAmuletUtils(AmuletUtils):
303 305
         self.log.debug('Checking if tenant exists ({})...'.format(tenant))
304 306
         return tenant in [t.name for t in keystone.tenants.list()]
305 307
 
308
+    @retry_on_exception(5, base_delay=10)
309
+    def keystone_wait_for_propagation(self, sentry_relation_pairs,
310
+                                      api_version):
311
+        """Iterate over list of sentry and relation tuples and verify that
312
+           api_version has the expected value.
313
+
314
+        :param sentry_relation_pairs: list of sentry, relation name tuples used
315
+                                      for monitoring propagation of relation
316
+                                      data
317
+        :param api_version: api_version to expect in relation data
318
+        :returns: None if successful.  Raise on error.
319
+        """
320
+        for (sentry, relation_name) in sentry_relation_pairs:
321
+            rel = sentry.relation('identity-service',
322
+                                  relation_name)
323
+            self.log.debug('keystone relation data: {}'.format(rel))
324
+            if rel['api_version'] != str(api_version):
325
+                raise Exception("api_version not propagated through relation"
326
+                                " data yet ('{}' != '{}')."
327
+                                "".format(rel['api_version'], api_version))
328
+
329
+    def keystone_configure_api_version(self, sentry_relation_pairs, deployment,
330
+                                       api_version):
331
+        """Configure preferred-api-version of keystone in deployment and
332
+           monitor provided list of relation objects for propagation
333
+           before returning to caller.
334
+
335
+        :param sentry_relation_pairs: list of sentry, relation tuples used for
336
+                                      monitoring propagation of relation data
337
+        :param deployment: deployment to configure
338
+        :param api_version: value preferred-api-version will be set to
339
+        :returns: None if successful.  Raise on error.
340
+        """
341
+        self.log.debug("Setting keystone preferred-api-version: '{}'"
342
+                       "".format(api_version))
343
+
344
+        config = {'preferred-api-version': api_version}
345
+        deployment.d.configure('keystone', config)
346
+        self.keystone_wait_for_propagation(sentry_relation_pairs, api_version)
347
+
306 348
     def authenticate_cinder_admin(self, keystone_sentry, username,
307 349
                                   password, tenant):
308 350
         """Authenticates admin user with cinder."""
@@ -311,6 +353,37 @@ class OpenStackAmuletUtils(AmuletUtils):
311 353
         ept = "http://{}:5000/v2.0".format(keystone_ip.strip().decode('utf-8'))
312 354
         return cinder_client.Client(username, password, tenant, ept)
313 355
 
356
+    def authenticate_keystone(self, keystone_ip, username, password,
357
+                              api_version=False, admin_port=False,
358
+                              user_domain_name=None, domain_name=None,
359
+                              project_domain_name=None, project_name=None):
360
+        """Authenticate with Keystone"""
361
+        self.log.debug('Authenticating with keystone...')
362
+        port = 5000
363
+        if admin_port:
364
+            port = 35357
365
+        base_ep = "http://{}:{}".format(keystone_ip.strip().decode('utf-8'),
366
+                                        port)
367
+        if not api_version or api_version == 2:
368
+            ep = base_ep + "/v2.0"
369
+            return keystone_client.Client(username=username, password=password,
370
+                                          tenant_name=project_name,
371
+                                          auth_url=ep)
372
+        else:
373
+            ep = base_ep + "/v3"
374
+            auth = keystone_id_v3.Password(
375
+                user_domain_name=user_domain_name,
376
+                username=username,
377
+                password=password,
378
+                domain_name=domain_name,
379
+                project_domain_name=project_domain_name,
380
+                project_name=project_name,
381
+                auth_url=ep
382
+            )
383
+            return keystone_client_v3.Client(
384
+                session=keystone_session.Session(auth=auth)
385
+            )
386
+
314 387
     def authenticate_keystone_admin(self, keystone_sentry, user, password,
315 388
                                     tenant=None, api_version=None,
316 389
                                     keystone_ip=None):
@@ -319,30 +392,28 @@ class OpenStackAmuletUtils(AmuletUtils):
319 392
         if not keystone_ip:
320 393
             keystone_ip = keystone_sentry.info['public-address']
321 394
 
322
-        base_ep = "http://{}:35357".format(keystone_ip.strip().decode('utf-8'))
323
-        if not api_version or api_version == 2:
324
-            ep = base_ep + "/v2.0"
325
-            return keystone_client.Client(username=user, password=password,
326
-                                          tenant_name=tenant, auth_url=ep)
327
-        else:
328
-            ep = base_ep + "/v3"
329
-            auth = keystone_id_v3.Password(
330
-                user_domain_name='admin_domain',
331
-                username=user,
332
-                password=password,
333
-                domain_name='admin_domain',
334
-                auth_url=ep,
335
-            )
336
-            sess = keystone_session.Session(auth=auth)
337
-            return keystone_client_v3.Client(session=sess)
395
+        user_domain_name = None
396
+        domain_name = None
397
+        if api_version == 3:
398
+            user_domain_name = 'admin_domain'
399
+            domain_name = user_domain_name
400
+
401
+        return self.authenticate_keystone(keystone_ip, user, password,
402
+                                          project_name=tenant,
403
+                                          api_version=api_version,
404
+                                          user_domain_name=user_domain_name,
405
+                                          domain_name=domain_name,
406
+                                          admin_port=True)
338 407
 
339 408
     def authenticate_keystone_user(self, keystone, user, password, tenant):
340 409
         """Authenticates a regular user with the keystone public endpoint."""
341 410
         self.log.debug('Authenticating keystone user ({})...'.format(user))
342 411
         ep = keystone.service_catalog.url_for(service_type='identity',
343 412
                                               endpoint_type='publicURL')
344
-        return keystone_client.Client(username=user, password=password,
345
-                                      tenant_name=tenant, auth_url=ep)
413
+        keystone_ip = urlparse.urlparse(ep).hostname
414
+
415
+        return self.authenticate_keystone(keystone_ip, user, password,
416
+                                          project_name=tenant)
346 417
 
347 418
     def authenticate_glance_admin(self, keystone):
348 419
         """Authenticates admin user with glance."""

+ 50
- 1
charmhelpers/contrib/openstack/context.py View File

@@ -14,6 +14,7 @@
14 14
 
15 15
 import glob
16 16
 import json
17
+import math
17 18
 import os
18 19
 import re
19 20
 import time
@@ -90,6 +91,8 @@ from charmhelpers.contrib.network.ip import (
90 91
 from charmhelpers.contrib.openstack.utils import (
91 92
     config_flags_parser,
92 93
     get_host_ip,
94
+    git_determine_usr_bin,
95
+    git_determine_python_path,
93 96
     enable_memcache,
94 97
 )
95 98
 from charmhelpers.core.unitdata import kv
@@ -1208,6 +1211,43 @@ class WorkerConfigContext(OSContextGenerator):
1208 1211
         return ctxt
1209 1212
 
1210 1213
 
1214
+class WSGIWorkerConfigContext(WorkerConfigContext):
1215
+
1216
+    def __init__(self, name=None, script=None, admin_script=None,
1217
+                 public_script=None, process_weight=1.00,
1218
+                 admin_process_weight=0.75, public_process_weight=0.25):
1219
+        self.service_name = name
1220
+        self.user = name
1221
+        self.group = name
1222
+        self.script = script
1223
+        self.admin_script = admin_script
1224
+        self.public_script = public_script
1225
+        self.process_weight = process_weight
1226
+        self.admin_process_weight = admin_process_weight
1227
+        self.public_process_weight = public_process_weight
1228
+
1229
+    def __call__(self):
1230
+        multiplier = config('worker-multiplier') or 1
1231
+        total_processes = self.num_cpus * multiplier
1232
+        ctxt = {
1233
+            "service_name": self.service_name,
1234
+            "user": self.user,
1235
+            "group": self.group,
1236
+            "script": self.script,
1237
+            "admin_script": self.admin_script,
1238
+            "public_script": self.public_script,
1239
+            "processes": int(math.ceil(self.process_weight * total_processes)),
1240
+            "admin_processes": int(math.ceil(self.admin_process_weight *
1241
+                                             total_processes)),
1242
+            "public_processes": int(math.ceil(self.public_process_weight *
1243
+                                              total_processes)),
1244
+            "threads": 1,
1245
+            "usr_bin": git_determine_usr_bin(),
1246
+            "python_path": git_determine_python_path(),
1247
+        }
1248
+        return ctxt
1249
+
1250
+
1211 1251
 class ZeroMQContext(OSContextGenerator):
1212 1252
     interfaces = ['zeromq-configuration']
1213 1253
 
@@ -1521,9 +1561,18 @@ class MemcacheContext(OSContextGenerator):
1521 1561
     This context provides options for configuring a local memcache client and
1522 1562
     server
1523 1563
     """
1564
+
1565
+    def __init__(self, package=None):
1566
+        """
1567
+        @param package: Package to examine to extrapolate OpenStack release.
1568
+                        Used when charms have no openstack-origin config
1569
+                        option (ie subordinates)
1570
+        """
1571
+        self.package = package
1572
+
1524 1573
     def __call__(self):
1525 1574
         ctxt = {}
1526
-        ctxt['use_memcache'] = enable_memcache(config('openstack-origin'))
1575
+        ctxt['use_memcache'] = enable_memcache(package=self.package)
1527 1576
         if ctxt['use_memcache']:
1528 1577
             # Trusty version of memcached does not support ::1 as a listen
1529 1578
             # address so use host file entry instead

+ 100
- 0
charmhelpers/contrib/openstack/templates/wsgi-openstack-api.conf View File

@@ -0,0 +1,100 @@
1
+# Configuration file maintained by Juju. Local changes may be overwritten.
2
+
3
+{% if port -%}
4
+Listen {{ port }}
5
+{% endif -%}
6
+
7
+{% if admin_port -%}
8
+Listen {{ admin_port }}
9
+{% endif -%}
10
+
11
+{% if public_port -%}
12
+Listen {{ public_port }}
13
+{% endif -%}
14
+
15
+{% if port -%}
16
+<VirtualHost *:{{ port }}>
17
+    WSGIDaemonProcess {{ service_name }} processes={{ processes }} threads={{ threads }} user={{ service_name }} group={{ service_name }} \
18
+{% if python_path -%}
19
+                      python-path={{ python_path }} \
20
+{% endif -%}
21
+                      display-name=%{GROUP}
22
+    WSGIProcessGroup {{ service_name }}
23
+    WSGIScriptAlias / {{ script }}
24
+    WSGIApplicationGroup %{GLOBAL}
25
+    WSGIPassAuthorization On
26
+    <IfVersion >= 2.4>
27
+      ErrorLogFormat "%{cu}t %M"
28
+    </IfVersion>
29
+    ErrorLog /var/log/apache2/{{ service_name }}_error.log
30
+    CustomLog /var/log/apache2/{{ service_name }}_access.log combined
31
+
32
+    <Directory {{ usr_bin }}>
33
+        <IfVersion >= 2.4>
34
+            Require all granted
35
+        </IfVersion>
36
+        <IfVersion < 2.4>
37
+            Order allow,deny
38
+            Allow from all
39
+        </IfVersion>
40
+    </Directory>
41
+</VirtualHost>
42
+{% endif -%}
43
+
44
+{% if admin_port -%}
45
+<VirtualHost *:{{ admin_port }}>
46
+    WSGIDaemonProcess {{ service_name }}-admin processes={{ admin_processes }} threads={{ threads }} user={{ service_name }} group={{ service_name }} \
47
+{% if python_path -%}
48
+                      python-path={{ python_path }} \
49
+{% endif -%}
50
+                      display-name=%{GROUP}
51
+    WSGIProcessGroup {{ service_name }}-admin
52
+    WSGIScriptAlias / {{ admin_script }}
53
+    WSGIApplicationGroup %{GLOBAL}
54
+    WSGIPassAuthorization On
55
+    <IfVersion >= 2.4>
56
+      ErrorLogFormat "%{cu}t %M"
57
+    </IfVersion>
58
+    ErrorLog /var/log/apache2/{{ service_name }}_error.log
59
+    CustomLog /var/log/apache2/{{ service_name }}_access.log combined
60
+
61
+    <Directory {{ usr_bin }}>
62
+        <IfVersion >= 2.4>
63
+            Require all granted
64
+        </IfVersion>
65
+        <IfVersion < 2.4>
66
+            Order allow,deny
67
+            Allow from all
68
+        </IfVersion>
69
+    </Directory>
70
+</VirtualHost>
71
+{% endif -%}
72
+
73
+{% if public_port -%}
74
+<VirtualHost *:{{ public_port }}>
75
+    WSGIDaemonProcess {{ service_name }}-public processes={{ public_processes }} threads={{ threads }} user={{ service_name }} group={{ service_name }} \
76
+{% if python_path -%}
77
+                      python-path={{ python_path }} \
78
+{% endif -%}
79
+                      display-name=%{GROUP}
80
+    WSGIProcessGroup {{ service_name }}-public
81
+    WSGIScriptAlias / {{ public_script }}
82
+    WSGIApplicationGroup %{GLOBAL}
83
+    WSGIPassAuthorization On
84
+    <IfVersion >= 2.4>
85
+      ErrorLogFormat "%{cu}t %M"
86
+    </IfVersion>
87
+    ErrorLog /var/log/apache2/{{ service_name }}_error.log
88
+    CustomLog /var/log/apache2/{{ service_name }}_access.log combined
89
+
90
+    <Directory {{ usr_bin }}>
91
+        <IfVersion >= 2.4>
92
+            Require all granted
93
+        </IfVersion>
94
+        <IfVersion < 2.4>
95
+            Order allow,deny
96
+            Allow from all
97
+        </IfVersion>
98
+    </Directory>
99
+</VirtualHost>
100
+{% endif -%}

+ 43
- 6
charmhelpers/contrib/openstack/utils.py View File

@@ -153,7 +153,7 @@ SWIFT_CODENAMES = OrderedDict([
153 153
     ('newton',
154 154
         ['2.8.0', '2.9.0', '2.10.0']),
155 155
     ('ocata',
156
-        ['2.11.0']),
156
+        ['2.11.0', '2.12.0']),
157 157
 ])
158 158
 
159 159
 # >= Liberty version->codename mapping
@@ -1119,6 +1119,35 @@ def git_generate_systemd_init_files(templates_dir):
1119 1119
                 shutil.copyfile(service_source, service_dest)
1120 1120
 
1121 1121
 
1122
+def git_determine_usr_bin():
1123
+    """Return the /usr/bin path for Apache2 config.
1124
+
1125
+    The /usr/bin path will be located in the virtualenv if the charm
1126
+    is configured to deploy from source.
1127
+    """
1128
+    if git_install_requested():
1129
+        projects_yaml = config('openstack-origin-git')
1130
+        projects_yaml = git_default_repos(projects_yaml)
1131
+        return os.path.join(git_pip_venv_dir(projects_yaml), 'bin')
1132
+    else:
1133
+        return '/usr/bin'
1134
+
1135
+
1136
+def git_determine_python_path():
1137
+    """Return the python-path for Apache2 config.
1138
+
1139
+    Returns 'None' unless the charm is configured to deploy from source,
1140
+    in which case the path of the virtualenv's site-packages is returned.
1141
+    """
1142
+    if git_install_requested():
1143
+        projects_yaml = config('openstack-origin-git')
1144
+        projects_yaml = git_default_repos(projects_yaml)
1145
+        return os.path.join(git_pip_venv_dir(projects_yaml),
1146
+                            'lib/python2.7/site-packages')
1147
+    else:
1148
+        return None
1149
+
1150
+
1122 1151
 def os_workload_status(configs, required_interfaces, charm_func=None):
1123 1152
     """
1124 1153
     Decorator to set workload status based on complete contexts
@@ -1927,16 +1956,24 @@ def os_application_version_set(package):
1927 1956
         application_version_set(application_version)
1928 1957
 
1929 1958
 
1930
-def enable_memcache(source=None, release=None):
1959
+def enable_memcache(source=None, release=None, package=None):
1931 1960
     """Determine if memcache should be enabled on the local unit
1932 1961
 
1933
-    @param source: source string for charm
1934 1962
     @param release: release of OpenStack currently deployed
1963
+    @param package: package to derive OpenStack version deployed
1935 1964
     @returns boolean Whether memcache should be enabled
1936 1965
     """
1937
-    if not release:
1938
-        release = get_os_codename_install_source(source)
1939
-    return release >= 'mitaka'
1966
+    _release = None
1967
+    if release:
1968
+        _release = release
1969
+    else:
1970
+        _release = os_release(package, base='icehouse')
1971
+    if not _release:
1972
+        _release = get_os_codename_install_source(source)
1973
+
1974
+    # TODO: this should be changed to a numeric comparison using a known list
1975
+    # of releases and comparing by index.
1976
+    return _release >= 'mitaka'
1940 1977
 
1941 1978
 
1942 1979
 def token_cache_pkgs(source=None, release=None):

+ 14
- 2
charmhelpers/contrib/storage/linux/ceph.py View File

@@ -40,6 +40,7 @@ from subprocess import (
40 40
 )
41 41
 from charmhelpers.core.hookenv import (
42 42
     config,
43
+    service_name,
43 44
     local_unit,
44 45
     relation_get,
45 46
     relation_ids,
@@ -1043,8 +1044,18 @@ class CephBrokerRq(object):
1043 1044
             self.request_id = str(uuid.uuid1())
1044 1045
         self.ops = []
1045 1046
 
1047
+    def add_op_request_access_to_group(self, name, namespace=None,
1048
+                                       permission=None):
1049
+        """
1050
+        Adds the requested permissions to the current service's Ceph key,
1051
+        allowing the key to access only the specified pools
1052
+        """
1053
+        self.ops.append({'op': 'add-permissions-to-key', 'group': name,
1054
+                         'namespace': namespace, 'name': service_name(),
1055
+                         'group-permission': permission})
1056
+
1046 1057
     def add_op_create_pool(self, name, replica_count=3, pg_num=None,
1047
-                           weight=None):
1058
+                           weight=None, group=None, namespace=None):
1048 1059
         """Adds an operation to create a pool.
1049 1060
 
1050 1061
         @param pg_num setting:  optional setting. If not provided, this value
@@ -1058,7 +1069,8 @@ class CephBrokerRq(object):
1058 1069
 
1059 1070
         self.ops.append({'op': 'create-pool', 'name': name,
1060 1071
                          'replicas': replica_count, 'pg_num': pg_num,
1061
-                         'weight': weight})
1072
+                         'weight': weight, 'group': group,
1073
+                         'group-namespace': namespace})
1062 1074
 
1063 1075
     def set_ops(self, ops):
1064 1076
         """Set request ops to provided value.

+ 45
- 0
charmhelpers/core/hookenv.py View File

@@ -616,6 +616,20 @@ def close_port(port, protocol="TCP"):
616 616
     subprocess.check_call(_args)
617 617
 
618 618
 
619
+def open_ports(start, end, protocol="TCP"):
620
+    """Opens a range of service network ports"""
621
+    _args = ['open-port']
622
+    _args.append('{}-{}/{}'.format(start, end, protocol))
623
+    subprocess.check_call(_args)
624
+
625
+
626
+def close_ports(start, end, protocol="TCP"):
627
+    """Close a range of service network ports"""
628
+    _args = ['close-port']
629
+    _args.append('{}-{}/{}'.format(start, end, protocol))
630
+    subprocess.check_call(_args)
631
+
632
+
619 633
 @cached
620 634
 def unit_get(attribute):
621 635
     """Get the unit ID for the remote unit"""
@@ -1021,3 +1035,34 @@ def network_get_primary_address(binding):
1021 1035
     '''
1022 1036
     cmd = ['network-get', '--primary-address', binding]
1023 1037
     return subprocess.check_output(cmd).decode('UTF-8').strip()
1038
+
1039
+
1040
+def add_metric(*args, **kwargs):
1041
+    """Add metric values. Values may be expressed with keyword arguments. For
1042
+    metric names containing dashes, these may be expressed as one or more
1043
+    'key=value' positional arguments. May only be called from the collect-metrics
1044
+    hook."""
1045
+    _args = ['add-metric']
1046
+    _kvpairs = []
1047
+    _kvpairs.extend(args)
1048
+    _kvpairs.extend(['{}={}'.format(k, v) for k, v in kwargs.items()])
1049
+    _args.extend(sorted(_kvpairs))
1050
+    try:
1051
+        subprocess.check_call(_args)
1052
+        return
1053
+    except EnvironmentError as e:
1054
+        if e.errno != errno.ENOENT:
1055
+            raise
1056
+    log_message = 'add-metric failed: {}'.format(' '.join(_kvpairs))
1057
+    log(log_message, level='INFO')
1058
+
1059
+
1060
+def meter_status():
1061
+    """Get the meter status, if running in the meter-status-changed hook."""
1062
+    return os.environ.get('JUJU_METER_STATUS')
1063
+
1064
+
1065
+def meter_info():
1066
+    """Get the meter status information, if running in the meter-status-changed
1067
+    hook."""
1068
+    return os.environ.get('JUJU_METER_INFO')

+ 197
- 30
charmhelpers/core/host.py View File

@@ -54,38 +54,138 @@ elif __platform__ == "centos":
54 54
         cmp_pkgrevno,
55 55
     )  # flake8: noqa -- ignore F401 for this import
56 56
 
57
+UPDATEDB_PATH = '/etc/updatedb.conf'
58
+
59
+def service_start(service_name, **kwargs):
60
+    """Start a system service.
61
+
62
+    The specified service name is managed via the system level init system.
63
+    Some init systems (e.g. upstart) require that additional arguments be
64
+    provided in order to directly control service instances whereas other init
65
+    systems allow for addressing instances of a service directly by name (e.g.
66
+    systemd).
67
+
68
+    The kwargs allow for the additional parameters to be passed to underlying
69
+    init systems for those systems which require/allow for them. For example,
70
+    the ceph-osd upstart script requires the id parameter to be passed along
71
+    in order to identify which running daemon should be reloaded. The follow-
72
+    ing example stops the ceph-osd service for instance id=4:
73
+
74
+    service_stop('ceph-osd', id=4)
75
+
76
+    :param service_name: the name of the service to stop
77
+    :param **kwargs: additional parameters to pass to the init system when
78
+                     managing services. These will be passed as key=value
79
+                     parameters to the init system's commandline. kwargs
80
+                     are ignored for systemd enabled systems.
81
+    """
82
+    return service('start', service_name, **kwargs)
83
+
84
+
85
+def service_stop(service_name, **kwargs):
86
+    """Stop a system service.
87
+
88
+    The specified service name is managed via the system level init system.
89
+    Some init systems (e.g. upstart) require that additional arguments be
90
+    provided in order to directly control service instances whereas other init
91
+    systems allow for addressing instances of a service directly by name (e.g.
92
+    systemd).
57 93
 
58
-def service_start(service_name):
59
-    """Start a system service"""
60
-    return service('start', service_name)
94
+    The kwargs allow for the additional parameters to be passed to underlying
95
+    init systems for those systems which require/allow for them. For example,
96
+    the ceph-osd upstart script requires the id parameter to be passed along
97
+    in order to identify which running daemon should be reloaded. The follow-
98
+    ing example stops the ceph-osd service for instance id=4:
99
+
100
+    service_stop('ceph-osd', id=4)
101
+
102
+    :param service_name: the name of the service to stop
103
+    :param **kwargs: additional parameters to pass to the init system when
104
+                     managing services. These will be passed as key=value
105
+                     parameters to the init system's commandline. kwargs
106
+                     are ignored for systemd enabled systems.
107
+    """
108
+    return service('stop', service_name, **kwargs)
61 109
 
62 110
 
63
-def service_stop(service_name):
64
-    """Stop a system service"""
65
-    return service('stop', service_name)
111
+def service_restart(service_name, **kwargs):
112
+    """Restart a system service.
66 113
 
114
+    The specified service name is managed via the system level init system.
115
+    Some init systems (e.g. upstart) require that additional arguments be
116
+    provided in order to directly control service instances whereas other init
117
+    systems allow for addressing instances of a service directly by name (e.g.
118
+    systemd).
67 119
 
68
-def service_restart(service_name):
69
-    """Restart a system service"""
120
+    The kwargs allow for the additional parameters to be passed to underlying
121
+    init systems for those systems which require/allow for them. For example,
122
+    the ceph-osd upstart script requires the id parameter to be passed along
123
+    in order to identify which running daemon should be restarted. The follow-
124
+    ing example restarts the ceph-osd service for instance id=4:
125
+
126
+    service_restart('ceph-osd', id=4)
127
+
128
+    :param service_name: the name of the service to restart
129
+    :param **kwargs: additional parameters to pass to the init system when
130
+                     managing services. These will be passed as key=value
131
+                     parameters to the  init system's commandline. kwargs
132
+                     are ignored for init systems not allowing additional
133
+                     parameters via the commandline (systemd).
134
+    """
70 135
     return service('restart', service_name)
71 136
 
72 137
 
73
-def service_reload(service_name, restart_on_failure=False):
138
+def service_reload(service_name, restart_on_failure=False, **kwargs):
74 139
     """Reload a system service, optionally falling back to restart if
75
-    reload fails"""
76
-    service_result = service('reload', service_name)
140
+    reload fails.
141
+
142
+    The specified service name is managed via the system level init system.
143
+    Some init systems (e.g. upstart) require that additional arguments be
144
+    provided in order to directly control service instances whereas other init
145
+    systems allow for addressing instances of a service directly by name (e.g.
146
+    systemd).
147
+
148
+    The kwargs allow for the additional parameters to be passed to underlying
149
+    init systems for those systems which require/allow for them. For example,
150
+    the ceph-osd upstart script requires the id parameter to be passed along
151
+    in order to identify which running daemon should be reloaded. The follow-
152
+    ing example restarts the ceph-osd service for instance id=4:
153
+
154
+    service_reload('ceph-osd', id=4)
155
+
156
+    :param service_name: the name of the service to reload
157
+    :param restart_on_failure: boolean indicating whether to fallback to a
158
+                               restart if the reload fails.
159
+    :param **kwargs: additional parameters to pass to the init system when
160
+                     managing services. These will be passed as key=value
161
+                     parameters to the  init system's commandline. kwargs
162
+                     are ignored for init systems not allowing additional
163
+                     parameters via the commandline (systemd).
164
+    """
165
+    service_result = service('reload', service_name, **kwargs)
77 166
     if not service_result and restart_on_failure:
78
-        service_result = service('restart', service_name)
167
+        service_result = service('restart', service_name, **kwargs)
79 168
     return service_result
80 169
 
81 170
 
82
-def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d"):
171
+def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d",
172
+                  **kwargs):
83 173
     """Pause a system service.
84 174
 
85
-    Stop it, and prevent it from starting again at boot."""
175
+    Stop it, and prevent it from starting again at boot.
176
+
177
+    :param service_name: the name of the service to pause
178
+    :param init_dir: path to the upstart init directory
179
+    :param initd_dir: path to the sysv init directory
180
+    :param **kwargs: additional parameters to pass to the init system when
181
+                     managing services. These will be passed as key=value
182
+                     parameters to the init system's commandline. kwargs
183
+                     are ignored for init systems which do not support
184
+                     key=value arguments via the commandline.
185
+    """
86 186
     stopped = True
87
-    if service_running(service_name):
88
-        stopped = service_stop(service_name)
187
+    if service_running(service_name, **kwargs):
188
+        stopped = service_stop(service_name, **kwargs)
89 189
     upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
90 190
     sysv_file = os.path.join(initd_dir, service_name)
91 191
     if init_is_systemd():
@@ -106,10 +206,19 @@ def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d"):
106 206
 
107 207
 
108 208
 def service_resume(service_name, init_dir="/etc/init",
109
-                   initd_dir="/etc/init.d"):
209
+                   initd_dir="/etc/init.d", **kwargs):
110 210
     """Resume a system service.
111 211
 
112
-    Reenable starting again at boot. Start the service"""
212
+    Reenable starting again at boot. Start the service.
213
+
214
+    :param service_name: the name of the service to resume
215
+    :param init_dir: the path to the init dir
216
+    :param initd dir: the path to the initd dir
217
+    :param **kwargs: additional parameters to pass to the init system when
218
+                     managing services. These will be passed as key=value
219
+                     parameters to the init system's commandline. kwargs
220
+                     are ignored for systemd enabled systems.
221
+    """
113 222
     upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
114 223
     sysv_file = os.path.join(initd_dir, service_name)
115 224
     if init_is_systemd():
@@ -126,19 +235,28 @@ def service_resume(service_name, init_dir="/etc/init",
126 235
             "Unable to detect {0} as SystemD, Upstart {1} or"
127 236
             " SysV {2}".format(
128 237
                 service_name, upstart_file, sysv_file))
238
+    started = service_running(service_name, **kwargs)
129 239
 
130
-    started = service_running(service_name)
131 240
     if not started:
132
-        started = service_start(service_name)
241
+        started = service_start(service_name, **kwargs)
133 242
     return started
134 243
 
135 244
 
136
-def service(action, service_name):
137
-    """Control a system service"""
245
+def service(action, service_name, **kwargs):
246
+    """Control a system service.
247
+
248
+    :param action: the action to take on the service
249
+    :param service_name: the name of the service to perform th action on
250
+    :param **kwargs: additional params to be passed to the service command in
251
+                    the form of key=value.
252
+    """
138 253
     if init_is_systemd():
139 254
         cmd = ['systemctl', action, service_name]
140 255
     else:
141 256
         cmd = ['service', service_name, action]
257
+        for key, value in six.iteritems(kwargs):
258
+            parameter = '%s=%s' % (key, value)
259
+            cmd.append(parameter)
142 260
     return subprocess.call(cmd) == 0
143 261
 
144 262
 
@@ -146,15 +264,26 @@ _UPSTART_CONF = "/etc/init/{}.conf"
146 264
 _INIT_D_CONF = "/etc/init.d/{}"
147 265
 
148 266
 
149
-def service_running(service_name):
150
-    """Determine whether a system service is running"""
267
+def service_running(service_name, **kwargs):
268
+    """Determine whether a system service is running.
269
+
270
+    :param service_name: the name of the service
271
+    :param **kwargs: additional args to pass to the service command. This is
272
+                     used to pass additional key=value arguments to the
273
+                     service command line for managing specific instance
274
+                     units (e.g. service ceph-osd status id=2). The kwargs
275
+                     are ignored in systemd services.
276
+    """
151 277
     if init_is_systemd():
152 278
         return service('is-active', service_name)
153 279
     else:
154 280
         if os.path.exists(_UPSTART_CONF.format(service_name)):
155 281
             try:
156
-                output = subprocess.check_output(
157
-                    ['status', service_name],
282
+                cmd = ['status', service_name]
283
+                for key, value in six.iteritems(kwargs):
284
+                    parameter = '%s=%s' % (key, value)
285
+                    cmd.append(parameter)
286
+                output = subprocess.check_output(cmd,
158 287
                     stderr=subprocess.STDOUT).decode('UTF-8')
159 288
             except subprocess.CalledProcessError:
160 289
                 return False
@@ -306,15 +435,17 @@ def add_user_to_group(username, group):
306 435
     subprocess.check_call(cmd)
307 436
 
308 437
 
309
-def rsync(from_path, to_path, flags='-r', options=None):
438
+def rsync(from_path, to_path, flags='-r', options=None, timeout=None):
310 439
     """Replicate the contents of a path"""
311 440
     options = options or ['--delete', '--executability']
312 441
     cmd = ['/usr/bin/rsync', flags]
442
+    if timeout:
443
+        cmd = ['timeout', str(timeout)] + cmd
313 444
     cmd.extend(options)
314 445
     cmd.append(from_path)
315 446
     cmd.append(to_path)
316 447
     log(" ".join(cmd))
317
-    return subprocess.check_output(cmd).decode('UTF-8').strip()
448
+    return subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode('UTF-8').strip()
318 449
 
319 450
 
320 451
 def symlink(source, destination):
@@ -684,7 +815,7 @@ def chownr(path, owner, group, follow_links=True, chowntopdir=False):
684 815
     :param str path: The string path to start changing ownership.
685 816
     :param str owner: The owner string to use when looking up the uid.
686 817
     :param str group: The group string to use when looking up the gid.
687
-    :param bool follow_links: Also Chown links if True
818
+    :param bool follow_links: Also follow and chown links if True
688 819
     :param bool chowntopdir: Also chown path itself if True
689 820
     """
690 821
     uid = pwd.getpwnam(owner).pw_uid
@@ -698,7 +829,7 @@ def chownr(path, owner, group, follow_links=True, chowntopdir=False):
698 829
         broken_symlink = os.path.lexists(path) and not os.path.exists(path)
699 830
         if not broken_symlink:
700 831
             chown(path, uid, gid)
701
-    for root, dirs, files in os.walk(path):
832
+    for root, dirs, files in os.walk(path, followlinks=follow_links):
702 833
         for name in dirs + files:
703 834
             full = os.path.join(root, name)
704 835
             broken_symlink = os.path.lexists(full) and not os.path.exists(full)
@@ -718,6 +849,20 @@ def lchownr(path, owner, group):
718 849
     chownr(path, owner, group, follow_links=False)
719 850
 
720 851
 
852
+def owner(path):
853
+    """Returns a tuple containing the username & groupname owning the path.
854
+
855
+    :param str path: the string path to retrieve the ownership
856
+    :return tuple(str, str): A (username, groupname) tuple containing the
857
+                             name of the user and group owning the path.
858
+    :raises OSError: if the specified path does not exist
859
+    """
860
+    stat = os.stat(path)
861
+    username = pwd.getpwuid(stat.st_uid)[0]
862
+    groupname = grp.getgrgid(stat.st_gid)[0]
863
+    return username, groupname
864
+
865
+
721 866
 def get_total_ram():
722 867
     """The total amount of system RAM in bytes.
723 868
 
@@ -749,3 +894,25 @@ def is_container():
749 894
     else:
750 895
         # Detect using upstart container file marker
751 896
         return os.path.exists(UPSTART_CONTAINER_TYPE)
897
+
898
+
899
+def add_to_updatedb_prunepath(path, updatedb_path=UPDATEDB_PATH):
900
+    with open(updatedb_path, 'r+') as f_id:
901
+        updatedb_text = f_id.read()
902
+        output = updatedb(updatedb_text, path)
903
+        f_id.seek(0)
904
+        f_id.write(output)
905
+        f_id.truncate()
906
+
907
+
908
+def updatedb(updatedb_text, new_path):
909
+    lines = [line for line in updatedb_text.split("\n")]
910
+    for i, line in enumerate(lines):
911
+        if line.startswith("PRUNEPATHS="):
912
+            paths_line = line.split("=")[1].replace('"', '')
913
+            paths = paths_line.split(" ")
914
+            if new_path not in paths:
915
+                paths.append(new_path)
916
+                lines[i] = 'PRUNEPATHS="{}"'.format(' '.join(paths))
917
+    output = "\n".join(lines)
918
+    return output

+ 6
- 0
charmhelpers/osplatform.py View File

@@ -8,12 +8,18 @@ def get_platform():
8 8
     will be returned (which is the name of the module).
9 9
     This string is used to decide which platform module should be imported.
10 10
     """
11
+    # linux_distribution is deprecated and will be removed in Python 3.7
12
+    # Warings *not* disabled, as we certainly need to fix this.
11 13
     tuple_platform = platform.linux_distribution()
12 14
     current_platform = tuple_platform[0]
13 15
     if "Ubuntu" in current_platform:
14 16
         return "ubuntu"
15 17
     elif "CentOS" in current_platform:
16 18
         return "centos"
19
+    elif "debian" in current_platform:
20
+        # Stock Python does not detect Ubuntu and instead returns debian.
21
+        # Or at least it does in some build environments like Travis CI
22
+        return "ubuntu"
17 23
     else:
18 24
         raise RuntimeError("This module is not supported on {}."
19 25
                            .format(current_platform))

+ 5
- 0
config.yaml View File

@@ -119,6 +119,11 @@ options:
119 119
       created for the pool. The number of placement groups for a pool can
120 120
       only be increased, never decreased - so it is important to identify the
121 121
       percent of data that will likely reside in the pool.
122
+  restrict-ceph-pools:
123
+    default: False
124
+    type: boolean
125
+    description: |
126
+      Optionally restrict Ceph key permissions to access pools as required.
122 127
   # HA configuration settings
123 128
   dns-ha:
124 129
     type: boolean

+ 8
- 1
hooks/glance_relations.py View File

@@ -303,7 +303,10 @@ def get_ceph_request():
303 303
     replicas = config('ceph-osd-replication-count')
304 304
     weight = config('ceph-pool-weight')
305 305
     rq.add_op_create_pool(name=service, replica_count=replicas,
306
-                          weight=weight)
306
+                          weight=weight, group='images')
307
+    if config('restrict-ceph-pools'):
308
+        rq.add_op_request_access_to_group(name="images",
309
+                                          permission='rwx')
307 310
     return rq
308 311
 
309 312
 
@@ -409,6 +412,10 @@ def config_changed():
409 412
     for r_id in relation_ids('ha'):
410 413
         ha_relation_joined(relation_id=r_id)
411 414
 
415
+    # NOTE(jamespage): trigger any configuration related changes
416
+    #                  for cephx permissions restrictions
417
+    ceph_changed()
418
+
412 419
 
413 420
 @hooks.hook('cluster-relation-joined')
414 421
 def cluster_joined(relation_id=None):

+ 89
- 18
tests/charmhelpers/contrib/openstack/amulet/utils.py View File

@@ -20,6 +20,7 @@ import re
20 20
 import six
21 21
 import time
22 22
 import urllib
23
+import urlparse
23 24
 
24 25
 import cinderclient.v1.client as cinder_client
25 26
 import glanceclient.v1.client as glance_client
@@ -37,6 +38,7 @@ import swiftclient
37 38
 from charmhelpers.contrib.amulet.utils import (
38 39
     AmuletUtils
39 40
 )
41
+from charmhelpers.core.decorators import retry_on_exception
40 42
 
41 43
 DEBUG = logging.DEBUG
42 44
 ERROR = logging.ERROR
@@ -303,6 +305,46 @@ class OpenStackAmuletUtils(AmuletUtils):
303 305
         self.log.debug('Checking if tenant exists ({})...'.format(tenant))
304 306
         return tenant in [t.name for t in keystone.tenants.list()]
305 307
 
308
+    @retry_on_exception(5, base_delay=10)
309
+    def keystone_wait_for_propagation(self, sentry_relation_pairs,
310
+                                      api_version):
311
+        """Iterate over list of sentry and relation tuples and verify that
312
+           api_version has the expected value.
313
+
314
+        :param sentry_relation_pairs: list of sentry, relation name tuples used
315
+                                      for monitoring propagation of relation
316
+                                      data
317
+        :param api_version: api_version to expect in relation data
318
+        :returns: None if successful.  Raise on error.
319
+        """
320
+        for (sentry, relation_name) in sentry_relation_pairs:
321
+            rel = sentry.relation('identity-service',
322
+                                  relation_name)
323
+            self.log.debug('keystone relation data: {}'.format(rel))
324
+            if rel['api_version'] != str(api_version):
325
+                raise Exception("api_version not propagated through relation"
326
+                                " data yet ('{}' != '{}')."
327
+                                "".format(rel['api_version'], api_version))
328
+
329
+    def keystone_configure_api_version(self, sentry_relation_pairs, deployment,
330
+                                       api_version):
331
+        """Configure preferred-api-version of keystone in deployment and
332
+           monitor provided list of relation objects for propagation
333
+           before returning to caller.
334
+
335
+        :param sentry_relation_pairs: list of sentry, relation tuples used for
336
+                                      monitoring propagation of relation data
337
+        :param deployment: deployment to configure
338
+        :param api_version: value preferred-api-version will be set to
339
+        :returns: None if successful.  Raise on error.
340
+        """
341
+        self.log.debug("Setting keystone preferred-api-version: '{}'"
342
+                       "".format(api_version))
343
+
344
+        config = {'preferred-api-version': api_version}
345
+        deployment.d.configure('keystone', config)
346
+        self.keystone_wait_for_propagation(sentry_relation_pairs, api_version)
347
+
306 348
     def authenticate_cinder_admin(self, keystone_sentry, username,
307 349
                                   password, tenant):
308 350
         """Authenticates admin user with cinder."""
@@ -311,6 +353,37 @@ class OpenStackAmuletUtils(AmuletUtils):
311 353
         ept = "http://{}:5000/v2.0".format(keystone_ip.strip().decode('utf-8'))
312 354
         return cinder_client.Client(username, password, tenant, ept)
313 355
 
356
+    def authenticate_keystone(self, keystone_ip, username, password,
357
+                              api_version=False, admin_port=False,
358
+                              user_domain_name=None, domain_name=None,
359
+                              project_domain_name=None, project_name=None):
360
+        """Authenticate with Keystone"""
361
+        self.log.debug('Authenticating with keystone...')
362
+        port = 5000
363
+        if admin_port:
364
+            port = 35357
365
+        base_ep = "http://{}:{}".format(keystone_ip.strip().decode('utf-8'),
366
+                                        port)
367
+        if not api_version or api_version == 2:
368
+            ep = base_ep + "/v2.0"
369
+            return keystone_client.Client(username=username, password=password,
370
+                                          tenant_name=project_name,
371
+                                          auth_url=ep)
372
+        else:
373
+            ep = base_ep + "/v3"
374
+            auth = keystone_id_v3.Password(
375
+                user_domain_name=user_domain_name,
376
+                username=username,
377
+                password=password,
378
+                domain_name=domain_name,
379
+                project_domain_name=project_domain_name,
380
+                project_name=project_name,
381
+                auth_url=ep
382
+            )
383
+            return keystone_client_v3.Client(
384
+                session=keystone_session.Session(auth=auth)
385
+            )
386
+
314 387
     def authenticate_keystone_admin(self, keystone_sentry, user, password,
315 388
                                     tenant=None, api_version=None,
316 389
                                     keystone_ip=None):
@@ -319,30 +392,28 @@ class OpenStackAmuletUtils(AmuletUtils):
319 392
         if not keystone_ip:
320 393
             keystone_ip = keystone_sentry.info['public-address']
321 394
 
322
-        base_ep = "http://{}:35357".format(keystone_ip.strip().decode('utf-8'))
323
-        if not api_version or api_version == 2:
324
-            ep = base_ep + "/v2.0"
325
-            return keystone_client.Client(username=user, password=password,
326
-                                          tenant_name=tenant, auth_url=ep)
327
-        else:
328
-            ep = base_ep + "/v3"
329
-            auth = keystone_id_v3.Password(
330
-                user_domain_name='admin_domain',
331
-                username=user,
332
-                password=password,
333
-                domain_name='admin_domain',
334
-                auth_url=ep,
335
-            )
336
-            sess = keystone_session.Session(auth=auth)
337
-            return keystone_client_v3.Client(session=sess)
395
+        user_domain_name = None
396
+        domain_name = None
397
+        if api_version == 3:
398
+            user_domain_name = 'admin_domain'
399
+            domain_name = user_domain_name
400
+
401
+        return self.authenticate_keystone(keystone_ip, user, password,
402
+                                          project_name=tenant,
403
+                                          api_version=api_version,
404
+                                          user_domain_name=user_domain_name,
405
+                                          domain_name=domain_name,
406
+                                          admin_port=True)
338 407
 
339 408
     def authenticate_keystone_user(self, keystone, user, password, tenant):
340 409
         """Authenticates a regular user with the keystone public endpoint."""
341 410
         self.log.debug('Authenticating keystone user ({})...'.format(user))
342 411
         ep = keystone.service_catalog.url_for(service_type='identity',
343 412
                                               endpoint_type='publicURL')
344
-        return keystone_client.Client(username=user, password=password,
345
-                                      tenant_name=tenant, auth_url=ep)
413
+        keystone_ip = urlparse.urlparse(ep).hostname
414
+
415
+        return self.authenticate_keystone(keystone_ip, user, password,
416
+                                          project_name=tenant)
346 417
 
347 418
     def authenticate_glance_admin(self, keystone):
348 419
         """Authenticates admin user with glance."""

+ 13
- 0
tests/charmhelpers/core/__init__.py View File

@@ -0,0 +1,13 @@
1
+# Copyright 2014-2015 Canonical Limited.
2
+#
3
+# Licensed under the Apache License, Version 2.0 (the "License");
4
+# you may not use this file except in compliance with the License.
5
+# You may obtain a copy of the License at
6
+#
7
+#  http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+# Unless required by applicable law or agreed to in writing, software
10
+# distributed under the License is distributed on an "AS IS" BASIS,
11
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+# See the License for the specific language governing permissions and
13
+# limitations under the License.

+ 55
- 0
tests/charmhelpers/core/decorators.py View File

@@ -0,0 +1,55 @@
1
+# Copyright 2014-2015 Canonical Limited.
2
+#
3
+# Licensed under the Apache License, Version 2.0 (the "License");
4
+# you may not use this file except in compliance with the License.
5
+# You may obtain a copy of the License at
6
+#
7
+#  http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+# Unless required by applicable law or agreed to in writing, software
10
+# distributed under the License is distributed on an "AS IS" BASIS,
11
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+# See the License for the specific language governing permissions and
13
+# limitations under the License.
14
+
15
+#
16
+# Copyright 2014 Canonical Ltd.
17
+#
18
+# Authors:
19
+#  Edward Hope-Morley <opentastic@gmail.com>
20
+#
21
+
22
+import time
23
+
24
+from charmhelpers.core.hookenv import (
25
+    log,
26
+    INFO,
27
+)
28
+
29
+
30
+def retry_on_exception(num_retries, base_delay=0, exc_type=Exception):
31
+    """If the decorated function raises exception exc_type, allow num_retries
32
+    retry attempts before raise the exception.
33
+    """
34
+    def _retry_on_exception_inner_1(f):
35
+        def _retry_on_exception_inner_2(*args, **kwargs):
36
+            retries = num_retries
37
+            multiplier = 1
38
+            while True:
39
+                try:
40
+                    return f(*args, **kwargs)
41
+                except exc_type:
42
+                    if not retries:
43
+                        raise
44
+
45
+                delay = base_delay * multiplier
46
+                multiplier += 1
47
+                log("Retrying '%s' %d more times (delay=%s)" %
48
+                    (f.__name__, retries, delay), level=INFO)
49
+                retries -= 1
50
+                if delay:
51
+                    time.sleep(delay)
52
+
53
+        return _retry_on_exception_inner_2
54
+
55
+    return _retry_on_exception_inner_1

+ 43
- 0
tests/charmhelpers/core/files.py View File

@@ -0,0 +1,43 @@
1
+#!/usr/bin/env python
2
+# -*- coding: utf-8 -*-
3
+
4
+# Copyright 2014-2015 Canonical Limited.
5
+#
6
+# Licensed under the Apache License, Version 2.0 (the "License");
7
+# you may not use this file except in compliance with the License.
8
+# You may obtain a copy of the License at
9
+#
10
+#  http://www.apache.org/licenses/LICENSE-2.0
11
+#
12
+# Unless required by applicable law or agreed to in writing, software
13
+# distributed under the License is distributed on an "AS IS" BASIS,
14
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+# See the License for the specific language governing permissions and
16
+# limitations under the License.
17
+
18
+__author__ = 'Jorge Niedbalski <niedbalski@ubuntu.com>'
19
+
20
+import os
21
+import subprocess
22
+
23
+
24
+def sed(filename, before, after, flags='g'):
25
+    """
26
+    Search and replaces the given pattern on filename.
27
+
28
+    :param filename: relative or absolute file path.
29
+    :param before: expression to be replaced (see 'man sed')
30
+    :param after: expression to replace with (see 'man sed')
31
+    :param flags: sed-compatible regex flags in example, to make
32
+    the  search and replace case insensitive, specify ``flags="i"``.
33
+    The ``g`` flag is always specified regardless, so you do not
34
+    need to remember to include it when overriding this parameter.
35
+    :returns: If the sed command exit code was zero then return,
36
+    otherwise raise CalledProcessError.
37
+    """
38
+    expression = r's/{0}/{1}/{2}'.format(before,
39
+                                         after, flags)
40
+
41
+    return subprocess.check_call(["sed", "-i", "-r", "-e",
42
+                                  expression,
43
+                                  os.path.expanduser(filename)])

+ 132
- 0
tests/charmhelpers/core/fstab.py View File

@@ -0,0 +1,132 @@
1
+#!/usr/bin/env python
2
+# -*- coding: utf-8 -*-
3
+
4
+# Copyright 2014-2015 Canonical Limited.
5
+#
6
+# Licensed under the Apache License, Version 2.0 (the "License");
7
+# you may not use this file except in compliance with the License.
8
+# You may obtain a copy of the License at
9
+#
10
+#  http://www.apache.org/licenses/LICENSE-2.0
11
+#
12
+# Unless required by applicable law or agreed to in writing, software
13
+# distributed under the License is distributed on an "AS IS" BASIS,
14
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+# See the License for the specific language governing permissions and
16
+# limitations under the License.
17
+
18
+import io
19
+import os
20
+
21
+__author__ = 'Jorge Niedbalski R. <jorge.niedbalski@canonical.com>'
22
+
23
+
24
+class Fstab(io.FileIO):
25
+    """This class extends file in order to implement a file reader/writer
26
+    for file `/etc/fstab`
27
+    """
28
+
29
+    class Entry(object):
30
+        """Entry class represents a non-comment line on the `/etc/fstab` file
31
+        """
32
+        def __init__(self, device, mountpoint, filesystem,
33
+                     options, d=0, p=0):
34
+            self.device = device
35
+            self.mountpoint = mountpoint
36
+            self.filesystem = filesystem
37
+
38
+            if not options:
39
+                options = "defaults"
40
+
41
+            self.options = options
42
+            self.d = int(d)
43
+            self.p = int(p)
44
+
45
+        def __eq__(self, o):
46
+            return str(self) == str(o)
47
+
48
+        def __str__(self):
49
+            return "{} {} {} {} {} {}".format(self.device,
50
+                                              self.mountpoint,
51
+                                              self.filesystem,
52
+                                              self.options,
53
+                                              self.d,
54
+                                              self.p)
55
+
56
+    DEFAULT_PATH = os.path.join(os.path.sep, 'etc', 'fstab')
57
+
58
+    def __init__(self, path=None):
59
+        if path:
60
+            self._path = path
61
+        else:
62
+            self._path = self.DEFAULT_PATH
63
+        super(Fstab, self).__init__(self._path, 'rb+')
64
+
65
+    def _hydrate_entry(self, line):
66
+        # NOTE: use split with no arguments to split on any
67
+        #       whitespace including tabs
68
+        return Fstab.Entry(*filter(
69
+            lambda x: x not in ('', None),
70
+            line.strip("\n").split()))
71
+
72
+    @property
73
+    def entries(self):
74
+        self.seek(0)
75
+        for line in self.readlines():
76
+            line = line.decode('us-ascii')
77
+            try:
78
+                if line.strip() and not line.strip().startswith("#"):
79
+                    yield self._hydrate_entry(line)
80
+            except ValueError:
81
+                pass
82
+
83
+    def get_entry_by_attr(self, attr, value):
84
+        for entry in self.entries:
85
+            e_attr = getattr(entry, attr)
86
+            if e_attr == value:
87
+                return entry
88
+        return None
89
+
90
+    def add_entry(self, entry):
91
+        if self.get_entry_by_attr('device', entry.device):
92
+            return False
93
+
94
+        self.write((str(entry) + '\n').encode('us-ascii'))
95
+        self.truncate()
96
+        return entry
97
+
98
+    def remove_entry(self, entry):
99
+        self.seek(0)
100
+
101
+        lines = [l.decode('us-ascii') for l in self.readlines()]
102
+
103
+        found = False
104
+        for index, line in enumerate(lines):
105
+            if line.strip() and not line.strip().startswith("#"):
106
+                if self._hydrate_entry(line) == entry:
107
+                    found = True
108
+                    break
109
+
110
+        if not found:
111
+            return False
112
+
113
+        lines.remove(line)
114
+
115
+        self.seek(0)
116
+        self.write(''.join(lines).encode('us-ascii'))
117
+        self.truncate()
118
+        return True
119
+
120
+    @classmethod
121
+    def remove_by_mountpoint(cls, mountpoint, path=None):
122
+        fstab = cls(path=path)
123
+        entry = fstab.get_entry_by_attr('mountpoint', mountpoint)
124
+        if entry:
125
+            return fstab.remove_entry(entry)
126
+        return False
127
+
128
+    @classmethod
129
+    def add(cls, device, mountpoint, filesystem, options=None, path=None):
130
+        return cls(path=path).add_entry(Fstab.Entry(device,
131
+                                                    mountpoint, filesystem,
132
+                                                    options=options))

+ 1068
- 0
tests/charmhelpers/core/hookenv.py
File diff suppressed because it is too large
View File


+ 918
- 0
tests/charmhelpers/core/host.py View File

@@ -0,0 +1,918 @@
1
+# Copyright 2014-2015 Canonical Limited.
2
+#
3
+# Licensed under the Apache License, Version 2.0 (the "License");
4
+# you may not use this file except in compliance with the License.
5
+# You may obtain a copy of the License at
6
+#
7
+#  http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+# Unless required by applicable law or agreed to in writing, software
10
+# distributed under the License is distributed on an "AS IS" BASIS,
11
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+# See the License for the specific language governing permissions and
13
+# limitations under the License.
14
+
15
+"""Tools for working with the host system"""
16
+# Copyright 2012 Canonical Ltd.
17
+#
18
+# Authors:
19
+#  Nick Moffitt <nick.moffitt@canonical.com>
20
+#  Matthew Wedgwood <matthew.wedgwood@canonical.com>
21
+
22
+import os
23
+import re
24
+import pwd
25
+import glob
26
+import grp
27
+import random
28
+import string
29
+import subprocess
30
+import hashlib
31
+import functools
32
+import itertools
33
+import six
34
+
35
+from contextlib import contextmanager
36
+from collections import OrderedDict
37
+from .hookenv import log
38
+from .fstab import Fstab
39
+from charmhelpers.osplatform import get_platform
40
+
41
+__platform__ = get_platform()
42
+if __platform__ == "ubuntu":
43
+    from charmhelpers.core.host_factory.ubuntu import (
44
+        service_available,
45
+        add_new_group,
46
+        lsb_release,
47
+        cmp_pkgrevno,
48
+    )  # flake8: noqa -- ignore F401 for this import
49
+elif __platform__ == "centos":
50
+    from charmhelpers.core.host_factory.centos import (
51
+        service_available,
52
+        add_new_group,
53
+        lsb_release,
54
+        cmp_pkgrevno,
55
+    )  # flake8: noqa -- ignore F401 for this import
56
+
57
+UPDATEDB_PATH = '/etc/updatedb.conf'
58
+
59
+def service_start(service_name, **kwargs):
60
+    """Start a system service.
61
+
62
+    The specified service name is managed via the system level init system.
63
+    Some init systems (e.g. upstart) require that additional arguments be
64
+    provided in order to directly control service instances whereas other init
65
+    systems allow for addressing instances of a service directly by name (e.g.
66
+    systemd).
67
+
68
+    The kwargs allow for the additional parameters to be passed to underlying
69
+    init systems for those systems which require/allow for them. For example,
70
+    the ceph-osd upstart script requires the id parameter to be passed along
71
+    in order to identify which running daemon should be reloaded. The follow-
72
+    ing example stops the ceph-osd service for instance id=4:
73
+
74
+    service_stop('ceph-osd', id=4)
75
+
76
+    :param service_name: the name of the service to stop
77
+    :param **kwargs: additional parameters to pass to the init system when
78
+                     managing services. These will be passed as key=value
79
+                     parameters to the init system's commandline. kwargs
80
+                     are ignored for systemd enabled systems.
81
+    """
82
+    return service('start', service_name, **kwargs)
83
+
84
+
85
+def service_stop(service_name, **kwargs):
86
+    """Stop a system service.
87
+
88
+    The specified service name is managed via the system level init system.
89
+    Some init systems (e.g. upstart) require that additional arguments be
90
+    provided in order to directly control service instances whereas other init
91
+    systems allow for addressing instances of a service directly by name (e.g.
92
+    systemd).
93
+
94
+    The kwargs allow for the additional parameters to be passed to underlying
95
+    init systems for those systems which require/allow for them. For example,
96
+    the ceph-osd upstart script requires the id parameter to be passed along
97
+    in order to identify which running daemon should be reloaded. The follow-
98
+    ing example stops the ceph-osd service for instance id=4:
99
+
100
+    service_stop('ceph-osd', id=4)
101
+
102
+    :param service_name: the name of the service to stop
103
+    :param **kwargs: additional parameters to pass to the init system when
104
+                     managing services. These will be passed as key=value
105
+                     parameters to the init system's commandline. kwargs
106
+                     are ignored for systemd enabled systems.
107
+    """
108
+    return service('stop', service_name, **kwargs)
109
+
110
+
111
+def service_restart(service_name, **kwargs):
112
+    """Restart a system service.
113
+
114
+    The specified service name is managed via the system level init system.
115
+    Some init systems (e.g. upstart) require that additional arguments be
116
+    provided in order to directly control service instances whereas other init
117
+    systems allow for addressing instances of a service directly by name (e.g.
118
+    systemd).
119
+
120
+    The kwargs allow for the additional parameters to be passed to underlying
121
+    init systems for those systems which require/allow for them. For example,
122
+    the ceph-osd upstart script requires the id parameter to be passed along
123
+    in order to identify which running daemon should be restarted. The follow-
124
+    ing example restarts the ceph-osd service for instance id=4:
125
+
126
+    service_restart('ceph-osd', id=4)
127
+
128
+    :param service_name: the name of the service to restart
129
+    :param **kwargs: additional parameters to pass to the init system when
130
+                     managing services. These will be passed as key=value
131
+                     parameters to the  init system's commandline. kwargs
132
+                     are ignored for init systems not allowing additional
133
+                     parameters via the commandline (systemd).
134
+    """
135
+    return service('restart', service_name)
136
+
137
+
138
+def service_reload(service_name, restart_on_failure=False, **kwargs):
139
+    """Reload a system service, optionally falling back to restart if
140
+    reload fails.
141
+
142
+    The specified service name is managed via the system level init system.
143
+    Some init systems (e.g. upstart) require that additional arguments be
144
+    provided in order to directly control service instances whereas other init
145
+    systems allow for addressing instances of a service directly by name (e.g.
146
+    systemd).
147
+
148
+    The kwargs allow for the additional parameters to be passed to underlying
149
+    init systems for those systems which require/allow for them. For example,
150
+    the ceph-osd upstart script requires the id parameter to be passed along
151
+    in order to identify which running daemon should be reloaded. The follow-
152
+    ing example restarts the ceph-osd service for instance id=4:
153
+
154
+    service_reload('ceph-osd', id=4)
155
+
156
+    :param service_name: the name of the service to reload
157
+    :param restart_on_failure: boolean indicating whether to fallback to a
158
+                               restart if the reload fails.
159
+    :param **kwargs: additional parameters to pass to the init system when
160
+                     managing services. These will be passed as key=value
161
+                     parameters to the  init system's commandline. kwargs
162
+                     are ignored for init systems not allowing additional
163
+                     parameters via the commandline (systemd).
164
+    """
165
+    service_result = service('reload', service_name, **kwargs)
166
+    if not service_result and restart_on_failure:
167
+        service_result = service('restart', service_name, **kwargs)
168
+    return service_result
169
+
170
+
171
+def service_pause(service_name, init_dir="/etc/init", initd_dir="/etc/init.d",
172
+                  **kwargs):
173
+    """Pause a system service.
174
+
175
+    Stop it, and prevent it from starting again at boot.
176
+
177
+    :param service_name: the name of the service to pause
178
+    :param init_dir: path to the upstart init directory
179
+    :param initd_dir: path to the sysv init directory
180
+    :param **kwargs: additional parameters to pass to the init system when
181
+                     managing services. These will be passed as key=value
182
+                     parameters to the init system's commandline. kwargs
183
+                     are ignored for init systems which do not support
184
+                     key=value arguments via the commandline.
185
+    """
186
+    stopped = True
187
+    if service_running(service_name, **kwargs):
188
+        stopped = service_stop(service_name, **kwargs)
189
+    upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
190
+    sysv_file = os.path.join(initd_dir, service_name)
191
+    if init_is_systemd():
192
+        service('disable', service_name)
193
+    elif os.path.exists(upstart_file):
194
+        override_path = os.path.join(
195
+            init_dir, '{}.override'.format(service_name))
196
+        with open(override_path, 'w') as fh:
197
+            fh.write("manual\n")
198
+    elif os.path.exists(sysv_file):
199
+        subprocess.check_call(["update-rc.d", service_name, "disable"])
200
+    else:
201
+        raise ValueError(
202
+            "Unable to detect {0} as SystemD, Upstart {1} or"
203
+            " SysV {2}".format(
204
+                service_name, upstart_file, sysv_file))
205
+    return stopped
206
+
207
+
208
+def service_resume(service_name, init_dir="/etc/init",
209
+                   initd_dir="/etc/init.d", **kwargs):
210
+    """Resume a system service.
211
+
212
+    Reenable starting again at boot. Start the service.
213
+
214
+    :param service_name: the name of the service to resume
215
+    :param init_dir: the path to the init dir
216
+    :param initd dir: the path to the initd dir
217
+    :param **kwargs: additional parameters to pass to the init system when
218
+                     managing services. These will be passed as key=value
219
+                     parameters to the init system's commandline. kwargs
220
+                     are ignored for systemd enabled systems.
221
+    """
222
+    upstart_file = os.path.join(init_dir, "{}.conf".format(service_name))
223
+    sysv_file = os.path.join(initd_dir, service_name)
224
+    if init_is_systemd():
225
+        service('enable', service_name)
226
+    elif os.path.exists(upstart_file):
227
+        override_path = os.path.join(
228
+            init_dir, '{}.override'.format(service_name))
229
+        if os.path.exists(override_path):
230
+            os.unlink(override_path)
231
+    elif os.path.exists(sysv_file):
232
+        subprocess.check_call(["update-rc.d", service_name, "enable"])
233
+    else:
234
+        raise ValueError(
235
+            "Unable to detect {0} as SystemD, Upstart {1} or"
236
+            " SysV {2}".format(
237
+                service_name, upstart_file, sysv_file))
238
+    started = service_running(service_name, **kwargs)
239
+
240
+    if not started:
241
+        started = service_start(service_name, **kwargs)
242
+    return started
243
+
244
+
245
+def service(action, service_name, **kwargs):
246
+    """Control a system service.
247
+
248
+    :param action: the action to take on the service
249
+    :param service_name: the name of the service to perform th action on
250
+    :param **kwargs: additional params to be passed to the service command in
251
+                    the form of key=value.
252
+    """
253
+    if init_is_systemd():
254
+        cmd = ['systemctl', action, service_name]
255
+    else:
256
+        cmd = ['service', service_name, action]
257
+        for key, value in six.iteritems(kwargs):
258
+            parameter = '%s=%s' % (key, value)
259
+            cmd.append(parameter)
260
+    return subprocess.call(cmd) == 0
261
+
262
+
263
+_UPSTART_CONF = "/etc/init/{}.conf"
264
+_INIT_D_CONF = "/etc/init.d/{}"
265
+
266
+
267
+def service_running(service_name, **kwargs):
268
+    """Determine whether a system service is running.
269
+
270
+    :param service_name: the name of the service
271
+    :param **kwargs: additional args to pass to the service command. This is
272
+                     used to pass additional key=value arguments to the
273
+                     service command line for managing specific instance
274
+                     units (e.g. service ceph-osd status id=2). The kwargs
275
+                     are ignored in systemd services.
276
+    """
277
+    if init_is_systemd():
278
+        return service('is-active', service_name)
279
+    else:
280
+        if os.path.exists(_UPSTART_CONF.format(service_name)):
281
+            try:
282
+                cmd = ['status', service_name]
283
+                for key, value in six.iteritems(kwargs):
284
+                    parameter = '%s=%s' % (key, value)
285
+                    cmd.append(parameter)
286
+                output = subprocess.check_output(cmd,
287
+                    stderr=subprocess.STDOUT).decode('UTF-8')
288
+            except subprocess.CalledProcessError:
289
+                return False
290
+            else:
291
+                # This works for upstart scripts where the 'service' command
292
+                # returns a consistent string to represent running
293
+                # 'start/running'
294
+                if ("start/running" in output or
295
+                        "is running" in output or
296
+                        "up and running" in output):
297
+                    return True
298
+        elif os.path.exists(_INIT_D_CONF.format(service_name)):
299
+            # Check System V scripts init script return codes
300
+            return service('status', service_name)
301
+        return False
302
+
303
+
304
+SYSTEMD_SYSTEM = '/run/systemd/system'
305
+
306
+
307
+def init_is_systemd():
308
+    """Return True if the host system uses systemd, False otherwise."""
309
+    return os.path.isdir(SYSTEMD_SYSTEM)
310
+
311
+
312
+def adduser(username, password=None, shell='/bin/bash',
313
+            system_user=False, primary_group=None,
314
+            secondary_groups=None, uid=None, home_dir=None):
315
+    """Add a user to the system.
316
+
317
+    Will log but otherwise succeed if the user already exists.
318
+
319
+    :param str username: Username to create
320
+    :param str password: Password for user; if ``None``, create a system user
321
+    :param str shell: The default shell for the user
322
+    :param bool system_user: Whether to create a login or system user
323
+    :param str primary_group: Primary group for user; defaults to username
324
+    :param list secondary_groups: Optional list of additional groups
325
+    :param int uid: UID for user being created
326
+    :param str home_dir: Home directory for user
327
+
328
+    :returns: The password database entry struct, as returned by `pwd.getpwnam`
329
+    """
330
+    try:
331
+        user_info = pwd.getpwnam(username)
332
+        log('user {0} already exists!'.format(username))
333
+        if uid:
334
+            user_info = pwd.getpwuid(int(uid))
335
+            log('user with uid {0} already exists!'.format(uid))
336
+    except KeyError:
337
+        log('creating user {0}'.format(username))
338
+        cmd = ['useradd']
339
+        if uid:
340
+            cmd.extend(['--uid', str(uid)])
341
+        if home_dir:
342
+            cmd.extend(['--home', str(home_dir)])
343
+        if system_user or password is None:
344
+            cmd.append('--system')
345
+        else:
346
+            cmd.extend([
347
+                '--create-home',
348
+                '--shell', shell,
349
+                '--password', password,
350
+            ])
351
+        if not primary_group:
352
+            try:
353
+                grp.getgrnam(username)
354
+                primary_group = username  # avoid "group exists" error
355
+            except KeyError:
356
+                pass
357
+        if primary_group:
358
+            cmd.extend(['-g', primary_group])
359
+        if secondary_groups:
360
+            cmd.extend(['-G', ','.join(secondary_groups)])
361
+        cmd.append(username)
362
+        subprocess.check_call(cmd)
363
+        user_info = pwd.getpwnam(username)
364
+    return user_info
365
+
366
+
367
+def user_exists(username):
368
+    """Check if a user exists"""
369
+    try:
370
+        pwd.getpwnam(username)
371
+        user_exists = True
372
+    except KeyError:
373
+        user_exists = False
374
+    return user_exists
375
+
376
+
377
+def uid_exists(uid):
378
+    """Check if a uid exists"""
379
+    try:
380
+        pwd.getpwuid(uid)
381
+        uid_exists = True
382
+    except KeyError:
383
+        uid_exists = False
384
+    return uid_exists
385
+
386
+
387
+def group_exists(groupname):
388
+    """Check if a group exists"""
389
+    try:
390
+        grp.getgrnam(groupname)
391
+        group_exists = True
392
+    except KeyError:
393
+        group_exists = False
394
+    return group_exists
395
+
396
+
397
+def gid_exists(gid):
398
+    """Check if a gid exists"""
399
+    try:
400
+        grp.getgrgid(gid)
401
+        gid_exists = True
402
+    except KeyError:
403
+        gid_exists = False
404
+    return gid_exists
405
+
406
+
407
+def add_group(group_name, system_group=False, gid=None):
408
+    """Add a group to the system
409
+
410
+    Will log but otherwise succeed if the group already exists.
411
+
412
+    :param str group_name: group to create
413
+    :param bool system_group: Create system group
414
+    :param int gid: GID for user being created
415
+
416
+    :returns: The password database entry struct, as returned by `grp.getgrnam`
417
+    """
418
+    try:
419
+        group_info = grp.getgrnam(group_name)
420
+        log('group {0} already exists!'.format(group_name))
421
+        if gid:
422
+            group_info = grp.getgrgid(gid)
423
+            log('group with gid {0} already exists!'.format(gid))
424
+    except KeyError:
425
+        log('creating group {0}'.format(group_name))
426
+        add_new_group(group_name, system_group, gid)
427
+        group_info = grp.getgrnam(group_name)
428
+    return group_info
429
+
430
+
431
+def add_user_to_group(username, group):
432
+    """Add a user to a group"""
433
+    cmd = ['gpasswd', '-a', username, group]
434
+    log("Adding user {} to group {}".format(username, group))
435
+    subprocess.check_call(cmd)
436
+
437
+
438
+def rsync(from_path, to_path, flags='-r', options=None, timeout=None):
439
+    """Replicate the contents of a path"""
440
+    options = options or ['--delete', '--executability']
441
+    cmd = ['/usr/bin/rsync', flags]
442
+    if timeout:
443
+        cmd = ['timeout', str(timeout)] + cmd
444
+    cmd.extend(options)
445
+    cmd.append(from_path)
446
+    cmd.append(to_path)
447
+    log(" ".join(cmd))
448
+    return subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode('UTF-8').strip()
449
+
450
+
451
+def symlink(source, destination):
452
+    """Create a symbolic link"""
453
+    log("Symlinking {} as {}".format(source, destination))
454
+    cmd = [
455
+        'ln',
456
+        '-sf',
457
+        source,
458
+        destination,
459
+    ]
460
+    subprocess.check_call(cmd)
461
+
462
+
463
+def mkdir(path, owner='root', group='root', perms=0o555, force=False):
464
+    """Create a directory"""
465
+    log("Making dir {} {}:{} {:o}".format(path, owner, group,
466
+                                          perms))
467
+    uid = pwd.getpwnam(owner).pw_uid
468
+    gid = grp.getgrnam(group).gr_gid
469
+    realpath = os.path.abspath(path)
470
+    path_exists = os.path.exists(realpath)
471
+    if path_exists and force:
472
+        if not os.path.isdir(realpath):
473
+            log("Removing non-directory file {} prior to mkdir()".format(path))
474
+            os.unlink(realpath)
475
+            os.makedirs(realpath, perms)
476
+    elif not path_exists:
477
+        os.makedirs(realpath, perms)
478
+    os.chown(realpath, uid, gid)
479
+    os.chmod(realpath, perms)
480
+
481
+
482
+def write_file(path, content, owner='root', group='root', perms=0o444):
483
+    """Create or overwrite a file with the contents of a byte string."""
484
+    log("Writing file {} {}:{} {:o}".format(path, owner, group, perms))
485
+    uid = pwd.getpwnam(owner).pw_uid
486
+    gid = grp.getgrnam(group).gr_gid
487
+    with open(path, 'wb') as target:
488
+        os.fchown(target.fileno(), uid, gid)
489
+        os.fchmod(target.fileno(), perms)
490
+        target.write(content)
491
+
492
+
493
+def fstab_remove(mp):
494
+    """Remove the given mountpoint entry from /etc/fstab"""
495
+    return Fstab.remove_by_mountpoint(mp)
496
+
497
+
498
+def fstab_add(dev, mp, fs, options=None):
499
+    """Adds the given device entry to the /etc/fstab file"""
500
+    return Fstab.add(dev, mp, fs, options=options)
501
+
502
+
503
+def mount(device, mountpoint, options=None, persist=False, filesystem="ext3"):
504
+    """Mount a filesystem at a particular mountpoint"""
505
+    cmd_args = ['mount']
506
+    if options is not None:
507
+        cmd_args.extend(['-o', options])
508
+    cmd_args.extend([device, mountpoint])
509
+    try:
510
+        subprocess.check_output(cmd_args)
511
+    except subprocess.CalledProcessError as e:
512
+        log('Error mounting {} at {}\n{}'.format(device, mountpoint, e.output))
513
+        return False
514
+
515
+    if persist:
516
+        return fstab_add(device, mountpoint, filesystem, options=options)
517
+    return True
518
+
519
+
520
+def umount(mountpoint, persist=False):
521
+    """Unmount a filesystem"""
522
+    cmd_args = ['umount', mountpoint]
523
+    try:
524
+        subprocess.check_output(cmd_args)
525
+    except subprocess.CalledProcessError as e:
526
+        log('Error unmounting {}\n{}'.format(mountpoint, e.output))
527
+        return False
528
+
529
+    if persist:
530
+        return fstab_remove(mountpoint)
531
+    return True
532
+
533
+
534
+def mounts():
535
+    """Get a list of all mounted volumes as [[mountpoint,device],[...]]"""
536
+    with open('/proc/mounts') as f:
537
+        # [['/mount/point','/dev/path'],[...]]
538
+        system_mounts = [m[1::-1] for m in [l.strip().split()
539
+                                            for l in f.readlines()]]
540
+    return system_mounts
541
+
542
+
543
+def fstab_mount(mountpoint):
544
+    """Mount filesystem using fstab"""
545
+    cmd_args = ['mount', mountpoint]
546
+    try:
547
+        subprocess.check_output(cmd_args)
548
+    except subprocess.CalledProcessError as e:
549
+        log('Error unmounting {}\n{}'.format(mountpoint, e.output))
550
+        return False
551
+    return True
552
+
553
+
554
+def file_hash(path, hash_type='md5'):
555
+    """Generate a hash checksum of the contents of 'path' or None if not found.
556
+
557
+    :param str hash_type: Any hash alrgorithm supported by :mod:`hashlib`,
558
+                          such as md5, sha1, sha256, sha512, etc.
559
+    """
560
+    if os.path.exists(path):
561
+        h = getattr(hashlib, hash_type)()
562
+        with open(path, 'rb') as source:
563
+            h.update(source.read())
564
+        return h.hexdigest()
565
+    else:
566
+        return None
567
+
568
+
569
+def path_hash(path):
570
+    """Generate a hash checksum of all files matching 'path'. Standard
571
+    wildcards like '*' and '?' are supported, see documentation for the 'glob'
572
+    module for more information.
573
+
574
+    :return: dict: A { filename: hash } dictionary for all matched files.
575
+                   Empty if none found.
576
+    """
577
+    return {
578
+        filename: file_hash(filename)
579
+        for filename in glob.iglob(path)
580
+    }
581
+
582
+
583
+def check_hash(path, checksum, hash_type='md5'):
584
+    """Validate a file using a cryptographic checksum.
585
+
586
+    :param str checksum: Value of the checksum used to validate the file.
587
+    :param str hash_type: Hash algorithm used to generate `checksum`.
588
+        Can be any hash alrgorithm supported by :mod:`hashlib`,
589
+        such as md5, sha1, sha256, sha512, etc.
590
+    :raises ChecksumError: If the file fails the checksum
591
+
592
+    """
593
+    actual_checksum = file_hash(path, hash_type)
594
+    if checksum != actual_checksum:
595
+        raise ChecksumError("'%s' != '%s'" % (checksum, actual_checksum))
596
+
597
+
598
+class ChecksumError(ValueError):
599
+    """A class derived from Value error to indicate the checksum failed."""
600
+    pass
601
+
602
+
603
+def restart_on_change(restart_map, stopstart=False, restart_functions=None):
604
+    """Restart services based on configuration files changing
605
+
606
+    This function is used a decorator, for example::
607
+
608
+        @restart_on_change({
609
+            '/etc/ceph/ceph.conf': [ 'cinder-api', 'cinder-volume' ]
610
+            '/etc/apache/sites-enabled/*': [ 'apache2' ]
611
+            })
612
+        def config_changed():
613
+            pass  # your code here
614
+
615
+    In this example, the cinder-api and cinder-volume services
616
+    would be restarted if /etc/ceph/ceph.conf is changed by the
617
+    ceph_client_changed function. The apache2 service would be
618
+    restarted if any file matching the pattern got changed, created
619
+    or removed. Standard wildcards are supported, see documentation
620
+    for the 'glob' module for more information.
621
+
622
+    @param restart_map: {path_file_name: [service_name, ...]
623
+    @param stopstart: DEFAULT false; whether to stop, start OR restart
624
+    @param restart_functions: nonstandard functions to use to restart services
625
+                              {svc: func, ...}
626
+    @returns result from decorated function
627
+    """
628
+    def wrap(f):
629
+        @functools.wraps(f)
630
+        def wrapped_f(*args, **kwargs):
631
+            return restart_on_change_helper(
632
+                (lambda: f(*args, **kwargs)), restart_map, stopstart,
633
+                restart_functions)
634
+        return wrapped_f
635
+    return wrap
636
+
637
+
638
+def restart_on_change_helper(lambda_f, restart_map, stopstart=False,
639
+                             restart_functions=None):
640
+    """Helper function to perform the restart_on_change function.
641
+
642
+    This is provided for decorators to restart services if files described
643
+    in the restart_map have changed after an invocation of lambda_f().
644
+
645
+    @param lambda_f: function to call.
646
+    @param restart_map: {file: [service, ...]}
647
+    @param stopstart: whether to stop, start or restart a service
648
+    @param restart_functions: nonstandard functions to use to restart services
649
+                              {svc: func, ...}
650
+    @returns result of lambda_f()
651
+    """
652
+    if restart_functions is None:
653
+        restart_functions = {}
654
+    checksums = {path: path_hash(path) for path in restart_map}
655
+    r = lambda_f()
656
+    # create a list of lists of the services to restart
657
+    restarts = [restart_map[path]
658
+                for path in restart_map
659
+                if path_hash(path) != checksums[path]]
660
+    # create a flat list of ordered services without duplicates from lists
661
+    services_list = list(OrderedDict.fromkeys(itertools.chain(*restarts)))
662
+    if services_list:
663
+        actions = ('stop', 'start') if stopstart else ('restart',)
664
+        for service_name in services_list:
665
+            if service_name in restart_functions:
666
+                restart_functions[service_name](service_name)
667
+            else:
668
+                for action in actions:
669
+                    service(action, service_name)
670
+    return r
671
+
672
+
673
+def pwgen(length=None):
674
+    """Generate a random pasword."""
675
+    if length is None:
676
+        # A random length is ok to use a weak PRNG
677
+        length = random.choice(range(35, 45))
678
+    alphanumeric_chars = [
679
+        l for l in (string.ascii_letters + string.digits)
680
+        if l not in 'l0QD1vAEIOUaeiou']
681
+    # Use a crypto-friendly PRNG (e.g. /dev/urandom) for making the
682
+    # actual password
683
+    random_generator = random.SystemRandom()
684
+    random_chars = [
685
+        random_generator.choice(alphanumeric_chars) for _ in range(length)]
686
+    return(''.join(random_chars))
687
+
688
+
689
+def is_phy_iface(interface):
690
+    """Returns True if interface is not virtual, otherwise False."""
691
+    if interface:
692
+        sys_net = '/sys/class/net'
693
+        if os.path.isdir(sys_net):
694
+            for iface in glob.glob(os.path.join(sys_net, '*')):
695
+                if '/virtual/' in os.path.realpath(iface):
696
+                    continue
697
+
698
+                if interface == os.path.basename(iface):
699
+                    return True
700
+
701
+    return False
702
+
703
+
704
+def get_bond_master(interface):
705
+    """Returns bond master if interface is bond slave otherwise None.
706
+
707
+    NOTE: the provided interface is expected to be physical
708
+    """
709
+    if interface:
710
+        iface_path = '/sys/class/net/%s' % (interface)
711
+        if os.path.exists(iface_path):
712
+            if '/virtual/' in os.path.realpath(iface_path):
713
+                return None
714
+
715
+            master = os.path.join(iface_path, 'master')
716
+            if os.path.exists(master):
717
+                master = os.path.realpath(master)
718
+                # make sure it is a bond master
719
+                if os.path.exists(os.path.join(master, 'bonding')):
720
+                    return os.path.basename(master)
721
+
722
+    return None
723
+
724
+
725
+def list_nics(nic_type=None):
726
+    """Return a list of nics of given type(s)"""
727
+    if isinstance(nic_type, six.string_types):
728
+        int_types = [nic_type]
729
+    else:
730
+        int_types = nic_type
731
+
732
+    interfaces = []
733
+    if nic_type:
734
+        for int_type in int_types:
735
+            cmd = ['ip', 'addr', 'show', 'label', int_type + '*']
736
+            ip_output = subprocess.check_output(cmd).decode('UTF-8')
737
+            ip_output = ip_output.split('\n')
738
+            ip_output = (line for line in ip_output if line)
739
+            for line in ip_output:
740
+                if line.split()[1].startswith(int_type):
741
+                    matched = re.search('.*: (' + int_type +
742
+                                        r'[0-9]+\.[0-9]+)@.*', line)
743
+                    if matched:
744
+                        iface = matched.groups()[0]
745
+                    else:
746
+                        iface = line.split()[1].replace(":", "")
747
+
748
+                    if iface not in interfaces:
749
+                        interfaces.append(iface)
750
+    else:
751
+        cmd = ['ip', 'a']
752
+        ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
753
+        ip_output = (line.strip() for line in ip_output if line)
754
+
755
+        key = re.compile('^[0-9]+:\s+(.+):')
756
+        for line in ip_output:
757
+            matched = re.search(key, line)
758
+            if matched:
759
+                iface = matched.group(1)
760
+                iface = iface.partition("@")[0]
761
+                if iface not in interfaces:
762
+                    interfaces.append(iface)
763
+
764
+    return interfaces
765
+
766
+
767
+def set_nic_mtu(nic, mtu):
768
+    """Set the Maximum Transmission Unit (MTU) on a network interface."""
769
+    cmd = ['ip', 'link', 'set', nic, 'mtu', mtu]
770
+    subprocess.check_call(cmd)
771
+
772
+
773
+def get_nic_mtu(nic):
774
+    """Return the Maximum Transmission Unit (MTU) for a network interface."""
775
+    cmd = ['ip', 'addr', 'show', nic]
776
+    ip_output = subprocess.check_output(cmd).decode('UTF-8').split('\n')
777
+    mtu = ""
778
+    for line in ip_output:
779
+        words = line.split()
780
+        if 'mtu' in words:
781
+            mtu = words[words.index("mtu") + 1]
782
+    return mtu
783
+
784
+
785
+def get_nic_hwaddr(nic):
786
+    """Return the Media Access Control (MAC) for a network interface."""
787
+    cmd = ['ip', '-o', '-0', 'addr', 'show', nic]
788
+    ip_output = subprocess.check_output(cmd).decode('UTF-8')
789
+    hwaddr = ""
790
+    words = ip_output.split()
791
+    if 'link/ether' in words:
792
+        hwaddr = words[words.index('link/ether') + 1]
793
+    return hwaddr
794
+
795
+
796
+@contextmanager
797
+def chdir(directory):
798
+    """Change the current working directory to a different directory for a code
799
+    block and return the previous directory after the block exits. Useful to
800
+    run commands from a specificed directory.
801
+
802
+    :param str directory: The directory path to change to for this context.
803
+    """
804
+    cur = os.getcwd()
805
+    try:
806
+        yield os.chdir(directory)
807
+    finally:
808
+        os.chdir(cur)
809
+
810
+
811
+def chownr(path, owner, group, follow_links=True, chowntopdir=False):
812
+    """Recursively change user and group ownership of files and directories
813
+    in given path. Doesn't chown path itself by default, only its children.
814
+
815
+    :param str path: The string path to start changing ownership.
816
+    :param str owner: The owner string to use when looking up the uid.
817
+    :param str group: The group string to use when looking up the gid.
818
+    :param bool follow_links: Also follow and chown links if True
819
+    :param bool chowntopdir: Also chown path itself if True
820
+    """
821
+    uid = pwd.getpwnam(owner).pw_uid
822
+    gid = grp.getgrnam(group).gr_gid
823
+    if follow_links:
824
+        chown = os.chown
825
+    else:
826
+        chown = os.lchown
827
+
828
+    if chowntopdir:
829
+        broken_symlink = os.path.lexists(path) and not os.path.exists(path)
830
+        if not broken_symlink:
831
+            chown(path, uid, gid)
832
+    for root, dirs, files in os.walk(path, followlinks=follow_links):
833
+        for name in dirs + files:
834
+            full = os.path.join(root, name)
835
+            broken_symlink = os.path.lexists(full) and not os.path.exists(full)
836
+            if not broken_symlink:
837
+                chown(full, uid, gid)
838
+
839
+
840
+def lchownr(path, owner, group):
841
+    """Recursively change user and group ownership of files and directories
842
+    in a given path, not following symbolic links. See the documentation for
843
+    'os.lchown' for more information.
844
+
845
+    :param str path: The string path to start changing ownership.
846
+    :param str owner: The owner string to use when looking up the uid.
847
+    :param str group: The group string to use when looking up the gid.
848
+    """
849
+    chownr(path, owner, group, follow_links=False)
850
+
851
+
852
+def owner(path):
853
+    """Returns a tuple containing the username & groupname owning the path.
854
+
855
+    :param str path: the string path to retrieve the ownership
856
+    :return tuple(str, str): A (username, groupname) tuple containing the
857
+                             name of the user and group owning the path.
858
+    :raises OSError: if the specified path does not exist
859
+    """
860
+    stat = os.stat(path)
861
+    username = pwd.getpwuid(stat.st_uid)[0]
862
+    groupname = grp.getgrgid(stat.st_gid)[0]
863
+    return username, groupname
864
+
865
+
866
+def get_total_ram():
867
+    """The total amount of system RAM in bytes.
868
+
869
+    This is what is reported by the OS, and may be overcommitted when
870
+    there are multiple containers hosted on the same machine.
871
+    """
872
+    with open('/proc/meminfo', 'r') as f:
873
+        for line in f.readlines():
874
+            if line:
875
+                key, value, unit = line.split()
876
+                if key == 'MemTotal:':
877
+                    assert unit == 'kB', 'Unknown unit'
878
+                    return int(value) * 1024  # Classic, not KiB.
879
+        raise NotImplementedError()
880
+
881
+
882
+UPSTART_CONTAINER_TYPE = '/run/container_type'
883
+
884
+
885
+def is_container():
886
+    """Determine whether unit is running in a container
887
+
888
+    @return: boolean indicating if unit is in a container
889
+    """
890
+    if init_is_systemd():
891
+        # Detect using systemd-detect-virt
892
+        return subprocess.call(['systemd-detect-virt',
893
+                                '--container']) == 0
894
+    else:
895
+        # Detect using upstart container file marker
896
+        return os.path.exists(UPSTART_CONTAINER_TYPE)
897
+
898
+
899
+def add_to_updatedb_prunepath(path, updatedb_path=UPDATEDB_PATH):
900
+    with open(updatedb_path, 'r+') as f_id:
901
+        updatedb_text = f_id.read()
902
+        output = updatedb(updatedb_text, path)
903
+        f_id.seek(0)
904
+        f_id.write(output)
905
+        f_id.truncate()
906
+
907
+
908
+def updatedb(updatedb_text, new_path):
909
+    lines = [line for line in updatedb_text.split("\n")]
910
+    for i, line in enumerate(lines):
911
+        if line.startswith("PRUNEPATHS="):
912
+            paths_line = line.split("=")[1].replace('"', '')
913
+            paths = paths_line.split(" ")
914
+            if new_path not in paths:
915
+                paths.append(new_path)
916
+                lines[i] = 'PRUNEPATHS="{}"'.format(' '.join(paths))
917
+    output = "\n".join(lines)
918
+    return output

+ 0
- 0
tests/charmhelpers/core/host_factory/__init__.py View File


+ 56
- 0
tests/charmhelpers/core/host_factory/centos.py View File

@@ -0,0 +1,56 @@
1
+import subprocess
2
+import yum
3
+import os
4
+
5
+
6
+def service_available(service_name):
7
+    # """Determine whether a system service is available."""
8
+    if os.path.isdir('/run/systemd/system'):
9
+        cmd = ['systemctl', 'is-enabled', service_name]
10
+    else:
11
+        cmd = ['service', service_name, 'is-enabled']
12
+    return subprocess.call(cmd) == 0
13
+
14
+
15
+def add_new_group(group_name, system_group=False, gid=None):
16
+    cmd = ['groupadd']
17
+    if gid:
18
+        cmd.extend(['--gid', str(gid)])
19
+    if system_group:
20
+        cmd.append('-r')
21
+    cmd.append(group_name)
22
+    subprocess.check_call(cmd)
23
+
24
+
25
+def lsb_release():
26
+    """Return /etc/os-release in a dict."""
27
+    d = {}
28
+    with open('/etc/os-release', 'r') as lsb:
29
+        for l in lsb:
30
+            s = l.split('=')
31
+            if len(s) != 2:
32
+                continue
33
+            d[s[0].strip()] = s[1].strip()
34
+    return d
35
+
36
+
37
+def cmp_pkgrevno(package, revno, pkgcache=None):
38
+    """Compare supplied revno with the revno of the installed package.
39
+
40
+    *  1 => Installed revno is greater than supplied arg
41
+    *  0 => Installed revno is the same as supplied arg
42
+    * -1 => Installed revno is less than supplied arg
43
+
44
+    This function imports YumBase function if the pkgcache argument
45
+    is None.
46
+    """
47
+    if not pkgcache:
48
+        y = yum.YumBase()
49
+        packages = y.doPackageLists()
50
+        pkgcache = {i.Name: i.version for i in packages['installed']}
51
+    pkg = pkgcache[package]
52
+    if pkg > revno:
53
+        return 1
54
+    if pkg < revno:
55
+        return -1
56
+    return 0

+ 56
- 0
tests/charmhelpers/core/host_factory/ubuntu.py View File

@@ -0,0 +1,56 @@
1
+import subprocess
2
+
3
+
4
+def service_available(service_name):
5
+    """Determine whether a system service is available"""
6
+    try:
7
+        subprocess.check_output(
8
+            ['service', service_name, 'status'],
9
+            stderr=subprocess.STDOUT).decode('UTF-8')
10
+    except subprocess.CalledProcessError as e:
11
+        return b'unrecognized service' not in e.output
12
+    else:
13
+        return True
14
+
15
+
16
+def add_new_group(group_name, system_group=False, gid=None):
17
+    cmd = ['addgroup']
18
+    if gid:
19
+        cmd.extend(['--gid', str(gid)])
20
+    if system_group:
21
+        cmd.append('--system')
22
+    else:
23
+        cmd.extend([
24
+            '--group',
25
+        ])
26
+    cmd.append(group_name)
27
+    subprocess.check_call(cmd)
28
+
29
+
30
+def lsb_release():
31
+    """Return /etc/lsb-release in a dict"""
32
+    d = {}
33
+    with open('/etc/lsb-release', 'r') as lsb:
34
+        for l in lsb:
35
+            k, v = l.split('=')
36
+            d[k.strip()] = v.strip()
37
+    return d
38
+
39
+
40
+def cmp_pkgrevno(package, revno, pkgcache=None):
41
+    """Compare supplied revno with the revno of the installed package.
42
+
43
+    *  1 => Installed revno is greater than supplied arg
44
+    *  0 => Installed revno is the same as supplied arg
45
+    * -1 => Installed revno is less than supplied arg
46
+
47
+    This function imports apt_cache function from charmhelpers.fetch if
48
+    the pkgcache argument is None. Be sure to add charmhelpers.fetch if
49
+    you call this function, or pass an apt_pkg.Cache() instance.
50
+    """
51
+    import apt_pkg
52
+    if not pkgcache:
53
+        from charmhelpers.fetch import apt_cache
54
+        pkgcache = apt_cache()
55
+    pkg = pkgcache[package]
56
+    return apt_pkg.version_compare(pkg.current_ver.ver_str, revno)

+ 69
- 0
tests/charmhelpers/core/hugepage.py View File

@@ -0,0 +1,69 @@
1
+# -*- coding: utf-8 -*-
2
+
3
+# Copyright 2014-2015 Canonical Limited.
4
+#
5
+# Licensed under the Apache License, Version 2.0 (the "License");
6
+# you may not use this file except in compliance with the License.
7
+# You may obtain a copy of the License at
8
+#
9
+#  http://www.apache.org/licenses/LICENSE-2.0
10
+#
11
+# Unless required by applicable law or agreed to in writing, software
12
+# distributed under the License is distributed on an "AS IS" BASIS,
13
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+# See the License for the specific language governing permissions and
15
+# limitations under the License.
16
+
17
+import yaml
18
+from charmhelpers.core import fstab
19
+from charmhelpers.core import sysctl
20
+from charmhelpers.core.host import (
21
+    add_group,
22
+    add_user_to_group,
23
+    fstab_mount,
24
+    mkdir,
25
+)
26
+from charmhelpers.core.strutils import bytes_from_string
27
+from subprocess import check_output
28
+
29
+
30
+def hugepage_support(user, group='hugetlb', nr_hugepages=256,
31
+                     max_map_count=65536, mnt_point='/run/hugepages/kvm',
32
+                     pagesize='2MB', mount=True, set_shmmax=False):
33
+    """Enable hugepages on system.
34
+
35
+    Args:
36
+    user (str)  -- Username to allow access to hugepages to
37
+    group (str) -- Group name to own hugepages
38
+    nr_hugepages (int) -- Number of pages to reserve
39
+    max_map_count (int) -- Number of Virtual Memory Areas a process can own
40
+    mnt_point (str) -- Directory to mount hugepages on
41
+    pagesize (str) -- Size of hugepages
42
+    mount (bool) -- Whether to Mount hugepages
43
+    """
44
+    group_info = add_group(group)
45
+    gid = group_info.gr_gid
46
+    add_user_to_group(user, group)
47
+    if max_map_count < 2 * nr_hugepages:
48
+        max_map_count = 2 * nr_hugepages
49
+    sysctl_settings = {
50
+        'vm.nr_hugepages': nr_hugepages,
51
+        'vm.max_map_count': max_map_count,
52
+        'vm.hugetlb_shm_group': gid,
53
+    }
54
+    if set_shmmax:
55
+        shmmax_current = int(check_output(['sysctl', '-n', 'kernel.shmmax']))
56
+        shmmax_minsize = bytes_from_string(pagesize) * nr_hugepages
57
+        if shmmax_minsize > shmmax_current:
58
+            sysctl_settings['kernel.shmmax'] = shmmax_minsize
59
+    sysctl.create(yaml.dump(sysctl_settings), '/etc/sysctl.d/10-hugepage.conf')
60
+    mkdir(mnt_point, owner='root', group='root', perms=0o755, force=False)
61
+    lfstab = fstab.Fstab()
62
+    fstab_entry = lfstab.get_entry_by_attr('mountpoint', mnt_point)
63
+    if fstab_entry:
64
+        lfstab.remove_entry(fstab_entry)
65
+    entry = lfstab.Entry('nodev', mnt_point, 'hugetlbfs',
66
+                         'mode=1770,gid={},pagesize={}'.format(gid, pagesize), 0, 0)
67
+    lfstab.add_entry(entry)
68
+    if mount:
69
+        fstab_mount(mnt_point)

+ 72
- 0
tests/charmhelpers/core/kernel.py View File

@@ -0,0 +1,72 @@
1
+#!/usr/bin/env python
2
+# -*- coding: utf-8 -*-
3
+
4
+# Copyright 2014-2015 Canonical Limited.
5
+#
6
+# Licensed under the Apache License, Version 2.0 (the "License");
7
+# you may not use this file except in compliance with the License.
8
+# You may obtain a copy of the License at
9
+#
10
+#  http://www.apache.org/licenses/LICENSE-2.0
11
+#
12
+# Unless required by applicable law or agreed to in writing, software
13
+# distributed under the License is distributed on an "AS IS" BASIS,
14
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+# See the License for the specific language governing permissions and
16
+# limitations under the License.
17
+
18
+import re
19
+import subprocess
20
+
21
+from charmhelpers.osplatform import get_platform
22
+from charmhelpers.core.hookenv import (
23
+    log,
24
+    INFO
25
+)
26
+
27
+__platform__ = get_platform()
28
+if __platform__ == "ubuntu":
29
+    from charmhelpers.core.kernel_factory.ubuntu import (
30
+        persistent_modprobe,
31
+        update_initramfs,
32
+    )  # flake8: noqa -- ignore F401 for this import
33
+elif __platform__ == "centos":
34
+    from charmhelpers.core.kernel_factory.centos import (
35
+        persistent_modprobe,
36
+        update_initramfs,
37
+    )  # flake8: noqa -- ignore F401 for this import
38
+
39
+__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>"
40
+
41
+
42
+def modprobe(module, persist=True):
43
+    """Load a kernel module and configure for auto-load on reboot."""
44
+    cmd = ['modprobe', module]
45
+
46
+    log('Loading kernel module %s' % module, level=INFO)
47
+
48
+    subprocess.check_call(cmd)
49
+    if persist:
50
+        persistent_modprobe(module)
51
+
52
+
53
+def rmmod(module, force=False):
54
+    """Remove a module from the linux kernel"""
55
+    cmd = ['rmmod']
56
+    if force:
57
+        cmd.append('-f')
58
+    cmd.append(module)
59
+    log('Removing kernel module %s' % module, level=INFO)
60
+    return subprocess.check_call(cmd)
61
+
62
+
63
+def lsmod():
64
+    """Shows what kernel modules are currently loaded"""
65
+    return subprocess.check_output(['lsmod'],
66
+                                   universal_newlines=True)
67
+
68
+
69
+def is_module_loaded(module):
70
+    """Checks if a kernel module is already loaded"""
71
+    matches = re.findall('^%s[ ]+' % module, lsmod(), re.M)
72
+    return len(matches) > 0

+ 0
- 0
tests/charmhelpers/core/kernel_factory/__init__.py View File


+ 17
- 0
tests/charmhelpers/core/kernel_factory/centos.py View File

@@ -0,0 +1,17 @@
1
+import subprocess
2
+import os
3
+
4
+
5
+def persistent_modprobe(module):
6
+    """Load a kernel module and configure for auto-load on reboot."""
7
+    if not os.path.exists('/etc/rc.modules'):
8
+        open('/etc/rc.modules', 'a')
9
+        os.chmod('/etc/rc.modules', 111)
10
+    with open('/etc/rc.modules', 'r+') as modules:
11
+        if module not in modules.read():
12
+            modules.write('modprobe %s\n' % module)
13
+
14
+
15
+def update_initramfs(version='all'):
16
+    """Updates an initramfs image."""
17
+    return subprocess.check_call(["dracut", "-f", version])

+ 13
- 0
tests/charmhelpers/core/kernel_factory/ubuntu.py View File

@@ -0,0 +1,13 @@
1
+import subprocess
2
+
3
+
4
+def persistent_modprobe(module):
5
+    """Load a kernel module and configure for auto-load on reboot."""
6
+    with open('/etc/modules', 'r+') as modules:
7
+        if module not in modules.read():
8
+            modules.write(module + "\n")
9
+
10
+
11
+def update_initramfs(version='all'):
12
+    """Updates an initramfs image."""
13
+    return subprocess.check_call(["update-initramfs", "-k", version, "-u"])

+ 16
- 0
tests/charmhelpers/core/services/__init__.py View File

@@ -0,0 +1,16 @@