Browse Source

add ulimit support for kolla_docker

By default, docker containers inherit ulimit from limits of docker
deamon. On CentOS 7, docker daemon default NOFILE is 1048576.
It can found in /usr/lib/systemd/system/docker.service.
The big limit will cause many problem. we should control it in
production environment.

Change-Id: Iab962446a94ef092977728259d9818b86cfa7f68
binhong.hua 4 months ago
parent
commit
3d3f5f1613

+ 59
- 13
ansible/library/kolla_docker.py View File

@@ -474,7 +474,8 @@ class DockerWorker(object):
474 474
             'memswap_limit': 'MemorySwap', 'cpu_period': 'CpuPeriod',
475 475
             'cpu_quota': 'CpuQuota', 'cpu_shares': 'CpuShares',
476 476
             'cpuset_cpus': 'CpusetCpus', 'cpuset_mems': 'CpusetMems',
477
-            'kernel_memory': 'KernelMemory', 'blkio_weight': 'BlkioWeight'}
477
+            'kernel_memory': 'KernelMemory', 'blkio_weight': 'BlkioWeight',
478
+            'ulimits': 'Ulimits'}
478 479
         unsupported = set(new_dimensions.keys()) - \
479 480
             set(dimension_map.keys())
480 481
         if unsupported:
@@ -486,13 +487,29 @@ class DockerWorker(object):
486 487
             # NOTE(mgoddard): If a resource has been explicitly requested,
487 488
             # check for a match. Otherwise, ensure is is set to the default.
488 489
             if key1 in new_dimensions:
489
-                if new_dimensions[key1] != current_dimensions[key2]:
490
+                if key1 == 'ulimits':
491
+                    if self.compare_ulimits(new_dimensions[key1],
492
+                                            current_dimensions[key2]):
493
+                        return True
494
+                elif new_dimensions[key1] != current_dimensions[key2]:
490 495
                     return True
491 496
             elif current_dimensions[key2]:
492 497
                 # The default values of all currently supported resources are
493 498
                 # '' or 0 - both falsey.
494 499
                 return True
495 500
 
501
+    def compare_ulimits(self, new_ulimits, current_ulimits):
502
+        # The new_ulimits is dict, we need make it to a list of Ulimit
503
+        # instance.
504
+        new_ulimits = self.build_ulimits(new_ulimits)
505
+
506
+        def key(ulimit):
507
+            return ulimit['Name']
508
+
509
+        if current_ulimits is None:
510
+            current_ulimits = []
511
+        return sorted(new_ulimits, key=key) != sorted(current_ulimits, key=key)
512
+
496 513
     def compare_command(self, container_info):
497 514
         new_command = self.params.get('command')
498 515
         if new_command is not None:
@@ -606,6 +623,40 @@ class DockerWorker(object):
606 623
 
607 624
         return vol_list, vol_dict
608 625
 
626
+    def parse_dimensions(self, dimensions):
627
+        # When the data object contains types such as
628
+        # docker.types.Ulimit, Ansible will fail when these are
629
+        # returned via exit_json or fail_json. HostConfig is derived from dict,
630
+        # but its constructor requires additional arguments.
631
+        # to avoid that, here do copy the dimensions and return a new one.
632
+        dimensions = dimensions.copy()
633
+
634
+        supported = {'cpu_period', 'cpu_quota', 'cpu_shares',
635
+                     'cpuset_cpus', 'cpuset_mems', 'mem_limit',
636
+                     'mem_reservation', 'memswap_limit',
637
+                     'kernel_memory', 'blkio_weight', 'ulimits'}
638
+        unsupported = set(dimensions) - supported
639
+        if unsupported:
640
+            self.module.exit_json(failed=True,
641
+                                  msg=repr("Unsupported dimensions"),
642
+                                  unsupported_dimensions=unsupported)
643
+
644
+        ulimits = dimensions.get('ulimits')
645
+        if ulimits:
646
+            dimensions['ulimits'] = self.build_ulimits(ulimits)
647
+
648
+        return dimensions
649
+
650
+    def build_ulimits(self, ulimits):
651
+        ulimits_opt = []
652
+        for key, value in ulimits.items():
653
+            soft = value.get('soft')
654
+            hard = value.get('hard')
655
+            ulimits_opt.append(docker.types.Ulimit(name=key,
656
+                                                   soft=soft,
657
+                                                   hard=hard))
658
+        return ulimits_opt
659
+
609 660
     def build_host_config(self, binds):
610 661
         options = {
611 662
             'network_mode': 'host',
@@ -617,17 +668,11 @@ class DockerWorker(object):
617 668
             'volumes_from': self.params.get('volumes_from')
618 669
         }
619 670
 
620
-        if self.params.get('dimensions'):
621
-            supported = {'cpu_period', 'cpu_quota', 'cpu_shares',
622
-                         'cpuset_cpus', 'cpuset_mems', 'mem_limit',
623
-                         'mem_reservation', 'memswap_limit',
624
-                         'kernel_memory', 'blkio_weight'}
625
-            unsupported = set(self.params.get('dimensions')) - supported
626
-            if unsupported:
627
-                self.module.exit_json(failed=True,
628
-                                      msg=repr("Unsupported dimensions"),
629
-                                      unsupported_dimensions=unsupported)
630
-            options.update(self.params.get('dimensions'))
671
+        dimensions = self.params.get('dimensions')
672
+
673
+        if dimensions:
674
+            dimensions = self.parse_dimensions(dimensions)
675
+            options.update(dimensions)
631 676
 
632 677
         if self.params.get('restart_policy') in ['on-failure',
633 678
                                                  'always',
@@ -940,5 +985,6 @@ def main():
940 985
         module.fail_json(changed=True, msg=repr(traceback.format_exc()),
941 986
                          **dw.result)
942 987
 
988
+
943 989
 if __name__ == '__main__':
944 990
     main()

+ 21
- 0
doc/source/reference/deployment-config/resource-constraints.rst View File

@@ -27,6 +27,7 @@ The resources currently supported by Kolla Ansible are:
27 27
     memswap_limit
28 28
     kernel_memory
29 29
     blkio_weight
30
+    ulimits
30 31
 
31 32
 Pre-deployment Configuration
32 33
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -67,6 +68,9 @@ Dimensions are defined as a mapping from a Docker resource name
67 68
    * - cpuset_mems
68 69
      - String
69 70
      - ''(Empty String)
71
+   * - ulimits
72
+     - Dict
73
+     - {}
70 74
 
71 75
 
72 76
 The variable ``default_container_dimensions`` sets the default dimensions
@@ -96,6 +100,23 @@ options section in ``/etc/kolla/globals.yml``:
96 100
    nova_libvirt_dimensions:
97 101
      cpuset_cpus: "2"
98 102
 
103
+How to config ulimits in kolla
104
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
105
+
106
+.. code-block:: yaml
107
+
108
+  <container_name>_dimensions:
109
+    ulimits:
110
+      nofile:
111
+        soft: 131072
112
+        hard: 131072
113
+      fsize:
114
+        soft: 131072
115
+        hard: 131072
116
+
117
+A list of valid names can be found [here]
118
+(https://github.com/docker/go-units/blob/d4a9b9617350c034730bc5051c605919943080bf/ulimit.go#L46-L63)
119
+
99 120
 Deployment
100 121
 ~~~~~~~~~~
101 122
 

+ 1
- 0
etc/kolla/globals.yml View File

@@ -142,6 +142,7 @@ kolla_internal_vip_address: "10.10.10.254"
142 142
 #    mem_reservation:
143 143
 #    memswap_limit:
144 144
 #    kernel_memory:
145
+#    ulimits:
145 146
 
146 147
 
147 148
 #############

+ 5
- 0
releasenotes/notes/add_ulimit_support-35e8799f29a44d12.yaml View File

@@ -0,0 +1,5 @@
1
+---
2
+features:
3
+  - |
4
+    Add ulimit support for kolla_docker which provides ability
5
+    of modify docker ulimits。

+ 39
- 6
tests/test_kolla_docker.py View File

@@ -23,6 +23,7 @@ try:
23 23
 except ImportError:
24 24
     import mock
25 25
 from docker import errors as docker_error
26
+from docker.types import Ulimit
26 27
 from oslotest import base
27 28
 
28 29
 this_dir = os.path.dirname(sys.modules[__name__].__file__)
@@ -904,7 +905,8 @@ class TestAttrComp(base.BaseTestCase):
904 905
         container_info['HostConfig'] = {
905 906
             'CpuPeriod': 0, 'KernelMemory': 0, 'Memory': 0, 'CpuQuota': 0,
906 907
             'CpusetCpus': '', 'CpuShares': 0, 'BlkioWeight': 0,
907
-            'CpusetMems': '', 'MemorySwap': 0, 'MemoryReservation': 0}
908
+            'CpusetMems': '', 'MemorySwap': 0, 'MemoryReservation': 0,
909
+            'Ulimits': []}
908 910
         self.dw = get_DockerWorker(self.fake_data['params'])
909 911
         self.assertTrue(self.dw.compare_dimensions(container_info))
910 912
 
@@ -915,7 +917,8 @@ class TestAttrComp(base.BaseTestCase):
915 917
         container_info['HostConfig'] = {
916 918
             'CpuPeriod': 0, 'KernelMemory': 0, 'Memory': 0, 'CpuQuota': 0,
917 919
             'CpusetCpus': '', 'CpuShares': 0, 'BlkioWeight': 10,
918
-            'CpusetMems': '', 'MemorySwap': 0, 'MemoryReservation': 0}
920
+            'CpusetMems': '', 'MemorySwap': 0, 'MemoryReservation': 0,
921
+            'Ulimits': []}
919 922
         self.dw = get_DockerWorker(self.fake_data['params'])
920 923
         self.assertFalse(self.dw.compare_dimensions(container_info))
921 924
 
@@ -926,7 +929,8 @@ class TestAttrComp(base.BaseTestCase):
926 929
         container_info['HostConfig'] = {
927 930
             'CpuPeriod': 0, 'KernelMemory': 0, 'Memory': 0, 'CpuQuota': 0,
928 931
             'CpusetCpus': '', 'CpuShares': 0, 'BlkioWeight': 0,
929
-            'CpusetMems': '', 'MemorySwap': 0, 'MemoryReservation': 0}
932
+            'CpusetMems': '', 'MemorySwap': 0, 'MemoryReservation': 0,
933
+            'Ulimits': []}
930 934
         self.dw = get_DockerWorker(self.fake_data['params'])
931 935
         self.dw.compare_dimensions(container_info)
932 936
         self.dw.module.exit_json.assert_called_once_with(
@@ -939,7 +943,8 @@ class TestAttrComp(base.BaseTestCase):
939 943
         container_info['HostConfig'] = {
940 944
             'CpuPeriod': 0, 'KernelMemory': 0, 'Memory': 0, 'CpuQuota': 0,
941 945
             'CpusetCpus': '1', 'CpuShares': 0, 'BlkioWeight': 0,
942
-            'CpusetMems': '', 'MemorySwap': 0, 'MemoryReservation': 0}
946
+            'CpusetMems': '', 'MemorySwap': 0, 'MemoryReservation': 0,
947
+            'Ulimits': []}
943 948
         self.dw = get_DockerWorker(self.fake_data['params'])
944 949
         self.assertTrue(self.dw.compare_dimensions(container_info))
945 950
 
@@ -954,7 +959,8 @@ class TestAttrComp(base.BaseTestCase):
954 959
         container_info['HostConfig'] = {
955 960
             'CpuPeriod': 0, 'KernelMemory': 0, 'Memory': 10, 'CpuQuota': 0,
956 961
             'CpusetCpus': '', 'CpuShares': 0, 'BlkioWeight': 0,
957
-            'CpusetMems': '', 'MemorySwap': 0, 'MemoryReservation': 10}
962
+            'CpusetMems': '', 'MemorySwap': 0, 'MemoryReservation': 10,
963
+            'Ulimits': []}
958 964
         self.dw = get_DockerWorker(self.fake_data['params'])
959 965
         self.assertTrue(self.dw.compare_dimensions(container_info))
960 966
 
@@ -969,7 +975,8 @@ class TestAttrComp(base.BaseTestCase):
969 975
         container_info['HostConfig'] = {
970 976
             'CpuPeriod': 0, 'KernelMemory': 0, 'Memory': 0, 'CpuQuota': 0,
971 977
             'CpusetCpus': '', 'CpuShares': 0, 'BlkioWeight': 0,
972
-            'CpusetMems': '', 'MemorySwap': 0, 'MemoryReservation': 0}
978
+            'CpusetMems': '', 'MemorySwap': 0, 'MemoryReservation': 0,
979
+            'Ulimits': []}
973 980
         self.dw = get_DockerWorker(self.fake_data['params'])
974 981
         self.assertFalse(self.dw.compare_dimensions(container_info))
975 982
 
@@ -977,3 +984,29 @@ class TestAttrComp(base.BaseTestCase):
977 984
         container_info = {'State': dict(Status='running')}
978 985
         self.dw = get_DockerWorker({'state': 'exited'})
979 986
         self.assertTrue(self.dw.compare_container_state(container_info))
987
+
988
+    def test_compare_ulimits_pos(self):
989
+        self.fake_data['params']['dimensions'] = {
990
+            'ulimits': {'nofile': {'soft': 131072, 'hard': 131072}}}
991
+        container_info = dict()
992
+        container_info['HostConfig'] = {
993
+            'CpuPeriod': 0, 'KernelMemory': 0, 'Memory': 0, 'CpuQuota': 0,
994
+            'CpusetCpus': '', 'CpuShares': 0, 'BlkioWeight': 0,
995
+            'CpusetMems': '', 'MemorySwap': 0, 'MemoryReservation': 0,
996
+            'Ulimits': []}
997
+        self.dw = get_DockerWorker(self.fake_data['params'])
998
+        self.assertTrue(self.dw.compare_dimensions(container_info))
999
+
1000
+    def test_compare_ulimits_neg(self):
1001
+        self.fake_data['params']['dimensions'] = {
1002
+            'ulimits': {'nofile': {'soft': 131072, 'hard': 131072}}}
1003
+        ulimits_nofile = Ulimit(name='nofile',
1004
+                                soft=131072, hard=131072)
1005
+        container_info = dict()
1006
+        container_info['HostConfig'] = {
1007
+            'CpuPeriod': 0, 'KernelMemory': 0, 'Memory': 0, 'CpuQuota': 0,
1008
+            'CpusetCpus': '', 'CpuShares': 0, 'BlkioWeight': 0,
1009
+            'CpusetMems': '', 'MemorySwap': 0, 'MemoryReservation': 0,
1010
+            'Ulimits': [ulimits_nofile]}
1011
+        self.dw = get_DockerWorker(self.fake_data['params'])
1012
+        self.assertFalse(self.dw.compare_dimensions(container_info))

Loading…
Cancel
Save