From 7c9e77906e5b3c97c6436e0eddc5436624c2ac52 Mon Sep 17 00:00:00 2001
From: Matt Fischer <matt@mattfischer.com>
Date: Fri, 23 Sep 2016 21:56:27 -0600
Subject: [PATCH] Make pid_max settable on OSD nodes.

On OSD nodes during recovery you can ovewhelm the default PID max of
32k. The docs recommend setting to the largest possible value, 4194303.
By default this will not be set.

http://docs.ceph.com/docs/jewel/rados/troubleshooting/troubleshooting-osd/

Change-Id: Id7714e8d237db2137fdc32002ab7ae373fb81357
Closes-Bug: #1362825
---
 manifests/osds.pp                             | 20 ++++++++++++++++++-
 manifests/profile/params.pp                   |  7 +++++++
 metadata.json                                 |  4 ++++
 ..._pidmax_on_osd_nodes-d1a98328f666a895.yaml |  6 ++++++
 spec/classes/ceph_osds_spec.rb                | 15 ++++++++++++--
 5 files changed, 49 insertions(+), 3 deletions(-)
 create mode 100644 releasenotes/notes/allow_changing_pidmax_on_osd_nodes-d1a98328f666a895.yaml

diff --git a/manifests/osds.pp b/manifests/osds.pp
index f20b6035..600ec991 100644
--- a/manifests/osds.pp
+++ b/manifests/osds.pp
@@ -27,6 +27,24 @@
 # [*defaults*] A config hash
 #   Optional. Defaults to a empty hash
 #
-class ceph::osds($args = {}, $defaults = {}) {
+# [*pid_max*] Value for pid_max. Defaults to undef. Optional.
+#   For OSD nodes it is recommended that you raise pid_max above the
+#   default value because you may hit the system max during
+#   recovery. The recommended value is the absolute max for pid_max: 4194303
+#   http://docs.ceph.com/docs/jewel/rados/troubleshooting/troubleshooting-osd/
+#
+class ceph::osds(
+  $args = {},
+  $defaults = {},
+  $pid_max = $::ceph::profile::params::pid_max,
+)
+{
   create_resources(ceph::osd, $args, $defaults)
+
+  if $pid_max {
+    $sysctl_settings = {
+      'kernel.pid_max' => { value => $pid_max },
+    }
+    ensure_resources(sysctl::value,$sysctl_settings)
+  }
 }
diff --git a/manifests/profile/params.pp b/manifests/profile/params.pp
index fc340c84..52c16eef 100644
--- a/manifests/profile/params.pp
+++ b/manifests/profile/params.pp
@@ -145,6 +145,12 @@
 #   Set to 0 to disable it.
 #   Optional. Default provided by Ceph
 #
+# [*pid_max*] Value for pid_max. Defaults to undef. Optional.
+#   For OSD nodes it is recommended that you raise pid_max above the
+#   default value because you may hit the system max during
+#   recovery. The recommended value is the absolute max for pid_max: 4194303
+#   http://docs.ceph.com/docs/jewel/rados/troubleshooting/troubleshooting-osd/
+#
 # [*rgw_keystone_version*] The api version for keystone.
 #   Possible values 'v2.0', 'v3'
 #   Optional. Default is 'v2.0'
@@ -199,6 +205,7 @@ class ceph::profile::params (
   $osd_recovery_max_single_start = undef,
   $osd_max_scrubs = undef,
   $osd_op_threads = undef,
+  $pid_max = undef,
   $rgw_keystone_version = 'v2.0',
   $rgw_keystone_admin_domain = undef,
   $rgw_keystone_admin_project = undef,
diff --git a/metadata.json b/metadata.json
index 3ee53ac8..fe9e2338 100644
--- a/metadata.json
+++ b/metadata.json
@@ -66,6 +66,10 @@
     {
       "name": "puppetlabs/stdlib",
       "version_requirement": ">=4.10.0 <5.0.0"
+    },
+    {
+      "name": "duritong/sysctl",
+      "version_requirement": ">=0.0.1 <1.0.0"
     }
   ]
 }
diff --git a/releasenotes/notes/allow_changing_pidmax_on_osd_nodes-d1a98328f666a895.yaml b/releasenotes/notes/allow_changing_pidmax_on_osd_nodes-d1a98328f666a895.yaml
new file mode 100644
index 00000000..96059cdb
--- /dev/null
+++ b/releasenotes/notes/allow_changing_pidmax_on_osd_nodes-d1a98328f666a895.yaml
@@ -0,0 +1,6 @@
+---
+features:
+  - On OSD nodes you can run out of PIDs during a recovery.
+    Therefore it is recommended that you raise pid_max. This
+    change makes that possible with an enablement flag and
+    the ability to set the value.
diff --git a/spec/classes/ceph_osds_spec.rb b/spec/classes/ceph_osds_spec.rb
index 33ba8028..5ca3deed 100644
--- a/spec/classes/ceph_osds_spec.rb
+++ b/spec/classes/ceph_osds_spec.rb
@@ -39,14 +39,25 @@ describe 'ceph::osds' do
       is_expected.to contain_ceph__osd('/dev/sdb').with(
         :ensure  => 'present',
         :journal => '/srv/journal',
-        :cluster => 'CLUSTER'
-    )
+        :cluster => 'CLUSTER')
       is_expected.to contain_ceph__osd('/srv/data').with(
         :ensure  => 'present',
         :cluster => 'CLUSTER')
+      is_expected.not_to contain_sysctl__value('kernel.pid_max')
     }
   end
 
+  context 'sets pid_max when enabled' do
+    let :params do
+    {
+      :pid_max => 123456,
+    }
+    end
+    it do
+      is_expected.to contain_sysctl__value('kernel.pid_max').with_value(123456)
+    end
+  end
+
   describe 'Ubuntu' do
     let :facts do
       {