Ceph-osd: Fix security context for pod/container

This changes the  user from root to the nobody user instead
in ceph-osd chart wherever needed

This also permits read-only filesystems to back the containers by setting
the default to true

Change-Id: Ia777bf212e0e3414909c70a4bd839e12d4919bb2

ceph-osd/templates/daemonset-osd.yaml

@@ -265,6 +265,8 @@ spec:
               mountPath: /run
             - name: pod-etc-ceph
               mountPath: /etc/ceph
+            - name: pod-forego
+              mountPath: /etc/forego
             - name: ceph-osd-bin
               mountPath: /tmp/osd-start.sh
               subPath: osd-start.sh
@@ -335,6 +337,8 @@ spec:
             medium: "Memory"
         - name: pod-etc-ceph
           emptyDir: {}
+        - name: pod-forego
+          emptyDir: {}
         - name: devices
           hostPath:
             path: /dev

ceph-osd/values.yaml

@@ -43,29 +43,35 @@ pod:
   security_context:
     osd:
       pod:
-        runAsUser: 0
+        runAsUser: 65534
       container:
         ceph_init_dirs:
+          runAsUser: 0
           readOnlyRootFilesystem: true
         ceph_log_ownership:
+          runAsUser: 0
           readOnlyRootFilesystem: true
         osd_init:
+          runAsUser: 0
           privileged: true
           readOnlyRootFilesystem: true
         osd_pod:
+          runAsUser: 0
           privileged: true
-          readOnlyRootFilesystem: false
+          readOnlyRootFilesystem: true
     bootstrap:
       pod:
-        runAsUser: 0
+        runAsUser: 65534
       container:
         ceph_osd_bootstrap:
+          allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true
     test:
       pod:
-        runAsUser: 0
+        runAsUser: 65534
       container:
         ceph_cluster_helm_test:
+          allowPrivilegeEscalation: false
           readOnlyRootFilesystem: true
   dns_policy: "ClusterFirstWithHostNet"
   affinity: