k8s_fedora_atomic: Add PodSecurityPolicy

For moving to 1.15.x and beyond we need to have PSP for privileged pods.
flannel, calico and node-problem-detector need it.

PSP
story: 2006515
task: 36513

Allow-priv
story: 2006252
task: 35867

Change-Id: I306a249afb275fdbd71354ed75043ffc4d466304
Signed-off-by: Spyros Trigazis <spyridon.trigazis@cern.ch>

magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh

@@ -14,6 +14,23 @@ if [ "$NETWORK_DRIVER" = "calico" ]; then
     mkdir -p $(dirname ${CALICO_DEPLOY})
     cat << EOF > ${CALICO_DEPLOY}
 ---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: magnum:podsecuritypolicy:calico
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+    kubernetes.io/cluster-service: "true"
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: magnum:podsecuritypolicy:privileged
+subjects:
+- kind: ServiceAccount
+  name: calico-node
+  namespace: kube-system
+---
 # Calico Version v3.3.6
 # https://docs.projectcalico.org/v3.3/releases#v3.3.6
 kind: ClusterRole

magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh

@@ -92,6 +92,7 @@ sed -i '
 ' /etc/kubernetes/config
 
 KUBE_API_ARGS="--runtime-config=api/all=true"
+KUBE_API_ARGS="$KUBE_API_ARGS --allow-privileged=$KUBE_ALLOW_PRIV"
 KUBE_API_ARGS="$KUBE_API_ARGS --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP"
 KUBE_API_ARGS="$KUBE_API_ARGS $KUBEAPI_OPTIONS"
 if [ "$TLS_DISABLED" == "True" ]; then

magnum/drivers/common/templates/kubernetes/fragments/enable-auto-healing.sh

@@ -27,6 +27,23 @@ metadata:
     addonmanager.kubernetes.io/mode: Reconcile
 ---
 apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: magnum:podsecuritypolicy:node-problem-detector
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+    kubernetes.io/cluster-service: "true"
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: magnum:podsecuritypolicy:privileged
+subjects:
+- kind: ServiceAccount
+  name: node-problem-detector
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
   name: npd-binding

magnum/drivers/common/templates/kubernetes/fragments/flannel-service.sh

@@ -1,9 +1,11 @@
-#!/bin/sh
+#!/bin/bash
 
+set -e
+set +x
 . /etc/sysconfig/heat-params
-
 set -x
 
+
 if [ "$NETWORK_DRIVER" = "flannel" ]; then
     _prefix=${CONTAINER_INFRA_PREFIX:-quay.io/coreos/}
     FLANNEL_DEPLOY=/srv/magnum/kubernetes/manifests/flannel-deploy.yaml
@@ -11,13 +13,65 @@ if [ "$NETWORK_DRIVER" = "flannel" ]; then
     [ -f ${FLANNEL_DEPLOY} ] || {
     echo "Writing File: $FLANNEL_DEPLOY"
     mkdir -p "$(dirname ${FLANNEL_DEPLOY})"
+    set +x
     cat << EOF > ${FLANNEL_DEPLOY}
 ---
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: psp.flannel.unprivileged
+  annotations:
+    seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default
+    seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default
+    apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default
+    apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default
+spec:
+  privileged: false
+  volumes:
+    - configMap
+    - secret
+    - emptyDir
+    - hostPath
+  allowedHostPaths:
+    - pathPrefix: "/etc/cni/net.d"
+    - pathPrefix: "/etc/kube-flannel"
+    - pathPrefix: "/run/flannel"
+  readOnlyRootFilesystem: false
+  # Users and groups
+  runAsUser:
+    rule: RunAsAny
+  supplementalGroups:
+    rule: RunAsAny
+  fsGroup:
+    rule: RunAsAny
+  # Privilege Escalation
+  allowPrivilegeEscalation: false
+  defaultAllowPrivilegeEscalation: false
+  # Capabilities
+  allowedCapabilities: ['NET_ADMIN']
+  defaultAddCapabilities: []
+  requiredDropCapabilities: []
+  # Host namespaces
+  hostPID: false
+  hostIPC: false
+  hostNetwork: true
+  hostPorts:
+  - min: 0
+    max: 65535
+  # SELinux
+  seLinux:
+    # SELinux is unsed in CaaSP
+    rule: 'RunAsAny'
+---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1beta1
 metadata:
   name: flannel
 rules:
+  - apiGroups: ['extensions']
+    resources: ['podsecuritypolicies']
+    verbs: ['use']
+    resourceNames: ['psp.flannel.unprivileged']
   - apiGroups:
       - ""
     resources:
@@ -101,7 +155,7 @@ data:
       echo "Wrote CNI binaries to /host/opt/cni/bin/";
     fi;
 ---
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: kube-flannel-ds-amd64
@@ -110,6 +164,9 @@ metadata:
     tier: node
     app: flannel
 spec:
+  selector:
+    matchLabels:
+      app: flannel
   template:
     metadata:
       labels:
@@ -120,14 +177,8 @@ spec:
       nodeSelector:
         beta.kubernetes.io/arch: amd64
       tolerations:
-        # Make sure flannel gets scheduled on all nodes.
-        - effect: NoSchedule
-          operator: Exists
-        # Mark the pod as a critical add-on for rescheduling.
-        - key: CriticalAddonsOnly
-          operator: Exists
-        - effect: NoExecute
-          operator: Exists
+      - operator: Exists
+        effect: NoSchedule
       serviceAccountName: flannel
       initContainers:
       - name: install-cni-plugins
@@ -170,7 +221,9 @@ spec:
             cpu: "100m"
             memory: "50Mi"
         securityContext:
-          privileged: true
+          privileged: false
+          capabilities:
+             add: ["NET_ADMIN"]
         env:
         - name: POD_NAME
           valueFrom:
@@ -182,7 +235,7 @@ spec:
               fieldPath: metadata.namespace
         volumeMounts:
         - name: run
-          mountPath: /run
+          mountPath: /run/flannel
         - name: flannel-cfg
           mountPath: /etc/kube-flannel/
       volumes:
@@ -191,7 +244,7 @@ spec:
             path: /opt/cni/bin
         - name: run
           hostPath:
-            path: /run
+            path: /run/flannel
         - name: cni
           hostPath:
             path: /etc/cni/net.d
@@ -200,6 +253,7 @@ spec:
             name: kube-flannel-cfg
 EOF
     }
+    set -x
 
     if [ "$MASTER_INDEX" = "0" ]; then
 

magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh

@@ -3,8 +3,8 @@
 step="kube-apiserver-to-kubelet-role"
 printf "Starting to run ${step}\n"
 
+set +x
 . /etc/sysconfig/heat-params
-
 set -x
 
 echo "Waiting for Kubernetes API..."
@@ -80,6 +80,67 @@ EOF
 }
 kubectl apply --validate=false -f ${ADMIN_RBAC}
 
+POD_SECURITY_POLICIES=/srv/magnum/kubernetes/podsecuritypolicies.yaml
+# Pod Security Policies
+[ -f ${POD_SECURITY_POLICIES} ] || {
+    echo "Writing File: $POD_SECURITY_POLICIES"
+    mkdir -p $(dirname ${POD_SECURITY_POLICIES})
+    cat > ${POD_SECURITY_POLICIES} <<EOF
+---
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: magnum.privileged
+  annotations:
+    kubernetes.io/description: 'privileged allows full unrestricted access to
+      pod features, as if the PodSecurityPolicy controller was not enabled.'
+    seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*'
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  privileged: true
+  allowPrivilegeEscalation: true
+  allowedCapabilities:
+  - '*'
+  volumes:
+  - '*'
+  hostNetwork: true
+  hostPorts:
+  - min: 0
+    max: 65535
+  hostIPC: true
+  hostPID: true
+  runAsUser:
+    rule: 'RunAsAny'
+  seLinux:
+    rule: 'RunAsAny'
+  supplementalGroups:
+    rule: 'RunAsAny'
+  fsGroup:
+    rule: 'RunAsAny'
+  readOnlyRootFilesystem: false
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: magnum:podsecuritypolicy:privileged
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+rules:
+- apiGroups:
+  - policy
+  resourceNames:
+  - magnum.privileged
+  resources:
+  - podsecuritypolicies
+  verbs:
+  - use
+EOF
+}
+kubectl apply -f ${POD_SECURITY_POLICIES}
+
 # Add the openstack trustee as a secret under kube-system
 kubectl -n kube-system create secret generic os-trustee \
     --from-literal=os-authURL=${AUTH_URL} \

magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml

@@ -382,7 +382,7 @@ parameters:
   flannel_tag:
     type: string
     description: tag of the flannel container
-    default: v0.10.0-amd64
+    default: v0.11.0-amd64
 
   flannel_cni_tag:
     type: string

releasenotes/notes/podsecuritypolicy-2400063d73524e06.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    k8s_fedora_atomic_v1 Add PodSecurityPolicy for privileged pods. Use
+    privileged PSP for calico and node-problem-detector. Add PSP for flannel
+    from upstream.