k8s_fedora_atomic: Add PodSecurityPolicy

For moving to 1.15.x and beyond we need to have PSP for privileged pods. flannel, calico and node-problem-detector need it. PSP story: 2006515 task: 36513 Allow-priv story: 2006252 task: 35867 Change-Id: I306a249afb275fdbd71354ed75043ffc4d466304 Signed-off-by: Spyros Trigazis <spyridon.trigazis@cern.ch>
2019-09-09 13:47:23 +00:00 · 2019-09-09 13:47:23 +00:00 · 7267c1ea43
commit 7267c1ea43
parent 6fe3d5d7f1
7 changed files with 172 additions and 16 deletions
--- a/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh
@ -14,6 +14,23 @@ if [ "$NETWORK_DRIVER" = "calico" ]; then
    mkdir -p $(dirname ${CALICO_DEPLOY})
    cat << EOF > ${CALICO_DEPLOY}
 ---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: magnum:podsecuritypolicy:calico
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+    kubernetes.io/cluster-service: "true"
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: magnum:podsecuritypolicy:privileged
+subjects:
+- kind: ServiceAccount
+  name: calico-node
+  namespace: kube-system
+---
 # Calico Version v3.3.6
 # https://docs.projectcalico.org/v3.3/releases#v3.3.6
 kind: ClusterRole
--- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh
@ -92,6 +92,7 @@ sed -i '
 ' /etc/kubernetes/config

 KUBE_API_ARGS="--runtime-config=api/all=true"
+KUBE_API_ARGS="$KUBE_API_ARGS --allow-privileged=$KUBE_ALLOW_PRIV"
 KUBE_API_ARGS="$KUBE_API_ARGS --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP"
 KUBE_API_ARGS="$KUBE_API_ARGS $KUBEAPI_OPTIONS"
 if [ "$TLS_DISABLED" == "True" ]; then
--- a/magnum/drivers/common/templates/kubernetes/fragments/enable-auto-healing.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-auto-healing.sh
@ -27,6 +27,23 @@ metadata:
    addonmanager.kubernetes.io/mode: Reconcile
 ---
 apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: magnum:podsecuritypolicy:node-problem-detector
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+    kubernetes.io/cluster-service: "true"
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: magnum:podsecuritypolicy:privileged
+subjects:
+- kind: ServiceAccount
+  name: node-problem-detector
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
  name: npd-binding
--- a/magnum/drivers/common/templates/kubernetes/fragments/flannel-service.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/flannel-service.sh
@ -1,9 +1,11 @@
-#!/bin/sh
+#!/bin/bash

+set -e
+set +x
 . /etc/sysconfig/heat-params
-
 set -x

+
 if [ "$NETWORK_DRIVER" = "flannel" ]; then
    _prefix=${CONTAINER_INFRA_PREFIX:-quay.io/coreos/}
    FLANNEL_DEPLOY=/srv/magnum/kubernetes/manifests/flannel-deploy.yaml
@ -11,13 +13,65 @@ if [ "$NETWORK_DRIVER" = "flannel" ]; then
    [ -f ${FLANNEL_DEPLOY} ] || {
    echo "Writing File: $FLANNEL_DEPLOY"
    mkdir -p "$(dirname ${FLANNEL_DEPLOY})"
+    set +x
    cat << EOF > ${FLANNEL_DEPLOY}
 ---
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: psp.flannel.unprivileged
+  annotations:
+    seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default
+    seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default
+    apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default
+    apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default
+spec:
+  privileged: false
+  volumes:
+    - configMap
+    - secret
+    - emptyDir
+    - hostPath
+  allowedHostPaths:
+    - pathPrefix: "/etc/cni/net.d"
+    - pathPrefix: "/etc/kube-flannel"
+    - pathPrefix: "/run/flannel"
+  readOnlyRootFilesystem: false
+  # Users and groups
+  runAsUser:
+    rule: RunAsAny
+  supplementalGroups:
+    rule: RunAsAny
+  fsGroup:
+    rule: RunAsAny
+  # Privilege Escalation
+  allowPrivilegeEscalation: false
+  defaultAllowPrivilegeEscalation: false
+  # Capabilities
+  allowedCapabilities: ['NET_ADMIN']
+  defaultAddCapabilities: []
+  requiredDropCapabilities: []
+  # Host namespaces
+  hostPID: false
+  hostIPC: false
+  hostNetwork: true
+  hostPorts:
+  - min: 0
+    max: 65535
+  # SELinux
+  seLinux:
+    # SELinux is unsed in CaaSP
+    rule: 'RunAsAny'
+---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1beta1
 metadata:
  name: flannel
 rules:
+  - apiGroups: ['extensions']
+    resources: ['podsecuritypolicies']
+    verbs: ['use']
+    resourceNames: ['psp.flannel.unprivileged']
  - apiGroups:
      - ""
    resources:
@ -101,7 +155,7 @@ data:
      echo "Wrote CNI binaries to /host/opt/cni/bin/";
    fi;
 ---
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: DaemonSet
 metadata:
  name: kube-flannel-ds-amd64
@ -110,6 +164,9 @@ metadata:
    tier: node
    app: flannel
 spec:
+  selector:
+    matchLabels:
+      app: flannel
  template:
    metadata:
      labels:
@ -120,14 +177,8 @@ spec:
      nodeSelector:
        beta.kubernetes.io/arch: amd64
      tolerations:
-        # Make sure flannel gets scheduled on all nodes.
-        - effect: NoSchedule
-          operator: Exists
-        # Mark the pod as a critical add-on for rescheduling.
-        - key: CriticalAddonsOnly
-          operator: Exists
-        - effect: NoExecute
-          operator: Exists
+      - operator: Exists
+        effect: NoSchedule
      serviceAccountName: flannel
      initContainers:
      - name: install-cni-plugins
@ -170,7 +221,9 @@ spec:
            cpu: "100m"
            memory: "50Mi"
        securityContext:
-          privileged: true
+          privileged: false
+          capabilities:
+             add: ["NET_ADMIN"]
        env:
        - name: POD_NAME
          valueFrom:
@ -182,7 +235,7 @@ spec:
              fieldPath: metadata.namespace
        volumeMounts:
        - name: run
-          mountPath: /run
+          mountPath: /run/flannel
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      volumes:
@ -191,7 +244,7 @@ spec:
            path: /opt/cni/bin
        - name: run
          hostPath:
-            path: /run
+            path: /run/flannel
        - name: cni
          hostPath:
            path: /etc/cni/net.d
@ -200,6 +253,7 @@ spec:
            name: kube-flannel-cfg
 EOF
    }
+    set -x

    if [ "$MASTER_INDEX" = "0" ]; then

--- a/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh
+++ b/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh
@ -3,8 +3,8 @@
 step="kube-apiserver-to-kubelet-role"
 printf "Starting to run ${step}\n"

+set +x
 . /etc/sysconfig/heat-params
-
 set -x

 echo "Waiting for Kubernetes API..."
@ -80,6 +80,67 @@ EOF
 }
 kubectl apply --validate=false -f ${ADMIN_RBAC}

+POD_SECURITY_POLICIES=/srv/magnum/kubernetes/podsecuritypolicies.yaml
+# Pod Security Policies
+[ -f ${POD_SECURITY_POLICIES} ] || {
+    echo "Writing File: $POD_SECURITY_POLICIES"
+    mkdir -p $(dirname ${POD_SECURITY_POLICIES})
+    cat > ${POD_SECURITY_POLICIES} <<EOF
+---
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: magnum.privileged
+  annotations:
+    kubernetes.io/description: 'privileged allows full unrestricted access to
+      pod features, as if the PodSecurityPolicy controller was not enabled.'
+    seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*'
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  privileged: true
+  allowPrivilegeEscalation: true
+  allowedCapabilities:
+  - '*'
+  volumes:
+  - '*'
+  hostNetwork: true
+  hostPorts:
+  - min: 0
+    max: 65535
+  hostIPC: true
+  hostPID: true
+  runAsUser:
+    rule: 'RunAsAny'
+  seLinux:
+    rule: 'RunAsAny'
+  supplementalGroups:
+    rule: 'RunAsAny'
+  fsGroup:
+    rule: 'RunAsAny'
+  readOnlyRootFilesystem: false
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: magnum:podsecuritypolicy:privileged
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+rules:
+- apiGroups:
+  - policy
+  resourceNames:
+  - magnum.privileged
+  resources:
+  - podsecuritypolicies
+  verbs:
+  - use
+EOF
+}
+kubectl apply -f ${POD_SECURITY_POLICIES}
+
 # Add the openstack trustee as a secret under kube-system
 kubectl -n kube-system create secret generic os-trustee \
    --from-literal=os-authURL=${AUTH_URL} \
--- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml
+++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml
@ -382,7 +382,7 @@ parameters:
  flannel_tag:
    type: string
    description: tag of the flannel container
-    default: v0.10.0-amd64
+    default: v0.11.0-amd64

  flannel_cni_tag:
    type: string
--- a/releasenotes/notes/podsecuritypolicy-2400063d73524e06.yaml
+++ b/releasenotes/notes/podsecuritypolicy-2400063d73524e06.yaml
@ -0,0 +1,6 @@
+---
+features:
+  - |
+    k8s_fedora_atomic_v1 Add PodSecurityPolicy for privileged pods. Use
+    privileged PSP for calico and node-problem-detector. Add PSP for flannel
+    from upstream.