diff --git a/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh b/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh index 2769c767c9..0d041e481b 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/calico-service.sh @@ -14,6 +14,23 @@ if [ "$NETWORK_DRIVER" = "calico" ]; then mkdir -p $(dirname ${CALICO_DEPLOY}) cat << EOF > ${CALICO_DEPLOY} --- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: magnum:podsecuritypolicy:calico + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/cluster-service: "true" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: magnum:podsecuritypolicy:privileged +subjects: +- kind: ServiceAccount + name: calico-node + namespace: kube-system +--- # Calico Version v3.3.6 # https://docs.projectcalico.org/v3.3/releases#v3.3.6 kind: ClusterRole diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh index fd07eb83f0..6b9014d405 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh @@ -97,6 +97,7 @@ sed -i ' ' /etc/kubernetes/config KUBE_API_ARGS="--runtime-config=api/all=true" +KUBE_API_ARGS="$KUBE_API_ARGS --allow-privileged=$KUBE_ALLOW_PRIV" KUBE_API_ARGS="$KUBE_API_ARGS --kubelet-preferred-address-types=InternalIP,Hostname,ExternalIP" KUBE_API_ARGS="$KUBE_API_ARGS $KUBEAPI_OPTIONS" if [ "$TLS_DISABLED" == "True" ]; then diff --git a/magnum/drivers/common/templates/kubernetes/fragments/enable-auto-healing.sh b/magnum/drivers/common/templates/kubernetes/fragments/enable-auto-healing.sh index 04a46bbb25..be442ea783 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/enable-auto-healing.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-auto-healing.sh @@ -27,6 +27,23 @@ metadata: addonmanager.kubernetes.io/mode: Reconcile --- apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: magnum:podsecuritypolicy:node-problem-detector + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/cluster-service: "true" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: magnum:podsecuritypolicy:privileged +subjects: +- kind: ServiceAccount + name: node-problem-detector + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: npd-binding diff --git a/magnum/drivers/common/templates/kubernetes/fragments/flannel-service.sh b/magnum/drivers/common/templates/kubernetes/fragments/flannel-service.sh index beb6f5e930..01e9077d76 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/flannel-service.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/flannel-service.sh @@ -1,9 +1,11 @@ -#!/bin/sh +#!/bin/bash +set -e +set +x . /etc/sysconfig/heat-params - set -x + if [ "$NETWORK_DRIVER" = "flannel" ]; then _prefix=${CONTAINER_INFRA_PREFIX:-quay.io/coreos/} FLANNEL_DEPLOY=/srv/magnum/kubernetes/manifests/flannel-deploy.yaml @@ -11,13 +13,65 @@ if [ "$NETWORK_DRIVER" = "flannel" ]; then [ -f ${FLANNEL_DEPLOY} ] || { echo "Writing File: $FLANNEL_DEPLOY" mkdir -p "$(dirname ${FLANNEL_DEPLOY})" + set +x cat << EOF > ${FLANNEL_DEPLOY} --- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: psp.flannel.unprivileged + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default + seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default + apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default + apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default +spec: + privileged: false + volumes: + - configMap + - secret + - emptyDir + - hostPath + allowedHostPaths: + - pathPrefix: "/etc/cni/net.d" + - pathPrefix: "/etc/kube-flannel" + - pathPrefix: "/run/flannel" + readOnlyRootFilesystem: false + # Users and groups + runAsUser: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + fsGroup: + rule: RunAsAny + # Privilege Escalation + allowPrivilegeEscalation: false + defaultAllowPrivilegeEscalation: false + # Capabilities + allowedCapabilities: ['NET_ADMIN'] + defaultAddCapabilities: [] + requiredDropCapabilities: [] + # Host namespaces + hostPID: false + hostIPC: false + hostNetwork: true + hostPorts: + - min: 0 + max: 65535 + # SELinux + seLinux: + # SELinux is unsed in CaaSP + rule: 'RunAsAny' +--- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: name: flannel rules: + - apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: ['psp.flannel.unprivileged'] - apiGroups: - "" resources: @@ -101,7 +155,7 @@ data: echo "Wrote CNI binaries to /host/opt/cni/bin/"; fi; --- -apiVersion: extensions/v1beta1 +apiVersion: apps/v1 kind: DaemonSet metadata: name: kube-flannel-ds-amd64 @@ -110,6 +164,9 @@ metadata: tier: node app: flannel spec: + selector: + matchLabels: + app: flannel template: metadata: labels: @@ -120,14 +177,8 @@ spec: nodeSelector: beta.kubernetes.io/arch: amd64 tolerations: - # Make sure flannel gets scheduled on all nodes. - - effect: NoSchedule - operator: Exists - # Mark the pod as a critical add-on for rescheduling. - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - operator: Exists + - operator: Exists + effect: NoSchedule serviceAccountName: flannel initContainers: - name: install-cni-plugins @@ -170,7 +221,9 @@ spec: cpu: "100m" memory: "50Mi" securityContext: - privileged: true + privileged: false + capabilities: + add: ["NET_ADMIN"] env: - name: POD_NAME valueFrom: @@ -182,7 +235,7 @@ spec: fieldPath: metadata.namespace volumeMounts: - name: run - mountPath: /run + mountPath: /run/flannel - name: flannel-cfg mountPath: /etc/kube-flannel/ volumes: @@ -191,7 +244,7 @@ spec: path: /opt/cni/bin - name: run hostPath: - path: /run + path: /run/flannel - name: cni hostPath: path: /etc/cni/net.d @@ -200,6 +253,7 @@ spec: name: kube-flannel-cfg EOF } + set -x if [ "$MASTER_INDEX" = "0" ]; then diff --git a/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh b/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh index f0d0f4623d..74bf6ef750 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/kube-apiserver-to-kubelet-role.sh @@ -3,8 +3,8 @@ step="kube-apiserver-to-kubelet-role" printf "Starting to run ${step}\n" +set +x . /etc/sysconfig/heat-params - set -x echo "Waiting for Kubernetes API..." @@ -80,6 +80,67 @@ EOF } kubectl apply --validate=false -f ${ADMIN_RBAC} +POD_SECURITY_POLICIES=/srv/magnum/kubernetes/podsecuritypolicies.yaml +# Pod Security Policies +[ -f ${POD_SECURITY_POLICIES} ] || { + echo "Writing File: $POD_SECURITY_POLICIES" + mkdir -p $(dirname ${POD_SECURITY_POLICIES}) + cat > ${POD_SECURITY_POLICIES} <