Support Keystone AuthN and AuthZ for k8s

Now cloud-provider-openstack of Kubernetes has a webhook to support
Keystone authorization and authentication. With this feature, user
can use a new label 'keystone-auth-enabled' to enable the keystone
authN and authZ.

DocImpact
Task: 21637
Story: 1755770

Change-Id: I3d21ad8f55c0d7308a302f62db9e9af147a604f8

doc/source/user/index.rst

@@ -369,6 +369,10 @@ the table are linked to more details elsewhere in the user guide.
 | `service_cluster_ip_range`            | IPv4 CIDR for k8s  | 10.254.0.0/16 |
 |                                       | service portals    |               |
 +---------------------------------------+--------------------+---------------+
+| `keystone_auth_enabled`_              | see below          | false         |
++---------------------------------------+--------------------+---------------+
+| `k8s_keystone_auth_tag`_              | see below          | see below     |
++---------------------------------------+--------------------+---------------+
 
 Cluster
 -------
@@ -1183,6 +1187,16 @@ _`cloud_provider_enabled`
   'volume_driver', it is implied that the cloud provider will be enabled since
   they are combined.
 
+_`keystone_auth_enabled`
+  If this label is set to True, Kubernetes will support use Keystone for
+  authorization and authentication.
+
+_`k8s_keystone_auth_tag`
+  This label allows users to select `a specific k8s_keystone_auth
+  version, based on its container tag
+  <https://hub.docker.com/r/k8scloudprovider/k8s-keystone-auth/tags/>`_.
+  Stein-default: 1.13.0
+
 External load balancer for services
 -----------------------------------
 
@@ -1253,6 +1267,39 @@ in a Cluster
 <https://kubernetes.io/docs/tasks/administer-cluster/dns-horizontal-autoscaling/#tuning-autoscaling-parameters>`_
 for more info.
 
+Keystone authN and authZ
+------------------------
+
+Now `cloud-provider-openstack
+<https://github.com/kubernetes/cloud-provider-openstack>`_
+provides a good webhook between OpenStack Keystone and Kubernetes, so that
+user can do authorization and authentication with a Keystone user/role against
+the Kubernetes cluster. If label `keystone-auth-enabled` is set True, then
+user can use their OpenStack credentials and roles to access resources in
+Kubernetes.
+
+Assume you have already got the configs with command
+`eval $(openstack coe cluster config <cluster ID>)`, then to configure the
+kubectl client, the following commands are needed:
+
+1. Run `kubectl config set-credentials openstackuser --auth-provider=openstack`
+
+2. Run `kubectl config set-context --cluster=<your cluster name>
+   --user=openstackuser openstackuser@kubernetes`
+
+3. Run `kubectl config use-context openstackuser@kubernetes` to activate the
+   context
+
+
+**NOTE:** Please make sure the version of kubectl is 1.8+ and make sure
+OS_DOMAIN_NAME is included in the rc file.
+
+Now try `kubectl get pods`, you should be able to see response from Kubernetes
+based on current user's role.
+
+Please refer the doc of `k8s-keystone-auth in cloud-provider-openstack
+<https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/using-keystone-webhook-authenticator-and-authorizer.md>`_
+for more information.
 
 Swarm
 =====

magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh

@@ -79,6 +79,36 @@ if [ -n "$TRUST_ID" ] && [ "$(echo "${CLOUD_PROVIDER_ENABLED}" | tr '[:upper:]'
     KUBE_API_ARGS="$KUBE_API_ARGS --cloud-provider=external"
 fi
 
+if [ "$KEYSTONE_AUTH_ENABLED" == "True" ]; then
+    KEYSTONE_WEBHOOK_CONFIG=/etc/kubernetes/keystone_webhook_config.yaml
+
+    [ -f ${KEYSTONE_WEBHOOK_CONFIG} ] || {
+echo "Writing File: $KEYSTONE_WEBHOOK_CONFIG"
+mkdir -p $(dirname ${KEYSTONE_WEBHOOK_CONFIG})
+cat << EOF > ${KEYSTONE_WEBHOOK_CONFIG}
+---
+apiVersion: v1
+kind: Config
+preferences: {}
+clusters:
+  - cluster:
+      insecure-skip-tls-verify: true
+      server: https://127.0.0.1:8443/webhook
+    name: webhook
+users:
+  - name: webhook
+contexts:
+  - context:
+      cluster: webhook
+      user: webhook
+    name: webhook
+current-context: webhook
+EOF
+}
+    KUBE_API_ARGS="$KUBE_API_ARGS --authentication-token-webhook-config-file=/etc/kubernetes/keystone_webhook_config.yaml --authorization-webhook-config-file=/etc/kubernetes/keystone_webhook_config.yaml"
+    webhook_auth="--authorization-mode=Node,Webhook,RBAC"
+    KUBE_API_ARGS=${KUBE_API_ARGS/--authorization-mode=Node,RBAC/$webhook_auth}
+fi
 
 sed -i '
     /^KUBE_API_ADDRESS=/ s/=.*/="'"${KUBE_API_ADDRESS}"'"/

magnum/drivers/common/templates/kubernetes/fragments/enable-keystone-auth.sh

@@ -0,0 +1,185 @@
+#!/bin/sh
+
+. /etc/sysconfig/heat-params
+
+step="enable-keystone-auth"
+printf "Starting to run ${step}\n"
+
+if [ "$(echo $KEYSTONE_AUTH_ENABLED | tr '[:upper:]' '[:lower:]')" != "false" ]; then
+    _prefix=${CONTAINER_INFRA_PREFIX:-docker.io/k8scloudprovider/}
+    CERT_DIR=/etc/kubernetes/certs
+
+    # Create policy configmap for keystone auth
+    KEYSTONE_AUTH_POLICY=/srv/magnum/kubernetes/keystone-auth-policy.yaml
+
+    [ -f ${KEYSTONE_AUTH_POLICY} ] || {
+        echo "Writing File: $KEYSTONE_AUTH_POLICY"
+        mkdir -p $(dirname ${KEYSTONE_AUTH_POLICY})
+        cat << EOF > ${KEYSTONE_AUTH_POLICY}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: k8s-keystone-auth
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  labels:
+    kubernetes.io/bootstrapping: rbac-defaults
+  name: system:k8s-keystone-auth
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  - services
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  annotations:
+    rbac.authorization.kubernetes.io/autoupdate: "true"
+  labels:
+    kubernetes.io/bootstrapping: rbac-defaults
+  name: system:k8s-keystone-auth
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:k8s-keystone-auth
+subjects:
+- kind: ServiceAccount
+  name: k8s-keystone-auth
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: k8s-keystone-auth-policy
+  namespace: kube-system
+data:
+  policies: |
+    [
+      {
+        "resource": {
+          "verbs": ["list"],
+          "resources": ["pods", "services", "deployments", "pvc"],
+          "version": "*",
+          "namespace": "default"
+        },
+        "match": [
+          {
+            "type": "role",
+            "values": ["member"]
+          },
+          {
+            "type": "project",
+            "values": ["$PROJECT_ID"]
+          }
+        ]
+      }
+    ]
+EOF
+    }
+
+    # Generate k8s-keystone-auth service manifest file
+    KEYSTONE_AUTH_DEPLOY=/srv/magnum/kubernetes/manifests/k8s-keystone-auth.yaml
+
+    [ -f ${KEYSTONE_AUTH_DEPLOY} ] || {
+        echo "Writing File: $KEYSTONE_AUTH_DEPLOY"
+        mkdir -p $(dirname ${KEYSTONE_AUTH_DEPLOY})
+        cat << EOF > ${KEYSTONE_AUTH_DEPLOY}
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  labels:
+    component: k8s-keystone-auth
+    tier: control-plane
+  name: k8s-keystone-auth
+  namespace: kube-system
+spec:
+  # The controllers can only have a single active instance.
+  template:
+    metadata:
+      name: k8s-keystone-auth
+      namespace: kube-system
+      labels:
+        k8s-app: k8s-keystone-auth
+    spec:
+      serviceAccountName: k8s-keystone-auth
+      tolerations:
+        # Make sure the pod can be scheduled on master kubelet.
+        - effect: NoSchedule
+          operator: Exists
+        # Mark the pod as a critical add-on for rescheduling.
+        - key: CriticalAddonsOnly
+          operator: Exists
+        - effect: NoExecute
+          operator: Exists
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      containers:
+        - name: k8s-keystone-auth
+          image: ${_prefix}k8s-keystone-auth:${K8S_KEYSTONE_AUTH_TAG}
+          imagePullPolicy: Always
+          args:
+            - ./bin/k8s-keystone-auth
+            - --tls-cert-file
+            - ${CERT_DIR}/server.crt
+            - --tls-private-key-file
+            - ${CERT_DIR}/server.key
+            - --policy-configmap-name
+            - k8s-keystone-auth-policy
+            - --keystone-url
+            - ${AUTH_URL}
+            - --keystone-ca-file
+            - /etc/kubernetes/ca-bundle.crt
+            - --listen
+            - 127.0.0.1:8443
+          volumeMounts:
+            - mountPath: ${CERT_DIR}
+              name: k8s-certs
+              readOnly: true
+            - mountPath: /etc/kubernetes
+              name: ca-certs
+              readOnly: true
+          resources:
+            requests:
+              cpu: 200m
+          ports:
+            - containerPort: 8443
+              hostPort: 8443
+              name: https
+              protocol: TCP
+      hostNetwork: true
+      volumes:
+        - hostPath:
+            path: ${CERT_DIR}
+            type: DirectoryOrCreate
+          name: k8s-certs
+        - hostPath:
+            path: /etc/kubernetes
+            type: DirectoryOrCreate
+          name: ca-certs
+EOF
+    }
+
+    until  [ "ok" = "$(curl --silent http://127.0.0.1:8080/healthz)" ]
+    do
+        echo "Waiting for Kubernetes API..."
+        sleep 5
+    done
+
+    /usr/bin/kubectl apply -f ${KEYSTONE_AUTH_POLICY}
+    /usr/bin/kubectl apply -f ${KEYSTONE_AUTH_DEPLOY}
+
+fi
+
+printf "Finished running ${step}\n"

magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml

@@ -78,3 +78,6 @@ write_files:
       PROMETHEUS_TAG="$PROMETHEUS_TAG"
       GRAFANA_TAG="$GRAFANA_TAG"
       HEAT_CONTAINER_AGENT_TAG="$HEAT_CONTAINER_AGENT_TAG"
+      KEYSTONE_AUTH_ENABLED="$KEYSTONE_AUTH_ENABLED"
+      K8S_KEYSTONE_AUTH_TAG="$K8S_KEYSTONE_AUTH_TAG"
+      PROJECT_ID="$PROJECT_ID"

magnum/drivers/heat/k8s_fedora_template_def.py

@@ -113,7 +113,8 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition):
                       'cloud_provider_tag',
                       'prometheus_tag',
                       'grafana_tag',
-                      'heat_container_agent_tag']
+                      'heat_container_agent_tag',
+                      'keystone_auth_enabled', 'k8s_keystone_auth_tag']
 
         for label in label_list:
             label_value = cluster.labels.get(label)
@@ -135,6 +136,8 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition):
                 ca_cert.get_private_key(),
                 ca_cert.get_private_key_passphrase()).replace("\n", "\\n")
 
+        extra_params['project_id'] = cluster.project_id
+
         return super(K8sFedoraTemplateDefinition,
                      self).get_params(context, cluster_template, cluster,
                                       extra_params=extra_params,

magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml

@@ -519,6 +519,23 @@ parameters:
     description: tag of the heat_container_agent system container
     default: stein-dev
 
+  keystone_auth_enabled:
+    type: boolean
+    description: >
+      true if the keystone authN and authZ should be enabled
+    default:
+      false
+
+  k8s_keystone_auth_tag:
+    type: string
+    description: tag of the k8s_keystone_auth container
+    default: 1.13.0
+
+  project_id:
+    type: string
+    description: >
+      project id of current project
+
 resources:
 
   ######################################################################
@@ -745,6 +762,9 @@ resources:
           prometheus_tag: {get_param: prometheus_tag}
           grafana_tag: {get_param: grafana_tag}
           heat_container_agent_tag: {get_param: heat_container_agent_tag}
+          keystone_auth_enabled: {get_param: keystone_auth_enabled}
+          k8s_keystone_auth_tag: {get_param: k8s_keystone_auth_tag}
+          project_id: {get_param: project_id}
 
   kube_cluster_config:
     type: OS::Heat::SoftwareConfig
@@ -770,6 +790,7 @@ resources:
                   $enable-ingress-traefik: {get_file: ../../common/templates/kubernetes/fragments/enable-ingress-traefik.sh}
                 template: {get_file: ../../common/templates/kubernetes/fragments/enable-ingress-controller.sh}
             - get_file: ../../common/templates/kubernetes/fragments/kube-dashboard-service.sh
+            - get_file: ../../common/templates/kubernetes/fragments/enable-keystone-auth.sh
 
   kube_cluster_deploy:
     type: OS::Heat::SoftwareDeployment

magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml

@@ -398,6 +398,22 @@ parameters:
     type: string
     description: tag of the heat_container_agent system container
 
+  keystone_auth_enabled:
+    type: boolean
+    description: >
+      true if the keystone authN and authZ should be enabled
+    default:
+      false
+
+  k8s_keystone_auth_tag:
+    type: string
+    description: tag of the k8s_keystone_auth container
+
+  project_id:
+    type: string
+    description: >
+      project id of current project
+
 resources:
   ######################################################################
   #
@@ -499,6 +515,9 @@ resources:
             "$PROMETHEUS_TAG": {get_param: prometheus_tag}
             "$GRAFANA_TAG": {get_param: grafana_tag}
             "$HEAT_CONTAINER_AGENT_TAG": {get_param: heat_container_agent_tag}
+            "$KEYSTONE_AUTH_ENABLED": {get_param: keystone_auth_enabled}
+            "$K8S_KEYSTONE_AUTH_TAG": {get_param: k8s_keystone_auth_tag}
+            "$PROJECT_ID": {get_param: project_id}
 
   install_openstack_ca:
     type: OS::Heat::SoftwareConfig

magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py

@@ -112,6 +112,7 @@ class TestClusterConductorWithK8s(base.TestCase):
                        'service_cluster_ip_range': '10.254.0.0/16'},
             'master_flavor_id': 'master_flavor_id',
             'flavor_id': 'flavor_id',
+            'project_id': 'project_id',
         }
         self.context.user_name = 'fake_user'
         self.context.project_id = 'fake_tenant'
@@ -290,6 +291,7 @@ class TestClusterConductorWithK8s(base.TestCase):
             'kube_service_account_key': 'public_key',
             'kube_service_account_private_key': 'private_key',
             'portal_network_cidr': '10.254.0.0/16',
+            'project_id': 'project_id'
         }
         if missing_attr is not None:
             expected.pop(mapping[missing_attr], None)
@@ -410,6 +412,7 @@ class TestClusterConductorWithK8s(base.TestCase):
             'kube_service_account_key': 'public_key',
             'kube_service_account_private_key': 'private_key',
             'portal_network_cidr': '10.254.0.0/16',
+            'project_id': 'project_id'
         }
 
         self.assertEqual(expected, definition)
@@ -517,6 +520,7 @@ class TestClusterConductorWithK8s(base.TestCase):
             'kube_service_account_key': 'public_key',
             'kube_service_account_private_key': 'private_key',
             'portal_network_cidr': '10.254.0.0/16',
+            'project_id': 'project_id'
         }
         self.assertEqual(expected, definition)
         self.assertEqual(
@@ -951,6 +955,7 @@ class TestClusterConductorWithK8s(base.TestCase):
             'kube_service_account_key': 'public_key',
             'kube_service_account_private_key': 'private_key',
             'portal_network_cidr': '10.254.0.0/16',
+            'project_id': 'project_id'
         }
         self.assertEqual(expected, definition)
         self.assertEqual(

magnum/tests/unit/drivers/test_template_definition.py

@@ -409,6 +409,11 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
             'grafana_tag')
         heat_container_agent_tag = mock_cluster.labels.get(
             'heat_container_agent_tag')
+        keystone_auth_enabled = mock_cluster.labels.get(
+            'keystone_auth_enabled')
+        k8s_keystone_auth_tag = mock_cluster.labels.get(
+            'k8s_keystone_auth_tag')
+        project_id = mock_cluster.project_id
 
         k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
 
@@ -461,6 +466,9 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
             'prometheus_tag': prometheus_tag,
             'grafana_tag': grafana_tag,
             'heat_container_agent_tag': heat_container_agent_tag,
+            'keystone_auth_enabled': keystone_auth_enabled,
+            'k8s_keystone_auth_tag': k8s_keystone_auth_tag,
+            'project_id': project_id,
         }}
         mock_get_params.assert_called_once_with(mock_context,
                                                 mock_cluster_template,
@@ -588,6 +596,11 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
             'grafana_tag')
         heat_container_agent_tag = mock_cluster.labels.get(
             'heat_container_agent_tag')
+        keystone_auth_enabled = mock_cluster.labels.get(
+            'keystone_auth_enabled')
+        k8s_keystone_auth_tag = mock_cluster.labels.get(
+            'k8s_keystone_auth_tag')
+        project_id = mock_cluster.project_id
 
         k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
 
@@ -642,6 +655,9 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
             'prometheus_tag': prometheus_tag,
             'grafana_tag': grafana_tag,
             'heat_container_agent_tag': heat_container_agent_tag,
+            'keystone_auth_enabled': keystone_auth_enabled,
+            'k8s_keystone_auth_tag': k8s_keystone_auth_tag,
+            'project_id': project_id,
         }}
         mock_get_params.assert_called_once_with(mock_context,
                                                 mock_cluster_template,

releasenotes/notes/k8s-keystone-auth-6c88c1a2d406fb61.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    Now cloud-provider-openstack of Kubernetes has a webhook to support
+    Keystone authorization and authentication. With this feature, user can use
+    a new label 'keystone-auth-enabled' to enable the keystone authN and authZ.
+