airship/treasuremap
Code Issues Proposed changes

Uplift k8s API server and fix 'No API token' issue

Browse Source 
After enablement of etcd encryption in https://review.opendev.org/628290 for
promenade, bootstrap-armada pod should be run with
'--experimental-encryption-provider-config' service parameter to avoid issue:
'ERROR: No API token found for service account "airship-ucp-ceph-osd-test"'

Change-Id: Ib9bf1fa7333874b2d88db84019b26a2691a7d18a
This commit is contained in:
Alexander Noskov 2019-07-03 18:06:40 +00:00 committed by Kaspars Skels
parent 00cbd30abf
commit f08a454b45
10 changed files with 222 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
global/profiles/genesis.yaml
View File

@ -68,26 +68,37 @@ metadata:
name: common-addresses
path: .kubernetes.service_cidr
dest:
path: .apiserver.command_prefix[1]
path: .apiserver.arguments[2]
pattern: SERVICE_CIDR
- src:
schema: pegleg/CommonAddresses/v1
name: common-addresses
path: .kubernetes.service_node_port_range
dest:
path: .apiserver.command_prefix[2]
path: .apiserver.arguments[3]
pattern: SERVICE_NODE_PORT_RANGE
# Set etcd encryption policy
- src:
schema: promenade/EncryptionPolicy/v1
name: encryption-policy
path: .etcd
dest:
path: .apiserver.encryption
data:
apiserver:
command_prefix:
- /apiserver
arguments:
- --authorization-mode=Node,RBAC
- --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit
- --service-cluster-ip-range=SERVICE_CIDR
- --service-node-port-range=SERVICE_NODE_PORT_RANGE
- --authorization-mode=Node,RBAC
- --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds
- --endpoint-reconciler-type=lease
- --feature-gates=PodShareProcessNamespace=true
- --v=3
- --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml
- --experimental-encryption-provider-config=/etc/kubernetes/apiserver/encryption_provider.yaml
- --requestheader-allowed-names='aggregator'
armada:
target_manifest: cluster-bootstrap
labels:
@ -118,3 +129,20 @@ data:
- path: /var/lib/anchor/calico-etcd-bootstrap
content: "# placeholder for triggering calico etcd bootstrapping\n# this file will be deleted"
mode: 0644
- path: /etc/genesis/apiserver/acconfig.yaml
mode: 0444
content: |
kind: AdmissionConfiguration
apiVersion: apiserver.k8s.io/v1alpha1
plugins:
- name: EventRateLimit
path: eventconfig.yaml
- path: /etc/genesis/apiserver/eventconfig.yaml
mode: 0444
content: |
kind: Configuration
apiVersion: eventratelimit.admission.k8s.io/v1alpha1
limits:
- type: Server
qps: 1000
burst: 10000

50
global/schemas/promenade/EncryptionPolicy/v1.yaml Normal file
View File

@ -0,0 +1,50 @@
---
schema: deckhand/DataSchema/v1
metadata:
schema: metadata/Control/v1
name: promenade/EncryptionPolicy/v1
labels:
application: promenade
data:
$schema: http://json-schema.org/schema#
definitions:
script_encryption:
oneof:
- { $ref: '#/definitions/encryption_method_gpg' }
etcd_encryption:
type: array
items:
type: object
additionalProperties: false
properties:
resources:
type: array
items:
type: string
providers:
type: array
items:
type: object
additionalProperties: true
encryption_method_gpg:
properties:
gpg:
type: object
additionalProperties: false
required:
- gpg
additionalProperties: false
properties:
etcd:
$ref: '#/definitions/etcd_encryption'
scripts:
properties:
genesis:
$ref: '#/definitions/script_encryption'
join:
$ref: '#/definitions/script_encryption'
additionalProperties: false
...

16
global/schemas/promenade/Genesis/v1.yaml
View File

@ -67,10 +67,24 @@ data:
apiserver:
type: object
properties:
command_prefix:
arguments:
type: array
items:
type: string
encryption:
type: array
items:
type: object
properties:
resources:
type: array
items:
type: string
providers:
type: array
items:
type: object
additionalProperties: true
additionalProperties: false
files:

55
global/software/charts/kubernetes/core/apiserver.yaml
View File

@ -44,7 +44,7 @@ metadata:
name: common-addresses
path: .kubernetes.service_cidr
dest:
path: .values.command_prefix[1]
path: .values.apiserver.arguments[1]
pattern: SERVICE_CIDR
# Kubernetes Port Range
@ -53,7 +53,7 @@ metadata:
name: common-addresses
path: .kubernetes.service_node_port_range
dest:
path: .values.command_prefix[2]
path: .values.apiserver.arguments[2]
pattern: SERVICE_NODE_PORT_RANGE
# CA
@ -102,6 +102,14 @@ metadata:
dest:
path: .values.secrets.service_account.public_key
# Encryption policy
- src:
schema: promenade/EncryptionPolicy/v1
name: encryption-policy
path: .etcd
dest:
path: .values.conf.encryption_provider.content.resources
data:
chart_name: apiserver
release: kubernetes-apiserver
@ -128,14 +136,41 @@ data:
# https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/
# Possible values: VersionTLS10, VersionTLS11, VersionTLS12
tls-min-version: 'VersionTLS12'
command_prefix:
- /apiserver
- --service-cluster-ip-range=SERVICE_CIDR
- --service-node-port-range=SERVICE_NODE_PORT_RANGE
- --authorization-mode=Node,RBAC
- --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds
- --endpoint-reconciler-type=lease
- --feature-gates=PodShareProcessNamespace=true
arguments:
- --authorization-mode=Node,RBAC
- --service-cluster-ip-range=SERVICE_CIDR
- --service-node-port-range=SERVICE_NODE_PORT_RANGE
- --endpoint-reconciler-type=lease
- --feature-gates=PodShareProcessNamespace=true
- --v=3
conf:
encryption_provider:
file: encryption_provider.yaml
command_options:
- '--experimental-encryption-provider-config=/etc/kubernetes/apiserver/encryption_provider.yaml'
content:
kind: EncryptionConfig
apiVersion: v1
eventconfig:
file: eventconfig.yaml
content:
kind: Configuration
apiVersion: eventratelimit.admission.k8s.io/v1alpha1
limits:
- type: Server
qps: 100
burst: 1000
acconfig:
file: acconfig.yaml
command_options:
- '--enable-admission-plugins=PodSecurityPolicy,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit'
- '--admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml'
content:
kind: AdmissionConfiguration
apiVersion: apiserver.k8s.io/v1alpha1
plugins:
- name: EventRateLimit
path: eventconfig.yaml
dependencies:
- kubernetes-apiserver-htk
---

27
global/software/config/encryption.yaml Normal file
View File

@ -0,0 +1,27 @@
---
schema: promenade/EncryptionPolicy/v1
metadata:
schema: metadata/Document/v1
name: encryption-policy
layeringDefinition:
abstract: false
layer: global
storagePolicy: cleartext
substitutions:
- src:
schema: deckhand/Passphrase/v1
name: apiserver-encryption-key-key1
path: .
dest:
path: .etcd[0].providers[0].secretbox.keys[0].secret
data:
etcd:
- resources:
- 'secrets'
providers:
- secretbox:
keys:
- name: key1
secret: null
- identity: {}
...

3
global/software/config/versions.yaml
View File

@ -4,7 +4,7 @@ data:
kubernetes:
apiserver:
location: https://opendev.org/airship/promenade
reference: 44b5fae04788c6a28de0f9a2e132204561474d47
reference: 32a6c15ffd6c283375bfd1cc9ae82f9232a9b501
subpath: charts/apiserver
type: git
apiserver-htk:
@ -560,6 +560,7 @@ data:
apiserver:
anchor: gcr.io/google-containers/hyperkube-amd64:v1.11.6
apiserver: gcr.io/google-containers/hyperkube-amd64:v1.11.6
key_rotate: gcr.io/google-containers/hyperkube-amd64:v1.11.6
controller-manager:
anchor: gcr.io/google-containers/hyperkube-amd64:v1.11.6
controller_manager: gcr.io/google-containers/hyperkube-amd64:v1.11.6

12
site/aiab/secrets/passphrases/apiserver-encryption-key-key1.yaml Normal file
View File

@ -0,0 +1,12 @@
---
schema: deckhand/Passphrase/v1
metadata:
schema: metadata/Document/v1
name: apiserver-encryption-key-key1
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
# head -c 32 /dev/urandom | base64
data: /Y8HgBo/rZywuyF3yE3c1mi4bOWanR6FeC+7f6fS8IE=
...

12
site/airskiff/secrets/passphrases/apiserver-encryption-key-key1.yaml Normal file
View File

@ -0,0 +1,12 @@
---
schema: deckhand/Passphrase/v1
metadata:
schema: metadata/Document/v1
name: apiserver-encryption-key-key1
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
# head -c 32 /dev/urandom | base64
data: AH/KZrduGOc8NRs5Dkp1maqaOrVY+HZ9pAD/fCweMqw=
...

12
site/airsloop/secrets/passphrases/apiserver-encryption-key-key1.yaml Normal file
View File

@ -0,0 +1,12 @@
---
schema: deckhand/Passphrase/v1
metadata:
schema: metadata/Document/v1
name: apiserver-encryption-key-key1
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
# head -c 32 /dev/urandom | base64
data: bL2mHd9Sf5hQvZPuDncZRugYYqYyR3cGcZKVJ9wjswg=
...

13
site/seaworthy/secrets/passphrases/apiserver-encryption-key-key1.yaml Normal file
View File

@ -0,0 +1,13 @@
---
schema: deckhand/Passphrase/v1
metadata:
schema: metadata/Document/v1
name: apiserver-encryption-key-key1
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
# https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/
# use head -c 32 /dev/urandom | base64
data: n9VBwseT/JjV7r9vbUR/MvCobe01Bdh9XtWgsNF5zLY=
...