Uplift k8s API server and fix 'No API token' issue
After enablement of etcd encryption in https://review.opendev.org/628290 for promenade, bootstrap-armada pod should be run with '--experimental-encryption-provider-config' service parameter to avoid issue: 'ERROR: No API token found for service account "airship-ucp-ceph-osd-test"' Change-Id: Ib9bf1fa7333874b2d88db84019b26a2691a7d18a
This commit is contained in:
parent
00cbd30abf
commit
f08a454b45
|
@ -68,26 +68,37 @@ metadata:
|
||||||
name: common-addresses
|
name: common-addresses
|
||||||
path: .kubernetes.service_cidr
|
path: .kubernetes.service_cidr
|
||||||
dest:
|
dest:
|
||||||
path: .apiserver.command_prefix[1]
|
path: .apiserver.arguments[2]
|
||||||
pattern: SERVICE_CIDR
|
pattern: SERVICE_CIDR
|
||||||
- src:
|
- src:
|
||||||
schema: pegleg/CommonAddresses/v1
|
schema: pegleg/CommonAddresses/v1
|
||||||
name: common-addresses
|
name: common-addresses
|
||||||
path: .kubernetes.service_node_port_range
|
path: .kubernetes.service_node_port_range
|
||||||
dest:
|
dest:
|
||||||
path: .apiserver.command_prefix[2]
|
path: .apiserver.arguments[3]
|
||||||
pattern: SERVICE_NODE_PORT_RANGE
|
pattern: SERVICE_NODE_PORT_RANGE
|
||||||
|
|
||||||
|
# Set etcd encryption policy
|
||||||
|
- src:
|
||||||
|
schema: promenade/EncryptionPolicy/v1
|
||||||
|
name: encryption-policy
|
||||||
|
path: .etcd
|
||||||
|
dest:
|
||||||
|
path: .apiserver.encryption
|
||||||
|
|
||||||
data:
|
data:
|
||||||
apiserver:
|
apiserver:
|
||||||
command_prefix:
|
arguments:
|
||||||
- /apiserver
|
- --authorization-mode=Node,RBAC
|
||||||
|
- --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit
|
||||||
- --service-cluster-ip-range=SERVICE_CIDR
|
- --service-cluster-ip-range=SERVICE_CIDR
|
||||||
- --service-node-port-range=SERVICE_NODE_PORT_RANGE
|
- --service-node-port-range=SERVICE_NODE_PORT_RANGE
|
||||||
- --authorization-mode=Node,RBAC
|
|
||||||
- --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds
|
|
||||||
- --endpoint-reconciler-type=lease
|
- --endpoint-reconciler-type=lease
|
||||||
- --feature-gates=PodShareProcessNamespace=true
|
- --feature-gates=PodShareProcessNamespace=true
|
||||||
|
- --v=3
|
||||||
|
- --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml
|
||||||
|
- --experimental-encryption-provider-config=/etc/kubernetes/apiserver/encryption_provider.yaml
|
||||||
|
- --requestheader-allowed-names='aggregator'
|
||||||
armada:
|
armada:
|
||||||
target_manifest: cluster-bootstrap
|
target_manifest: cluster-bootstrap
|
||||||
labels:
|
labels:
|
||||||
|
@ -118,3 +129,20 @@ data:
|
||||||
- path: /var/lib/anchor/calico-etcd-bootstrap
|
- path: /var/lib/anchor/calico-etcd-bootstrap
|
||||||
content: "# placeholder for triggering calico etcd bootstrapping\n# this file will be deleted"
|
content: "# placeholder for triggering calico etcd bootstrapping\n# this file will be deleted"
|
||||||
mode: 0644
|
mode: 0644
|
||||||
|
- path: /etc/genesis/apiserver/acconfig.yaml
|
||||||
|
mode: 0444
|
||||||
|
content: |
|
||||||
|
kind: AdmissionConfiguration
|
||||||
|
apiVersion: apiserver.k8s.io/v1alpha1
|
||||||
|
plugins:
|
||||||
|
- name: EventRateLimit
|
||||||
|
path: eventconfig.yaml
|
||||||
|
- path: /etc/genesis/apiserver/eventconfig.yaml
|
||||||
|
mode: 0444
|
||||||
|
content: |
|
||||||
|
kind: Configuration
|
||||||
|
apiVersion: eventratelimit.admission.k8s.io/v1alpha1
|
||||||
|
limits:
|
||||||
|
- type: Server
|
||||||
|
qps: 1000
|
||||||
|
burst: 10000
|
||||||
|
|
|
@ -0,0 +1,50 @@
|
||||||
|
---
|
||||||
|
schema: deckhand/DataSchema/v1
|
||||||
|
metadata:
|
||||||
|
schema: metadata/Control/v1
|
||||||
|
name: promenade/EncryptionPolicy/v1
|
||||||
|
labels:
|
||||||
|
application: promenade
|
||||||
|
data:
|
||||||
|
$schema: http://json-schema.org/schema#
|
||||||
|
|
||||||
|
definitions:
|
||||||
|
script_encryption:
|
||||||
|
oneof:
|
||||||
|
- { $ref: '#/definitions/encryption_method_gpg' }
|
||||||
|
|
||||||
|
etcd_encryption:
|
||||||
|
type: array
|
||||||
|
items:
|
||||||
|
type: object
|
||||||
|
additionalProperties: false
|
||||||
|
properties:
|
||||||
|
resources:
|
||||||
|
type: array
|
||||||
|
items:
|
||||||
|
type: string
|
||||||
|
providers:
|
||||||
|
type: array
|
||||||
|
items:
|
||||||
|
type: object
|
||||||
|
additionalProperties: true
|
||||||
|
encryption_method_gpg:
|
||||||
|
properties:
|
||||||
|
gpg:
|
||||||
|
type: object
|
||||||
|
additionalProperties: false
|
||||||
|
required:
|
||||||
|
- gpg
|
||||||
|
additionalProperties: false
|
||||||
|
|
||||||
|
properties:
|
||||||
|
etcd:
|
||||||
|
$ref: '#/definitions/etcd_encryption'
|
||||||
|
scripts:
|
||||||
|
properties:
|
||||||
|
genesis:
|
||||||
|
$ref: '#/definitions/script_encryption'
|
||||||
|
join:
|
||||||
|
$ref: '#/definitions/script_encryption'
|
||||||
|
additionalProperties: false
|
||||||
|
...
|
|
@ -67,10 +67,24 @@ data:
|
||||||
apiserver:
|
apiserver:
|
||||||
type: object
|
type: object
|
||||||
properties:
|
properties:
|
||||||
command_prefix:
|
arguments:
|
||||||
type: array
|
type: array
|
||||||
items:
|
items:
|
||||||
type: string
|
type: string
|
||||||
|
encryption:
|
||||||
|
type: array
|
||||||
|
items:
|
||||||
|
type: object
|
||||||
|
properties:
|
||||||
|
resources:
|
||||||
|
type: array
|
||||||
|
items:
|
||||||
|
type: string
|
||||||
|
providers:
|
||||||
|
type: array
|
||||||
|
items:
|
||||||
|
type: object
|
||||||
|
additionalProperties: true
|
||||||
additionalProperties: false
|
additionalProperties: false
|
||||||
|
|
||||||
files:
|
files:
|
||||||
|
|
|
@ -44,7 +44,7 @@ metadata:
|
||||||
name: common-addresses
|
name: common-addresses
|
||||||
path: .kubernetes.service_cidr
|
path: .kubernetes.service_cidr
|
||||||
dest:
|
dest:
|
||||||
path: .values.command_prefix[1]
|
path: .values.apiserver.arguments[1]
|
||||||
pattern: SERVICE_CIDR
|
pattern: SERVICE_CIDR
|
||||||
|
|
||||||
# Kubernetes Port Range
|
# Kubernetes Port Range
|
||||||
|
@ -53,7 +53,7 @@ metadata:
|
||||||
name: common-addresses
|
name: common-addresses
|
||||||
path: .kubernetes.service_node_port_range
|
path: .kubernetes.service_node_port_range
|
||||||
dest:
|
dest:
|
||||||
path: .values.command_prefix[2]
|
path: .values.apiserver.arguments[2]
|
||||||
pattern: SERVICE_NODE_PORT_RANGE
|
pattern: SERVICE_NODE_PORT_RANGE
|
||||||
|
|
||||||
# CA
|
# CA
|
||||||
|
@ -102,6 +102,14 @@ metadata:
|
||||||
dest:
|
dest:
|
||||||
path: .values.secrets.service_account.public_key
|
path: .values.secrets.service_account.public_key
|
||||||
|
|
||||||
|
# Encryption policy
|
||||||
|
- src:
|
||||||
|
schema: promenade/EncryptionPolicy/v1
|
||||||
|
name: encryption-policy
|
||||||
|
path: .etcd
|
||||||
|
dest:
|
||||||
|
path: .values.conf.encryption_provider.content.resources
|
||||||
|
|
||||||
data:
|
data:
|
||||||
chart_name: apiserver
|
chart_name: apiserver
|
||||||
release: kubernetes-apiserver
|
release: kubernetes-apiserver
|
||||||
|
@ -128,14 +136,41 @@ data:
|
||||||
# https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/
|
# https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/
|
||||||
# Possible values: VersionTLS10, VersionTLS11, VersionTLS12
|
# Possible values: VersionTLS10, VersionTLS11, VersionTLS12
|
||||||
tls-min-version: 'VersionTLS12'
|
tls-min-version: 'VersionTLS12'
|
||||||
command_prefix:
|
arguments:
|
||||||
- /apiserver
|
- --authorization-mode=Node,RBAC
|
||||||
- --service-cluster-ip-range=SERVICE_CIDR
|
- --service-cluster-ip-range=SERVICE_CIDR
|
||||||
- --service-node-port-range=SERVICE_NODE_PORT_RANGE
|
- --service-node-port-range=SERVICE_NODE_PORT_RANGE
|
||||||
- --authorization-mode=Node,RBAC
|
- --endpoint-reconciler-type=lease
|
||||||
- --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds
|
- --feature-gates=PodShareProcessNamespace=true
|
||||||
- --endpoint-reconciler-type=lease
|
- --v=3
|
||||||
- --feature-gates=PodShareProcessNamespace=true
|
conf:
|
||||||
|
encryption_provider:
|
||||||
|
file: encryption_provider.yaml
|
||||||
|
command_options:
|
||||||
|
- '--experimental-encryption-provider-config=/etc/kubernetes/apiserver/encryption_provider.yaml'
|
||||||
|
content:
|
||||||
|
kind: EncryptionConfig
|
||||||
|
apiVersion: v1
|
||||||
|
eventconfig:
|
||||||
|
file: eventconfig.yaml
|
||||||
|
content:
|
||||||
|
kind: Configuration
|
||||||
|
apiVersion: eventratelimit.admission.k8s.io/v1alpha1
|
||||||
|
limits:
|
||||||
|
- type: Server
|
||||||
|
qps: 100
|
||||||
|
burst: 1000
|
||||||
|
acconfig:
|
||||||
|
file: acconfig.yaml
|
||||||
|
command_options:
|
||||||
|
- '--enable-admission-plugins=PodSecurityPolicy,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit'
|
||||||
|
- '--admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml'
|
||||||
|
content:
|
||||||
|
kind: AdmissionConfiguration
|
||||||
|
apiVersion: apiserver.k8s.io/v1alpha1
|
||||||
|
plugins:
|
||||||
|
- name: EventRateLimit
|
||||||
|
path: eventconfig.yaml
|
||||||
dependencies:
|
dependencies:
|
||||||
- kubernetes-apiserver-htk
|
- kubernetes-apiserver-htk
|
||||||
---
|
---
|
||||||
|
|
|
@ -0,0 +1,27 @@
|
||||||
|
---
|
||||||
|
schema: promenade/EncryptionPolicy/v1
|
||||||
|
metadata:
|
||||||
|
schema: metadata/Document/v1
|
||||||
|
name: encryption-policy
|
||||||
|
layeringDefinition:
|
||||||
|
abstract: false
|
||||||
|
layer: global
|
||||||
|
storagePolicy: cleartext
|
||||||
|
substitutions:
|
||||||
|
- src:
|
||||||
|
schema: deckhand/Passphrase/v1
|
||||||
|
name: apiserver-encryption-key-key1
|
||||||
|
path: .
|
||||||
|
dest:
|
||||||
|
path: .etcd[0].providers[0].secretbox.keys[0].secret
|
||||||
|
data:
|
||||||
|
etcd:
|
||||||
|
- resources:
|
||||||
|
- 'secrets'
|
||||||
|
providers:
|
||||||
|
- secretbox:
|
||||||
|
keys:
|
||||||
|
- name: key1
|
||||||
|
secret: null
|
||||||
|
- identity: {}
|
||||||
|
...
|
|
@ -4,7 +4,7 @@ data:
|
||||||
kubernetes:
|
kubernetes:
|
||||||
apiserver:
|
apiserver:
|
||||||
location: https://opendev.org/airship/promenade
|
location: https://opendev.org/airship/promenade
|
||||||
reference: 44b5fae04788c6a28de0f9a2e132204561474d47
|
reference: 32a6c15ffd6c283375bfd1cc9ae82f9232a9b501
|
||||||
subpath: charts/apiserver
|
subpath: charts/apiserver
|
||||||
type: git
|
type: git
|
||||||
apiserver-htk:
|
apiserver-htk:
|
||||||
|
@ -560,6 +560,7 @@ data:
|
||||||
apiserver:
|
apiserver:
|
||||||
anchor: gcr.io/google-containers/hyperkube-amd64:v1.11.6
|
anchor: gcr.io/google-containers/hyperkube-amd64:v1.11.6
|
||||||
apiserver: gcr.io/google-containers/hyperkube-amd64:v1.11.6
|
apiserver: gcr.io/google-containers/hyperkube-amd64:v1.11.6
|
||||||
|
key_rotate: gcr.io/google-containers/hyperkube-amd64:v1.11.6
|
||||||
controller-manager:
|
controller-manager:
|
||||||
anchor: gcr.io/google-containers/hyperkube-amd64:v1.11.6
|
anchor: gcr.io/google-containers/hyperkube-amd64:v1.11.6
|
||||||
controller_manager: gcr.io/google-containers/hyperkube-amd64:v1.11.6
|
controller_manager: gcr.io/google-containers/hyperkube-amd64:v1.11.6
|
||||||
|
|
|
@ -0,0 +1,12 @@
|
||||||
|
---
|
||||||
|
schema: deckhand/Passphrase/v1
|
||||||
|
metadata:
|
||||||
|
schema: metadata/Document/v1
|
||||||
|
name: apiserver-encryption-key-key1
|
||||||
|
layeringDefinition:
|
||||||
|
abstract: false
|
||||||
|
layer: site
|
||||||
|
storagePolicy: cleartext
|
||||||
|
# head -c 32 /dev/urandom | base64
|
||||||
|
data: /Y8HgBo/rZywuyF3yE3c1mi4bOWanR6FeC+7f6fS8IE=
|
||||||
|
...
|
|
@ -0,0 +1,12 @@
|
||||||
|
---
|
||||||
|
schema: deckhand/Passphrase/v1
|
||||||
|
metadata:
|
||||||
|
schema: metadata/Document/v1
|
||||||
|
name: apiserver-encryption-key-key1
|
||||||
|
layeringDefinition:
|
||||||
|
abstract: false
|
||||||
|
layer: site
|
||||||
|
storagePolicy: cleartext
|
||||||
|
# head -c 32 /dev/urandom | base64
|
||||||
|
data: AH/KZrduGOc8NRs5Dkp1maqaOrVY+HZ9pAD/fCweMqw=
|
||||||
|
...
|
|
@ -0,0 +1,12 @@
|
||||||
|
---
|
||||||
|
schema: deckhand/Passphrase/v1
|
||||||
|
metadata:
|
||||||
|
schema: metadata/Document/v1
|
||||||
|
name: apiserver-encryption-key-key1
|
||||||
|
layeringDefinition:
|
||||||
|
abstract: false
|
||||||
|
layer: site
|
||||||
|
storagePolicy: cleartext
|
||||||
|
# head -c 32 /dev/urandom | base64
|
||||||
|
data: bL2mHd9Sf5hQvZPuDncZRugYYqYyR3cGcZKVJ9wjswg=
|
||||||
|
...
|
|
@ -0,0 +1,13 @@
|
||||||
|
---
|
||||||
|
schema: deckhand/Passphrase/v1
|
||||||
|
metadata:
|
||||||
|
schema: metadata/Document/v1
|
||||||
|
name: apiserver-encryption-key-key1
|
||||||
|
layeringDefinition:
|
||||||
|
abstract: false
|
||||||
|
layer: site
|
||||||
|
storagePolicy: cleartext
|
||||||
|
# https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/
|
||||||
|
# use head -c 32 /dev/urandom | base64
|
||||||
|
data: n9VBwseT/JjV7r9vbUR/MvCobe01Bdh9XtWgsNF5zLY=
|
||||||
|
...
|
Loading…
Reference in New Issue