Browse Source

Uplift k8s API server and fix 'No API token' issue

After enablement of etcd encryption in https://review.opendev.org/628290 for
promenade, bootstrap-armada pod should be run with
'--experimental-encryption-provider-config' service parameter to avoid issue:
'ERROR: No API token found for service account "airship-ucp-ceph-osd-test"'

Change-Id: Ib9bf1fa7333874b2d88db84019b26a2691a7d18a
tags/v1.2
Alexander Noskov 3 months ago
parent
commit
f08a454b45

+ 34
- 6
global/profiles/genesis.yaml View File

@@ -68,26 +68,37 @@ metadata:
68 68
         name: common-addresses
69 69
         path: .kubernetes.service_cidr
70 70
       dest:
71
-        path: .apiserver.command_prefix[1]
71
+        path: .apiserver.arguments[2]
72 72
         pattern: SERVICE_CIDR
73 73
     - src:
74 74
         schema: pegleg/CommonAddresses/v1
75 75
         name: common-addresses
76 76
         path: .kubernetes.service_node_port_range
77 77
       dest:
78
-        path: .apiserver.command_prefix[2]
78
+        path: .apiserver.arguments[3]
79 79
         pattern: SERVICE_NODE_PORT_RANGE
80 80
 
81
+    # Set etcd encryption policy
82
+    - src:
83
+        schema: promenade/EncryptionPolicy/v1
84
+        name: encryption-policy
85
+        path: .etcd
86
+      dest:
87
+        path: .apiserver.encryption
88
+
81 89
 data:
82 90
   apiserver:
83
-    command_prefix:
84
-      - /apiserver
91
+    arguments:
92
+      - --authorization-mode=Node,RBAC
93
+      - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit
85 94
       - --service-cluster-ip-range=SERVICE_CIDR
86 95
       - --service-node-port-range=SERVICE_NODE_PORT_RANGE
87
-      - --authorization-mode=Node,RBAC
88
-      - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds
89 96
       - --endpoint-reconciler-type=lease
90 97
       - --feature-gates=PodShareProcessNamespace=true
98
+      - --v=3
99
+      - --admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml
100
+      - --experimental-encryption-provider-config=/etc/kubernetes/apiserver/encryption_provider.yaml
101
+      - --requestheader-allowed-names='aggregator'
91 102
   armada:
92 103
     target_manifest: cluster-bootstrap
93 104
   labels:
@@ -118,3 +129,20 @@ data:
118 129
     - path: /var/lib/anchor/calico-etcd-bootstrap
119 130
       content: "# placeholder for triggering calico etcd bootstrapping\n# this file will be deleted"
120 131
       mode: 0644
132
+    - path: /etc/genesis/apiserver/acconfig.yaml
133
+      mode: 0444
134
+      content: |
135
+        kind: AdmissionConfiguration
136
+        apiVersion: apiserver.k8s.io/v1alpha1
137
+        plugins:
138
+          - name: EventRateLimit
139
+            path: eventconfig.yaml
140
+    - path: /etc/genesis/apiserver/eventconfig.yaml
141
+      mode: 0444
142
+      content: |
143
+        kind: Configuration
144
+        apiVersion: eventratelimit.admission.k8s.io/v1alpha1
145
+        limits:
146
+          - type: Server
147
+            qps: 1000
148
+            burst: 10000

+ 50
- 0
global/schemas/promenade/EncryptionPolicy/v1.yaml View File

@@ -0,0 +1,50 @@
1
+---
2
+schema: deckhand/DataSchema/v1
3
+metadata:
4
+  schema: metadata/Control/v1
5
+  name: promenade/EncryptionPolicy/v1
6
+  labels:
7
+    application: promenade
8
+data:
9
+  $schema: http://json-schema.org/schema#
10
+
11
+  definitions:
12
+    script_encryption:
13
+      oneof:
14
+        - { $ref: '#/definitions/encryption_method_gpg' }
15
+
16
+    etcd_encryption:
17
+      type: array
18
+      items:
19
+        type: object
20
+        additionalProperties: false
21
+        properties:
22
+          resources:
23
+            type: array
24
+            items:
25
+              type: string
26
+          providers:
27
+            type: array
28
+            items:
29
+              type: object
30
+              additionalProperties: true
31
+    encryption_method_gpg:
32
+      properties:
33
+        gpg:
34
+          type: object
35
+          additionalProperties: false
36
+      required:
37
+        - gpg
38
+      additionalProperties: false
39
+
40
+  properties:
41
+    etcd:
42
+      $ref: '#/definitions/etcd_encryption'
43
+    scripts:
44
+      properties:
45
+        genesis:
46
+          $ref: '#/definitions/script_encryption'
47
+        join:
48
+          $ref: '#/definitions/script_encryption'
49
+      additionalProperties: false
50
+...

+ 15
- 1
global/schemas/promenade/Genesis/v1.yaml View File

@@ -67,10 +67,24 @@ data:
67 67
     apiserver:
68 68
       type: object
69 69
       properties:
70
-        command_prefix:
70
+        arguments:
71 71
           type: array
72 72
           items:
73 73
             type: string
74
+        encryption:
75
+          type: array
76
+          items:
77
+            type: object
78
+            properties:
79
+              resources:
80
+                type: array
81
+                items:
82
+                  type: string
83
+              providers:
84
+                type: array
85
+                items:
86
+                  type: object
87
+                  additionalProperties: true
74 88
       additionalProperties: false
75 89
 
76 90
     files:

+ 45
- 10
global/software/charts/kubernetes/core/apiserver.yaml View File

@@ -44,7 +44,7 @@ metadata:
44 44
         name: common-addresses
45 45
         path: .kubernetes.service_cidr
46 46
       dest:
47
-        path: .values.command_prefix[1]
47
+        path: .values.apiserver.arguments[1]
48 48
         pattern: SERVICE_CIDR
49 49
 
50 50
     # Kubernetes Port Range
@@ -53,7 +53,7 @@ metadata:
53 53
         name: common-addresses
54 54
         path: .kubernetes.service_node_port_range
55 55
       dest:
56
-        path: .values.command_prefix[2]
56
+        path: .values.apiserver.arguments[2]
57 57
         pattern: SERVICE_NODE_PORT_RANGE
58 58
 
59 59
     # CA
@@ -102,6 +102,14 @@ metadata:
102 102
       dest:
103 103
         path: .values.secrets.service_account.public_key
104 104
 
105
+    # Encryption policy
106
+    - src:
107
+        schema: promenade/EncryptionPolicy/v1
108
+        name: encryption-policy
109
+        path: .etcd
110
+      dest:
111
+        path: .values.conf.encryption_provider.content.resources
112
+
105 113
 data:
106 114
   chart_name: apiserver
107 115
   release: kubernetes-apiserver
@@ -128,14 +136,41 @@ data:
128 136
         # https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/
129 137
         # Possible values: VersionTLS10, VersionTLS11, VersionTLS12
130 138
         tls-min-version: 'VersionTLS12'
131
-    command_prefix:
132
-      - /apiserver
133
-      - --service-cluster-ip-range=SERVICE_CIDR
134
-      - --service-node-port-range=SERVICE_NODE_PORT_RANGE
135
-      - --authorization-mode=Node,RBAC
136
-      - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds
137
-      - --endpoint-reconciler-type=lease
138
-      - --feature-gates=PodShareProcessNamespace=true
139
+      arguments:
140
+        - --authorization-mode=Node,RBAC
141
+        - --service-cluster-ip-range=SERVICE_CIDR
142
+        - --service-node-port-range=SERVICE_NODE_PORT_RANGE
143
+        - --endpoint-reconciler-type=lease
144
+        - --feature-gates=PodShareProcessNamespace=true
145
+        - --v=3
146
+    conf:
147
+      encryption_provider:
148
+        file: encryption_provider.yaml
149
+        command_options:
150
+          - '--experimental-encryption-provider-config=/etc/kubernetes/apiserver/encryption_provider.yaml'
151
+        content:
152
+          kind: EncryptionConfig
153
+          apiVersion: v1
154
+      eventconfig:
155
+        file: eventconfig.yaml
156
+        content:
157
+          kind: Configuration
158
+          apiVersion: eventratelimit.admission.k8s.io/v1alpha1
159
+          limits:
160
+          - type: Server
161
+            qps: 100
162
+            burst: 1000
163
+      acconfig:
164
+        file: acconfig.yaml
165
+        command_options:
166
+          - '--enable-admission-plugins=PodSecurityPolicy,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds,NodeRestriction,EventRateLimit'
167
+          - '--admission-control-config-file=/etc/kubernetes/apiserver/acconfig.yaml'
168
+        content:
169
+          kind: AdmissionConfiguration
170
+          apiVersion: apiserver.k8s.io/v1alpha1
171
+          plugins:
172
+          - name: EventRateLimit
173
+            path: eventconfig.yaml
139 174
   dependencies:
140 175
     - kubernetes-apiserver-htk
141 176
 ---

+ 27
- 0
global/software/config/encryption.yaml View File

@@ -0,0 +1,27 @@
1
+---
2
+schema: promenade/EncryptionPolicy/v1
3
+metadata:
4
+  schema: metadata/Document/v1
5
+  name: encryption-policy
6
+  layeringDefinition:
7
+    abstract: false
8
+    layer: global
9
+  storagePolicy: cleartext
10
+  substitutions:
11
+    - src:
12
+        schema: deckhand/Passphrase/v1
13
+        name: apiserver-encryption-key-key1
14
+        path: .
15
+      dest:
16
+        path: .etcd[0].providers[0].secretbox.keys[0].secret
17
+data:
18
+  etcd:
19
+    - resources:
20
+        - 'secrets'
21
+      providers:
22
+        - secretbox:
23
+            keys:
24
+             - name: key1
25
+               secret: null
26
+        - identity: {}
27
+...

+ 2
- 1
global/software/config/versions.yaml View File

@@ -4,7 +4,7 @@ data:
4 4
     kubernetes:
5 5
       apiserver:
6 6
         location: https://opendev.org/airship/promenade
7
-        reference: 44b5fae04788c6a28de0f9a2e132204561474d47
7
+        reference: 32a6c15ffd6c283375bfd1cc9ae82f9232a9b501
8 8
         subpath: charts/apiserver
9 9
         type: git
10 10
       apiserver-htk:
@@ -560,6 +560,7 @@ data:
560 560
       apiserver:
561 561
         anchor: gcr.io/google-containers/hyperkube-amd64:v1.11.6
562 562
         apiserver: gcr.io/google-containers/hyperkube-amd64:v1.11.6
563
+        key_rotate: gcr.io/google-containers/hyperkube-amd64:v1.11.6
563 564
       controller-manager:
564 565
         anchor: gcr.io/google-containers/hyperkube-amd64:v1.11.6
565 566
         controller_manager: gcr.io/google-containers/hyperkube-amd64:v1.11.6

+ 12
- 0
site/aiab/secrets/passphrases/apiserver-encryption-key-key1.yaml View File

@@ -0,0 +1,12 @@
1
+---
2
+schema: deckhand/Passphrase/v1
3
+metadata:
4
+  schema: metadata/Document/v1
5
+  name: apiserver-encryption-key-key1
6
+  layeringDefinition:
7
+    abstract: false
8
+    layer: site
9
+  storagePolicy: cleartext
10
+# head -c 32 /dev/urandom | base64
11
+data: /Y8HgBo/rZywuyF3yE3c1mi4bOWanR6FeC+7f6fS8IE=
12
+...

+ 12
- 0
site/airskiff/secrets/passphrases/apiserver-encryption-key-key1.yaml View File

@@ -0,0 +1,12 @@
1
+---
2
+schema: deckhand/Passphrase/v1
3
+metadata:
4
+  schema: metadata/Document/v1
5
+  name: apiserver-encryption-key-key1
6
+  layeringDefinition:
7
+    abstract: false
8
+    layer: site
9
+  storagePolicy: cleartext
10
+# head -c 32 /dev/urandom | base64
11
+data: AH/KZrduGOc8NRs5Dkp1maqaOrVY+HZ9pAD/fCweMqw=
12
+...

+ 12
- 0
site/airsloop/secrets/passphrases/apiserver-encryption-key-key1.yaml View File

@@ -0,0 +1,12 @@
1
+---
2
+schema: deckhand/Passphrase/v1
3
+metadata:
4
+  schema: metadata/Document/v1
5
+  name: apiserver-encryption-key-key1
6
+  layeringDefinition:
7
+    abstract: false
8
+    layer: site
9
+  storagePolicy: cleartext
10
+# head -c 32 /dev/urandom | base64
11
+data: bL2mHd9Sf5hQvZPuDncZRugYYqYyR3cGcZKVJ9wjswg=
12
+...

+ 13
- 0
site/seaworthy/secrets/passphrases/apiserver-encryption-key-key1.yaml View File

@@ -0,0 +1,13 @@
1
+---
2
+schema: deckhand/Passphrase/v1
3
+metadata:
4
+  schema: metadata/Document/v1
5
+  name: apiserver-encryption-key-key1
6
+  layeringDefinition:
7
+    abstract: false
8
+    layer: site
9
+  storagePolicy: cleartext
10
+# https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/
11
+# use head -c 32 /dev/urandom | base64
12
+data: n9VBwseT/JjV7r9vbUR/MvCobe01Bdh9XtWgsNF5zLY=
13
+...

Loading…
Cancel
Save