Browse Source

Merge "Setup mirrored queues for celery in rabbit"

changes/86/628286/1
Zuul 6 months ago
parent
commit
4dd6b484d1

+ 2
- 0
charts/shipyard/templates/configmap-airflow-bin.yaml View File

@@ -21,6 +21,8 @@ kind: ConfigMap
21 21
 metadata:
22 22
   name: airflow-bin
23 23
 data:
24
+  rabbit-init.sh: |+
25
+{{- include "helm-toolkit.scripts.rabbit_init" . | indent 4 }}
24 26
   airflow-db-init.sh: |+
25 27
 {{ tuple "bin/_airflow-db-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
26 28
   airflow-db-sync.sh: |+

+ 20
- 0
charts/shipyard/templates/job-rabbit-init.yaml View File

@@ -0,0 +1,20 @@
1
+{{/*
2
+Copyright 2018 The Openstack-Helm Authors.
3
+
4
+Licensed under the Apache License, Version 2.0 (the "License");
5
+you may not use this file except in compliance with the License.
6
+You may obtain a copy of the License at
7
+
8
+   http://www.apache.org/licenses/LICENSE-2.0
9
+
10
+Unless required by applicable law or agreed to in writing, software
11
+distributed under the License is distributed on an "AS IS" BASIS,
12
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+See the License for the specific language governing permissions and
14
+limitations under the License.
15
+*/}}
16
+
17
+{{- if .Values.manifests.job_rabbit_init }}
18
+{{- $rmqJob := dict "envAll" . "serviceName" "airflow" -}}
19
+{{ $rmqJob | include "helm-toolkit.manifests.job_rabbit_init" }}
20
+{{- end }}

+ 28
- 0
charts/shipyard/templates/secret-rabbitmq.yaml View File

@@ -0,0 +1,28 @@
1
+{{/*
2
+Copyright 2018 The Openstack-Helm Authors.
3
+
4
+Licensed under the Apache License, Version 2.0 (the "License");
5
+you may not use this file except in compliance with the License.
6
+You may obtain a copy of the License at
7
+
8
+   http://www.apache.org/licenses/LICENSE-2.0
9
+
10
+Unless required by applicable law or agreed to in writing, software
11
+distributed under the License is distributed on an "AS IS" BASIS,
12
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+See the License for the specific language governing permissions and
14
+limitations under the License.
15
+*/}}
16
+
17
+{{- if .Values.manifests.secret_rabbitmq }}
18
+{{- $envAll := . }}
19
+{{- $secretName := index $envAll.Values.secrets.oslo_messaging "admin" }}
20
+---
21
+apiVersion: v1
22
+kind: Secret
23
+metadata:
24
+  name: {{ $secretName }}
25
+type: Opaque
26
+data:
27
+  RABBITMQ_CONNECTION: {{ tuple "oslo_messaging" "internal" "admin" "http" $envAll | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" | b64enc }}
28
+{{- end }}

+ 28
- 0
charts/shipyard/templates/secret-user-rabbitmq.yaml View File

@@ -0,0 +1,28 @@
1
+{{/*
2
+Copyright 2018 The Openstack-Helm Authors.
3
+
4
+Licensed under the Apache License, Version 2.0 (the "License");
5
+you may not use this file except in compliance with the License.
6
+You may obtain a copy of the License at
7
+
8
+   http://www.apache.org/licenses/LICENSE-2.0
9
+
10
+Unless required by applicable law or agreed to in writing, software
11
+distributed under the License is distributed on an "AS IS" BASIS,
12
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+See the License for the specific language governing permissions and
14
+limitations under the License.
15
+*/}}
16
+
17
+{{- if .Values.manifests.secret_rabbitmq }}
18
+{{- $envAll := . }}
19
+{{- $secretName := index $envAll.Values.secrets.oslo_messaging "airflow" }}
20
+---
21
+apiVersion: v1
22
+kind: Secret
23
+metadata:
24
+  name: {{ $secretName }}
25
+type: Opaque
26
+data:
27
+  RABBITMQ_CONNECTION: {{ tuple "oslo_messaging" "internal" "user" "amqp" $envAll | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" | b64enc }}
28
+{{- end }}

+ 35
- 1
charts/shipyard/values.yaml View File

@@ -37,6 +37,7 @@ images:
37 37
     shipyard_db_init: docker.io/postgres:9.5
38 38
     shipyard_db_sync: quay.io/airshipit/shipyard:latest
39 39
     airflow_db_init: docker.io/postgres:9.5
40
+    rabbit_init: docker.io/rabbitmq:3.7-management
40 41
     airflow_db_sync: quay.io/airshipit/airflow:latest
41 42
     ks_user: docker.io/openstackhelm/heat:ocata
42 43
     ks_service: docker.io/openstackhelm/heat:ocata
@@ -105,6 +106,10 @@ dependencies:
105 106
       services:
106 107
         - service: postgresql_airflow_db
107 108
           endpoint: internal
109
+    rabbit_init:
110
+      services:
111
+        - service: oslo_messaging
112
+          endpoint: internal
108 113
     airflow_db_sync:
109 114
       jobs:
110 115
         - airflow-db-init
@@ -143,6 +148,7 @@ dependencies:
143 148
           endpoint: internal
144 149
     airflow_server:
145 150
       jobs:
151
+        - airflow-rabbit-init
146 152
         - airflow-db-init
147 153
         - airflow-db-sync
148 154
       services:
@@ -316,17 +322,22 @@ endpoints:
316 322
   oslo_messaging:
317 323
     auth:
318 324
       user:
325
+        username: airflow
326
+        password: password
327
+      admin:
319 328
         username: rabbitmq
320 329
         password: password
321 330
     hosts:
322 331
       default: rabbitmq
323 332
     host_fqdn_override:
324 333
       default: null
325
-    path: /
334
+    path: /airflow
326 335
     scheme: amqp
327 336
     port:
328 337
       amqp:
329 338
         default: 5672
339
+      http:
340
+        default: 15672
330 341
   oslo_cache:
331 342
     hosts:
332 343
       default: memcached
@@ -340,6 +351,9 @@ secrets:
340 351
   identity:
341 352
     admin: shipyard-keystone-admin
342 353
     shipyard: shipyard-keystone-user
354
+  oslo_messaging:
355
+    admin: airflow-rabbitmq-admin
356
+    airflow: airflow-rabbitmq-user
343 357
   postgresql_shipyard_db:
344 358
     admin: shipyard-db-admin
345 359
     user: shipyard-db-user
@@ -381,6 +395,17 @@ conf:
381 395
     workflow_orchestrator:action_update_software: rule:admin_create
382 396
     workflow_orchestrator:action_redeploy_server: rule:admin_create
383 397
     workflow_orchestrator:action_relabel_nodes: rule:admin_create
398
+  rabbitmq:
399
+    # adding rmq policy to mirror messages from celery queues
400
+    policies:
401
+      - vhost: "/"
402
+        name: "ha_celery"
403
+        definition:
404
+          ha-mode: "all"
405
+          ha-sync-mode: "automatic"
406
+        priority: 0
407
+        apply-to: all
408
+        pattern: 'celery.*'
384 409
   paste:
385 410
     app:shipyard-api:
386 411
       paste.app_factory: shipyard_airflow.shipyard_api:paste_start_shipyard
@@ -785,6 +810,13 @@ pod:
785 810
         memory: "128Mi"
786 811
         cpu: "100m"
787 812
     jobs:
813
+      rabbit_init:
814
+        limits:
815
+          memory: "128Mi"
816
+          cpu: "500m"
817
+        requests:
818
+          memory: "128Mi"
819
+          cpu: "500m"
788 820
       airflow_db_init:
789 821
         limits:
790 822
           memory: "128Mi"
@@ -864,6 +896,7 @@ manifests:
864 896
   ingress_shipyard_api: true
865 897
   job_shipyard_db_init: true
866 898
   job_shipyard_db_sync: true
899
+  job_rabbit_init: true
867 900
   job_airflow_db_init: true
868 901
   job_airflow_db_sync: true
869 902
   job_ks_endpoints: true
@@ -873,6 +906,7 @@ manifests:
873 906
   secret_shipyard_db: true
874 907
   secret_ingress_tls: true
875 908
   secret_keystone: true
909
+  secret_rabbitmq: true
876 910
   service_airflow_ingress: true
877 911
   service_airflow_flower: true
878 912
   service_shipyard: true

+ 1
- 1
tools/gate/playbooks/zuul-linter.yaml View File

@@ -15,6 +15,6 @@
15 15
 - hosts: primary
16 16
   tasks:
17 17
     - name: Execute a Whitespace Linter check
18
-      command: find . -not -path "*/\.*" -not -path "*/doc/build/*" -not -name "*.tgz" -type f -exec egrep -l " +$" {} \;
18
+      command: find . -not -path "*/\.*" -not -path "*/doc/build/*" -not -name "*.tgz" -not -name "*.png" -type f -exec egrep -l " +$" {} \;
19 19
       register: result
20 20
       failed_when: result.stdout != ""

Loading…
Cancel
Save