Heat: Add helm test

This PS adds helm test functionaility to heat. This test is taxing
om the oslo.db backend and can expose clustering issues effectively
when run with high concurrency.

Change-Id: I5d6074cedbc870b9536d85859381c3dba1ad4f61

heat/templates/configmap-bin.yaml

@@ -16,6 +16,7 @@ limitations under the License.
 
 {{- if .Values.manifests.configmap_bin }}
 {{- $envAll := . }}
+{{- $rallyTests := .Values.conf.rally_tests }}
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -30,6 +31,8 @@ data:
   bootstrap.sh: |
 {{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
 {{- end }}
+  rally-test.sh: |
+{{ tuple $rallyTests | include "helm-toolkit.scripts.rally_test" | indent 4 }}
   db-init.py: |
 {{- include "helm-toolkit.scripts.db_init" . | indent 4 }}
   db-sync.sh: |

heat/templates/configmap-etc.yaml

@@ -126,10 +126,16 @@ kind: ConfigMap
 metadata:
   name: heat-etc
 data:
+  rally_tests.yaml: |
+{{ toYaml .Values.conf.rally_tests.tests | indent 4 }}
   heat.conf: |
 {{ include "helm-toolkit.utils.to_oslo_conf" .Values.conf.heat | indent 4 }}
   api-paste.ini: |
 {{ include "helm-toolkit.utils.to_ini" .Values.conf.paste | indent 4 }}
   policy.json: |
 {{  toJson .Values.conf.policy | indent 4 }}
+{{- range $key, $value := $envAll.Values.conf.rally_tests.templates }}
+  {{ printf "test_template_%d" $key }}: |
+{{ $value.template | indent 4 }}
+{{- end }}
 {{- end }}

heat/templates/pod-rally-test.yaml

@@ -0,0 +1,107 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.manifests.pod_rally_test }}
+{{- $envAll := . }}
+
+{{- $mounts_tests := .Values.pod.mounts.heat_tests.heat_tests }}
+{{- $mounts_tests_init := .Values.pod.mounts.heat_tests.init_container }}
+
+{{- $serviceAccountName := print $envAll.Release.Name "-test" }}
+{{ tuple $envAll "tests" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }}
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: {{ print $envAll.Release.Name "-test" }}
+  annotations:
+    "helm.sh/hook": test-success
+spec:
+  nodeSelector:
+    {{ .Values.labels.test.node_selector_key }}: {{ .Values.labels.test.node_selector_value }}
+  restartPolicy: Never
+  serviceAccountName: {{ $serviceAccountName }}
+  initContainers:
+{{ tuple $envAll "tests" $mounts_tests_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 4 }}
+    - name: {{ .Release.Name }}-test-ks-user
+      image: {{ .Values.images.tags.ks_user }}
+      imagePullPolicy: {{ .Values.images.pull_policy }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.ks_user | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }}
+      command:
+        - /tmp/ks-user.sh
+      volumeMounts:
+        - name: heat-bin
+          mountPath: /tmp/ks-user.sh
+          subPath: ks-user.sh
+          readOnly: true
+      env:
+{{- with $env := dict "ksUserSecret" .Values.secrets.identity.admin }}
+{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 8 }}
+{{- end }}
+        - name: SERVICE_OS_SERVICE_NAME
+          value: "test"
+{{- with $env := dict "ksUserSecret" .Values.secrets.identity.test }}
+{{- include "helm-toolkit.snippets.keystone_user_create_env_vars" $env | indent 8 }}
+{{- end }}
+        - name: SERVICE_OS_ROLE
+          value: {{ .Values.endpoints.identity.auth.test.role | quote }}
+  containers:
+    - name: {{ .Release.Name }}-test
+      image: {{ .Values.images.tags.test }}
+      imagePullPolicy: {{ .Values.images.pull_policy }}
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.tests | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }}
+      env:
+{{- with $env := dict "ksUserSecret" .Values.secrets.identity.admin }}
+{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 8 }}
+{{- end }}
+{{- with $env := dict "ksUserSecret" .Values.secrets.identity.test }}
+{{- include "helm-toolkit.snippets.keystone_user_create_env_vars" $env | indent 8 }}
+{{- end }}
+        - name: RALLY_ENV_NAME
+          value: {{.Release.Name}}
+      command:
+        - /tmp/rally-test.sh
+      volumeMounts:
+        - name: heat-etc
+          mountPath: /etc/rally/rally_tests.yaml
+          subPath: rally_tests.yaml
+          readOnly: true
+        - name: heat-bin
+          mountPath: /tmp/rally-test.sh
+          subPath: rally-test.sh
+          readOnly: true
+        - name: rally-db
+          mountPath: /var/lib/rally
+        {{- range $key, $value := $envAll.Values.conf.rally_tests.templates }}
+        - name: heat-etc
+          mountPath: {{ $value.name }}
+          subPath: {{ printf "test_template_%d" $key }}
+          readOnly: true
+        {{- end }}
+{{ if $mounts_tests.volumeMounts }}{{ toYaml $mounts_tests.volumeMounts | indent 8 }}{{ end }}
+  volumes:
+    - name: heat-etc
+      configMap:
+        name: heat-etc
+        defaultMode: 0444
+    - name: heat-bin
+      configMap:
+        name: heat-bin
+        defaultMode: 0555
+    - name: rally-db
+      emptyDir: {}
+{{ if $mounts_tests.volumes }}{{ toYaml $mounts_tests.volumes | indent 4 }}{{ end }}
+{{- end }}

heat/templates/secret-keystone.yaml

@@ -16,7 +16,7 @@ limitations under the License.
 
 {{- if .Values.manifests.secret_keystone }}
 {{- $envAll := . }}
-{{- range $key1, $userClass := tuple "admin" "heat" "heat_trustee" }}
+{{- range $key1, $userClass := tuple "admin" "heat" "heat_trustee" "test" }}
 {{- $secretName := index $envAll.Values.secrets.identity $userClass }}
 ---
 apiVersion: v1

heat/values.yaml

@@ -35,9 +35,13 @@ labels:
   job:
     node_selector_key: openstack-control-plane
     node_selector_value: enabled
+  test:
+    node_selector_key: openstack-control-plane
+    node_selector_value: enabled
 
 images:
   tags:
+    test: docker.io/kolla/ubuntu-source-rally:4.0.0
     bootstrap: docker.io/openstackhelm/heat:newton
     db_init: docker.io/openstackhelm/heat:newton
     heat_db_sync: docker.io/openstackhelm/heat:newton
@@ -68,6 +72,216 @@ jobs:
       failed: 1
 
 conf:
+  rally_tests:
+    run_tempest: false
+    tests:
+      HeatStacks.create_and_delete_stack:
+        - args:
+            template_path: /tmp/rally-jobs/default.yaml
+          runner:
+            concurrency: 1
+            times: 1
+            type: constant
+          sla:
+            failure_rate:
+              max: 0
+      HeatStacks.create_update_delete_stack:
+        - args:
+            template_path: /tmp/rally-jobs/random_strings.yaml
+            updated_template_path: /tmp/rally-jobs/updated_random_strings_replace.yaml
+          runner:
+            concurrency: 1
+            times: 1
+            type: constant
+          sla:
+            failure_rate:
+              max: 0
+      HeatStacks.create_check_delete_stack:
+        - args:
+            template_path: /tmp/rally-jobs/random_strings.yaml
+          runner:
+            concurrency: 1
+            times: 1
+            type: constant
+          sla:
+            failure_rate:
+              max: 0
+      HeatStacks.create_and_delete_stack:
+        - args:
+            template_path: /tmp/rally-jobs/resource_group_with_constraint.yaml
+          runner:
+            concurrency: 1
+            times: 1
+            type: constant
+          sla:
+            failure_rate:
+              max: 0
+      HeatStacks.create_and_list_stack:
+        - args:
+            template_path: /tmp/rally-jobs/default.yaml
+          runner:
+            concurrency: 1
+            times: 1
+            type: constant
+          sla:
+            failure_rate:
+              max: 0
+      HeatStacks.create_snapshot_restore_delete_stack:
+        - args:
+            template_path: /tmp/rally-jobs/random_strings.yaml
+          runner:
+            concurrency: 1
+            times: 1
+            type: constant
+          sla:
+            failure_rate:
+              max: 0
+      HeatStacks.create_stack_and_list_output:
+        - args:
+            template_path: /tmp/rally-jobs/resource_group_with_outputs.yaml
+          runner:
+            concurrency: 1
+            times: 1
+            type: constant
+          sla:
+            failure_rate:
+              max: 0
+      HeatStacks.create_stack_and_list_output_via_API:
+        - args:
+            template_path: /tmp/rally-jobs/resource_group_with_outputs.yaml
+          runner:
+            concurrency: 1
+            times: 1
+            type: constant
+          sla:
+            failure_rate:
+              max: 0
+    templates:
+      - name: /tmp/rally-jobs/default.yaml
+        template: |
+          heat_template_version: 2014-10-16
+      - name: /tmp/rally-jobs/random_strings.yaml
+        template: |
+          heat_template_version: 2014-10-16
+          description: Test template for rally create-update-delete scenario
+          resources:
+            test_string_one:
+              type: OS::Heat::RandomString
+              properties:
+                length: 20
+            test_string_two:
+              type: OS::Heat::RandomString
+              properties:
+                length: 20
+      - name: /tmp/rally-jobs/resource_group_with_constraint.yaml
+        template: |
+          heat_template_version: 2013-05-23
+          description: Template for testing caching.
+          parameters:
+            count:
+              type: number
+              default: 40
+            delay:
+              type: number
+              default: 0.1
+          resources:
+            rg:
+              type: OS::Heat::ResourceGroup
+              properties:
+                count:
+                  get_param: count
+                resource_def:
+                    type: OS::Heat::TestResource
+                    properties:
+                      constraint_prop_secs:
+                        get_param: delay
+      - name: /tmp/rally-jobs/resource_group_with_outputs.yaml
+        template: |
+          heat_template_version: 2013-05-23
+          parameters:
+            attr_wait_secs:
+              type: number
+              default: 0.5
+          resources:
+            rg:
+              type: OS::Heat::ResourceGroup
+              properties:
+                count: 10
+                resource_def:
+                  type: OS::Heat::TestResource
+                  properties:
+                    attr_wait_secs:
+                      get_param: attr_wait_secs
+          outputs:
+            val1:
+              value:
+                get_attr:
+                  - rg
+                  - resource.0.output
+            val2:
+              value:
+                get_attr:
+                  - rg
+                  - resource.1.output
+            val3:
+              value:
+                get_attr:
+                  - rg
+                  - resource.2.output
+            val4:
+              value:
+                get_attr:
+                  - rg
+                  - resource.3.output
+            val5:
+              value:
+                get_attr:
+                  - rg
+                  - resource.4.output
+            val6:
+              value:
+                get_attr:
+                  - rg
+                  - resource.5.output
+            val7:
+              value:
+                get_attr:
+                  - rg
+                  - resource.6.output
+            val8:
+              value:
+                get_attr:
+                  - rg
+                  - resource.7.output
+            val9:
+              value:
+                get_attr:
+                  - rg
+                  - resource.8.output
+            val10:
+              value:
+                get_attr:
+                  - rg
+                  - resource.9.output
+      - name: /tmp/rally-jobs/updated_random_strings_replace.yaml
+        template: |
+          heat_template_version: 2014-10-16
+          description: |
+            Test template for create-update-delete-stack scenario in rally.
+            The template deletes one resource from the stack defined by
+            random-strings.yaml.template and re-creates it with the updated parameters
+            (so-called update-replace). That happens because some parameters cannot be
+            changed without resource re-creation. The template allows to measure performance
+            of update-replace operation.
+          resources:
+            test_string_one:
+              type: OS::Heat::RandomString
+              properties:
+                length: 20
+            test_string_two:
+              type: OS::Heat::RandomString
+              properties:
+                length: 40
   paste:
     pipeline:heat-api:
       pipeline: cors request_id faultwrap http_proxy_to_wsgi versionnegotiation osprofiler authurl authtoken context apiv1app
@@ -429,6 +643,12 @@ dependencies:
       services:
         - endpoint: internal
           service: local_image_registry
+    tests:
+      services:
+        - endpoint: internal
+          service: identity
+        - endpoint: internal
+          service: orchestration
 
 # Names of secrets used by bootstrap and environmental checks
 secrets:
@@ -437,6 +657,7 @@ secrets:
     heat: heat-keystone-user
     heat_trustee: heat-keystone-trustee
     heat_stack_user: heat-keystone-stack-user
+    test: heat-keystone-test
   oslo_db:
     admin: heat-db-admin
     heat: heat-db-user
@@ -495,6 +716,14 @@ endpoints:
         username: heat-domain
         password: password
         domain_name: heat
+      test:
+        role: admin
+        region_name: RegionOne
+        username: test
+        password: password
+        project_name: test
+        user_domain_name: default
+        project_domain_name: default
     hosts:
       default: keystone-api
       public: keystone
@@ -640,6 +869,9 @@ pod:
     heat_engine_cleaner:
       init_container: null
       heat_engine_cleaner:
+    heat_tests:
+      init_container: null
+      heat_tests:
   replicas:
     api: 1
     cfn: 1
@@ -810,6 +1042,7 @@ manifests:
   pdb_api: true
   pdb_cfn: true
   pdb_cloudwatch: false
+  pod_rally_test: true
   secret_db: true
   secret_keystone: true
   secret_rabbitmq: true

tools/deployment/multinode/150-heat.sh

@@ -38,3 +38,4 @@ export OS_CLOUD=openstack_helm
 openstack service list
 sleep 30 #NOTE(portdirect): Wait for ingress controller to update rules and restart Nginx
 openstack orchestration service list
+helm test heat --timeout 900