From b492ee54c3eb4249889a6294f38cffa667b6f980 Mon Sep 17 00:00:00 2001 From: Steve Wilkerson Date: Thu, 22 Feb 2018 15:15:18 -0600 Subject: [PATCH] OSH-infra multinode gate rework This moves to using bash scripts for deploying the charts for the multinode gates in osh-infra as opposed to using the ansible chart builder Change-Id: I2951ccf57250a5e6e272872f2d6f7a81bd11f184 --- .zuul.yaml | 2 +- playbooks/osh-infra-multinode-deploy.yaml | 88 +++++++ .../common/010-deploy-docker-registry.sh | 2 +- .../multinode/010-deploy-docker-registry.sh | 1 + .../multinode/020-lma-nfs-provisioner.sh | 1 + tools/deployment/multinode/030-prometheus.sh | 35 +++ .../deployment/multinode/040-alertmanager.sh | 31 +++ .../multinode/050-kube-state-metrics.sh | 1 + .../deployment/multinode/060-node-exporter.sh | 1 + .../multinode/070-openstack-exporter.sh | 1 + tools/deployment/multinode/080-grafana.sh | 54 ++++ tools/deployment/multinode/090-nagios.sh | 32 +++ .../deployment/multinode/100-elasticsearch.sh | 46 ++++ .../multinode/110-fluent-logging.sh | 34 +++ tools/deployment/multinode/120-kibana.sh | 1 + tools/gate/chart-deploys/default.yaml | 241 ------------------ 16 files changed, 328 insertions(+), 243 deletions(-) create mode 100644 playbooks/osh-infra-multinode-deploy.yaml create mode 120000 tools/deployment/multinode/010-deploy-docker-registry.sh create mode 120000 tools/deployment/multinode/020-lma-nfs-provisioner.sh create mode 100755 tools/deployment/multinode/030-prometheus.sh create mode 100755 tools/deployment/multinode/040-alertmanager.sh create mode 120000 tools/deployment/multinode/050-kube-state-metrics.sh create mode 120000 tools/deployment/multinode/060-node-exporter.sh create mode 120000 tools/deployment/multinode/070-openstack-exporter.sh create mode 100755 tools/deployment/multinode/080-grafana.sh create mode 100755 tools/deployment/multinode/090-nagios.sh create mode 100755 tools/deployment/multinode/100-elasticsearch.sh create mode 100755 tools/deployment/multinode/110-fluent-logging.sh create mode 120000 tools/deployment/multinode/120-kibana.sh delete mode 100644 tools/gate/chart-deploys/default.yaml diff --git a/.zuul.yaml b/.zuul.yaml index d78167283..83b9539fe 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -178,7 +178,7 @@ - playbooks/osh-infra-build.yaml - playbooks/osh-infra-pull-images.yaml - playbooks/osh-infra-deploy-k8s.yaml - run: playbooks/osh-infra-deploy-charts.yaml + run: playbooks/osh-infra-multinode-deploy.yaml post-run: playbooks/osh-infra-collect-logs.yaml - job: diff --git a/playbooks/osh-infra-multinode-deploy.yaml b/playbooks/osh-infra-multinode-deploy.yaml new file mode 100644 index 000000000..68c0564db --- /dev/null +++ b/playbooks/osh-infra-multinode-deploy.yaml @@ -0,0 +1,88 @@ +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: primary + tasks: + - name: Deploy Registry NFS, Redis, and Docker Registry + shell: | + set -xe; + ./tools/deployment/developer/010-deploy-docker-registry.sh + args: + chdir: "{{ zuul.project.src_dir }}" + - name: Deploy NFS for Logging, Monitoring and Alerting Components + shell: | + set -xe; + ./tools/deployment/developer/020-lma-nfs-provisioner.sh + args: + chdir: "{{ zuul.project.src_dir }}" + - name: Deploy Prometheus + shell: | + set -xe; + ./tools/deployment/multinode/030-prometheus.sh + args: + chdir: "{{ zuul.project.src_dir }}" + - name: Deploy Alertmanager + shell: | + set -xe; + ./tools/deployment/multinode/040-alertmanager.sh + args: + chdir: "{{ zuul.project.src_dir }}" + - name: Deploy Kube-State-Metrics + shell: | + set -xe; + ./tools/deployment/multinode/050-kube-state-metrics.sh + args: + chdir: "{{ zuul.project.src_dir }}" + - name: Deploy Node Exporter + shell: | + set -xe; + ./tools/deployment/multinode/060-node-exporter.sh + args: + chdir: "{{ zuul.project.src_dir }}" + - name: Deploy Prometheus OpenStack Exporter + shell: | + set -xe; + ./tools/deployment/multinode/070-openstack-exporter.sh + args: + chdir: "{{ zuul.project.src_dir }}" + - name: Deploy Grafana + shell: | + set -xe; + ./tools/deployment/multinode/080-grafana.sh + args: + chdir: "{{ zuul.project.src_dir }}" + - name: Deploy Nagios + shell: | + set -xe; + ./tools/deployment/multinode/090-nagios.sh + args: + chdir: "{{ zuul.project.src_dir }}" + - name: Deploy Elasticsearch + shell: | + set -xe; + ./tools/deployment/multinode/100-elasticsearch.sh + args: + chdir: "{{ zuul.project.src_dir }}" + - name: Deploy Fluent-Logging + shell: | + set -xe; + ./tools/deployment/multinode/110-fluent-logging.sh + args: + chdir: "{{ zuul.project.src_dir }}" + - name: Deploy Kibana + shell: | + set -xe; + ./tools/deployment/multinode/120-kibana.sh + args: + chdir: "{{ zuul.project.src_dir }}" diff --git a/tools/deployment/common/010-deploy-docker-registry.sh b/tools/deployment/common/010-deploy-docker-registry.sh index c6fbfa0a3..d39add9e3 100755 --- a/tools/deployment/common/010-deploy-docker-registry.sh +++ b/tools/deployment/common/010-deploy-docker-registry.sh @@ -16,7 +16,7 @@ set -xe -#NOTE: Pull images and lint charts for deploying the docker registry +#NOTE: Pull images and lint charts for deploying a local docker registry make pull-images nfs-provisioner make pull-images redis make pull-images registry diff --git a/tools/deployment/multinode/010-deploy-docker-registry.sh b/tools/deployment/multinode/010-deploy-docker-registry.sh new file mode 120000 index 000000000..7360ae428 --- /dev/null +++ b/tools/deployment/multinode/010-deploy-docker-registry.sh @@ -0,0 +1 @@ +../common/010-deploy-docker-registry.sh \ No newline at end of file diff --git a/tools/deployment/multinode/020-lma-nfs-provisioner.sh b/tools/deployment/multinode/020-lma-nfs-provisioner.sh new file mode 120000 index 000000000..afcbfbe42 --- /dev/null +++ b/tools/deployment/multinode/020-lma-nfs-provisioner.sh @@ -0,0 +1 @@ +../common/020-lma-nfs-provisioner.sh \ No newline at end of file diff --git a/tools/deployment/multinode/030-prometheus.sh b/tools/deployment/multinode/030-prometheus.sh new file mode 100755 index 000000000..3114e6942 --- /dev/null +++ b/tools/deployment/multinode/030-prometheus.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Pull images and lint chart +make pull-images prometheus + +#NOTE: Deploy command +helm upgrade --install prometheus ./prometheus \ + --namespace=openstack \ + --set storage.storage_class=openstack-helm-lma-nfs \ + --set pod.replicas.prometheus=2 + +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh openstack + +#NOTE: Validate Deployment info +helm status prometheus + +#NOTE: Run helm tests +helm test prometheus diff --git a/tools/deployment/multinode/040-alertmanager.sh b/tools/deployment/multinode/040-alertmanager.sh new file mode 100755 index 000000000..6de8295b9 --- /dev/null +++ b/tools/deployment/multinode/040-alertmanager.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Pull images and lint chart +make pull-images alertmanager + +#NOTE: Deploy command +helm upgrade --install alertmanager ./prometheus-alertmanager \ + --namespace=openstack \ + --set storage.storage_class=openstack-helm-lma-nfs + +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh openstack + +#NOTE: Validate Deployment info +helm status alertmanager diff --git a/tools/deployment/multinode/050-kube-state-metrics.sh b/tools/deployment/multinode/050-kube-state-metrics.sh new file mode 120000 index 000000000..c1537e38c --- /dev/null +++ b/tools/deployment/multinode/050-kube-state-metrics.sh @@ -0,0 +1 @@ +../common/050-kube-state-metrics.sh \ No newline at end of file diff --git a/tools/deployment/multinode/060-node-exporter.sh b/tools/deployment/multinode/060-node-exporter.sh new file mode 120000 index 000000000..5c4daa1b9 --- /dev/null +++ b/tools/deployment/multinode/060-node-exporter.sh @@ -0,0 +1 @@ +../common/060-node-exporter.sh \ No newline at end of file diff --git a/tools/deployment/multinode/070-openstack-exporter.sh b/tools/deployment/multinode/070-openstack-exporter.sh new file mode 120000 index 000000000..cb0b54753 --- /dev/null +++ b/tools/deployment/multinode/070-openstack-exporter.sh @@ -0,0 +1 @@ +../common/070-openstack-exporter.sh \ No newline at end of file diff --git a/tools/deployment/multinode/080-grafana.sh b/tools/deployment/multinode/080-grafana.sh new file mode 100755 index 000000000..cbb5b8371 --- /dev/null +++ b/tools/deployment/multinode/080-grafana.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Pull images and lint chart +make pull-images grafana + +#NOTE: Deploy command +tee /tmp/grafana.yaml << EOF +dependencies: + static: + grafana: + jobs: null + services: null +manifests: + job_db_init: false + job_db_init_session: false + job_db_session_sync: false + secret_db: false + secret_db_session: false +conf: + grafana: + database: + type: sqlite3 + session: + provider: file + provider_config: sessions +pod: + replicas: + grafana: 2 +EOF +helm upgrade --install grafana ./grafana \ + --namespace=openstack \ + --values=/tmp/grafana.yaml + +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh openstack + +#NOTE: Validate Deployment info +helm status grafana diff --git a/tools/deployment/multinode/090-nagios.sh b/tools/deployment/multinode/090-nagios.sh new file mode 100755 index 000000000..edf098349 --- /dev/null +++ b/tools/deployment/multinode/090-nagios.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Pull images and lint chart +make pull-images nagios + +#NOTE: Deploy command +helm upgrade --install nagios ./nagios \ + --namespace=openstack \ + --set network.nagios.ingress.public=false \ + --set pod.replicas.nagios=3 + +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh openstack + +#NOTE: Validate Deployment info +helm status nagios diff --git a/tools/deployment/multinode/100-elasticsearch.sh b/tools/deployment/multinode/100-elasticsearch.sh new file mode 100755 index 000000000..8b378e536 --- /dev/null +++ b/tools/deployment/multinode/100-elasticsearch.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Pull images and lint chart +make pull-images elasticsearch + +#NOTE: Deploy command +tee /tmp/elasticsearch.yaml << EOF +storage: + elasticsearch: + storage_class: openstack-helm-lma-nfs +conf: + elasticsearch: + env: + java_opts: "-Xms512m -Xmx512m" +monitoring: + prometheus: + enabled: true +EOF +helm upgrade --install elasticsearch ./elasticsearch \ + --namespace=openstack \ + --values=/tmp/elasticsearch.yaml + +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh openstack + +#NOTE: Validate Deployment info +helm status elasticsearch + +#NOTE: Run helm tests +helm test elasticsearch diff --git a/tools/deployment/multinode/110-fluent-logging.sh b/tools/deployment/multinode/110-fluent-logging.sh new file mode 100755 index 000000000..519f58c5d --- /dev/null +++ b/tools/deployment/multinode/110-fluent-logging.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Pull images and lint chart +make pull-images fluent-logging + +#NOTE: Deploy command +helm upgrade --install fluent-logging ./fluent-logging \ + --namespace=openstack \ + --set monitoring.prometheus.enabled=true + +#NOTE: Wait for deploy +./tools/deployment/common/wait-for-pods.sh openstack + +#NOTE: Validate Deployment info +helm status fluent-logging + +#NOTE: Run helm tests +helm test fluent-logging diff --git a/tools/deployment/multinode/120-kibana.sh b/tools/deployment/multinode/120-kibana.sh new file mode 120000 index 000000000..8f9030c60 --- /dev/null +++ b/tools/deployment/multinode/120-kibana.sh @@ -0,0 +1 @@ +../common/120-kibana.sh \ No newline at end of file diff --git a/tools/gate/chart-deploys/default.yaml b/tools/gate/chart-deploys/default.yaml deleted file mode 100644 index 3d6215da2..000000000 --- a/tools/gate/chart-deploys/default.yaml +++ /dev/null @@ -1,241 +0,0 @@ -# Copyright 2017 The Openstack-Helm Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -chart_groups: - - name: docker_registry - timeout: 600 - charts: - - docker_registry_nfs_provisioner - - docker_registry_redis - - docker_registry - - - name: infra_monitoring - timeout: 600 - charts: - - prometheus - - nagios - - prometheus_node_exporter - - prometheus_kube_state_metrics - - prometheus_alertmanager - - grafana - - - name: openstack_infra_exporters - timeout: 600 - charts: - - prometheus_openstack_exporter - - - name: openstack_infra_logging - timeout: 1200 - charts: - - openstack_elasticsearch - - fluent_logging - - kibana - -charts: - docker_registry_nfs_provisioner: - chart_name: nfs-provisioner - release: docker-registry-nfs-provisioner - namespace: docker-registry - upgrade: - pre: - delete: - - name: docker-bootstrap - type: job - labels: - application: docker - component: bootstrap - values: - labels: - node_selector_key: openstack-helm-node-class - node_selector_value: primary - storageclass: - name: openstack-helm-bootstrap - - docker_registry_redis: - chart_name: redis - release: docker-registry-redis - namespace: docker-registry - values: - labels: - node_selector_key: openstack-helm-node-class - node_selector_value: primary - - docker_registry: - chart_name: registry - release: docker-registry - namespace: docker-registry - values: - labels: - node_selector_key: openstack-helm-node-class - node_selector_value: primary - volume: - class_name: openstack-helm-bootstrap - - prometheus: - chart_name: prometheus - release: prometheus - namespace: openstack - timeout: 300 - test: - enabled: true - timeout: 300 - output: false - values: - storage: - enabled: false - manifests: - pvc: false - network: - prometheus: - ingress: - public: false - - prometheus_kube_state_metrics: - chart_name: prometheus-kube-state-metrics - release: prometheus-kube-metrics - namespace: kube-system - test: - enabled: false - timeout: 300 - output: false - - prometheus_node_exporter: - chart_name: prometheus-node-exporter - release: prometheus-node-exporter - namespace: kube-system - test: - enabled: false - timeout: 300 - output: false - - prometheus_alertmanager: - chart_name: prometheus-alertmanager - release: prometheus-alertmanager - namespace: openstack - test: - enabled: false - timeout: 300 - output: false - values: - storage: - enabled: false - manifests: - pvc: false - network: - alertmanager: - ingress: - public: false - - nagios: - chart_name: nagios - release: nagios - namespace: openstack - values: - network: - nagios: - ingress: - public: false - - prometheus_openstack_exporter: - chart_name: prometheus-openstack-exporter - release: prometheus-openstack-exporter - namespace: openstack - timeout: 300 - test: - enabled: false - timeout: 300 - output: false - values: - # NOTE(portdirect): Keystone Management is disabled here, as keystone is - # not deployed in the OSH infra gates. - manifests: - job_ks_user: false - dependencies: - static: - prometheus_openstack_exporter: - jobs: null - services: null - - grafana: - chart_name: grafana - release: prometheus-grafana - namespace: openstack - test: - enabled: false - timeout: 300 - output: false - values: - dependencies: - static: - grafana: - jobs: null - services: null - manifests: - ingress: false - job_db_init: false - job_db_init_session: false - job_db_session_sync: false - secret_db: false - secret_db_session: false - service_ingress: false - conf: - grafana: - database: - type: sqlite3 - session: - provider: file - provider_config: sessions - network: - grafana: - ingress: - public: false - - openstack_elasticsearch: - chart_name: elasticsearch - release: elasticsearch - namespace: openstack - timeout: 600 - test: - enabled: true - timeout: 600 - output: false - values: - storage: - elasticsearch: - storage_class: openstack-helm-bootstrap - filesystem_repository: - storage_class: openstack-helm-bootstrap - monitoring: - prometheus: - enabled: true - - fluent_logging: - chart_name: fluent-logging - release: fluent-logging - namespace: openstack - timeout: 600 - test: - enabled: true - timeout: 600 - output: false - values: - monitoring: - prometheus: - enabled: true - - - kibana: - chart_name: kibana - release: kibana - namespace: openstack