airskiff: Use global Airship charts

Currently, Airskiff deploys a custom manifest defined in
tools/deployment/airskiff/manifests/airship.yaml to boostrap Airship
components. This exempts Airskiff from the daily version uplift process
and causes a divergence from the global UCP charts. This commit updates
the Airskiff bootstrap process to use the existing UCP charts at the
global layer.

Change-Id: I68dc405fda3cb1dbf5865104b657bdd75812e9bc

site/airskiff/networks/common-addresses.yaml

@@ -39,7 +39,7 @@ data:
     # NEWSITE-CHANGEME: FQDN for ingress (i.e. "publicly facing" access point)
     # Choose FQDN according to the ingress/public FQDN naming conventions at
     # the top of this document.
-    ingress_domain: openstack.svc.cluster.local
+    ingress_domain: ucp.svc.cluster.local
 
   genesis:
     # NEWSITE-CHANGEME: Update with the hostname for the node which will take on

site/airskiff/software/charts/ucp/armada/armada.yaml

@@ -0,0 +1,29 @@
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp-armada
+  replacement: true
+  layeringDefinition:
+    abstract: false
+    layer: site
+    parentSelector:
+      name: ucp-armada-global
+    actions:
+      - method: replace
+        path: .source
+      - method: merge
+        path: .
+  storagePolicy: cleartext
+data:
+  wait:
+    timeout: 1800
+  source:
+    type: local
+    location: /airship-components/armada
+    subpath: charts/armada
+  values:
+    pod:
+      replicas:
+        api: 1
+...

site/airskiff/software/charts/ucp/armada/chart-group.yaml

@@ -0,0 +1,19 @@
+---
+schema: armada/ChartGroup/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp-armada
+  replacement: true
+  layeringDefinition:
+    abstract: false
+    layer: site
+    parentSelector:
+      name: ucp-armada-chart-group-global
+    actions:
+      - method: replace
+        path: .chart_group
+  storagePolicy: cleartext
+data:
+  chart_group:
+    - ucp-armada
+...

site/airskiff/software/charts/ucp/core/ingress.yaml

@@ -0,0 +1,22 @@
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp-ingress
+  replacement: true
+  layeringDefinition:
+    abstract: false
+    layer: site
+    parentSelector:
+      name: ucp-ingress-global
+    actions:
+      - method: merge
+        path: .
+  storagePolicy: cleartext
+data:
+  values:
+    pod:
+      replicas:
+        ingress: 1
+        error_page: 1
+...

site/airskiff/software/charts/ucp/core/mariadb.yaml

@@ -0,0 +1,24 @@
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp-mariadb
+  replacement: true
+  layeringDefinition:
+    abstract: false
+    layer: site
+    parentSelector:
+      name: ucp-mariadb-global
+    actions:
+      - method: merge
+        path: .
+  storagePolicy: cleartext
+data:
+  wait:
+    timeout: 1800
+  values:
+    pod:
+      replicas:
+        server: 1
+        ingress: 1
+...

site/airskiff/software/charts/ucp/core/rabbitmq.yaml

@@ -0,0 +1,26 @@
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp-rabbitmq
+  replacement: true
+  layeringDefinition:
+    abstract: false
+    layer: site
+    parentSelector:
+      name: ucp-rabbitmq-global
+    actions:
+      - method: replace
+        path: .test
+      - method: merge
+        path: .
+  storagePolicy: cleartext
+data:
+  timeout: 1800
+  test:
+    enabled: false
+  values:
+    pod:
+      replicas:
+        server: 1
+...

site/airskiff/software/charts/ucp/deckhand/barbican.yaml

@@ -0,0 +1,21 @@
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp-barbican
+  replacement: true
+  layeringDefinition:
+    abstract: false
+    layer: site
+    parentSelector:
+      name: ucp-barbican-global
+    actions:
+      - method: merge
+        path: .
+  storagePolicy: cleartext
+data:
+  values:
+    pod:
+      replicas:
+        api: 1
+...

site/airskiff/software/charts/ucp/deckhand/deckhand.yaml

@@ -0,0 +1,29 @@
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp-deckhand
+  replacement: true
+  layeringDefinition:
+    abstract: false
+    layer: site
+    parentSelector:
+      name: ucp-deckhand-global
+    actions:
+      - method: replace
+        path: .source
+      - method: merge
+        path: .
+  storagePolicy: cleartext
+data:
+  wait:
+    timeout: 1800
+  source:
+    type: local
+    location: /airship-components/deckhand
+    subpath: charts/deckhand
+  values:
+    pod:
+      replicas:
+        deckhand: 1
+...

site/airskiff/software/charts/ucp/keystone/keystone.yaml

@@ -0,0 +1,21 @@
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp-keystone
+  replacement: true
+  layeringDefinition:
+    abstract: false
+    layer: site
+    parentSelector:
+      name: ucp-keystone-global
+    actions:
+      - method: merge
+        path: .
+  storagePolicy: cleartext
+data:
+  values:
+    pod:
+      replicas:
+        api: 1
+...

site/airskiff/software/charts/ucp/shipyard/shipyard.yaml

@@ -0,0 +1,41 @@
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp-shipyard
+  replacement: true
+  layeringDefinition:
+    abstract: false
+    layer: site
+    parentSelector:
+      name: ucp-shipyard-global
+    actions:
+      - method: replace
+        path: .source
+      - method: merge
+        path: .
+  storagePolicy: cleartext
+data:
+  wait:
+    timeout: 1800
+  source:
+    type: local
+    location: /airship-components/shipyard
+    subpath: charts/shipyard
+  values:
+    pod:
+      replicas:
+        shipyard:
+          api: 1
+        airflow:
+          worker: 1
+          scheduler: 1
+    conf:
+      shipyard:
+        # NOTE(drewwalters96): Since Drydock and Promenade are not deployed,
+        # temporarily alias those validations to Armada.
+        drydock:
+          service_type: armada
+        promenade:
+          service_type: armada
+...

site/airskiff/software/config/endpoints.yaml

@@ -20,12 +20,6 @@ metadata:
           pattern: DOMAIN
         - path: .ucp.identity.host_fqdn_override.admin.host
           pattern: DOMAIN
-        - path: .ucp.shipyard.host_fqdn_override.public.host
-          pattern: DOMAIN
-        - path: .ucp.physicalprovisioner.host_fqdn_override.public.host
-          pattern: DOMAIN
-        - path: .ucp.maas_region.host_fqdn_override.public.host
-          pattern: DOMAIN
         - path: .ceph.object_store.host_fqdn_override.public.host
           pattern: DOMAIN
         - path: .ceph.ceph_object_store.host_fqdn_override.public.host
@@ -71,17 +65,17 @@ data:
       host_fqdn_override:
         default: null
         public:
-          host: iam-sw.DOMAIN
+          host: keystone-api.DOMAIN
         admin:
-          host: iam-sw.DOMAIN
+          host: keystone-api.DOMAIN
       path:
         default: /v3
       scheme:
-        default: "https"
+        default: http
-        internal: "http"
+        internal: http
       port:
         api:
-          default: 443
+          default: 5000
           internal: 5000
     armada:
       name: armada
@@ -273,16 +267,12 @@ data:
       port:
         api:
           default: 9000
-          public: 443
       path:
         default: /api/v1.0
       scheme:
         default: "http"
-        public: "https"
       host_fqdn_override:
         default: null
-        public:
-          host: shipyard-sw.DOMAIN
     prometheus_openstack_exporter:
       namespace: ucp
       hosts:
@@ -592,9 +582,9 @@ data:
       host_fqdn_override:
         default: null
         public:
-          host: identity-sw.DOMAIN
+          host: identity.DOMAIN
         admin:
-          host: identity-sw.DOMAIN
+          host: identity.DOMAIN
       path:
         default: /v3
       scheme:
@@ -638,7 +628,7 @@ data:
       host_fqdn_override:
         default: null
         public:
-          host: image-sw.DOMAIN
+          host: image.DOMAIN
       path:
         default: null
       scheme:
@@ -777,7 +767,7 @@ data:
       host_fqdn_override:
         default: null
         public:
-          host: orchestration-sw.DOMAIN
+          host: orchestration.DOMAIN
       path:
         default: "/v1/%(project_id)s"
       scheme:
@@ -795,7 +785,7 @@ data:
       host_fqdn_override:
         default: null
         public:
-          host: cloudformation-sw.DOMAIN
+          host: cloudformation.DOMAIN
       path:
         default: /v1
       scheme:
@@ -855,7 +845,7 @@ data:
       host_fqdn_override:
         default: null
         public:
-          host: network-sw.DOMAIN
+          host: network.DOMAIN
       path:
         default: null
       scheme:
@@ -899,7 +889,7 @@ data:
       host_fqdn_override:
         default: null
         public:
-          host: compute-sw.DOMAIN
+          host: compute.DOMAIN
       path:
         default: "/v2/%(tenant_id)s"
       scheme:
@@ -934,7 +924,7 @@ data:
       host_fqdn_override:
         default: null
         public:
-          host: nova-novncproxy-sw.DOMAIN
+          host: nova-novncproxy.DOMAIN
       path:
         default: /vnc_auto.html
       scheme:
@@ -965,7 +955,7 @@ data:
       host_fqdn_override:
         default: null
         public:
-          host: placement-sw.DOMAIN
+          host: placement.DOMAIN
       path:
         default: /
       scheme:

site/airskiff/software/manifests/ucp-bootstrap.yaml

@@ -0,0 +1,19 @@
+---
+schema: armada/Manifest/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp-bootstrap
+  layeringDefinition:
+    abstract: false
+    layer: site
+  storagePolicy: cleartext
+data:
+  release_prefix: airship
+  chart_groups:
+    - osh-infra-nfs-provisioner
+    - ucp-core
+    - ucp-keystone
+    - ucp-shipyard
+    - ucp-armada
+    - ucp-deckhand
+...

tools/deployment/airskiff/common/os-env.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+#
+# Copyright 2019, AT&T Intellectual Property
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export OS_PROJECT_DOMAIN_NAME=`grep -m 1 project_domain_name /etc/openstack/clouds.yaml | cut -d "'" -f 2`
+export OS_USER_DOMAIN_NAME=`grep -m 1 user_domain_name /etc/openstack/clouds.yaml | cut -d "'" -f 2`
+export OS_PROJECT_NAME=`grep -m 1 project_name /etc/openstack/clouds.yaml | cut -d "'" -f 2`
+export OS_USERNAME=`grep -m 1 username /etc/openstack/clouds.yaml | cut -d "'" -f 2`
+export OS_PASSWORD=`grep -m 1 password /etc/openstack/clouds.yaml | cut -d "'" -f 2`
+export OS_AUTH_URL=`grep -m 1 auth_url /etc/openstack/clouds.yaml | cut -d "'" -f 2`

tools/deployment/airskiff/common/setup-proxy.sh

@@ -24,8 +24,4 @@ CURRENT_DIR="$(pwd)"
 sed -i -e "/\s      type:/a\        proxy_server: ${PROXY}" \
   global/software/config/versions.yaml
 
-# Update Armada bootstrap manifest
-sed -i -e "/type: git/a\    proxy_server: ${PROXY}" \
-  tools/deployment/airskiff/manifests/airship.yaml
-
 cd "${CURRENT_DIR}"

tools/deployment/airskiff/developer/020-setup-client.sh

@@ -30,11 +30,11 @@ clouds:
     identity_api_version: 3
     auth:
       username: 'admin'
-      password: 'password'
+      password: 'password123'
       project_name: 'admin'
       project_domain_name: 'default'
       user_domain_name: 'default'
-      auth_url: 'http://keystone.ucp.svc.cluster.local/v3'
+      auth_url: 'http://keystone-api.ucp.svc.cluster.local:5000/v3'
   openstack:
     region_name: RegionOne
     identity_api_version: 3

tools/deployment/airskiff/developer/030-armada-bootstrap.sh

@@ -17,12 +17,24 @@
 
 set -xe
 
+CURRENT_DIR="$(pwd)"
 : "${INSTALL_PATH:="$(pwd)/../"}"
 
+: "${PL_PATH:="../pegleg"}"
+
+# NOTE: Image to use for all Pegleg operations
+: "${PL_IMAGE:=quay.io/airshipit/pegleg:latest}"
+
+: "${PEGLEG:="${PL_PATH}/tools/pegleg.sh"}"
+: "${PL_SITE:="airskiff"}"
+
+# Render documents
+IMAGE=${PL_IMAGE} TERM_OPTS=" " ${PEGLEG} site -r . render "${PL_SITE}" -o airskiff.yaml
+
 # Download latest Armada image and deploy Airship components
 docker run --rm --net host -p 8000:8000 --name armada \
     -v ~/.kube/config:/armada/.kube/config \
-    -v "$(pwd)"/tools/deployment/airskiff/manifests/:/manifests \
+    -v "$(pwd)"/airskiff.yaml:/airskiff.yaml \
     -v "${INSTALL_PATH}":/airship-components \
     quay.io/airshipit/armada:latest \
-    apply /manifests/airship.yaml
+    apply /airskiff.yaml --target-manifest ucp-bootstrap

tools/deployment/airskiff/developer/100-deploy-osh.sh

@@ -29,6 +29,9 @@ CURRENT_DIR="$(pwd)"
 : "${PEGLEG:="${PL_PATH}/tools/pegleg.sh"}"
 : "${PL_SITE:="airskiff"}"
 
+# Source OpenStack credentials for Airship utility scripts
+. tools/deployment/airskiff/common/os-env.sh
+
 # NOTE(drewwalters96): Disable Pegleg linting errors P001 and P009; a
 #  a cleartext storage policy is acceptable for non-production use cases
 #  and maintain consistency with other treasuremap sites.

tools/deployment/airskiff/manifests/airship.yaml

@@ -1,358 +0,0 @@
----
-schema: armada/Chart/v1
-metadata:
-  schema: metadata/Document/v1
-  name: osh-helm-toolkit
-data:
-  chart_name: helm-toolkit
-  release: helm-toolkit
-  namespace: helm-toolkit
-  wait:
-    timeout: 1800
-    labels:
-      release_group: airship-helm-toolkit
-  values: {}
-  source:
-    type: git
-    location: https://opendev.org/openstack/openstack-helm-infra.git
-    reference: 6b17525b9312a0c9681cce2d22ab0e182362133d
-    subpath: helm-toolkit
-  dependencies: []
----
-schema: armada/Chart/v1
-metadata:
-  schema: metadata/Document/v1
-  name: ingress-kube-system
-data:
-  chart_name: ingress
-  release: ingress-kube-system
-  namespace: kube-system
-  wait:
-    timeout: 1800
-    labels:
-      release_group: airship-ingress-kube-system
-  values:
-    deployment:
-      mode: cluster
-      type: DaemonSet
-    network:
-      host_namespace: True
-  source:
-    type: git
-    location: https://opendev.org/openstack/openstack-helm-infra.git
-    reference: 8662018a4dceb82a5d699d9e118caff9e5edb156
-    subpath: ingress
-  dependencies:
-  - osh-helm-toolkit
----
-schema: armada/Chart/v1
-metadata:
-  schema: metadata/Document/v1
-  name: ingress-ucp
-data:
-  chart_name: ingress
-  release: ingress-ucp
-  namespace: ucp
-  wait:
-    timeout: 1800
-    labels:
-      release_group: airship-ingress-ucp
-  values: {}
-  source:
-    type: git
-    location: https://opendev.org/openstack/openstack-helm-infra.git
-    reference: 8662018a4dceb82a5d699d9e118caff9e5edb156
-    subpath: ingress
-  dependencies:
-    - osh-helm-toolkit
----
-schema: armada/Chart/v1
-metadata:
-  schema: metadata/Document/v1
-  name: nfs-provisioner
-data:
-  chart_name: nfs-provisioner
-  release: nfs-provisioner
-  namespace: nfs
-  wait:
-    timeout: 1800
-    labels:
-      release_group: airship-nfs-provisioner
-  values:
-    storageclass:
-      name: general
-  source:
-    type: git
-    location: https://opendev.org/openstack/openstack-helm-infra.git
-    reference: 8662018a4dceb82a5d699d9e118caff9e5edb156
-    subpath: nfs-provisioner
-  dependencies:
-    - osh-helm-toolkit
----
-schema: armada/Chart/v1
-metadata:
-  schema: metadata/Document/v1
-  name: mariadb
-data:
-  chart_name: mariadb
-  release: mariadb
-  namespace: ucp
-  wait:
-    timeout: 1800
-    labels:
-      release_group: airship-mariadb
-  values:
-    pod:
-      replicas:
-        server: 1
-  source:
-    type: git
-    location: https://opendev.org/openstack/openstack-helm-infra.git
-    reference: 8662018a4dceb82a5d699d9e118caff9e5edb156
-    subpath: mariadb
-  dependencies:
-    - osh-helm-toolkit
----
-schema: armada/Chart/v1
-metadata:
-  schema: metadata/Document/v1
-  name: rabbitmq
-data:
-  chart_name: rabbitmq
-  release: rabbitmq
-  namespace: ucp
-  wait:
-    timeout: 1800
-    labels:
-      release_group: airship-rabbitmq
-  test:
-    enabled: false
-  values:
-    pod:
-      replicas:
-        server: 1
-  source:
-    type: git
-    location: https://opendev.org/openstack/openstack-helm-infra.git
-    reference: 8662018a4dceb82a5d699d9e118caff9e5edb156
-    subpath: rabbitmq
-  dependencies:
-    - osh-helm-toolkit
----
-schema: armada/Chart/v1
-metadata:
-  schema: metadata/Document/v1
-  name: memcached
-data:
-  chart_name: memcached
-  release: memcached
-  namespace: ucp
-  wait:
-    timeout: 1800
-    labels:
-      release_group: airship-memcached
-  values: {}
-  source:
-    type: git
-    location: https://opendev.org/openstack/openstack-helm-infra.git
-    reference: 8662018a4dceb82a5d699d9e118caff9e5edb156
-    subpath: memcached
-  dependencies:
-    - osh-helm-toolkit
----
-schema: armada/Chart/v1
-metadata:
-  schema: metadata/Document/v1
-  name: keystone-ucp
-data:
-  chart_name: keystone
-  release: keystone-ucp
-  namespace: ucp
-  wait:
-    timeout: 1800
-    labels:
-      release_group: airship-keystone-ucp
-  values: {}
-  source:
-    type: git
-    location: https://opendev.org/openstack/openstack-helm.git
-    reference: 112e15b1655285194593a7afca4352ce046c7a47
-    subpath: keystone
-  dependencies:
-    - osh-helm-toolkit
----
-schema: armada/Chart/v1
-metadata:
-  schema: metadata/Document/v1
-  name: postgresql
-data:
-  chart_name: postgresql
-  release: postgresql
-  namespace: ucp
-  wait:
-    timeout: 1800
-    labels:
-      release_group: airship-postgresql
-  values:
-    pod:
-      replicas:
-        server: 1
-  source:
-    type: git
-    location: https://opendev.org/openstack/openstack-helm-infra.git
-    reference: 8662018a4dceb82a5d699d9e118caff9e5edb156
-    subpath: postgresql
-  dependencies:
-    - osh-helm-toolkit
----
-schema: armada/Chart/v1
-metadata:
-  schema: metadata/Document/v1
-  name: barbican
-data:
-  chart_name: barbican
-  release: barbican
-  namespace: ucp
-  wait:
-    timeout: 1800
-    labels:
-      release_group: airship-barbican
-  values: {}
-  source:
-    type: git
-    location: https://opendev.org/openstack/openstack-helm.git
-    reference: 112e15b1655285194593a7afca4352ce046c7a47
-    subpath: barbican
-  dependencies:
-    - osh-helm-toolkit
----
-schema: armada/Chart/v1
-metadata:
-  schema: metadata/Document/v1
-  name: armada
-data:
-  chart_name: armada
-  release: armada
-  namespace: ucp
-  wait:
-    timeout: 1800
-    labels:
-      release_group: airship-armada
-  values:
-    conf:
-      armada:
-        keystone_authtoken:
-          timeout: null
-  source:
-    type: local
-    location: /airship-components/armada
-    subpath: charts/armada
-  dependencies:
-    - osh-helm-toolkit
----
-schema: armada/Chart/v1
-metadata:
-  schema: metadata/Document/v1
-  name: deckhand
-data:
-  chart_name: deckhand
-  release: deckhand
-  namespace: ucp
-  wait:
-    timeout: 1800
-    labels:
-      release_group: airship-deckhand
-  values: {}
-  source:
-    type: local
-    location: /airship-components/deckhand
-    subpath: charts/deckhand
-  dependencies:
-    - osh-helm-toolkit
----
-schema: armada/Chart/v1
-metadata:
-  schema: metadata/Document/v1
-  name: shipyard
-data:
-  chart_name: shipyard
-  release: shipyard
-  namespace: ucp
-  wait:
-    timeout: 1800
-    labels:
-      release_group: airship-shipyard
-  values:
-    pod:
-      replicas:
-        shipyard:
-          api: 1
-        airflow:
-          web: 1
-          worker: 1
-          flower: 1
-          scheduler: 1
-    conf:
-      shipyard:
-        # NOTE(drewwalters96): Since Drydock and Promenade are not deployed,
-        #    alias those validations to Armada.
-        drydock:
-          service_type: armada
-        promenade:
-          service_type: armada
-  source:
-    type: local
-    location: /airship-components/shipyard
-    subpath: charts/shipyard
-  dependencies:
-    - osh-helm-toolkit
----
-schema: armada/ChartGroup/v1
-metadata:
-  schema: metadata/Document/v1
-  name: kube-system
-data:
-  description: Kube system ingress controller
-  sequenced: False
-  chart_group:
-    - ingress-kube-system
----
-schema: armada/ChartGroup/v1
-metadata:
-  schema: metadata/Document/v1
-  name: airship-infra
-data:
-  description: Services required for the operation of Airship components
-  sequenced: True
-  chart_group:
-    - ingress-ucp
-    - nfs-provisioner
-    - mariadb
-    - rabbitmq
-    - memcached
-    - keystone-ucp
-    - postgresql
-    - barbican
----
-schema: armada/ChartGroup/v1
-metadata:
-  schema: metadata/Document/v1
-  name: airship-components
-data:
-  description: Airship software delivery components
-  sequenced: True
-  chart_group:
-    - shipyard
-    - deckhand
-    - armada
----
-schema: armada/Manifest/v1
-metadata:
-  schema: metadata/Document/v1
-  name: armada-manifest
-data:
-  release_prefix: airship
-  chart_groups:
-    - kube-system
-    - airship-infra
-    - airship-components

tools/gate/wait-for-shipyard.sh

@@ -18,6 +18,9 @@ set -e
 
 : "${SHIPYARD:=../shipyard/tools/shipyard.sh}"
 
+# Source OpenStack credentials for Airship utility scripts
+. tools/deployment/airskiff/common/os-env.sh
+
 ACTION=$(${SHIPYARD} get actions | grep -i "Processing" | awk '{ print $2 }')
 
 echo -e "\nWaiting for $ACTION..."