Add control plane clouds to nodepool builder clouds.yaml

In order to have nodepool build images and upload them to control
plane clouds, add them to the clouds.yaml on the nodepool-builder
hosts. Keep them out of the launcher configs by splitting the config
templates. So that we can keep our copies of things to a minimum,
create a group called "control-plane-clouds" and put bridge and nb0*
in it.

There are clouds mentions in here that we no longer use, a followup
patch will clean those up.

NOTE: Requires shifting the clouds config dict from
host_vars/bridge.openstack.org.yaml to group_vars/control-plane-clouds.yaml
in the secrets on bridge.

Needed-By: https://review.opendev.org/640044
Change-Id: Id1161bca8f23129202599dba299c288a6aa29212

inventory/groups.yaml

@@ -13,6 +13,9 @@ groups:
   afsdb: afsdb[0-9]*.open*.org
   ask: ask*.open*.org
   cacti: cacti[0-9]*.open*.org
+  control-plane-clouds:
+    - bridge.openstack.org
+    - nb[0-9]*.open*.org
   disabled:
     - ask-staging.openstack.org
     - ask-staging01.openstack.org

playbooks/group_vars/nodepool-builder.yaml

@@ -1,4 +1,4 @@
 openstacksdk_config_dir: /home/nodepool/.config/openstack
 openstacksdk_config_owner: nodepool
 openstacksdk_config_group: nodepool
-openstacksdk_config_template: clouds/nodepool_clouds.yaml.j2
+openstacksdk_config_template: clouds/nodepool_builder_clouds.yaml.j2

playbooks/group_vars/nodepool-launcher.yaml

@@ -1,4 +1,4 @@
 openstacksdk_config_dir: /home/nodepool/.config/openstack
 openstacksdk_config_owner: nodepool
 openstacksdk_config_group: nodepool
-openstacksdk_config_template: clouds/nodepool_clouds.yaml.j2
+openstacksdk_config_template: clouds/nodepool_launcher_clouds.yaml.j2

playbooks/roles/install-ansible/files/inventory_plugins/test-fixtures/results.yaml

@@ -65,6 +65,7 @@ results:
     - puppet
     - puppet4
     - webservers
+    - control-plane-clouds
 
   ze01.openstack.org:
     - afs-client

playbooks/templates/clouds/nodepool_builder_clouds.yaml.j2

@@ -0,0 +1,171 @@
+#
+# Nodepool openstacksdk configuration
+#
+# This file is deployed to nodepool builder hosts as
+#
+#   ~nodepool/.config/openstack/config/clouds.yaml
+#
+# and is used there to authenticate nodepool operations to clouds.
+# The naming should correspond that used in nodepool configuration
+# files.
+#
+
+cache:
+  expiration:
+    server: 5
+    port: 5
+    floating-ip: 5
+clouds:
+  rax:
+    profile: rackspace
+    regions:
+      - DFW
+      - ORD
+      - IAD
+    api_timeout: 60
+    auth:
+      username: '{{ nodepool_rackspace_username }}'
+      password: '{{ nodepool_rackspace_password }}'
+      project_id: '{{ nodepool_rackspace_project }}'
+    force_ipv4: true
+  rax-control-plane:
+    regions:
+      - DFW
+      - ORD
+      - IAD
+    profile: rackspace
+    auth:
+      username: '{{ clouds.openstackci_rax_username }}'
+      password: '{{ clouds.openstackci_rax_password }}'
+      project_id: '{{ clouds.openstackci_rax_project_id }}'
+    force_ipv4: true
+  ovh:
+    profile: ovh
+    # OVH has a weird new ipv6 setup that we can't handle properly
+    # for now ignore ipv6
+    force_ipv4: true
+    regions:
+      - BHS1
+      - GRA1
+    api_timeout: 60
+    auth:
+      username: '{{ nodepool_ovh_username }}'
+      password: '{{ nodepool_ovh_password }}'
+      project_name: '{{ nodepool_ovh_project }}'
+  inap:
+    profile: internap
+    api_timeout: 60
+    auth:
+      username: '{{ nodepool_internap_username }}'
+      password: '{{ nodepool_internap_password }}'
+      project_name: '{{ nodepool_internap_project }}'
+    regions:
+      - name: mtl01
+        values:
+          networks:
+            - name: inap-17304-WAN1101
+              routes_externally: True
+  vexxhost:
+    profile: vexxhost
+    regions:
+      - ca-ymq-1
+      - sjc1
+    api_timeout: 60
+    auth_type: password
+    auth:
+      username: '{{ nodepool_vexxhost_username }}'
+      password: '{{ nodepool_vexxhost_password }}'
+      project_name: '{{ nodepool_vexxhost_project }}'
+      project_domain_name: default
+      user_domain_name: default
+    image_format: 'raw'
+    force_ipv4: true
+  vexxhost-control-plane:
+    regions:
+      - ca-ymq-1
+      - sjc1
+    profile: vexxhost
+    auth:
+      username: '{{ clouds.openstackci_vexxhost_username }}'
+      password: '{{ clouds.openstackci_vexxhost_password }}'
+      project_name: '{{ clouds.openstackci_vexxhost_project_name }}'
+      project_domain_name: default
+      user_domain_name: default
+    image_format: 'raw'
+    force_ipv4: true
+  citycloud:
+    regions:
+      - Lon1
+      - Kna1
+      - La1
+      - Sto2
+    profile: citycloud
+    api_timeout: 60
+    auth:
+      # TODO(mordred) This auth_url entry is only needed until openstacksdk
+      # 0.18.2 or 0.19.0 is released.
+      auth_url: "https://{region_name}.citycloud.com:5000/v3/"
+      username: '{{ nodepool_citycloud_username }}'
+      password: '{{ nodepool_citycloud_password }}'
+      project_name: 'OpenStack CI Project'
+      project_domain_name: CCP_Domain_27611
+      user_domain_name: CCP_Domain_27611
+  linaro-london:
+    regions:
+      - London
+    identity_api_version: '3'
+    auth:
+      auth_url: https://uk.linaro.cloud:5000
+      username: '{{ nodepool_linaro_london_username }}'
+      password: '{{ nodepool_linaro_london_password }}'
+      project_name: '{{ nodepool_linaro_london_project }}'
+      project_domain_name: default
+      user_domain_name: default
+  limestone:
+    regions:
+      - name: RegionOne
+        values:
+          networks:
+            - name: "Gateway Network"
+              routes_externally: false
+              routes_ipv6_externally: true
+              nat_destination: true
+    auth:
+      auth_url: https://osa.continuous.pw:5000
+      username: '{{ nodepool_limestone_username }}'
+      password: '{{ nodepool_limestone_password }}'
+      project_name: '{{ nodepool_limestone_project }}'
+      user_domain_name: default
+      project_domain_name: default
+    api_timeout: 60
+    identity_api_version: 3
+    floating_ip_source: None
+    cacert: /home/nodepool/.config/openstack/limestone_cacert.pem
+  packethost:
+    regions:
+      - name: us-west-1
+        values:
+          networks:
+            - name: openstackzuul-net
+              default_interface: True
+    auth:
+      auth_url: https://packet.platform9.net/keystone
+      username: '{{ nodepool_packethost_username }}'
+      password: '{{ nodepool_packethost_password }}'
+      project_name: '{{ nodepool_packethost_project }}'
+      user_domain_name: default
+      project_domain_name: default
+    identity_api_version: '3'
+    image_format: 'raw'
+  arm64ci:
+    regions:
+      - nrt1
+    identity_api_version: '3'
+    auth:
+      auth_url: https://arm64ci.cloud:5000
+      username: '{{ nodepool_arm64ci_username }}'
+      password: '{{ nodepool_arm64ci_password }}'
+      project_name: '{{ nodepool_arm64ci_project }}'
+      project_domain_name: openstack-infra
+      user_domain_name: openstack-infra
+    interface: public

playbooks/templates/clouds/nodepool_launcher_clouds.yaml.j2

@@ -1,7 +1,7 @@
 #
 # Nodepool openstacksdk configuration
 #
-# This file is deployed to nodepool launcher and builder hosts as
+# This file is deployed to nodepool launcher hosts as
 #
 #   ~nodepool/.config/openstack/config/clouds.yaml
 #

playbooks/zuul/run-base.yaml

@@ -78,6 +78,7 @@
         - group_vars/gitea-lb.yaml
         - group_vars/letsencrypt.yaml
         - group_vars/registry.yaml
+        - group_vars/control-plane-clouds.yaml
         - host_vars/bridge.openstack.org.yaml
         - host_vars/letsencrypt01.opendev.org.yaml
         - host_vars/letsencrypt02.opendev.org.yaml

playbooks/zuul/templates/group_vars/control-plane-clouds.yaml.j2

@@ -0,0 +1,60 @@
+# Necessary for fake clouds.yaml to be written
+clouds:
+  openstackci_internap_username: user
+  openstackci_internap_password: password
+  openstackci_internap_project_name: project
+  openstackjenkins_internap_username: user
+  openstackjenkins_internap_password: password
+  openstackjenkins_internap_project_name: project
+  openstackci_ovh_username: user
+  openstackci_ovh_password: password
+  openstackci_ovh_project_name: project
+  openstackci_rax_username: user
+  openstackci_rax_password: password
+  openstackci_rax_project_id: project
+  openstackjenkins_rax_username: user
+  openstackjenkins_rax_password: password
+  openstackjenkins_rax_project_id: project
+  openstackjenkins_ovh_username: user
+  openstackjenkins_ovh_password: password
+  openstackjenkins_ovh_project_name: project
+  openstackjenkins_vexxhost_username: user
+  openstackjenkins_vexxhost_password: password
+  openstackjenkins_vexxhost_project_name: project
+  openstackci_vexxhost_username: user
+  openstackci_vexxhost_password: password
+  openstackci_vexxhost_project_name: project
+  openstackci_citycloud_username: user
+  openstackci_citycloud_password: password
+  openstackzuul_citycloud_username: user
+  openstackzuul_citycloud_password: password
+  openstackci_linaro_username: user
+  openstackci_linaro_password: password
+  openstackci_linaro_project_name: project
+  openstackzuul_linaro_username: user
+  openstackzuul_linaro_password: password
+  openstackzuul_linaro_project_name: project
+  openstackci_linaro_london_username: user
+  openstackci_linaro_london_password: password
+  openstackci_linaro_london_project_name: project
+  openstackzuul_linaro_london_username: user
+  openstackzuul_linaro_london_password: password
+  openstackzuul_linaro_london_project_name: project
+  openstackci_limestone_username: user
+  openstackci_limestone_password: password
+  openstackci_limestone_project_name: project
+  openstackzuul_limestone_username: user
+  openstackzuul_limestone_password: password
+  openstackzuul_limestone_project_name: project
+  openstackci_packethost_username: user
+  openstackci_packethost_password: password
+  openstackci_packethost_project_name: project
+  openstackzuul_packethost_username: user
+  openstackzuul_packethost_password: password
+  openstackzuul_packethost_project_name: project
+  openstackci_arm64ci_username: user
+  openstackci_arm64ci_password: password
+  openstackci_arm64ci_project_name: project
+  openstackzuul_arm64ci_username: user
+  openstackzuul_arm64ci_password: password
+  openstackzuul_arm64ci_project_name: project

tools/check_clouds_yaml.py

@@ -21,7 +21,8 @@ import sys
 import tempfile
 
 FILES_TO_CHECK = (
-    'playbooks/templates/clouds/nodepool_clouds.yaml.j2',
+    'playbooks/templates/clouds/nodepool_builder_clouds.yaml.j2',
+    'playbooks/templates/clouds/nodepool_launcher_clouds.yaml.j2',
     'playbooks/templates/clouds/bridge_all_clouds.yaml.j2',
     'playbooks/templates/clouds/bridge_clouds.yaml.j2',
 )