Run nodepool launchers with ansible and containers

We don't run start in prod normally but we do need to run
it in the gate.

Change-Id: Iec50684280409eb978bf5638bf74ae16fad8aa26

.zuul.yaml

@@ -2232,7 +2232,7 @@
       - playbooks/roles/configure-kubectl/
       - playbooks/roles/configure-openstacksdk/
       - playbooks/roles/install-docker/
-      - playbooks/roles/nodepool-
+      - playbooks/roles/nodepool
       - playbooks/templates/clouds/nodepool_
 
 - job:

inventory/groups.yaml

@@ -101,8 +101,7 @@ groups:
   nodepool-builder_opendev:
     - nb[0-9]*.opendev.org
   nodepool-launcher:
-    - nl[0-9]*.openstack.org
-    - nl[0-8]*.opendev.org
+    - nl[0-9]*.open*.org
   ns:
     - ns[0-9]*.open*.org
   openstackid-dev:
@@ -137,7 +136,6 @@ groups:
     - mirror-update[0-9]*.openstack.org
     - mirror[0-9]*.openstack.org
     - nb[0-9]*.openstack.org
-    - nl[0-9]*.openstack.org
     - openstackid-dev*.openstack.org
     - openstackid.org
     - openstackid[0-9]*.openstack.org
@@ -177,7 +175,6 @@ groups:
     - mirror-update[0-9]*.openstack.org
     - ^mirror[0-9].*\..*\.(?!linaro|linaro-london|linaro-us).*\.openstack\.org
     - ^nb(?!03)[0-9]*\.openstack\.org
-    - nl[0-9]*.open*.org
     - openstackid[0-9]*.openstack.org
     - openstackid-dev[0-9]*.openstack.org
     - paste[0-9]*.open*.org

manifests/site.pp

@@ -369,28 +369,6 @@ node /^survey\d+\.open.*\.org$/ {
   }
 }
 
-# Node-OS: xenial
-node /^nl\d+\.open.*\.org$/ {
-  $group = 'nodepool'
-
-  # NOTE(ianw) From 09-2018 (https://review.opendev.org/#/c/598329/)
-  # the cloud credentials are deployed with ansible via the
-  # configure-openstacksdk role and are no longer configured here
-
-  class { 'openstack_project::server': }
-
-  include openstack_project
-
-  class { '::openstackci::nodepool_launcher':
-    nodepool_ssh_private_key => hiera('zuul_worker_ssh_private_key_contents'),
-    project_config_repo      => 'https://opendev.org/openstack/project-config',
-    statsd_host              => 'graphite.opendev.org',
-    revision                 => 'master',
-    python_version           => 3,
-    enable_webapp            => true,
-  }
-}
-
 # Node-OS: xenial
 node /^nb\d+\.open.*\.org$/ {
   $group = 'nodepool'

playbooks/group_vars/nodepool-launcher.yaml

@@ -1,4 +1,4 @@
-openstacksdk_config_dir: /home/nodepool/.config/openstack
+openstacksdk_config_dir: /etc/openstack
 openstacksdk_config_owner: nodepool
 openstacksdk_config_group: nodepool
 openstacksdk_config_template: clouds/nodepool_clouds.yaml.j2

playbooks/roles/nodepool-builder/templates/docker-compose.yaml.j2

@@ -10,6 +10,8 @@ services:
 
     environment:
       - DEBUG=1
+      - STATSD_HOST=graphite.opendev.org
+      - STATSD_PORT=8125
 
     volumes:
       # nodepool config

playbooks/roles/nodepool-launcher/README.rst

@@ -0,0 +1 @@
+Deploy nodepool launchers

playbooks/roles/nodepool-launcher/defaults/main.yaml

@@ -0,0 +1 @@
+nodepool_launcher_start: false

playbooks/roles/nodepool-launcher/files/logging.conf

@@ -0,0 +1,54 @@
+[loggers]
+keys=root,nodepool,requests,openstack,kazoo
+
+[handlers]
+keys=console,debug,normal
+
+[formatters]
+keys=simple
+
+[logger_root]
+level=WARNING
+handlers=console
+
+[logger_requests]
+level=WARNING
+handlers=debug,normal
+qualname=requests
+
+[logger_openstack]
+level=WARNING
+handlers=debug,normal
+qualname=openstack
+
+[logger_kazoo]
+level=INFO
+handlers=debug,normal
+qualname=kazoo
+
+[logger_nodepool]
+level=DEBUG
+handlers=debug,normal
+qualname=nodepool
+
+[handler_console]
+level=WARNING
+class=StreamHandler
+formatter=simple
+args=(sys.stdout,)
+
+[handler_debug]
+level=DEBUG
+class=logging.handlers.TimedRotatingFileHandler
+formatter=simple
+args=('/var/log/nodepool/launcher-debug.log', 'H', 8, 30,)
+
+[handler_normal]
+level=INFO
+class=logging.handlers.TimedRotatingFileHandler
+formatter=simple
+args=('/var/log/nodepool/nodepool-launcher.log', 'H', 8, 30,)
+
+[formatter_simple]
+format=%(asctime)s %(levelname)s %(name)s: %(message)s
+datefmt=

playbooks/roles/nodepool-launcher/handlers/main.yaml

@@ -0,0 +1,4 @@
+- name: nodepool-launcher Reload apache2
+  service:
+    name: apache2
+    state: reloaded

playbooks/roles/nodepool-launcher/tasks/main.yaml

@@ -0,0 +1,55 @@
+- name: Copy logging config
+  copy:
+    src: logging.conf
+    dest: /etc/nodepool/launcher-logging.conf
+
+- name: Install apache2
+  apt:
+    name:
+      - apache2
+      - apache2-utils
+    state: present
+
+- name: Apache modules
+  apache2_module:
+    state: present
+    name: "{{ item }}"
+  loop:
+    - rewrite
+    - proxy
+    - proxy_http
+
+- name: Copy apache config
+  template:
+    src: launcher.vhost.j2
+    dest: /etc/apache2/sites-enabled/000-default.conf
+    owner: root
+    group: root
+    mode: 0644
+  notify: nodepool-launcher Reload apache2
+
+# Do this until 719589/ passes
+- name: Install docker-compose
+  pip:
+    name: docker-compose
+    state: present
+    executable: pip3
+
+- name: Ensure docker compose dir
+  file:
+    state: directory
+    path: /etc/nodepool-docker
+
+- name: Copy docker compose file
+  template:
+    src: docker-compose.yaml.j2
+    dest: /etc/nodepool-docker/docker-compose.yaml
+
+- name: Run docker-compose pull
+  shell:
+    cmd: docker-compose pull
+    chdir: /etc/nodepool-docker/
+
+- name: Start nodepool launcher
+  include_tasks: start.yaml
+  when: nodepool_launcher_start | bool

playbooks/roles/nodepool-launcher/tasks/start.yaml

@@ -0,0 +1,8 @@
+- name: Run docker-compose up
+  shell:
+    cmd: docker-compose up -d
+    chdir: /etc/nodepool-docker/
+
+- name: Run docker prune to cleanup unneeded images
+  shell:
+    cmd: docker image prune -f

playbooks/roles/nodepool-launcher/templates/docker-compose.yaml.j2

@@ -0,0 +1,20 @@
+version: '2'
+services:
+  nodepool-launcher:
+    image: docker.io/zuul/nodepool-launcher:{{ nodepool_launcher_container_tag|default('latest') }}
+    user: nodepool
+    network_mode: host
+    restart: always
+
+    environment:
+      - DEBUG=1
+      - STATSD_HOST=graphite.opendev.org
+      - STATSD_PORT=8125
+
+    volumes:
+      # nodepool config
+      - /etc/nodepool:/etc/nodepool:ro
+      # openstacksdk config
+      - /etc/openstack:/etc/openstack:ro
+      # logs
+      - /var/log/nodepool:/var/log/nodepool:rw

playbooks/roles/nodepool-launcher/templates/launcher.vhost.j2

@@ -0,0 +1,19 @@
+<VirtualHost *:80>
+  ServerName {{ inventory_hostname }}
+
+  ErrorLog ${APACHE_LOG_DIR}/nodepool-error.log
+  LogLevel warn
+  CustomLog ${APACHE_LOG_DIR}/nodepool-access.log combined
+  ServerSignature Off
+
+  <IfModule mod_deflate.c>
+      SetOutputFilter DEFLATE
+  </IfModule>
+
+  RewriteEngine on
+  RewriteRule ^/image-list$ http://127.0.0.1:8005/image-list [P]
+  RewriteRule ^/dib-image-list$ http://127.0.0.1:8005/dib-image-list [P]
+  RewriteRule ^/image-list.json$ http://127.0.0.1:8005/image-list.json [P]
+  RewriteRule ^/dib-image-list.json$ http://127.0.0.1:8005/dib-image-list.json [P]
+
+</VirtualHost>

playbooks/service-nodepool.yaml

@@ -1,11 +1,3 @@
-- hosts: nodepool-launcher:nodepool-builder:!disabled
-  name: "Base: configure OpenStackSDK on nodepool legacy hosts"
-  strategy: free
-  roles:
-    - nodepool-base-legacy
-    - configure-openstacksdk
-    - configure-kubectl
-
 - hosts: nodepool-builder_opendev:!disabled
   name: "Configure nodepool builders"
   strategy: free
@@ -19,10 +11,19 @@
   name: "run puppet on all older servers"
   strategy: free
   roles:
+    - nodepool-base-legacy
+    - configure-openstacksdk
+    - configure-kubectl
     - puppet-install
     - disable-puppet-agent
     - puppet
 
-# TODO(ianw) 2020-03-03 : watch this space...
-#- hosts: nodepool-launcher_opendev:!disabled
-#  name: "Configure nodepool launchers"
+- hosts: nodepool-launcher:!disabled
+  name: "Configure nodepool launchers"
+  strategy: free
+  roles:
+    - install-docker
+    - nodepool-base
+    - configure-openstacksdk
+    - configure-kubectl
+    - nodepool-launcher

playbooks/zuul/run-base.yaml

@@ -53,6 +53,7 @@
         - group_vars/gitea-lb.yaml
         - group_vars/letsencrypt.yaml
         - group_vars/meetpad.yaml
+        - group_vars/nodepool-launcher.yaml
         - group_vars/registry.yaml
         - group_vars/review.yaml
         - group_vars/review-dev.yaml

playbooks/zuul/templates/group_vars/nodepool-launcher.yaml.j2

@@ -0,0 +1 @@
+nodepool_launcher_start: true

testinfra/test_nodepool.py

@@ -19,7 +19,7 @@ testinfra_hosts = ['nl01.openstack.org', 'nb01.openstack.org',
 
 
 def test_clouds_yaml(host):
-    if host.backend.get_hostname().endswith('openstack.org'):
+    if host.backend.get_hostname() == 'nb01.openstack.org':
         cfg_file = '/home/nodepool/.config/openstack/clouds.yaml'
     else:
         cfg_file = '/etc/openstack/clouds.yaml'
@@ -30,13 +30,20 @@ def test_clouds_yaml(host):
     assert b'password' in clouds_yaml.content
 
 def test_kube_config(host):
-    if not host.backend.get_hostname().endswith('openstack.org'):
+    if not host.backend.get_hostname().startswith('nl'):
         pytest.skip()
     kubeconfig = host.file('/home/nodepool/.kube/config')
     assert kubeconfig.exists
 
     assert b'nodepool_k8s_key' in kubeconfig.content
 
+def test_launcher_container_running(host):
+    if host.backend.get_hostname() != 'nl01.openstack.org':
+        pytest.skip()
+
+    cmd = host.run("docker ps -a --format '{{ .Names }}'")
+    assert 'nodepool-docker_nodepool-launcher_1' in cmd.stdout
+
 def test_builder_container_running(host):
     if host.backend.get_hostname() != 'nb04.opendev.org':
         pytest.skip()