From 381c1bad3cef56a06a858731ccb6058d7a5a7fdc Mon Sep 17 00:00:00 2001 From: Paul Belanger Date: Wed, 25 Oct 2017 15:18:54 -0400 Subject: [PATCH] Remove zuul-launcher support With the migration to zuulv3, there is no more zuul-launcher. This has become zuul-executor, which has been moved into production. Servers have already been deleted, lets also remove it from puppet. Change-Id: Id2b53decdc63712460049f5fa9ed751e049d17ff Signed-off-by: Paul Belanger --- doc/source/zuul.rst | 16 --- hiera/common.yaml | 7 -- manifests/site.pp | 63 ----------- .../manifests/zuul_launcher.pp | 107 ------------------ playbooks/hard_restart_zuul_launchers.yaml | 23 ---- playbooks/restart_zuul_launchers.yaml | 23 ---- start_all_zuul.yaml | 9 -- stop_all_zuul.yaml | 13 --- 8 files changed, 261 deletions(-) delete mode 100644 modules/openstack_project/manifests/zuul_launcher.pp delete mode 100644 playbooks/hard_restart_zuul_launchers.yaml delete mode 100644 playbooks/restart_zuul_launchers.yaml diff --git a/doc/source/zuul.rst b/doc/source/zuul.rst index 781d4d9758..be7020570d 100644 --- a/doc/source/zuul.rst +++ b/doc/source/zuul.rst @@ -16,7 +16,6 @@ At a Glance * http://zuul.openstack.org * http://zuul-dev.openstack.org * zm*.openstack.org - * zl*.openstack.org :Puppet: * https://git.openstack.org/cgit/openstack-infra/puppet-zuul/tree/ * :file:`modules/openstack_project/manifests/zuul_prod.pp` @@ -162,18 +161,3 @@ necessary to accommodate load. If you remove a merger, be sure to leave Apache running for several hours until the last job that may have been launched with instructions to fetch from that merger has completed. - -Launchers ---------- - -We use an Ansible based launcher in Zuul to actually run jobs. This -component runs on a horizontally scalable set of servers named -zl*.openstack.org. It reads job configuration from Jenkins Job -Builder files in the project-config repository and translates that -into Ansible playbooks which it runs on our workers. Our jobs are -configured to upload as much information as possible along with their -logs, but if there is an error which can not be diagnosed in that -manner, Ansible logs are available in the launcher-debug log file on -the launcher host. You may use the Zuul build UUID to track -assignment of a given job from the Zuul scheduler to the Zuul launcher -used by that job. diff --git a/hiera/common.yaml b/hiera/common.yaml index 6018e0e517..61a33356c3 100644 --- a/hiera/common.yaml +++ b/hiera/common.yaml @@ -476,13 +476,6 @@ cacti_hosts: - ze08.openstack.org - ze09.openstack.org - ze10.openstack.org -- zlstatic01.openstack.org -- zl01.openstack.org -- zl02.openstack.org -- zl03.openstack.org -- zl04.openstack.org -- zl05.openstack.org -- zl06.openstack.org - zm01.openstack.org - zm02.openstack.org - zm03.openstack.org diff --git a/manifests/site.pp b/manifests/site.pp index 7b75cee135..860eaeddbc 100644 --- a/manifests/site.pp +++ b/manifests/site.pp @@ -1277,7 +1277,6 @@ node 'zuulv3-dev.openstack.org' { manage_common_zuul => false, } # TODO(pabelanger): Add zuul_scheduler support - # TODO(pabelanger): Add zuul_launcher support } # Node-OS: xenial @@ -1373,13 +1372,6 @@ node 'zuulv3.openstack.org' { node 'zuul.openstack.org' { $gearman_workers = [ 'nodepool.openstack.org', - 'zlstatic01.openstack.org', - 'zl01.openstack.org', - 'zl02.openstack.org', - 'zl03.openstack.org', - 'zl04.openstack.org', - 'zl05.openstack.org', - 'zl06.openstack.org', ] $iptables_rules = regsubst ($gearman_workers, '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 4730 -s \1 -j ACCEPT') @@ -1405,61 +1397,6 @@ node 'zuul.openstack.org' { } } -# Node-OS: trusty -node /^zlstatic\d+\.openstack\.org$/ { - $group = "zuul-merger" - $zmq_event_receivers = ['logstash.openstack.org', - 'nodepool.openstack.org'] - $zmq_iptables_rule = regsubst($zmq_event_receivers, - '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 8888 -s \1 -j ACCEPT') - $iptables_rule = flatten([$zmq_iptables_rule]) - class { 'openstack_project::server': - iptables_rules6 => $iptables_rule, - iptables_rules4 => $iptables_rule, - sysadmins => hiera('sysadmins', []), - afs => true, - } - class { 'openstack_project::zuul_launcher': - gearman_server => 'zuul.openstack.org', - gerrit_server => 'review.openstack.org', - gerrit_user => 'jenkins', - gerrit_ssh_host_key => hiera('gerrit_ssh_rsa_pubkey_contents'), - zuul_ssh_private_key => hiera('jenkins_ssh_private_key_contents'), - project_config_repo => 'https://git.openstack.org/openstack-infra/project-config', - sysadmins => hiera('sysadmins', []), - sites => hiera('zuul_sites', []), - nodes => hiera('zuul_nodes', []), - accept_nodes => false, - } -} - -# Node-OS: trusty -node /^zl\d+\.openstack\.org$/ { - $group = "zuul-merger" - $zmq_event_receivers = ['logstash.openstack.org', - 'nodepool.openstack.org'] - $zmq_iptables_rule = regsubst($zmq_event_receivers, - '^(.*)$', '-m state --state NEW -m tcp -p tcp --dport 8888 -s \1 -j ACCEPT') - $iptables_rule = flatten([$zmq_iptables_rule]) - class { 'openstack_project::server': - iptables_rules6 => $iptables_rule, - iptables_rules4 => $iptables_rule, - sysadmins => hiera('sysadmins', []), - afs => true, - } - class { 'openstack_project::zuul_launcher': - gearman_server => 'zuul.openstack.org', - gerrit_server => 'review.openstack.org', - gerrit_user => 'jenkins', - gerrit_ssh_host_key => hiera('gerrit_ssh_rsa_pubkey_contents'), - zuul_ssh_private_key => hiera('jenkins_ssh_private_key_contents'), - project_config_repo => 'https://git.openstack.org/openstack-infra/project-config', - sysadmins => hiera('sysadmins', []), - sites => hiera('zuul_sites', []), - zuul_launcher_keytab => hiera('zuul_launcher_keytab'), - } -} - # Node-OS: xenial node /^zm\d+.openstack\.org$/ { $group = "zuul-merger" diff --git a/modules/openstack_project/manifests/zuul_launcher.pp b/modules/openstack_project/manifests/zuul_launcher.pp deleted file mode 100644 index 0a836d0b3c..0000000000 --- a/modules/openstack_project/manifests/zuul_launcher.pp +++ /dev/null @@ -1,107 +0,0 @@ -# == Class: openstack_project::zuul_launcher -# -class openstack_project::zuul_launcher( - $vhost_name = $::fqdn, - $gearman_server = '127.0.0.1', - $gerrit_server = '', - $gerrit_user = '', - $gerrit_ssh_host_key = '', - $zuul_ssh_private_key = '', - $url_pattern = '', - $zuul_url = '', - $status_url = 'http://status.openstack.org/zuul/', - $swift_authurl = '', - $swift_auth_version = '', - $swift_user = '', - $swift_key = '', - $swift_tenant_name = '', - $swift_region_name = '', - $swift_default_container = '', - $swift_default_logserver_prefix = '', - $swift_default_expiry = 7200, - $proxy_ssl_cert_file_contents = '', - $proxy_ssl_key_file_contents = '', - $proxy_ssl_chain_file_contents = '', - $sysadmins = [], - $statsd_host = '', - $project_config_repo = '', - $project_config_base = '', - $git_email = 'jenkins@openstack.org', - $git_name = 'OpenStack Jenkins', - $workspace_root = '/home/jenkins/workspace', - $worker_private_key_file = '/var/lib/zuul/ssh/id_rsa', - $worker_username = 'jenkins', - $sites = [], - $nodes = [], - $accept_nodes = '', - $zuul_launcher_keytab = '', -) { - - class { '::project_config': - url => $project_config_repo, - base => $project_config_base, - } - - file { '/etc/zuul-launcher.keytab': - owner => 'zuul', - group => 'zuul', - mode => '0400', - content => $zuul_launcher_keytab, - } - - file { '/etc/jenkins_jobs': - ensure => directory, - } - - file { '/etc/jenkins_jobs/config': - ensure => directory, - owner => 'root', - group => 'root', - mode => '0755', - recurse => true, - purge => true, - force => true, - source => $::project_config::jenkins_job_builder_config_dir, - require => [File['/etc/jenkins_jobs'], - $::project_config::config_dir], - notify => Exec['zuul-launcher-reload'], - } - - file { '/home/zuul/.ssh': - ensure => directory, - owner => 'zuul', - group => 'zuul', - mode => '0700', - require => User['zuul'], - } - - file { '/home/zuul/.ssh/config': - ensure => present, - source => 'puppet:///modules/openstack_project/zuul/launcher_ssh_config', - owner => 'zuul', - group => 'zuul', - require => File['/home/zuul/.ssh'], - } - - class { '::zuul': - vhost_name => $vhost_name, - gearman_server => $gearman_server, - gerrit_server => $gerrit_server, - gerrit_user => $gerrit_user, - zuul_ssh_private_key => $zuul_ssh_private_key, - zuul_url => $zuul_url, - git_email => $git_email, - git_name => $git_name, - revision => $revision, - git_source_repo => $git_source_repo, - jenkins_jobs => '/etc/jenkins_jobs/config', - workspace_root => $workspace_root, - worker_private_key_file => $worker_private_key_file, - worker_username => $worker_username, - sites => $sites, - nodes => $nodes, - accept_nodes => $accept_nodes, - } - - class { 'zuul::launcher': } -} diff --git a/playbooks/hard_restart_zuul_launchers.yaml b/playbooks/hard_restart_zuul_launchers.yaml deleted file mode 100644 index bff1fed767..0000000000 --- a/playbooks/hard_restart_zuul_launchers.yaml +++ /dev/null @@ -1,23 +0,0 @@ ---- -- hosts: 'zlstatic0*.openstack.org:zl0*.openstack.org' - # Do the entire play completely for one host at a time - serial: 1 - any_errors_fatal: true - tasks: - - name: Hard stop zuul-launcher - shell: 'zuul-launcher stop' - become: true - become_user: zuul - - - name: Wait for (3h10m) to stop zuul-launcher - wait_for: - path: /var/run/zuul-launcher/zuul-launcher.pid - state: absent - timeout: 11400 - - - name: Restart zuul-launcher - service: - name: zuul-launcher - state: restarted - become: true - become_user: root diff --git a/playbooks/restart_zuul_launchers.yaml b/playbooks/restart_zuul_launchers.yaml deleted file mode 100644 index 306d11cd1d..0000000000 --- a/playbooks/restart_zuul_launchers.yaml +++ /dev/null @@ -1,23 +0,0 @@ ---- -- hosts: 'zlstatic0*.openstack.org:zl0*.openstack.org' - # Do the entire play completely for one host at a time - serial: 1 - any_errors_fatal: true - tasks: - - name: Graceful stop zuul-launcher - shell: 'zuul-launcher graceful' - become: true - become_user: zuul - - - name: Wait for (3h10m) to stop zuul-launcher - wait_for: - path: /var/run/zuul-launcher/zuul-launcher.pid - state: absent - timeout: 11400 - - - name: Restart zuul-launcher - service: - name: zuul-launcher - state: restarted - become: true - become_user: root diff --git a/start_all_zuul.yaml b/start_all_zuul.yaml index fe4c855025..e5857cf4b1 100644 --- a/start_all_zuul.yaml +++ b/start_all_zuul.yaml @@ -9,15 +9,6 @@ become: true become_user: root -- hosts: 'zlstatic0*.openstack.org:zl0*.openstack.org' - tasks: - - name: Start zuul-launcher - service: - name: zuul-launcher - state: started - become: true - become_user: root - - hosts: 'zm0*.openstack.org' tasks: - name: Start zuul-merger diff --git a/stop_all_zuul.yaml b/stop_all_zuul.yaml index 9d0ce2ba03..b2cb8c476a 100644 --- a/stop_all_zuul.yaml +++ b/stop_all_zuul.yaml @@ -14,19 +14,6 @@ state: absent timeout: 11400 -- hosts: 'zlstatic0*.openstack.org:zl0*.openstack.org' - tasks: - - name: Hard stop zuul-launcher - shell: 'zuul-launcher stop' - become: true - become_user: zuul - - - name: Wait for (3h10m) to stop zuul-launcher - wait_for: - path: /var/run/zuul-launcher/zuul-launcher.pid - state: absent - timeout: 11400 - - hosts: 'zm0*.openstack.org' tasks: - name: Hard stop zuul-merger