Add upgrade stuff for new versions

Following items were added to perform environment update:

  1. Add task to perform upgrade with call of apt-get dist-upgrade.
     This commands force to keep old configuration files for all
     services.

  2. If MU upgrade is enabled, we trigger all services to restart.
     The exception is for Ceph, MySQL and RabbitMQ.

Some workarounds were implemented for following issues:

  MySQL and RabbitMQ:
    MySQL and RabbitMQ restart is managed separately from other
    services and it is disabled by default. To enable it you
    should set mu_upgrade['restart_mysql'] and mu_upgrade['restart_rabbitmq']
    to true in astute.yaml.

  Pacemaker service issue:
    If corosync package was upgraded APT (or dpkg) restarts corosync
    service and this restart kills pacemaker service. So we need to
    start it again. Also we should deny APT to stop pacemaker service
    during upgrade because it leads to unload of all pacemaker resources
    (all services under it will be stopped). That's not appropriate
    behaviour during the update process. Puppet will manage pacemaker
    service itself.

  Ceph:
    Ceph should be upgraded by following next guide [1], otherwise simple
    restart of services might break the cluster or cause a data loss.

    [1] http://docs.ceph.com/docs/master/install/upgrading-ceph/#upgrade-procedures

DocImpact

Related-bug: #1614893

Change-Id: I0d7231aa5900318f75f71c698f3e1c07f8e5cfbe

deployment/puppet/openstack_tasks/examples/aodh/aodh.pp

@@ -1 +1,2 @@
 class { '::openstack_tasks::aodh::aodh' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/openstack_tasks/examples/ceilometer/compute.pp

@@ -1 +1,2 @@
 class { '::openstack_tasks::ceilometer::compute' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/openstack_tasks/examples/ceilometer/controller.pp

@@ -1 +1,2 @@
 class { '::openstack_tasks::ceilometer::controller' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/openstack_tasks/examples/ceilometer/radosgw_user.pp

@@ -1 +1,2 @@
 class { '::openstack_tasks::ceilometer::radosgw_user' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/openstack_tasks/examples/glance/glance.pp

@@ -1 +1,2 @@
 class { '::openstack_tasks::glance::glance' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/openstack_tasks/examples/heat/heat.pp

@@ -1,5 +1,5 @@
 class { '::openstack_tasks::heat::heat' :}
-
+class { '::osnailyfacter::upgrade::restart_services' :}
 class mysql::server {}
 class mysql::config {}
 include mysql::server

deployment/puppet/openstack_tasks/examples/horizon/horizon.pp

@@ -1 +1,2 @@
 class { '::openstack_tasks::horizon::horizon' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/openstack_tasks/examples/ironic/ironic-compute.pp

@@ -1 +1,2 @@
 class { '::openstack_tasks::ironic::ironic_compute' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/openstack_tasks/examples/ironic/ironic.pp

@@ -1 +1,2 @@
 class { '::openstack_tasks::ironic::ironic' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/openstack_tasks/examples/keystone/keystone.pp

@@ -1 +1,2 @@
 class { '::openstack_tasks::keystone::keystone' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/openstack_tasks/examples/murano/cfapi.pp

@@ -1,4 +1,4 @@
 class { '::openstack_tasks::murano::cfapi' :}
-
+class { '::osnailyfacter::upgrade::restart_services' :}
 class openstack::firewall {}
 include openstack::firewall

deployment/puppet/openstack_tasks/examples/murano/murano.pp

@@ -1,4 +1,4 @@
 class { '::openstack_tasks::murano::murano' :}
-
+class { '::osnailyfacter::upgrade::restart_services' :}
 class openstack::firewall {}
 include openstack::firewall

deployment/puppet/openstack_tasks/examples/murano/rabbitmq.pp

@@ -1 +1,2 @@
 class { '::openstack_tasks::murano::rabbitmq' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/openstack_tasks/examples/openstack-cinder/openstack-cinder.pp

@@ -1 +1,2 @@
 class { '::openstack_tasks::openstack_cinder::openstack_cinder' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/openstack_tasks/examples/openstack-controller/openstack-controller.pp

@@ -1 +1,2 @@
 class { '::openstack_tasks::openstack_controller::openstack_controller' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/openstack_tasks/examples/openstack-network/agents/dhcp.pp

@@ -1,4 +1,4 @@
 class neutron {}
 class { 'neutron' :}
-
 class { '::openstack_tasks::openstack_network::agents::dhcp' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/openstack_tasks/examples/openstack-network/agents/l3.pp

@@ -1,4 +1,4 @@
 class neutron {}
 class { 'neutron' :}
-
 class { '::openstack_tasks::openstack_network::agents::l3' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/openstack_tasks/examples/openstack-network/agents/metadata.pp

@@ -1,4 +1,4 @@
 class neutron {}
 class { 'neutron' :}
-
 class { '::openstack_tasks::openstack_network::agents::metadata' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/openstack_tasks/examples/openstack-network/agents/sriov.pp

@@ -1,4 +1,4 @@
 class neutron {}
 class { 'neutron' :}
-
 class { '::openstack_tasks::openstack_network::agents::sriov' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/openstack_tasks/examples/roles/cinder.pp

@@ -1 +1,2 @@
 class { '::openstack_tasks::roles::cinder' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/openstack_tasks/examples/roles/compute.pp

@@ -1 +1,2 @@
 class { '::openstack_tasks::roles::compute' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/openstack_tasks/examples/roles/controller.pp

@@ -1 +1,2 @@
 class { '::openstack_tasks::roles::controller' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/openstack_tasks/examples/roles/ironic-conductor.pp

@@ -1 +1,2 @@
 class { '::openstack_tasks::roles::ironic_conductor' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/openstack_tasks/examples/roles/mongo.pp

@@ -1 +1,2 @@
 class { '::openstack_tasks::roles::mongo' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/openstack_tasks/examples/sahara/sahara.pp

@@ -1,4 +1,4 @@
 class { '::openstack_tasks::sahara::sahara' :}
-
+class { '::osnailyfacter::upgrade::restart_services' :}
 class openstack::firewall {}
 include openstack::firewall

deployment/puppet/openstack_tasks/examples/swift/proxy_storage.pp

@@ -1,5 +1,5 @@
 class { '::openstack_tasks::swift::proxy_storage' :}
-
+class { '::osnailyfacter::upgrade::restart_services' :}
 # 'ceilometer' class is being declared inside openstack::ceilometer class
 # which is declared inside openstack::controller class in the other task.
 # So we need a stub here for dependency from swift::proxy::ceilometer

deployment/puppet/openstack_tasks/examples/swift/swift.pp

@@ -1,7 +1,6 @@
 class { '::openstack_tasks::swift::storage' :}
 class { '::openstack_tasks::swift::proxy' :}
-
-
+class { '::osnailyfacter::upgrade::restart_services' :}
 # 'ceilometer' class is being declared inside openstack::ceilometer class
 # which is declared inside openstack::controller class in the other task.
 # So we need a stub here for dependency from swift::proxy::ceilometer

deployment/puppet/osnailyfacter/manifests/upgrade/ceph_service.pp

@@ -0,0 +1,11 @@
+class osnailyfacter::upgrade::ceph_service {
+
+  $mu_upgrade = hiera_hash('mu_upgrade', {})
+
+  if $mu_upgrade['enabled'] and $mu_upgrade['restart_ceph'] {
+    # Restart all services in puppet catalog only if restart_ceph
+    # is true. If we try to restart non-ceph services they could
+    # trigger Ceph to restart as well.
+    notify { 'restarting Ceph': } ~> Service <||>
+  }
+}

deployment/puppet/osnailyfacter/manifests/upgrade/mysql_service.pp

@@ -0,0 +1,11 @@
+class osnailyfacter::upgrade::mysql_service {
+
+  $mu_upgrade = hiera_hash('mu_upgrade', {})
+
+  if $mu_upgrade['enabled'] and $mu_upgrade['restart_mysql'] {
+    # Restart all services in puppet catalog only if restart_mysql
+    # is true. If we try to restart non-mysql services they could
+    # trigger MySQL to restart as well.
+    notify { 'restarting MySQL DB services': } ~> Service <||>
+  }
+}

deployment/puppet/osnailyfacter/manifests/upgrade/pkg_upgrade.pp

@@ -0,0 +1,40 @@
+class osnailyfacter::upgrade::pkg_upgrade {
+  # hardcode with retries and sleeps for resolving lock issue
+  # should be rewritten
+  exec { 'do_upgrade':
+    command     => 'apt-get dist-upgrade -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold"',
+    environment => [ 'DEBIAN_FRONTEND=noninteractive' ],
+    path        => ['/usr/bin', '/usr/local/sbin', '/usr/sbin', '/sbin', '/bin' ],
+    timeout     => 1700,
+    try_sleep   => 10,
+    tries       => 5,
+    logoutput   => true,
+  }
+
+  $corosync_roles = hiera('corosync_roles', ['primary-controller', 'controller'])
+  if roles_include($corosync_roles) {
+    $content_policy = "#!/bin/bash\n[[ \"\$1\" == \"pacemaker\" ]] && exit 101\n"
+    $policyrc_file  = '/usr/sbin/policy-rc.d'
+
+    ensure_resource('file', 'create-policy-rc.d', {
+      ensure  => present,
+      path    => $policyrc_file,
+      content => $content_policy,
+      mode    => '0755',
+      owner   => 'root',
+      group   => 'root',
+      before  => Exec['do_upgrade']
+    })
+
+    exec { 'remove_policy':
+      command => "rm -rf ${policyrc_file}",
+      path    => '/bin',
+      require => Exec['do_upgrade'],
+    }
+
+    ensure_resource('service', 'pacemaker', {
+      ensure  => running,
+      require => Exec['remove_policy']
+    })
+  }
+}

deployment/puppet/osnailyfacter/manifests/upgrade/rabbitmq_service.pp

@@ -0,0 +1,11 @@
+class osnailyfacter::upgrade::rabbitmq_service {
+
+  $mu_upgrade = hiera_hash('mu_upgrade', {})
+
+  if $mu_upgrade['enabled'] and $mu_upgrade['restart_rabbitmq'] {
+    # Restart all services in puppet catalog only if restart_rabbitmq
+    # is true. If we try to restart non-rabbitmq services they could
+    # trigger RabbitMQ to restart as well.
+    notify { 'restarting RabbitMQ': } ~> Service <||>
+  }
+}

deployment/puppet/osnailyfacter/manifests/upgrade/restart_services.pp

@@ -0,0 +1,6 @@
+class osnailyfacter::upgrade::restart_services {
+  $mu_upgrade = hiera_hash('mu_upgrade', {})
+  if $mu_upgrade['enabled'] {
+    notify { 'restarting services': } ~> Service<||>
+  }
+}

deployment/puppet/osnailyfacter/modular/apache/apache.pp

@@ -1 +1,2 @@
 class { '::osnailyfacter::apache::apache' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/osnailyfacter/modular/api-proxy/api-proxy.pp

@@ -1 +1,2 @@
 class { '::osnailyfacter::api_proxy::api_proxy' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/osnailyfacter/modular/ceph/ceph-osd.pp

@@ -1 +1,2 @@
 class { '::osnailyfacter::ceph::ceph_osd' :}
+class { '::osnailyfacter::upgrade::ceph_service' :}

deployment/puppet/osnailyfacter/modular/ceph/ceph_compute.pp

@@ -1 +1,2 @@
 class { '::osnailyfacter::ceph::ceph_compute' :}
+class { '::osnailyfacter::upgrade::ceph_service' :}

deployment/puppet/osnailyfacter/modular/ceph/ceph_pools.pp

@@ -1 +1,2 @@
 class { '::osnailyfacter::ceph::ceph_pools' :}
+class { '::osnailyfacter::upgrade::ceph_service' :}

deployment/puppet/osnailyfacter/modular/ceph/mon.pp

@@ -1 +1,2 @@
 class { '::osnailyfacter::ceph::mon' :}
+class { '::osnailyfacter::upgrade::ceph_service' :}

deployment/puppet/osnailyfacter/modular/ceph/radosgw.pp

@@ -1 +1,2 @@
 class { '::osnailyfacter::ceph::radosgw' :}
+class { '::osnailyfacter::upgrade::ceph_service' :}

deployment/puppet/osnailyfacter/modular/cgroups/cgroups.pp

@@ -1 +1,2 @@
 class { '::osnailyfacter::cgroups::cgroups' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/osnailyfacter/modular/cluster-haproxy/cluster-haproxy.pp

@@ -1 +1,2 @@
 class { '::osnailyfacter::cluster_haproxy::cluster_haproxy' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/osnailyfacter/modular/cluster-vrouter/cluster-vrouter.pp

@@ -1 +1,2 @@
 class { '::osnailyfacter::cluster_vrouter::cluster_vrouter' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/osnailyfacter/modular/cluster/cluster.pp

@@ -1 +1,2 @@
 class { '::osnailyfacter::cluster::cluster' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/osnailyfacter/modular/cluster/health.pp

@@ -1 +1,2 @@
 class { '::osnailyfacter::cluster::health' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/osnailyfacter/modular/logging/logging.pp

@@ -1 +1,2 @@
 class { '::osnailyfacter::logging::logging' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/osnailyfacter/modular/memcached/memcached.pp

@@ -1 +1,2 @@
 class { '::osnailyfacter::memcached::memcached' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/osnailyfacter/modular/ntp/ntp-client.pp

@@ -1 +1,2 @@
 class { '::osnailyfacter::ntp::ntp_client' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/osnailyfacter/modular/ntp/ntp-server.pp

@@ -1 +1,2 @@
 class { '::osnailyfacter::ntp::ntp_server' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/osnailyfacter/modular/tools/tools.pp

@@ -1 +1,2 @@
 class { '::osnailyfacter::tools::tools' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/osnailyfacter/modular/upgrade/pkg_upgrade.pp

@@ -0,0 +1 @@
+include ::osnailyfacter::upgrade::pkg_upgrade

deployment/puppet/osnailyfacter/modular/upgrade/tasks.yaml

@@ -0,0 +1,12 @@
+- id: pkg_upgrade
+  type: puppet
+  version: 2.1.0
+  groups: [primary-controller, controller, cinder, cinder-block-device, cinder-vmware, compute, ceph-osd, primary-mongo, mongo, ironic]
+  requires: [setup_repositories]
+  required_for: [fuel_pkgs]
+  condition:
+    yaql_exp: $.get('mu_upgrade', {}).get('enabled')
+  parameters:
+    puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/upgrade/pkg_upgrade.pp
+    puppet_modules: /etc/puppet/modules
+    timeout: 1800

deployment/puppet/osnailyfacter/modular/vmware/cinder-vmware.pp

@@ -1 +1,2 @@
 class { '::osnailyfacter::vmware::cinder_vmware' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/osnailyfacter/modular/vmware/compute-vmware.pp

@@ -1 +1,2 @@
 class { '::osnailyfacter::vmware::compute_vmware' :}
+class { '::osnailyfacter::upgrade::restart_services' :}

deployment/puppet/osnailyfacter/modular/vmware/vcenter.pp

@@ -1 +1,2 @@
 class { '::osnailyfacter::vmware::vcenter' :}
+class { '::osnailyfacter::upgrade::restart_services' :}