Merge "Implementation of preupgrade-compute as task graph"

deployment/puppet/octane_tasks/graphs/orig/preupgrade-compute.yaml

@@ -0,0 +1,96 @@
+# GROUPS
+- id: compute
+  type: group
+  role: [compute]
+  fault_tolerance: 0
+
+# TASKS
+- id: rsync_octane
+  type: sync
+  version: 2.1.0
+  groups: [compute]
+  requires: []
+  required_for: []
+  parameters:
+    src: rsync://{MASTER_IP}:/octane_code/puppet
+    dst: /etc/fuel/octane/
+    timeout: 180
+
+- id: override_repos_in_hiera
+  type: upload_file
+  version: 2.1.0
+  groups: [compute]
+  requires: []
+  required_for: []
+  parameters:
+    path: /etc/hiera/override/common.yaml
+    data:
+      yaql_exp: >
+        ({"repo_setup" => {"repos" => $.repo_setup.preupgrade_compute},
+          "preupgrade_packages" => $.preupgrade_packages}.toYaml())
+
+- id: cleanup_existing_repos
+  type: shell
+  version: 2.1.0
+  groups: [compute]
+  requires: []
+  required_for: []
+  parameters:
+    cmd: >
+      tar zcf /root/sources.list.d-backup-$(date +%F-%H%M).tar.gz /etc/apt/sources.list.d;
+      rm /etc/apt/sources.list.d/*.list || true
+    timeout: 60
+
+- id: rsync_latest_puppet
+  type: sync
+  version: 2.1.0
+  groups: [compute]
+  requires: []
+  required_for: []
+  parameters:
+    src: rsync://{MASTER_IP}:/puppet/modules/
+    dst: /etc/fuel/octane/latest_modules
+    timeout: 180
+
+- id: setup_new_repositories
+  type: puppet
+  version: 2.1.0
+  groups: [compute]
+  requires: [cleanup_existing_repos, rsync_latest_puppet, override_repos_in_hiera]
+  required_for: []
+  parameters:
+    puppet_manifest: /etc/fuel/octane/latest_modules/osnailyfacter/modular/fuel_pkgs/setup_repositories.pp
+    puppet_modules: /etc/fuel/octane/latest_modules
+    timeout: 600
+
+- id: stop_compute_services
+  type: puppet
+  version: 2.1.0
+  groups: [compute]
+  requires: [setup_new_repositories, rsync_octane]
+  required_for: []
+  parameters:
+    puppet_manifest: /etc/fuel/octane/puppet/octane_tasks/modular/stop_compute_services.pp
+    puppet_modules:  /etc/fuel/octane/puppet:/etc/puppet/modules
+    timeout: 360
+
+- id: preupgrade_compute
+  type: puppet
+  version: 2.1.0
+  groups: [compute]
+  requires: [stop_compute_services, rsync_octane, setup_new_repositories]
+  required_for: []
+  parameters:
+    puppet_manifest: /etc/fuel/octane/puppet/octane_tasks/modular/preupgrade_compute.pp
+    puppet_modules:  /etc/fuel/octane/puppet:/etc/puppet/modules
+    timeout: 360
+
+- id: remove_hiera_override
+  type: shell
+  version: 2.1.0
+  groups: [compute]
+  requires: [preupgrade_compute]
+  required_for: []
+  parameters:
+    cmd: rm /etc/hiera/override/common.yaml || true
+    timeout: 60

deployment/puppet/octane_tasks/manifests/preupgrade_compute.pp

@@ -0,0 +1,47 @@
+# == Class: octane_tasks::preupgrade_compute
+#
+# This class upgrades required packages on compute node
+# inplace. See magic_consts.COMPUTE_PREUPGRADE_PACKAGES
+# for the complete list.
+#
+class octane_tasks::preupgrade_compute {
+  $preupgrade_packages = hiera('preupgrade_packages')
+  $preupgrade_packages_str = join($preupgrade_packages, ' ')
+
+  # As much as I would love to use package type, it just won't
+  # cut it. The to-be-updated packages have dependencies between
+  # each other, so it would take a strict ordering of the package
+  # list to do so. Assuming that passing the list to the resource
+  # makes Puppet realize these resources in the same order, they
+  # are present in the list.
+  Exec {
+    path    => '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin',
+  }
+
+  exec { 'pre-upgrade-apt-get-update':
+    command => 'apt-get update',
+    before  => Exec['upgrade-packages'],
+  }
+
+  exec { 'upgrade-packages':
+    path        => '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin',
+    command     => "apt-get install --only-upgrade --yes --force-yes \
+                    -o Dpkg::Options::=\"--force-confdef\" \
+                    -o Dpkg::Options::=\"--force-confold\" \
+                    ${preupgrade_packages_str}",
+    environment => ['DEBIAN_FRONTEND=noninteractive'],
+    before      => Anchor['packages-are-updated'],
+  }
+
+  anchor { 'packages-are-updated': }
+
+  service { 'nova-compute':
+    ensure    => running,
+    subscribe => Anchor['packages-are-updated'],
+  }
+
+  service { 'neutron-plugin-openvswitch-agent':
+    ensure    => running,
+    subscribe => Anchor['packages-are-updated'],
+  }
+}

deployment/puppet/octane_tasks/manifests/stop_compute_services.pp

@@ -0,0 +1,14 @@
+# == Class: octane_tasks::stop_compute_service
+#
+# This class stops compute services to prepare
+# for inplace package updates.
+#
+class octane_tasks::stop_compute_services {
+  service { 'nova-compute':
+    ensure    => stopped,
+  }
+
+  service { 'neutron-plugin-openvswitch-agent':
+    ensure    => stopped,
+  }
+}

deployment/puppet/octane_tasks/modular/preupgrade_compute.pp

@@ -0,0 +1,3 @@
+notice('MODULAR: octane_tasks::preupgrade_compute')
+
+include ::octane_tasks::preupgrade_compute

deployment/puppet/octane_tasks/modular/stop_compute_services.pp

@@ -0,0 +1,3 @@
+notice('MODULAR: octane_tasks::stop_compute_services')
+
+include ::octane_tasks::stop_compute_services

octane/commands/preupgrade_compute.py

@@ -18,6 +18,7 @@ from fuelclient import objects
 
 from octane import magic_consts
 from octane.util import apt
+from octane.util import deployment as deploy
 from octane.util import helpers
 from octane.util import ssh
 
@@ -80,20 +81,34 @@ def get_repos(release, master_ip=''):
     return repos
 
 
+def get_package_list(release):
+    version = release.data['version']
+    try:
+        packages = magic_consts.COMPUTE_PREUPGRADE_PACKAGES[version]
+    except KeyError:
+        LOG.exception("Info about packages for a release with id {0} "
+                      "does not exist".format(release.id))
+        raise
+    return packages
+
+
+def add_upgrade_attrs_to_settings(env, repos, packages):
+    attrs = env.get_settings_data()
+    attrs['editable']['repo_setup']['preupgrade_compute'] = {'value': repos,
+                                                             'type': 'hidden'}
+    attrs['editable']['common']['preupgrade_packages'] = {'value': packages,
+                                                          'type': 'hidden'}
+    env.set_settings_data(attrs)
+
+
 def preupgrade_compute(release_id, node_ids):
     nodes = [objects.node.Node(node_id) for node_id in node_ids]
     release = objects.Release(release_id)
     check_sanity(nodes, release)
     master_ip = helpers.get_astute_dict()["ADMIN_NETWORK"]['ipaddress']
 
-    version = release.data['version']
     repos = get_repos(release, master_ip)
-    try:
+    packages = get_package_list(release)
-        packages = magic_consts.COMPUTE_PREUPGRADE_PACKAGES[version]
-    except KeyError:
-        LOG.exception("Info about packages for a release with id {0} "
-                      "is not exist".format(release_id))
-        raise
 
     for node in nodes:
         change_repositories(node, repos)
@@ -101,6 +116,26 @@ def preupgrade_compute(release_id, node_ids):
         apt.upgrade_packages(node, packages)
 
 
+def preupgrade_compute_with_graph(release_id, node_ids):
+    nodes = [objects.node.Node(node_id) for node_id in node_ids]
+    release = objects.Release(release_id)
+    check_sanity(nodes, release)
+    master_ip = helpers.get_astute_dict()["ADMIN_NETWORK"]['ipaddress']
+
+    repos = get_repos(release, master_ip)
+    packages = get_package_list(release)
+    env_id = nodes[0].env.id
+    env = objects.environment.Environment(env_id)
+
+    # Add following data to cluster attributes:
+    # - new repositories
+    # - list of packages to be updated
+    add_upgrade_attrs_to_settings(env, repos, packages)
+
+    deploy.upload_graph(env_id, "orig")
+    deploy.execute_graph_and_wait('preupgrade-compute', env_id, node_ids)
+
+
 class PreupgradeComputeCommand(cmd.Command):
     """Preupgrade compute"""
 
@@ -118,7 +153,15 @@ class PreupgradeComputeCommand(cmd.Command):
             metavar='NODE_ID',
             help="IDs of compute nodes to be preupgraded",
             nargs="+")
+        parser.add_argument(
+            '--with-graph', action='store_true',
+            help='EXPERIMENTAL: Use Fuel deployment graphs'
+                 ' instead of python-based commands.')
         return parser
 
     def take_action(self, parsed_args):
-        preupgrade_compute(parsed_args.release_id, parsed_args.node_ids)
+        if parsed_args.with_graph:
+            preupgrade_compute_with_graph(parsed_args.release_id,
+                                          parsed_args.node_ids)
+        else:
+            preupgrade_compute(parsed_args.release_id, parsed_args.node_ids)