Ensure present/latest for puppet driven package updates

This change updates yum_update.sh so that we set set a boolean
output when "managed" packages should get updated. The
output is named 'update_managed_packages' and for the
puppet implementation it is wired up so that it
directly sets tripleo::packages::enable_upgrade to
control whether packages are updated.

It also modifies yum_update.sh to build a yum update excludes list for
packages managed by puppet. The exclude lists are being
generated via puppet-tripleo as well via the new 'write_package_names'
function that is now wired into all the role manifests.

This change does not actually trigger the puppet apply. The fix for
Related-Bug: #1463092 will be used to trigger the puppet run when the
hiera changes. As a minor tweak to this logic we append the
UpdateIdentifier to the config_identifier so that we ensure
puppet gets executed on an update where other (non-related)
hiera changes also occur.

Co-Authored-By: Dan Prince <dprince@redhat.com>

Change-Id: I343c3959517eae38bbcd43648ed56f610272864d

extraconfig/tasks/yum_update.sh

@@ -8,6 +8,7 @@
 #   command_arguments - yum command arguments, defaults to ""
 
 echo "Started yum_update.sh on server $deploy_server_id at `date`"
+echo -n "false" > $heat_outputs_path.update_managed_packages
 
 if [[ -z "$update_identifier" ]]; then
     echo "Not running due to unset update_identifier"
@@ -17,6 +18,12 @@ fi
 timestamp_dir=/var/lib/overcloud-yum-update
 mkdir -p $timestamp_dir
 
+command_arguments=${command_arguments:-}
+# exclude upgrading packages that are handled by config management tooling
+for exclude in $(cat /var/lib/tripleo/installed-packages/* | sort -u); do
+  command_arguments="$command_arguments --exclude $exclude"
+done
+
 # sanitise to remove unusual characters
 update_identifier=${update_identifier//[^a-zA-Z0-9-_]/}
 
@@ -36,6 +43,8 @@ return_code=$?
 echo "$result"
 echo "yum return code: $return_code"
 
+echo -n "true" > $heat_outputs_path.update_managed_packages
+
 echo "Finished yum_update.sh on server $deploy_server_id at `date`"
 
 exit $return_code

extraconfig/tasks/yum_update.yaml

@@ -20,6 +20,9 @@ resources:
       - name: command_arguments
         description: yum command arguments, defaults to ""
         default: ''
+      outputs:
+      - name: update_managed_packages
+        description: boolean value indicating whether to upgrade managed packages
 
 outputs:
   OS::stack_id:

puppet/ceph-storage.yaml

@@ -139,6 +139,7 @@ resources:
             params:
               server: {get_param: NtpServer}
         enable_package_install: {get_param: EnablePackageInstall}
+        enable_package_upgrade: {get_attr: [UpdateDeployment, update_managed_packages]}
         ceph_cluster_network: {get_attr: [NetIpSubnetMap, net_ip_subnet_map, {get_param: [ServiceNetMap, CephClusterNetwork]}]}
         ceph_public_network: {get_attr: [NetIpSubnetMap, net_ip_subnet_map, {get_param: [ServiceNetMap, CephPublicNetwork]}]}
 
@@ -169,6 +170,7 @@ resources:
               mapped_data:
                 ntp::servers: {get_input: ntp_servers}
                 tripleo::packages::enable_install: {get_input: enable_package_install}
+                tripleo::packages::enable_upgrade: {get_input: enable_package_upgrade}
                 ceph::profile::params::cluster_network: {get_input: ceph_cluster_network}
                 ceph::profile::params::public_network: {get_input: ceph_public_network}
 
@@ -216,3 +218,4 @@ outputs:
       - ','
       - - {get_attr: [CephStorageDeployment, deploy_stdout]}
         - {get_attr: [CephStorageExtraConfigPre, deploy_stdout]}
+        - {get_param: UpdateIdentifier}

puppet/cinder-storage.yaml

@@ -217,6 +217,7 @@ resources:
             params:
               server: {get_param: NtpServer}
         enable_package_install: {get_param: EnablePackageInstall}
+        enable_package_upgrade: {get_attr: [UpdateDeployment, update_managed_packages]}
 
   # Map heat metadata into hiera datafiles
   BlockStorageConfig:
@@ -258,6 +259,7 @@ resources:
                 cinder::glance::glance_api_servers: {get_input: glance_api_servers}
                 ntp::servers: {get_input: ntp_servers}
                 tripleo::packages::enable_install: {get_input: enable_package_install}
+                tripleo::packages::enable_upgrade: {get_input: enable_package_upgrade}
                 snmpd_readonly_user_name: {get_input: snmpd_readonly_user_name}
                 snmpd_readonly_user_password: {get_input: snmpd_readonly_user_password}
 
@@ -296,4 +298,8 @@ outputs:
     value: {get_attr: [StorageMgmtPort, ip_address]}
   config_identifier:
     description: identifier which changes if the node configuration may need re-applying
-    value: {get_attr: [BlockStorageDeployment, deploy_stdout]}
+    value:
+      list_join:
+        - ''
+        - - {get_attr: [BlockStorageDeployment, deploy_stdout]}
+          - {get_param: UpdateIdentifier}

puppet/compute.yaml

@@ -429,6 +429,7 @@ resources:
                 admin_password: {get_input: admin_password}
                 ntp::servers: {get_input: ntp_servers}
                 tripleo::packages::enable_install: {get_input: enable_package_install}
+                tripleo::packages::enable_upgrade: {get_input: enable_package_upgrade}
 
   NovaComputeDeployment:
     type: OS::TripleO::SoftwareDeployment
@@ -544,6 +545,7 @@ resources:
             params:
               server: {get_param: NtpServer}
         enable_package_install: {get_param: EnablePackageInstall}
+        enable_package_upgrade: {get_attr: [UpdateDeployment, update_managed_packages]}
 
   # Hook for site-specific additional pre-deployment config, e.g extra hieradata
   ComputeExtraConfigPre:
@@ -600,3 +602,4 @@ outputs:
       - ','
       - - {get_attr: [NovaComputeDeployment, deploy_stdout]}
         - {get_attr: [ComputeExtraConfigPre, deploy_stdout]}
+        - {get_param: UpdateIdentifier}

puppet/controller.yaml

@@ -888,6 +888,7 @@ resources:
         swift_min_part_hours: {get_param: SwiftMinPartHours}
         swift_mount_check: {get_param: SwiftMountCheck}
         enable_package_install: {get_param: EnablePackageInstall}
+        enable_package_upgrade: {get_attr: [UpdateDeployment, update_managed_packages]}
         swift_proxy_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, SwiftProxyNetwork]}]}
         swift_management_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, SwiftMgmtNetwork]}]}
         cinder_iscsi_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, CinderIscsiNetwork]}]}
@@ -1196,6 +1197,7 @@ resources:
                 tripleo::loadbalancer::control_virtual_interface: {get_input: control_virtual_interface}
                 tripleo::loadbalancer::public_virtual_interface: {get_input: public_virtual_interface}
                 tripleo::packages::enable_install: {get_input: enable_package_install}
+                tripleo::packages::enable_upgrade: {get_input: enable_package_upgrade}
 
   # Hook for site-specific additional pre-deployment config, e.g extra hieradata
   ControllerExtraConfigPre:
@@ -1277,6 +1279,7 @@ outputs:
     description: identifier which changes if the controller configuration may need re-applying
     value:
       list_join:
-      - ','
+        - ','
-      - - {get_attr: [ControllerDeployment, deploy_stdout]}
+        - - {get_attr: [ControllerDeployment, deploy_stdout]}
-        - {get_attr: [ControllerExtraConfigPre, deploy_stdout]}
+          - {get_attr: [ControllerExtraConfigPre, deploy_stdout]}
+          - {get_param: UpdateIdentifier}

puppet/swift-storage.yaml

@@ -189,6 +189,7 @@ resources:
                 snmpd_readonly_user_name: {get_input: snmpd_readonly_user_name}
                 snmpd_readonly_user_password: {get_input: snmpd_readonly_user_password}
                 tripleo::packages::enable_install: {get_input: enable_package_install}
+                tripleo::packages::enable_upgrade: {get_input: enable_package_upgrade}
 
 
   SwiftStorageHieraDeploy:
@@ -212,6 +213,7 @@ resources:
             params:
               server: {get_param: NtpServer}
         enable_package_install: {get_param: EnablePackageInstall}
+        enable_package_upgrade: {get_attr: [UpdateDeployment, update_managed_packages]}
         swift_management_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, SwiftMgmtNetwork]}]}
 
   UpdateConfig:
@@ -256,4 +258,8 @@ outputs:
     value: {get_attr: [StorageMgmtPort, ip_address]}
   config_identifier:
     description: identifier which changes if the node configuration may need re-applying
-    value: {get_attr: [SwiftStorageHieraDeploy, deploy_stdout]}
+    value:
+      list_join:
+        - ','
+        - - {get_attr: [SwiftStorageHieraDeploy, deploy_stdout]}
+          - {get_param: UpdateIdentifier}