Keystone liberty mitaka upgrade step

This step needs to be run after the major-upgrade-pacemaker-init.yaml
step and after the liberty->mitaka aodh upgrade step.
It is split in more parts:

1) Here we run a small puppet manifest that configures keystone to be
run under httpd. This part does not restart httpd at all because doing
so before turning off the openstack-keystone resource would fail
due to both services wanting to listen on the same ports.

2) In this step we add a dummy openstack-core resource, delete the
openstack-keystone resource and make sure that all the resources that
were previously depending on keystone now depend on the dummy resource.

3) We restart httpd to make sure the changes are taking place.

The rationale for doing the whole keystone migration in this single step
is that it will be verifiable on its own. We also need to add both
the aodh and the horizon apache configuration to make sure we do not
remove any services from httpd during the upgrade.

Partial-Bug: #1579705

Depends-On: I8cb4cb3193e6b823aad48ad7dbbbb227364d2a58
Depends-On: I22a348c298ff44f616b2e898f4872eddea040239

Change-Id: If85672afa8a92de2a1c0a222d28682fd100bcf7d

environments/major-upgrade-keystone-liberty-mitaka.yaml

@@ -0,0 +1,14 @@
+resource_registry:
+  OS::TripleO::Tasks::PackageUpdate:  ../extraconfig/tasks/yum_update_noop.yaml
+
+  # Task that prepares the keystone configuration file to be run under http
+  # but does not yet restart neither keystone nor httpd. This will happen
+  # in the major_upgrade_pacemaker step
+  OS::TripleO::Tasks::UpdateWorkflow: ../extraconfig/tasks/major_upgrade_keystone_liberty_mitaka.yaml
+
+  # no-op the rest
+  OS::TripleO::ControllerPostDeployment: OS::Heat::None
+  OS::TripleO::ComputePostDeployment: OS::Heat::None
+  OS::TripleO::ObjectStoragePostDeployment: OS::Heat::None
+  OS::TripleO::BlockStoragePostDeployment: OS::Heat::None
+  OS::TripleO::CephStoragePostDeployment: OS::Heat::None

extraconfig/tasks/liberty_to_mitaka_keystone_upgrade.pp

@@ -0,0 +1,92 @@
+# Copyright 2015 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# This puppet manifest is to be used only during a Liberty->Mitaka upgrade
+# It configures keystone to be run under httpd but it makes sure to not
+# restart any services. This snippet needs to be called after the
+# major-upgrade-pacemaker-init.yaml step because that pushes new hiera data
+# on the nodes and before the major-upgrade-pacemaker.yaml
+
+Service <|
+  tag == 'keystone-service'
+|> {
+  hasrestart => true,
+  restart    => '/bin/true',
+  start      => '/bin/true',
+  stop       => '/bin/true',
+}
+
+if $::hostname == downcase(hiera('bootstrap_nodeid')) {
+  $pacemaker_master = true
+  $sync_db = true
+} else {
+  $pacemaker_master = false
+  $sync_db = false
+}
+
+include ::tripleo::packages
+
+class { '::keystone':
+  sync_db        => $sync_db,
+  manage_service => false,
+  enabled        => false,
+  # This parameter does not exist in liberty puppet modules
+  #enable_bootstrap => $pacemaker_master,
+}
+
+include ::keystone::config
+
+file { [ '/etc/keystone/ssl', '/etc/keystone/ssl/certs', '/etc/keystone/ssl/private' ]:
+  ensure  => 'directory',
+  owner   => 'keystone',
+  group   => 'keystone',
+  require => Package['keystone'],
+}
+file { '/etc/keystone/ssl/certs/signing_cert.pem':
+  content => hiera('keystone_signing_certificate'),
+  owner   => 'keystone',
+  group   => 'keystone',
+  notify  => Service['keystone'],
+  require => File['/etc/keystone/ssl/certs'],
+}
+file { '/etc/keystone/ssl/private/signing_key.pem':
+  content => hiera('keystone_signing_key'),
+  owner   => 'keystone',
+  group   => 'keystone',
+  notify  => Service['keystone'],
+  require => File['/etc/keystone/ssl/private'],
+}
+file { '/etc/keystone/ssl/certs/ca.pem':
+  content => hiera('keystone_ca_certificate'),
+  owner   => 'keystone',
+  group   => 'keystone',
+  notify  => Service['keystone'],
+  require => File['/etc/keystone/ssl/certs'],
+}
+
+class { '::apache' :
+  service_enable  => false,
+  # This needs to be true otherwise keystone_config won't find service
+  service_manage  => true,
+  # we must not restart httpd at this stage of the upgrade
+  service_restart => '/bin/true',
+  purge_configs   => false,
+  purge_vhost_dir => false,
+}
+
+
+# Needed to make sure we do not disable the aodh ports
+include ::aodh::wsgi::apache
+include ::keystone::wsgi::apache

extraconfig/tasks/major_upgrade_keystone_liberty_mitaka.yaml

@@ -0,0 +1,73 @@
+heat_template_version: 2014-10-16
+
+description: >
+  Software-config for keystone configuration under httpd during upgrades
+
+parameters:
+  controller_servers:
+    type: json
+  compute_servers:
+    type: json
+  blockstorage_servers:
+    type: json
+  objectstorage_servers:
+    type: json
+  cephstorage_servers:
+    type: json
+  input_values:
+    type: json
+    description: input values for the software deployments
+resources:
+  KeystoneLibertyMitakaPreUpgradeConfig:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: puppet
+      config:
+        get_file: liberty_to_mitaka_keystone_upgrade.pp
+    type: OS::Heat::SoftwareConfig
+
+  KeystoneLibertyMitakaUpgradeConfig:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      config:
+        list_join:
+          - ''
+          - - "#!/bin/bash\n\n"
+            - get_file: pacemaker_common_functions.sh
+            - get_file: major_upgrade_pacemaker_migrations.sh
+            - "liberty_to_mitaka_keystone\n"
+            - "add_missing_openstack_core_constraints\n\n"
+
+  KeystoneLibertyMitakaPostUpgradeConfig:
+    type: OS::Heat::SoftwareConfig
+    properties:
+      group: script
+      config:
+        list_join:
+          - ''
+          - - "#!/bin/bash\nset -e\n"
+            - "/usr/bin/systemctl reload httpd"
+
+  KeystoneLibertyMitakaPreUpgradeDeployment:
+    type: OS::Heat::SoftwareDeploymentGroup
+    properties:
+      name: KeystoneLibertyMitakaPreUpgradeDeployment
+      servers: {get_param: controller_servers}
+      config: {get_resource: KeystoneLibertyMitakaPreUpgradeConfig}
+
+  KeystoneLibertyMitakaUpgradeConfigDeployment:
+    type: OS::Heat::SoftwareDeploymentGroup
+    depends_on: KeystoneLibertyMitakaPreUpgradeDeployment
+    properties:
+      name: KeystoneLibertyMitakaUpgradeConfigDeployment
+      servers: {get_param: controller_servers}
+      config: {get_resource: KeystoneLibertyMitakaUpgradeConfig}
+
+  KeystoneLibertyMitakaPostUpgradeDeployment:
+    type: OS::Heat::SoftwareDeploymentGroup
+    depends_on: KeystoneLibertyMitakaUpgradeConfigDeployment
+    properties:
+      name: KeystoneLibertyMitakaPostUpgradeDeployment
+      servers: {get_param: controller_servers}
+      config: {get_resource: KeystoneLibertyMitakaPostUpgradeConfig}

extraconfig/tasks/major_upgrade_pacemaker_migrations.sh

@@ -13,39 +13,97 @@
 # been already applied, it should be possible to call the function
 # again without damaging the deployment or failing the upgrade.
 
+# This function will be called during a liberty->mitaka upgrade after init and
+# after the aodh upgrade. It assumes that a special puppet snippet configuring
+# keystone under wsgi has alread been run (i.e. /etc/httpd/conf.d/10*keystone*.conf
+# files are already set).
+function liberty_to_mitaka_keystone {
+    # If the "openstack-core-clone" resource already exists we do not need to make this transition
+    # as the function needs to be idempotent
+    if pcs resource show "openstack-core-clone"; then
+        return 0
+    fi
+    # Only run this on the bootstrap node
+    if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)" ]; then
+        CIB="/root/liberty-cib.xml"
+        CIB_BACKUP="/root/liberty-cib-orig.xml"
+        rm -f $CIB $CIB_BACKUP || /bin/true
+
+        pcs cluster cib $CIB
+
+        cp -f $CIB $CIB_BACKUP || /bin/true
+        PCS="pcs -f $CIB"
+
+        # Create dummy resource
+        $PCS resource create openstack-core ocf:heartbeat:Dummy --clone
+
+        # change all constraints from keystone to dummy
+        CONSTR="$($PCS config | grep keystone | grep start | grep then)"
+        echo "$CONSTR" | {
+            while read i; do
+                ACT=$(echo "$i" | awk '{print $1}')
+                SRC=$(echo "$i" | awk '{print $2}')
+                DST=$(echo "$i" | awk '{print $5}')
+                CID=$(echo "$i" | awk '{print $7}' | sed -e 's/.*id\://g' -e 's/)//g')
+                if [ "$SRC" == "openstack-keystone-clone" ]; then
+                    $PCS constraint order $ACT openstack-core-clone then $DST
+                else
+                    $PCS constraint order $ACT $SRC then openstack-core-clone
+                fi
+                $PCS constraint remove $CID
+            done;
+        }
+        # We push the CIB after removing the keystone resource as we want
+        # to be sure that the httpd resource is untouched. Otherwise we risk
+        # httpd being restarted before keystone is stopped which would give
+        # us a conflicting listening port, because during this step httpd already
+        # has the keystone wsgi configuration but was not restarted
+        $PCS resource delete openstack-keystone-clone
+        pcs cluster cib-push $CIB
+
+        # make sure httpd (which provides keystone now) are started after dummy
+        pcs constraint order start openstack-core-clone then httpd-clone
+    fi
+}
+
 function add_missing_openstack_core_constraints {
     # The CIBs are saved under /root as they might contain sensitive data
-    CIB="/root/migration.cib"
-    CIB_BACKUP="/root/backup.cib"
-    CIB_PUSH_NEEDED=n
+    if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)" ]; then
+        CIB="/root/migration.cib"
+        CIB_BACKUP="/root/backup.cib"
+        CIB_PUSH_NEEDED=n
 
-    rm -f "$CIB" "$CIB_BACKUP" || /bin/true
-    pcs cluster cib "$CIB"
-    cp "$CIB" "$CIB_BACKUP"
+        rm -f "$CIB" "$CIB_BACKUP" || /bin/true
+        pcs cluster cib "$CIB"
+        cp "$CIB" "$CIB_BACKUP"
 
-    if ! pcs -f "$CIB" constraint --full | grep 'start openstack-sahara-api-clone then start openstack-sahara-engine-clone'; then
-        pcs -f "$CIB" constraint order start openstack-sahara-api-clone then start openstack-sahara-engine-clone
-        CIB_PUSH_NEEDED=y
-    fi
+        # sahara is not necessarily always present
+        if pcs -f "$CIB" resource | grep 'openstack-sahara-api-clone'; then
+            if ! pcs -f "$CIB" constraint --full | grep 'start openstack-sahara-api-clone then start openstack-sahara-engine-clone'; then
+                pcs -f "$CIB" constraint order start openstack-sahara-api-clone then start openstack-sahara-engine-clone
+                CIB_PUSH_NEEDED=y
+            fi
+        fi
 
-    if ! pcs -f "$CIB" constraint --full | grep 'start openstack-core-clone then start openstack-ceilometer-notification-clone'; then
-        pcs -f "$CIB" constraint order start openstack-core-clone then start openstack-ceilometer-notification-clone
-        CIB_PUSH_NEEDED=y
-    fi
+        if ! pcs -f "$CIB" constraint --full | grep 'start openstack-core-clone then start openstack-ceilometer-notification-clone'; then
+            pcs -f "$CIB" constraint order start openstack-core-clone then start openstack-ceilometer-notification-clone
+            CIB_PUSH_NEEDED=y
+        fi
 
-    if ! pcs -f "$CIB" constraint --full | grep 'start openstack-aodh-evaluator-clone then start openstack-aodh-listener-clone'; then
-        pcs -f "$CIB" constraint order start openstack-aodh-evaluator-clone then start openstack-aodh-listener-clone
-        CIB_PUSH_NEEDED=y
-    fi
+        if ! pcs -f "$CIB" constraint --full | grep 'start openstack-aodh-evaluator-clone then start openstack-aodh-listener-clone'; then
+            pcs -f "$CIB" constraint order start openstack-aodh-evaluator-clone then start openstack-aodh-listener-clone
+            CIB_PUSH_NEEDED=y
+        fi
 
-    if pcs -f "$CIB" constraint --full | grep 'start openstack-core-clone then start openstack-heat-api-clone'; then
-        CID=$(pcs -f "$CIB" constraint --full | grep 'start openstack-core-clone then start openstack-heat-api-clone' | sed -e 's/.*id\://g' -e 's/)//g')
-        pcs -f "$CIB" constraint remove $CID
-        CIB_PUSH_NEEDED=y
-    fi
+        if pcs -f "$CIB" constraint --full | grep 'start openstack-core-clone then start openstack-heat-api-clone'; then
+            CID=$(pcs -f "$CIB" constraint --full | grep 'start openstack-core-clone then start openstack-heat-api-clone' | sed -e 's/.*id\://g' -e 's/)//g')
+            pcs -f "$CIB" constraint remove $CID
+            CIB_PUSH_NEEDED=y
+        fi
 
-    if [ "$CIB_PUSH_NEEDED" = 'y' ]; then
-        pcs cluster cib-push "$CIB"
+        if [ "$CIB_PUSH_NEEDED" = 'y' ]; then
+            pcs cluster cib-push "$CIB"
+        fi
     fi
 }