Merge "Add Grenade upgrade support for Cyborg"
This commit is contained in:
Zuul
+53
@@ -1,3 +1,52 @@
|
||||
- job:
|
||||
name: cyborg-grenade
|
||||
description: |
|
||||
Grenade job that performs upgrade from previous release(N-1) to
|
||||
current release(N).
|
||||
parent: grenade
|
||||
required-projects: &base_required_projects_master
|
||||
- openstack/cyborg
|
||||
- openstack/python-cyborgclient
|
||||
- openstack/cyborg-tempest-plugin
|
||||
vars: &grenade_vars
|
||||
devstack_plugins:
|
||||
cyborg: https://opendev.org/openstack/cyborg
|
||||
devstack_services:
|
||||
tempest: true
|
||||
s-account: false
|
||||
s-container: false
|
||||
s-object: false
|
||||
s-proxy: false
|
||||
c-api: false
|
||||
c-bak: false
|
||||
c-sch: false
|
||||
c-vol: false
|
||||
cinder: false
|
||||
tempest_plugins:
|
||||
- cyborg-tempest-plugin
|
||||
tempest_test_regex: cyborg_tempest_plugin
|
||||
tox_envlist: all
|
||||
irrelevant-files: &base_irrelevant_files
|
||||
- ^.*\.rst$
|
||||
- ^doc/.*$
|
||||
- ^setup.cfg$
|
||||
- ^releasenotes/.*$
|
||||
- ^(test-|)requirements.txt$
|
||||
- ^.git.*$
|
||||
- ^tox.ini$
|
||||
|
||||
- job:
|
||||
name: cyborg-grenade-skip-level-always
|
||||
description: |
|
||||
Grenade job that skips the previous release, validating that projects
|
||||
can support upgrades from N-2 to N release. It will always upgrade
|
||||
to the current branch. For example: If N-2 is 2025.1 release then N
|
||||
would be 2026.1.
|
||||
parent: grenade-skip-level-always
|
||||
required-projects: *base_required_projects_master
|
||||
vars: *grenade_vars
|
||||
irrelevant-files: *base_irrelevant_files
|
||||
|
||||
- project:
|
||||
templates:
|
||||
- openstack-cover-jobs
|
||||
@@ -9,6 +58,10 @@
|
||||
jobs:
|
||||
- cyborg-tempest
|
||||
- cyborg-tempest-ipv6-only
|
||||
- cyborg-grenade
|
||||
- cyborg-grenade-skip-level-always
|
||||
gate:
|
||||
jobs:
|
||||
- cyborg-tempest
|
||||
- cyborg-grenade
|
||||
- cyborg-grenade-skip-level-always
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
=============================
|
||||
Upgrade Testing using grenade
|
||||
=============================
|
||||
|
||||
Run a Grenade upgrade: deploy a base DevStack, then upgrade to a target
|
||||
release and verify Cyborg device profiles survive the upgrade.
|
||||
|
||||
Requirements
|
||||
============
|
||||
|
||||
- 8 GB RAM, 8 vCPUs (or reduce services in the configs).
|
||||
|
||||
Setup
|
||||
=====
|
||||
|
||||
1. Clone Grenade::
|
||||
|
||||
git clone https://opendev.org/openstack/grenade
|
||||
|
||||
2. Configure Grenade. Use Grenade's base configuration files with minimal
|
||||
Cyborg-specific overrides.
|
||||
|
||||
a. **Create localrc** with Cyborg-specific branch settings::
|
||||
|
||||
cat > localrc <<EOF
|
||||
# Set base release branch (update to match desired base release)
|
||||
BASE_DEVSTACK_BRANCH=stable/2025.2
|
||||
TARGET_DEVSTACK_BRANCH=master
|
||||
EOF
|
||||
|
||||
b. **Create pluginrc** to enable Cyborg grenade plugin::
|
||||
|
||||
cat > pluginrc <<EOF
|
||||
enable_grenade_plugin cyborg https://opendev.org/openstack/cyborg
|
||||
EOF
|
||||
|
||||
3. (Optional) Customize devstack configs for resource-constrained environments.
|
||||
|
||||
Edit ``devstack.local.conf.base`` and ``devstack.local.conf.target`` to
|
||||
disable unneeded services::
|
||||
|
||||
[[local|localrc]]
|
||||
disable_service s-account s-container s-object s-proxy
|
||||
disable_service etcd
|
||||
disable_service c-api c-bak c-sch c-vol cinder
|
||||
disable_service horizon
|
||||
|
||||
Run
|
||||
===
|
||||
|
||||
From the Grenade repo root::
|
||||
|
||||
cd grenade
|
||||
./grenade.sh
|
||||
|
||||
Clean up after test failure::
|
||||
|
||||
./clean.sh
|
||||
./grenade.sh
|
||||
|
||||
Configuration Details
|
||||
=====================
|
||||
|
||||
**Required Files** (in Grenade repo root):
|
||||
|
||||
- ``localrc`` — Grenade variables (branches, paths). Minimum requirement:
|
||||
set ``BASE_DEVSTACK_BRANCH`` to the release you want to upgrade from.
|
||||
|
||||
- ``pluginrc`` — Enables the Cyborg grenade plugin.
|
||||
|
||||
**Optional Files** (customize as needed):
|
||||
|
||||
- ``devstack.local.conf.base`` — Base (old) release DevStack configuration.
|
||||
- ``devstack.local.conf.target`` — Target (new) release DevStack configuration.
|
||||
|
||||
**Cyborg Grenade Plugin** (``cyborg/devstack/upgrade/``):
|
||||
|
||||
Automatically used by Grenade when the plugin is enabled in ``pluginrc``:
|
||||
|
||||
- ``resources.sh`` — Create/verify/destroy device profiles during upgrade.
|
||||
- ``shutdown.sh`` — Stop Cyborg services before upgrade.
|
||||
- ``upgrade.sh`` — Install target Cyborg, run DB migrations, start services.
|
||||
- ``settings`` — Register Cyborg project/DB with Grenade.
|
||||
|
||||
Configuration Examples
|
||||
=======================
|
||||
|
||||
**Minimal localrc**::
|
||||
|
||||
BASE_DEVSTACK_BRANCH=stable/2025.2
|
||||
TARGET_DEVSTACK_BRANCH=master
|
||||
|
||||
**Reduce resource usage** (add to devstack.local.conf files)::
|
||||
|
||||
disable_service s-account s-container s-object s-proxy etcd cinder horizon
|
||||
|
||||
**Custom passwords** (add to devstack.local.conf files)::
|
||||
|
||||
ADMIN_PASSWORD=password
|
||||
DATABASE_PASSWORD=password
|
||||
MYSQL_PASSWORD=password
|
||||
RABBIT_PASSWORD=password
|
||||
SERVICE_PASSWORD=password
|
||||
|
||||
**Tuning for CI/stability**::
|
||||
|
||||
ENABLE_SYSCTL_MEM_TUNING="True"
|
||||
ENABLE_SYSCTL_NET_TUNING="True"
|
||||
ENABLE_ZSWAP="True"
|
||||
|
||||
**Fixed network range for tenant networks**::
|
||||
|
||||
NETWORK_GATEWAY=10.0.0.1
|
||||
FIXED_RANGE=10.0.0.0/22
|
||||
|
||||
Notes
|
||||
=====
|
||||
|
||||
- Grenade provides default settings for passwords, networking, paths, and
|
||||
service configurations. Only override settings specific to your test
|
||||
environment.
|
||||
|
||||
- The ``BASE_DEVSTACK_BRANCH`` should match the OpenStack release you want
|
||||
to upgrade from (e.g., stable/2025.2, stable/2026.1).
|
||||
|
||||
- Grenade automatically detects the Cyborg upgrade plugin when
|
||||
``devstack_plugins`` includes Cyborg in CI jobs. For local testing,
|
||||
use ``pluginrc`` to enable the plugin.
|
||||
|
||||
- In CI job, use `grenade_devstack_localrc` var to pass extra things
|
||||
in devstack localrc on the base or target node::
|
||||
|
||||
grenade_devstack_localrc:
|
||||
old:
|
||||
<setting for base devstack localrc>
|
||||
new:
|
||||
<setting for target devstack localrc>
|
||||
shared:
|
||||
<setting which are applicable for both base and target>
|
||||
Executable
+103
@@ -0,0 +1,103 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -o errexit
|
||||
|
||||
source $GRENADE_DIR/grenaderc
|
||||
source $GRENADE_DIR/functions
|
||||
|
||||
TOP_DIR=${TOP_DIR:-$TARGET_DEVSTACK_DIR}
|
||||
|
||||
source $TOP_DIR/openrc admin admin
|
||||
|
||||
set -o xtrace
|
||||
|
||||
# Device profile groups JSON requesting one fake FPGA device
|
||||
DEVICE_PROFILE_GROUPS='[{"resources:FPGA": "1", "trait:CUSTOM_FAKE_DEVICE": "required"}]'
|
||||
|
||||
# Create a device profile that requests one FPGA from the fake driver
|
||||
function create_device_profile {
|
||||
local dp_uuid
|
||||
dp_uuid=$(openstack accelerator device profile create \
|
||||
cyborg-grenade-dp "$DEVICE_PROFILE_GROUPS" -f value -c uuid)
|
||||
resource_save cyborg dp_uuid "$dp_uuid"
|
||||
}
|
||||
|
||||
# Verify the pre-upgrade device profile is still listable after upgrade
|
||||
function verify_device_profile {
|
||||
local dp_uuid=$(resource_get cyborg dp_uuid)
|
||||
openstack accelerator device profile show "$dp_uuid"
|
||||
}
|
||||
|
||||
# Create a new device profile post-upgrade to confirm new resources work.
|
||||
# If it already exists (e.g. previous run didn't reach destroy), use its uuid.
|
||||
function create_post_upgrade_device_profile {
|
||||
local dp_uuid
|
||||
dp_uuid=$(openstack accelerator device profile list -f value -c uuid -c name \
|
||||
2>/dev/null | awk -v name="cyborg-grenade-post-dp" '$2==name {print $1; exit}')
|
||||
if [[ -z "$dp_uuid" ]]; then
|
||||
dp_uuid=$(openstack accelerator device profile create \
|
||||
cyborg-grenade-post-dp "$DEVICE_PROFILE_GROUPS" -f value -c uuid)
|
||||
fi
|
||||
resource_save cyborg post_dp_uuid "$dp_uuid"
|
||||
}
|
||||
|
||||
# Verify the post-upgrade device profile is listable
|
||||
function verify_post_upgrade_device_profile {
|
||||
local dp_uuid=$(resource_get cyborg post_dp_uuid)
|
||||
openstack accelerator device profile show "$dp_uuid"
|
||||
}
|
||||
|
||||
# Remove the pre-upgrade device profile
|
||||
function delete_device_profile {
|
||||
local dp_uuid=$(resource_get cyborg dp_uuid)
|
||||
openstack accelerator device profile delete "$dp_uuid"
|
||||
}
|
||||
|
||||
# Remove the post-upgrade device profile
|
||||
function delete_post_upgrade_device_profile {
|
||||
local dp_uuid=$(resource_get cyborg post_dp_uuid)
|
||||
openstack accelerator device profile delete "$dp_uuid"
|
||||
}
|
||||
|
||||
# Currently no good way to verify without the API
|
||||
function verify_noapi {
|
||||
:
|
||||
}
|
||||
|
||||
# Create cyborg resources before upgrade
|
||||
function create {
|
||||
create_device_profile
|
||||
}
|
||||
|
||||
# Verify pre-upgrade resources survive, then create and verify new ones
|
||||
function verify {
|
||||
verify_device_profile
|
||||
create_post_upgrade_device_profile
|
||||
verify_post_upgrade_device_profile
|
||||
}
|
||||
|
||||
# Clean up all cyborg resources in reverse order
|
||||
function destroy {
|
||||
delete_post_upgrade_device_profile
|
||||
delete_device_profile
|
||||
}
|
||||
|
||||
# Dispatcher
|
||||
case $1 in
|
||||
"create")
|
||||
create
|
||||
;;
|
||||
"verify_noapi")
|
||||
verify_noapi
|
||||
;;
|
||||
"verify")
|
||||
verify
|
||||
;;
|
||||
"destroy")
|
||||
destroy
|
||||
;;
|
||||
"force_destroy")
|
||||
set +o errexit
|
||||
destroy
|
||||
;;
|
||||
esac
|
||||
@@ -0,0 +1,11 @@
|
||||
# Register Cyborg with Grenade and include its DB in the saved state for upgrade.
|
||||
register_project_for_upgrade cyborg
|
||||
register_db_to_save cyborg
|
||||
|
||||
# Enable the Cyborg DevStack plugin on base (pinned branch) and target (current tree).
|
||||
devstack_localrc base enable_plugin cyborg https://opendev.org/openstack/cyborg $BASE_DEVSTACK_BRANCH
|
||||
devstack_localrc target enable_plugin cyborg https://opendev.org/openstack/cyborg
|
||||
|
||||
# Start Cyborg API, conductor, and agent on both base and target stacks.
|
||||
devstack_localrc base enable_service cyborg-api cyborg-cond cyborg-agent
|
||||
devstack_localrc target enable_service cyborg-api cyborg-cond cyborg-agent
|
||||
Executable
+24
@@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -o errexit
|
||||
|
||||
source $GRENADE_DIR/grenaderc
|
||||
source $GRENADE_DIR/functions
|
||||
|
||||
# We need base DevStack functions for this
|
||||
source $BASE_DEVSTACK_DIR/functions
|
||||
source $BASE_DEVSTACK_DIR/stackrc # needed for status directory
|
||||
source $BASE_DEVSTACK_DIR/lib/tls
|
||||
source $BASE_DEVSTACK_DIR/lib/apache
|
||||
|
||||
CYBORG_DEVSTACK_DIR=$(dirname $(dirname $0))
|
||||
source $CYBORG_DEVSTACK_DIR/settings
|
||||
source $CYBORG_DEVSTACK_DIR/plugin.sh
|
||||
source $CYBORG_DEVSTACK_DIR/lib/cyborg
|
||||
|
||||
set -o xtrace
|
||||
|
||||
stop_cyborg
|
||||
|
||||
# sanity check that service is actually down
|
||||
ensure_services_stopped cyborg-api cyborg-cond cyborg-agent
|
||||
Executable
+108
@@ -0,0 +1,108 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# ``upgrade-cyborg``
|
||||
|
||||
echo "*********************************************************************"
|
||||
echo "Begin $0"
|
||||
echo "*********************************************************************"
|
||||
|
||||
# Clean up any resources that may be in use
|
||||
cleanup() {
|
||||
set +o errexit
|
||||
|
||||
echo "********************************************************************"
|
||||
echo "ERROR: Abort $0"
|
||||
echo "********************************************************************"
|
||||
|
||||
# Kill ourselves to signal any calling process
|
||||
trap 2; kill -2 $$
|
||||
}
|
||||
|
||||
trap cleanup SIGHUP SIGINT SIGTERM
|
||||
|
||||
# Keep track of the grenade directory
|
||||
RUN_DIR=$(cd $(dirname "$0") && pwd)
|
||||
|
||||
# Source params
|
||||
source $GRENADE_DIR/grenaderc
|
||||
|
||||
# Import common functions
|
||||
source $GRENADE_DIR/functions
|
||||
|
||||
# This script exits on an error so that errors don't compound and you see
|
||||
# only the first error that occurred.
|
||||
set -o errexit
|
||||
|
||||
# Upgrade cyborg
|
||||
# ==============
|
||||
|
||||
# Get functions from current DevStack
|
||||
source $TARGET_DEVSTACK_DIR/stackrc
|
||||
source $TARGET_DEVSTACK_DIR/lib/apache
|
||||
source $TARGET_DEVSTACK_DIR/lib/tls
|
||||
source $TARGET_DEVSTACK_DIR/lib/keystone
|
||||
source $TARGET_DEVSTACK_DIR/lib/placement
|
||||
|
||||
TOP_DIR=${TOP_DIR:-$TARGET_DEVSTACK_DIR}
|
||||
|
||||
source $TOP_DIR/openrc admin admin
|
||||
|
||||
CYBORG_DEVSTACK_DIR=$(dirname $(dirname $0))
|
||||
source $CYBORG_DEVSTACK_DIR/settings
|
||||
source $CYBORG_DEVSTACK_DIR/lib/cyborg
|
||||
|
||||
# Print the commands being run so that we can see the command that triggers
|
||||
# an error. It is also useful for following allowing as the install occurs.
|
||||
set -o xtrace
|
||||
|
||||
# Save current config files for posterity
|
||||
[[ -d $SAVE_DIR/etc.cyborg ]] || cp -pr $CYBORG_CONF_DIR $SAVE_DIR/etc.cyborg
|
||||
|
||||
# Install the target upgrade
|
||||
install_cyborg
|
||||
|
||||
# calls upgrade-cyborg for specific release
|
||||
upgrade_project cyborg $RUN_DIR $BASE_DEVSTACK_BRANCH $TARGET_DEVSTACK_BRANCH
|
||||
|
||||
# Make sure it is OK to do an upgrade. We aren't parsing the output of this
|
||||
# command because the output could change based on the checks it makes.
|
||||
$CYBORG_BIN_DIR/cyborg-status upgrade check && ret_val=$? || ret_val=$?
|
||||
if [ $ret_val -gt 1 ] ; then
|
||||
# Warnings are permissible and returned as status code 1, errors are
|
||||
# returned as greater than 1 which means there is a major upgrade
|
||||
# stopping issue which needs to be addressed.
|
||||
echo "WARNING: Status check failed, we're going to attempt to apply the schema update and then re-evaluate."
|
||||
$CYBORG_BIN_DIR/cyborg-dbsync --config-file=$CYBORG_CONF_FILE upgrade
|
||||
$CYBORG_BIN_DIR/cyborg-status upgrade check && ret_val=$? || ret_val=$?
|
||||
if [ $ret_val -gt 1 ] ; then
|
||||
die $LINENO "Cyborg DB Status check failed, returned: $ret_val"
|
||||
fi
|
||||
fi
|
||||
|
||||
$CYBORG_BIN_DIR/cyborg-dbsync --config-file=$CYBORG_CONF_FILE upgrade || die $LINENO "DB migration error"
|
||||
|
||||
# When using uWSGI: create config if missing; re-enable Apache site (stop_cyborg
|
||||
# disables it, and we only call write_uwsgi_config when conf is missing).
|
||||
if [[ "$CYBORG_USE_UWSGI" == "True" ]]; then
|
||||
if [[ ! -f "$CYBORG_UWSGI_CONF" ]]; then
|
||||
write_uwsgi_config "$CYBORG_UWSGI_CONF" "$CYBORG_UWSGI" "/accelerator"
|
||||
endpoints=$(openstack endpoint list --service cyborg -c ID -f value)
|
||||
for id in $endpoints; do
|
||||
openstack endpoint delete $id
|
||||
done
|
||||
create_cyborg_accounts
|
||||
fi
|
||||
enable_apache_site cyborg-api
|
||||
restart_apache_server
|
||||
fi
|
||||
|
||||
start_cyborg
|
||||
|
||||
# Don't succeed unless the services come up
|
||||
ensure_services_started cyborg-api cyborg-cond cyborg-agent
|
||||
|
||||
set +o xtrace
|
||||
echo "*********************************************************************"
|
||||
echo "SUCCESS: End $0"
|
||||
echo "*********************************************************************"
|
||||
|
||||
Reference in New Issue
Block a user