B&R: Fix openstack helper scripts for B&R
The helper script will be used to backup and restore the glance images. This script was previously deleted by https://review.opendev.org/#/c/631312/. In order to be able to access the containerized openstack, the tidy storage script was modified to use the configuration data from the '/etc/openstack/clouds.yaml' file. Also, the numpy dependency was removed because all the numpy code was easily rewritten in plain python code. Moreover, the v1 glance client was removed and the code was updated accordingly. Depends-on: https://review.opendev.org/#/c/698478/ Story: 2006770 Task: 37286 Change-Id: I2918b4220b1bd3aafde20263e6cef756ac06c6db Signed-off-by: Mihnea Saracin <Mihnea.Saracin@windriver.com>
This commit is contained in:
Mihnea Saracin
parent
c002ad15ee
commit
516369277d
@ -59,6 +59,7 @@ install -m 644 dist/*.whl $RPM_BUILD_ROOT/wheels/
|
||||
install -d -m 755 %{buildroot}%{local_bindir}
|
||||
install -p -D -m 700 scripts/openstack_update_admin_password %{buildroot}%{local_bindir}/openstack_update_admin_password
|
||||
install -p -D -m 700 scripts/upgrade_swact_migration.py %{buildroot}%{local_bindir}/upgrade_swact_migration.py
|
||||
install -p -D -m 755 scripts/image-backup.sh %{buildroot}%{local_bindir}/image-backup.sh
|
||||
|
||||
install -d -m 755 %{buildroot}%{local_goenabledd}
|
||||
install -p -D -m 700 scripts/config_goenabled_check.sh %{buildroot}%{local_goenabledd}/config_goenabled_check.sh
|
||||
|
||||
@ -1,17 +1,17 @@
|
||||
"""
|
||||
|
||||
Copyright (c) 2015-2018 Wind River Systems, Inc.
|
||||
Copyright (c) 2015-2020 Wind River Systems, Inc.
|
||||
|
||||
SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import textwrap
|
||||
import time
|
||||
import yaml
|
||||
|
||||
from keystoneclient.auth.identity import v3
|
||||
from keystoneauth1 import session as ksc_session
|
||||
@ -39,8 +39,8 @@ class OpenStack(object):
|
||||
self.admin_token = None
|
||||
self.conf = {}
|
||||
self.cinder_client = None
|
||||
self.glance_client_v1 = None
|
||||
self.glance_client_v2 = None
|
||||
self.glance_client = None
|
||||
openstack_keystone_address = 'keystone.openstack.svc.cluster.local'
|
||||
|
||||
try:
|
||||
self.conf['admin_user'] = os.environ['OS_USERNAME']
|
||||
@ -51,8 +51,43 @@ class OpenStack(object):
|
||||
self.conf['user_domain'] = os.environ['OS_USER_DOMAIN_NAME']
|
||||
self.conf['project_domain'] = os.environ['OS_PROJECT_DOMAIN_NAME']
|
||||
except KeyError:
|
||||
LOG.error("Please source openstack service credentials file.")
|
||||
raise TidyStorageFail("Please source openstack credentials file.")
|
||||
openstack_config_file_path = \
|
||||
os.path.join("/", "etc", "openstack", "clouds.yaml")
|
||||
try:
|
||||
with open(openstack_config_file_path) as f:
|
||||
openstack_config_data = \
|
||||
yaml.load(f)['clouds']['openstack_helm']
|
||||
self.conf['admin_user'] = \
|
||||
openstack_config_data['auth']['username']
|
||||
self.conf['admin_pwd'] = \
|
||||
openstack_config_data['auth']['password']
|
||||
self.conf['admin_tenant'] = \
|
||||
openstack_config_data['auth']['project_name']
|
||||
self.conf['auth_url'] = \
|
||||
openstack_config_data['auth']['auth_url']
|
||||
self.conf['region_name'] = \
|
||||
openstack_config_data['region_name']
|
||||
self.conf['user_domain'] = \
|
||||
openstack_config_data['auth']['user_domain_name']
|
||||
self.conf['project_domain'] = \
|
||||
openstack_config_data['auth']['project_domain_name']
|
||||
except KeyError:
|
||||
msg = "Openstack config file has missing values"
|
||||
LOG.error(msg)
|
||||
raise TidyStorageFail(msg)
|
||||
except IOError:
|
||||
msg = "Openstack config file does not exist (%s)" \
|
||||
% openstack_config_file_path
|
||||
LOG.error(msg)
|
||||
raise TidyStorageFail(msg)
|
||||
|
||||
if openstack_keystone_address not in self.conf['auth_url']:
|
||||
msg = 'The auth_url conf variable is expected to be using ' \
|
||||
'the %s service. ' \
|
||||
'Please confirm the provided access credentials.'\
|
||||
% openstack_keystone_address
|
||||
LOG.error(msg)
|
||||
raise TidyStorageFail(msg)
|
||||
|
||||
def __enter__(self):
|
||||
if not self._connect():
|
||||
@ -109,7 +144,7 @@ class OpenStack(object):
|
||||
|
||||
@property
|
||||
def get_glance_client(self):
|
||||
if not self.glance_client_v1 or not self.glance_client_v2:
|
||||
if not self.glance_client:
|
||||
auth = v3.Password(auth_url=self.conf['auth_url'],
|
||||
username=self.conf['admin_user'],
|
||||
password=self.conf['admin_pwd'],
|
||||
@ -117,12 +152,10 @@ class OpenStack(object):
|
||||
project_name=self.conf['admin_tenant'],
|
||||
project_domain_name=self.conf['project_domain'])
|
||||
|
||||
self.glance_client_v1 = Client(
|
||||
'1', session=ksc_session.Session(auth=auth))
|
||||
self.glance_client_v2 = Client(
|
||||
self.glance_client = Client(
|
||||
'2', session=ksc_session.Session(auth=auth))
|
||||
|
||||
return self.glance_client_v1, self.glance_client_v2
|
||||
return self.glance_client
|
||||
|
||||
|
||||
def show_help():
|
||||
@ -163,8 +196,8 @@ def tidy_storage(result_file):
|
||||
# Check Glance images
|
||||
print("Scanning Glance images in DB and rbd images pool...\n")
|
||||
try:
|
||||
g_client_v1, g_client_v2 = client.get_glance_client
|
||||
image_l = g_client_v2.images.list()
|
||||
g_client = client.get_glance_client
|
||||
image_l = g_client.images.list()
|
||||
image_id_l = [image['id'].encode('utf-8') for image in image_l]
|
||||
|
||||
output = subprocess.check_output(
|
||||
@ -185,13 +218,13 @@ def tidy_storage(result_file):
|
||||
print("Images in Glance images DB: %s \n" % image_id_l)
|
||||
print("Images in rbd images pool: %s \n" % rbd_image_l)
|
||||
|
||||
in_glance_only = np.setdiff1d(image_id_l, rbd_image_l)
|
||||
in_rbd_image_only = np.setdiff1d(rbd_image_l, image_id_l)
|
||||
in_glance_only = list(set(image_id_l) - set(rbd_image_l))
|
||||
in_rbd_image_only = list(set(rbd_image_l) - set(image_id_l))
|
||||
|
||||
print("Images in Glance images DB only: %s \n" % in_glance_only)
|
||||
print("Images in rbd images pool only: %s \n" % in_rbd_image_only)
|
||||
|
||||
if in_rbd_image_only.size != 0:
|
||||
if in_rbd_image_only:
|
||||
output = subprocess.check_output(
|
||||
["grep",
|
||||
"fsid",
|
||||
@ -217,12 +250,12 @@ def tidy_storage(result_file):
|
||||
fields['name'] = 'found-image-%s' % image
|
||||
fields['id'] = image
|
||||
fields['container_format'] = 'bare'
|
||||
fields['location'] = \
|
||||
'rbd://{}/images/{}/snap'.format(ceph_cluster[0],
|
||||
image)
|
||||
url = 'rbd://{}/images/{}/snap'.format(ceph_cluster[0],
|
||||
image)
|
||||
|
||||
print("Creating a Glance image %s ...\n " % fields['name'])
|
||||
g_client_v1.images.create(**fields)
|
||||
g_client.images.create(**fields)
|
||||
g_client.images.add_location(image, url, {})
|
||||
except subprocess.CalledProcessError:
|
||||
LOG.error("Failed to access rbd image %s" % image)
|
||||
raise TidyStorageFail("Failed to access rbd image")
|
||||
@ -261,7 +294,7 @@ def tidy_storage(result_file):
|
||||
break
|
||||
|
||||
if found_vol:
|
||||
volume = 'cinder-volumes/volume-{}'.format(snap.volume_id)
|
||||
volume = 'cinder-volumes/{}'.format(snap.volume_id)
|
||||
try:
|
||||
output = subprocess.check_output(
|
||||
["rbd", "snap", "list", volume],
|
||||
@ -330,14 +363,14 @@ def tidy_storage(result_file):
|
||||
LOG.error("Failed to access rbd cinder-volumes pool")
|
||||
raise TidyStorageFail("Failed to access rbd cinder-volumes pool")
|
||||
|
||||
rbd_volume_l = [i[7:] for i in output.split('\n') if i != ""]
|
||||
rbd_volume_l = [i for i in output.split('\n') if i != ""]
|
||||
|
||||
print("Volumes in Cinder volumes DB: %s \n" % cinder_volume_l)
|
||||
print("Volumes in rbd pool: %s \n" % rbd_volume_l)
|
||||
|
||||
in_cinder_only = np.setdiff1d(cinder_volume_l, rbd_volume_l)
|
||||
in_rbd_volume_only = np.setdiff1d(rbd_volume_l, cinder_volume_l)
|
||||
in_cinder_and_rbd = np.intersect1d(cinder_volume_l, rbd_volume_l)
|
||||
in_cinder_only = list(set(cinder_volume_l) - set(rbd_volume_l))
|
||||
in_rbd_volume_only = list(set(rbd_volume_l) - set(cinder_volume_l))
|
||||
in_cinder_and_rbd = list(set(cinder_volume_l) & set(rbd_volume_l))
|
||||
|
||||
print("Volumes in Cinder volumes DB only: %s \n" % in_cinder_only)
|
||||
print("Volumes in rbd pool only: %s \n" % in_rbd_volume_only)
|
||||
@ -345,7 +378,7 @@ def tidy_storage(result_file):
|
||||
% in_cinder_and_rbd)
|
||||
|
||||
for vol_id in in_rbd_volume_only:
|
||||
volume = 'cinder-volumes/volume-{}'.format(vol_id)
|
||||
volume = 'cinder-volumes/{}'.format(vol_id)
|
||||
try:
|
||||
# Find out if the volume is a bootable one
|
||||
output = subprocess.check_output(
|
||||
@ -429,7 +462,7 @@ def tidy_storage(result_file):
|
||||
raise TidyStorageFail("Failed to get Cinder snapshots")
|
||||
|
||||
for vol_id in in_cinder_and_rbd:
|
||||
volume = 'cinder-volumes/volume-{}'.format(vol_id)
|
||||
volume = 'cinder-volumes/{}'.format(vol_id)
|
||||
try:
|
||||
# Find out if the volume has any snapshots.
|
||||
print("Checking if volume %s has snapshots...\n" % vol_id)
|
||||
@ -469,7 +502,7 @@ def tidy_storage(result_file):
|
||||
"\"glance image-delete <id>\" command.", 80))
|
||||
f.write("\n\n")
|
||||
f.write('{0[0]:<40}{0[1]:<50}\n'.format(['ID', 'NAME']))
|
||||
image_l = g_client_v2.images.list()
|
||||
image_l = g_client.images.list()
|
||||
for image in image_l:
|
||||
if image['name'].find("found-image") != -1:
|
||||
f.write('{0[0]:<40}{0[1]:<50}\n'.format(
|
||||
@ -485,9 +518,9 @@ def tidy_storage(result_file):
|
||||
"document to restore the image.", 80))
|
||||
f.write("\n\n")
|
||||
f.write('{0[0]:<40}{0[1]:<50}\n'.format(['ID', 'NAME']))
|
||||
image_l = g_client_v2.images.list()
|
||||
image_l = g_client.images.list()
|
||||
for image in image_l:
|
||||
if (in_glance_only.size != 0 and
|
||||
if (in_glance_only and
|
||||
image['id'].encode('utf-8') in in_glance_only):
|
||||
f.write('{0[0]:<40}{0[1]:<50}\n'.format(
|
||||
[image['id'].encode('utf-8'), image['name']]))
|
||||
@ -520,8 +553,7 @@ def tidy_storage(result_file):
|
||||
f.write('{0[0]:<40}{0[1]:<50}\n'.format(['ID', 'NAME']))
|
||||
volume_l = c_client.volumes.list(search_opts=search_opts)
|
||||
for volume in volume_l:
|
||||
if (in_cinder_only.size != 0 and
|
||||
volume.id in in_cinder_only):
|
||||
if in_cinder_only and volume.id in in_cinder_only:
|
||||
f.write('{0[0]:<40}{0[1]:<50}\n'.format(
|
||||
[volume.id.encode('utf-8'), volume.name]))
|
||||
|
||||
|
||||
202
controllerconfig/controllerconfig/scripts/image-backup.sh
Normal file
202
controllerconfig/controllerconfig/scripts/image-backup.sh
Normal file
@ -0,0 +1,202 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Copyright (c) 2016-2020 Wind River Systems, Inc.
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
backup_dir="/opt/backups"
|
||||
tmp_dir="${backup_dir}/image_temp"
|
||||
|
||||
function usage {
|
||||
cat <<"EOF"
|
||||
Helper tool for backing up Glance images
|
||||
Usage:
|
||||
image-backup export <uuid> - export the image with <uuid> into backup file /opt/backups/image_<uuid>.tgz
|
||||
image-backup import image_<uuid>.tgz - import the image from the backup source file at /opt/backups/image_<uuid>.tgz
|
||||
into the corresponding image.
|
||||
|
||||
Temporary files are stored in /opt/backups/image_temp
|
||||
|
||||
Please consult the OpenStack application backup and restore section of the Backup and Restore Guide.
|
||||
EOF
|
||||
}
|
||||
|
||||
function create_tmp {
|
||||
if [ ! -d ${backup_dir} ]; then
|
||||
echo "Error: backup directory ${backup_dir} does not exist"
|
||||
exit 1
|
||||
fi
|
||||
# Create temporary directory
|
||||
if [ ! -d ${tmp_dir} ]; then
|
||||
mkdir ${tmp_dir}
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
function remove_tmp {
|
||||
# Remove temporary files and directory if not empty
|
||||
local uuid=$1
|
||||
rm -f ${tmp_dir}/${uuid}*
|
||||
rmdir --ignore-fail-on-non-empty ${tmp_dir} &>/dev/null
|
||||
}
|
||||
|
||||
function export_file_from_rbd_image {
|
||||
local file=$1
|
||||
rbd export -p images ${file} ${tmp_dir}/${file}
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Error: Failed to export image ${file} from Ceph images pool, please check status of storage cluster"
|
||||
remove_tmp; exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
function export_image {
|
||||
local uuid=$1
|
||||
|
||||
# Check if the corresponding image is present in the RBD pool
|
||||
rbd -p images ls | grep -q -e "^${uuid}$"
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Error: Corresponding file for image with id: ${uuid} was not found in the RBD images pool"
|
||||
remove_tmp; exit 1
|
||||
fi
|
||||
|
||||
# Export original image
|
||||
export_file_from_rbd_image ${uuid}
|
||||
|
||||
# Export raw cache if present
|
||||
rbd -p images ls | grep -q ${uuid}_raw
|
||||
if [ $? -eq 0 ]; then
|
||||
export_file_from_rbd_image ${uuid}_raw
|
||||
raw="${uuid}_raw"
|
||||
fi
|
||||
|
||||
echo -n "Creating backup archive..."
|
||||
archive="${backup_dir}/image_${uuid}.tgz"
|
||||
tar czf ${archive} -C ${tmp_dir} ${uuid} ${raw}
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Error: Failed to create archive ${archive}"
|
||||
remove_tmp; exit 1
|
||||
else
|
||||
echo "done"
|
||||
fi
|
||||
|
||||
echo "Backup archive ${archive} created"
|
||||
}
|
||||
|
||||
function import_file_to_rbd_image {
|
||||
local file=$1
|
||||
local snap="images/${file}@snap"
|
||||
rbd import --image-format 2 ${tmp_dir}/${file} images/${file}
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Error: Failed to import image ${file} into Ceph images pool, please check status of storage cluster"
|
||||
remove_tmp; exit 1
|
||||
fi
|
||||
rbd snap create ${snap} 1>/dev/null
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Error: Failed to create snapshot ${snap}, please check status of storage cluster"
|
||||
remove_tmp; exit 1
|
||||
fi
|
||||
rbd snap protect ${snap} 1>/dev/null
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Error: Failed to protect snapshot ${snap}, please check status of storage cluster"
|
||||
remove_tmp; exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
function import_image {
|
||||
local uuid=$1
|
||||
|
||||
# Storage cluster must be healthy before starting the import
|
||||
if [ ! "$(ceph health)" = "HEALTH_OK" ]; then
|
||||
echo "Error: The storage cluster health must be HEALTH_OK before proceding"
|
||||
remove_tmp; exit 1
|
||||
fi
|
||||
|
||||
# Check if the corresponding image is already present in the RBD pool
|
||||
rbd -p images ls | grep -q -e "^${uuid}$"
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "Error: Image with id: ${uuid} is already imported"
|
||||
remove_tmp; exit 1
|
||||
fi
|
||||
|
||||
# Import original image
|
||||
import_file_to_rbd_image ${uuid}
|
||||
|
||||
# Import raw cache
|
||||
if [ -f "${tmp_dir}/${uuid}_raw" ]; then
|
||||
import_file_to_rbd_image ${uuid}_raw
|
||||
fi
|
||||
}
|
||||
|
||||
if [ $EUID -ne 0 ]; then
|
||||
echo "This script must be executed as root"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ $# -ne 2 ]; then
|
||||
usage
|
||||
exit 0
|
||||
fi
|
||||
|
||||
source /etc/platform/openrc
|
||||
export OS_AUTH_URL=http://keystone.openstack.svc.cluster.local/v3
|
||||
|
||||
if [ "$1" = "export" ]; then
|
||||
# Check that glance image is present in glance
|
||||
openstack image list -f value -c ID | grep -q $2
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Error: Glance image with id: $2 not found. Please try with an existing image id."
|
||||
remove_tmp; exit 1
|
||||
fi
|
||||
|
||||
# Only allow backup of images that use rbd as backend.
|
||||
openstack image show $2 -c properties | grep -q -F "direct_url='rbd://"
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Image with id: $2 is not stored in Ceph RBD. Backup using image-backup tool is not needed."
|
||||
echo "Please consult the Software Management Manual for more details."
|
||||
remove_tmp; exit 1
|
||||
fi
|
||||
|
||||
create_tmp
|
||||
export_image $2
|
||||
remove_tmp
|
||||
|
||||
elif [ "$1" = "import" ]; then
|
||||
# Check that the input file format is correct
|
||||
if [[ ! $2 =~ ^image_.*\.tgz$ ]]; then
|
||||
echo "Error: Source file name must conform to image_<uuid>.tgz format and exist in /opt/backups"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check that the source file exists
|
||||
if [ ! -f ${backup_dir}/$2 ]; then
|
||||
echo "Error: File $2 does not exists in ${backup_dir}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Get glance uuid from filename
|
||||
uuid=$(echo $2 | sed "s/^image_\(.*\)\.tgz/\1/g")
|
||||
|
||||
# Check that glance has this image in the database
|
||||
openstack image show $uuid
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Error: Glance image with id: ${uuid} not found. Please try with an existing image id."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
create_tmp
|
||||
|
||||
# Extract the files that need to be imported into the temp directory
|
||||
echo -n "Extracting files..."
|
||||
tar xfz ${backup_dir}/$2 -C ${tmp_dir} 1>/dev/null
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Error: Failed to extract archive ${backup_dir}/$2 into ${tmp_dir}."
|
||||
remove_tmp; exit 1
|
||||
fi
|
||||
echo "done"
|
||||
|
||||
# Importing images into RBD
|
||||
import_image $uuid
|
||||
remove_tmp
|
||||
else
|
||||
usage
|
||||
fi
|
||||
Loading…
x
Reference in New Issue
Block a user