openstack/tripleo-common
Code Issues Proposed changes

Merge "Update artifact tooling for swiftless underclouds"

Browse Source
This commit is contained in:
Zuul 2021-04-28 23:39:53 +00:00 committed by Gerrit Code Review
commit 795cd51a91
4 changed files with 151 additions and 181 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
releasenotes/notes/DeployArtifactURLs-Simplified-e3993493022653ab.yaml Normal file
View File

@ -0,0 +1,9 @@
---
deprecations:
- The DeployArtifactURL(s) server is no longer makes any use of Swift when
using built-in scripts to upload artifacts. The old
`upload-swift-artifacts` file is now a symlink pointed to the updated
`upload-artifacts` script. While the file has changed, the API remains the
same. The symlink will ensure that legacy automation remains functional
while moving over to the new script. The symlink will be removed in a
future release.

133
scripts/upload-artifacts Normal file
View File

@ -0,0 +1,133 @@
#!/bin/bash
#set -x
# Copyright 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# Script to upload files to a Artifact container for deployment via
# TripleO Heat Templates.
#
set -eu
set -o pipefail
SCRIPT_NAME=$(basename $0)
SCRIPT_HOME=$(cd $(dirname $0); pwd)
function show_options {
echo "Usage: $SCRIPT_NAME"
echo
echo "Options:"
echo " -h, --help -- print this help."
echo " -f <file> -- File(s) to upload."
echo " -c <container> -- Artifact container to use."
echo " Default: overcloud-artifacts"
echo " --environment <env> -- Generate this heat <env>"
echo " Default: $HOME/.tripleo/environments/deployment-artifacts.yaml"
echo " --seconds <seconds> -- Number of seconds for temp URLs."
echo " Default: 31536000 (one year)"
echo
echo "Upload a set of files to Artifact and generate a Heat environment"
echo "file containing the required DeployArtifactURLs parameter(s) "
echo "so that it is used with TripleO Heat Templates."
echo
exit $1
}
function check_file {
local FILE=$1
supported_files=(" RPM " "gzip compressed data")
array_length=${#supported_files[@]}
local test_type=`file $FILE`
local i=0
local matched=0
while [ $i -ne $array_length -a $matched -eq 0 ]; do
if [[ "$test_type" =~ ${supported_files[$i]} ]]; then
matched=1
fi
i=$((i+1))
done
if [ $matched -eq 0 ]; then
echo "Not a supported file type: $FILE"
exit 1
fi
}
TEMP=`getopt -o he:f:c:s: -l help,environment:,file:,container:,seconds: -n $SCRIPT_NAME -- "$@"`
if [ $? != 0 ]; then
echo "Terminating..." >&2
exit 1
fi
# Note the quotes around `$TEMP': they are essential!
eval set -- "$TEMP"
ENVIRONMENT_FILE="$HOME/.tripleo/environments/deployment-artifacts.yaml"
FILES=
CONTAINER_NAME=overcloud-artifacts
SECONDS=31536000
BINDPORT="${BINDPORT:-8080}"
BINDADDRESS=$(ip addr show dev br-ctlplane | awk -F'\\s|/' '/inet\s/ {print $6}' | head -n 1)
while true ; do
case "$1" in
-h|--help) show_options 0 >&2;;
-e|--environment) ENVIRONMENT_FILE=$2 ; shift 2;;
-f|--file) FILES+=" $2" ; shift 2;;
-c|--container) CONTAINER_NAME=$2 ; shift 2;;
-s|--seconds) SECONDS=$2 ; shift 2;;
--) shift ; break;;
*) echo "Error: unsupported option $1." ; exit 1;;
esac
done
if [ -z "${FILES:-}" ]; then
echo "Error: No files were specified."
exit 1
fi
if [ -z "${CONTAINER_NAME:-}" ]; then
echo "Error: No Artifact --container was specified."
exit 1
fi
# Create artifact archive
sudo mkdir -p "/var/lib/tripleo/artifacts/${CONTAINER_NAME}"
for FILE in ${FILES[@]}; do
check_file "$FILE"
sudo mv -v "$FILE" "/var/lib/tripleo/artifacts/${CONTAINER_NAME}/"
done
if [ -n "${ENVIRONMENT_FILE:-}" ]; then
echo "Creating heat environment file: ${ENVIRONMENT_FILE}"
mkdir -p $(dirname "${ENVIRONMENT_FILE}")
/bin/python3 <<EOC
import os
import yaml
key = 'DeployArtifactFILEs'
urls = [
"/var/lib/tripleo/artifacts/${CONTAINER_NAME}/{}".format(i)
for i in files
]
content = {'parameter_defaults': {key: []}}
content['parameter_defaults'][key].extend(urls)
with open("${ENVIRONMENT_FILE}", 'w') as f:
yaml.dump(content, f, default_flow_style=False)
EOC
fi

16
scripts/upload-puppet-modules
View File

@ -14,7 +14,7 @@
# under the License.
#
# Script to upload directory tarball of puppet modules to a swift container.
# Script to upload directory tarball of puppet modules to a artifact container.
#
set -eu
set -o pipefail
@ -34,14 +34,14 @@ function show_options {
echo "Options:"
echo " -h, --help -- print this help."
echo " -d <directory> -- Puppet modules directory. Required."
echo " -c <container> -- Swift container to use."
echo " -c <container> -- Artifact container to use."
echo " Default: overcloud-artifacts"
echo " --environment <env> -- Generate this heat <env>"
echo " Default: $HOME/.tripleo/environments/puppet-modules-url.yaml"
echo " --seconds <seconds> -- Number of seconds for temp URL."
echo " Default: 31536000 (one year)"
echo
echo "Upload a directory tarball of puppet modules to a swift container"
echo "Upload a directory tarball of puppet modules to a artifact container"
echo "and generate a heat environment file containing the required"
echo "DeployArtifactURLs parameter so that it is used for TripleO deployment."
echo
@ -79,7 +79,7 @@ done
modules_directory=${MODULES_DIRECTORY%/}
modules_directory=${modules_directory#/}
: ${CONTAINER_NAME:?No swift --container was specified}
: ${CONTAINER_NAME:?No artifact --container was specified}
echo "Creating tarball..."
PUPPET_MODULES_TMP=$(mktemp -d -t 'puppet-modules-XXXXXXX')
@ -87,7 +87,7 @@ tar --transform "s|${modules_directory}|etc/puppet/modules|" \
-czf "$PUPPET_MODULES_TMP/puppet-modules.tar.gz" $MODULES_DIRECTORY
echo "Tarball created."
upload-swift-artifacts -f "$PUPPET_MODULES_TMP/puppet-modules.tar.gz" \
-c "$CONTAINER_NAME" \
--environment "$ENVIRONMENT_FILE" \
--seconds "$SECONDS"
upload-artifacts -f "$PUPPET_MODULES_TMP/puppet-modules.tar.gz" \
-c "$CONTAINER_NAME" \
--environment "$ENVIRONMENT_FILE" \
--seconds "$SECONDS"

173
scripts/upload-swift-artifacts
View File

@ -1,173 +0,0 @@
#!/bin/bash
#set -x
# Copyright 2015 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# Script to upload files to a Swift container for deployment via
# TripleO Heat Templates.
#
set -eu
set -o pipefail
SCRIPT_NAME=$(basename $0)
SCRIPT_HOME=$(cd $(dirname $0); pwd)
function show_options {
echo "Usage: $SCRIPT_NAME"
echo
echo "Options:"
echo " -h, --help -- print this help."
echo " -f <file> -- File(s) to upload."
echo " -c <container> -- Swift container to use."
echo " Default: overcloud-artifacts"
echo " --environment <env> -- Generate this heat <env>"
echo " Default: $HOME/.tripleo/environments/deployment-artifacts.yaml"
echo " --seconds <seconds> -- Number of seconds for temp URLs."
echo " Default: 31536000 (one year)"
echo
echo "Upload a set of files to Swift and generate a Heat environment"
echo "file containing the required DeployArtifactURLs parameter(s) "
echo "so that it is used with TripleO Heat Templates."
echo
exit $1
}
function check_file {
local FILE=$1
supported_files=(" RPM " "gzip compressed data")
array_length=${#supported_files[@]}
local test_type=`file $FILE`
local i=0
local matched=0
while [ $i -ne $array_length -a $matched -eq 0 ]; do
if [[ "$test_type" =~ ${supported_files[$i]} ]]; then
matched=1
fi
i=$((i+1))
done
if [ $matched -eq 0 ]; then
echo "Not a supported file type: $FILE"
exit 1
fi
}
TEMP=`getopt -o he:f:c:s: -l help,environment:,file:,container:,seconds: -n $SCRIPT_NAME -- "$@"`
if [ $? != 0 ]; then
echo "Terminating..." >&2
exit 1
fi
# Note the quotes around `$TEMP': they are essential!
eval set -- "$TEMP"
ENVIRONMENT_FILE="$HOME/.tripleo/environments/deployment-artifacts.yaml"
FILES=
CONTAINER_NAME=overcloud-artifacts
SECONDS=31536000
while true ; do
case "$1" in
-h|--help) show_options 0 >&2;;
-e|--environment) ENVIRONMENT_FILE=$2 ; shift 2;;
-f|--file) FILES+=" $2" ; shift 2;;
-c|--container) CONTAINER_NAME=$2 ; shift 2;;
-s|--seconds) SECONDS=$2 ; shift 2;;
--) shift ; break;;
*) echo "Error: unsupported option $1." ; exit 1;;
esac
done
if [ -z "${FILES:-}" ]; then
echo "Error: No files were specified."
exit 1
fi
if [ -z "${CONTAINER_NAME:-}" ]; then
echo "Error: No Swift --container was specified."
exit 1
fi
function create_heat_env {
local heat_env_file=$1
mkdir -p $(dirname "$heat_env_file")
cat > "$heat_env_file" <<-EOF_CAT
# Heat environment to deploy artifacts via Swift Temp URL(s)
parameter_defaults:
DeployArtifactURLs:
EOF_CAT
}
# create the container (doing it more than once won't hurt...) and grab the
# account
SWIFT_ACCOUNT=$(openstack container create $CONTAINER_NAME -f value -c account)
# This works with newer openstackclient which displays the endpoints in list
SWIFT_INTERNAL_URL=$(openstack endpoint list | grep swift | grep internal | sed -e "s|.*\(http.*\)://\([^/]*\)/.*|\1://\2|" || true)
if [ -z "$SWIFT_INTERNAL_URL" ]; then
# fallback to old openstack client 'show' which displays all the URLs
SWIFT_INTERNAL_URL=$(openstack endpoint show swift | grep internalurl | sed -e "s|.*\(http.*\)://\([^/]*\)/.*|\1://\2|")
fi
# Does the Temp-URL-Key exist on this container?
# jq will output "null" if it doesn't
# If not set it...
SWIFT_TEMP_URL_KEY=$(openstack container show $CONTAINER_NAME -f json 2>/dev/null | jq -r '.properties."Temp-Url-Key"' || true)
if [ "$SWIFT_TEMP_URL_KEY" == "null" ]; then
echo "Creating new Swift Temp-Url-Key for container: $CONTAINER_NAME"
SWIFT_TEMP_URL_KEY=$(uuidgen | sha1sum | awk '{print $1}')
openstack container set --property "Temp-Url-Key=$SWIFT_TEMP_URL_KEY" "${CONTAINER_NAME}"
fi
if [ -n "${ENVIRONMENT_FILE:-}" ]; then
echo "Creating heat environment file: $ENVIRONMENT_FILE"
create_heat_env "$ENVIRONMENT_FILE"
else
echo "No environment file specified... skipping creation of Heat environment."
fi
function get_tempurl {
# https://docs.openstack.org/swift/latest/api/temporary_url_middleware.html#hmac-sha1-signature-for-temporary-urls
local FILE="$1"
local SWIFT_METHOD='GET'
local SWIFT_PATH="/v1/${SWIFT_ACCOUNT}/${CONTAINER_NAME}/$(basename $FILE)"
local SWIFT_EXPIRES=$(( $(date '+%s') + $SECONDS ))
local SWIFT_SIG=$(printf '%s\n%s\n%s' $SWIFT_METHOD $SWIFT_EXPIRES $SWIFT_PATH | openssl sha1 -hmac $SWIFT_TEMP_URL_KEY | sed 's/^.* //')
echo -n "${SWIFT_PATH}?temp_url_sig=${SWIFT_SIG}&temp_url_expires=${SWIFT_EXPIRES}"
}
function upload_file {
local FILE="$1"
echo "Uploading file to swift: $1"
pushd $(dirname $FILE) &>/dev/null
openstack object create "$CONTAINER_NAME" $(basename $FILE)
popd &>/dev/null
echo "Upload complete."
local URL=$(get_tempurl "$FILE")
echo " - '${SWIFT_INTERNAL_URL}$URL'" >> $ENVIRONMENT_FILE
}
for FILE in ${FILES[@]}; do
check_file "$FILE"
upload_file "$FILE"
done

1
scripts/upload-swift-artifacts Symbolic link
View File

@ -0,0 +1 @@
upload-artifacts