Merge "Update artifact tooling for swiftless underclouds"
This commit is contained in:
Zuul
Gerrit Code Review
commit
795cd51a91
@ -0,0 +1,9 @@
|
|||||||
|
---
|
||||||
|
deprecations:
|
||||||
|
- The DeployArtifactURL(s) server is no longer makes any use of Swift when
|
||||||
|
using built-in scripts to upload artifacts. The old
|
||||||
|
`upload-swift-artifacts` file is now a symlink pointed to the updated
|
||||||
|
`upload-artifacts` script. While the file has changed, the API remains the
|
||||||
|
same. The symlink will ensure that legacy automation remains functional
|
||||||
|
while moving over to the new script. The symlink will be removed in a
|
||||||
|
future release.
|
||||||
133
scripts/upload-artifacts
Normal file
133
scripts/upload-artifacts
Normal file
@ -0,0 +1,133 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
#set -x
|
||||||
|
# Copyright 2015 Red Hat, Inc.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
#
|
||||||
|
# Script to upload files to a Artifact container for deployment via
|
||||||
|
# TripleO Heat Templates.
|
||||||
|
#
|
||||||
|
set -eu
|
||||||
|
set -o pipefail
|
||||||
|
SCRIPT_NAME=$(basename $0)
|
||||||
|
SCRIPT_HOME=$(cd $(dirname $0); pwd)
|
||||||
|
|
||||||
|
function show_options {
|
||||||
|
echo "Usage: $SCRIPT_NAME"
|
||||||
|
echo
|
||||||
|
echo "Options:"
|
||||||
|
echo " -h, --help -- print this help."
|
||||||
|
echo " -f <file> -- File(s) to upload."
|
||||||
|
echo " -c <container> -- Artifact container to use."
|
||||||
|
echo " Default: overcloud-artifacts"
|
||||||
|
echo " --environment <env> -- Generate this heat <env>"
|
||||||
|
echo " Default: $HOME/.tripleo/environments/deployment-artifacts.yaml"
|
||||||
|
echo " --seconds <seconds> -- Number of seconds for temp URLs."
|
||||||
|
echo " Default: 31536000 (one year)"
|
||||||
|
echo
|
||||||
|
echo "Upload a set of files to Artifact and generate a Heat environment"
|
||||||
|
echo "file containing the required DeployArtifactURLs parameter(s) "
|
||||||
|
echo "so that it is used with TripleO Heat Templates."
|
||||||
|
echo
|
||||||
|
exit $1
|
||||||
|
}
|
||||||
|
|
||||||
|
function check_file {
|
||||||
|
local FILE=$1
|
||||||
|
supported_files=(" RPM " "gzip compressed data")
|
||||||
|
array_length=${#supported_files[@]}
|
||||||
|
|
||||||
|
local test_type=`file $FILE`
|
||||||
|
|
||||||
|
local i=0
|
||||||
|
local matched=0
|
||||||
|
|
||||||
|
while [ $i -ne $array_length -a $matched -eq 0 ]; do
|
||||||
|
if [[ "$test_type" =~ ${supported_files[$i]} ]]; then
|
||||||
|
matched=1
|
||||||
|
fi
|
||||||
|
i=$((i+1))
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ $matched -eq 0 ]; then
|
||||||
|
echo "Not a supported file type: $FILE"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
TEMP=`getopt -o he:f:c:s: -l help,environment:,file:,container:,seconds: -n $SCRIPT_NAME -- "$@"`
|
||||||
|
if [ $? != 0 ]; then
|
||||||
|
echo "Terminating..." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Note the quotes around `$TEMP': they are essential!
|
||||||
|
eval set -- "$TEMP"
|
||||||
|
|
||||||
|
ENVIRONMENT_FILE="$HOME/.tripleo/environments/deployment-artifacts.yaml"
|
||||||
|
FILES=
|
||||||
|
CONTAINER_NAME=overcloud-artifacts
|
||||||
|
SECONDS=31536000
|
||||||
|
BINDPORT="${BINDPORT:-8080}"
|
||||||
|
BINDADDRESS=$(ip addr show dev br-ctlplane | awk -F'\\s|/' '/inet\s/ {print $6}' | head -n 1)
|
||||||
|
|
||||||
|
while true ; do
|
||||||
|
case "$1" in
|
||||||
|
-h|--help) show_options 0 >&2;;
|
||||||
|
-e|--environment) ENVIRONMENT_FILE=$2 ; shift 2;;
|
||||||
|
-f|--file) FILES+=" $2" ; shift 2;;
|
||||||
|
-c|--container) CONTAINER_NAME=$2 ; shift 2;;
|
||||||
|
-s|--seconds) SECONDS=$2 ; shift 2;;
|
||||||
|
--) shift ; break;;
|
||||||
|
*) echo "Error: unsupported option $1." ; exit 1;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ -z "${FILES:-}" ]; then
|
||||||
|
echo "Error: No files were specified."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -z "${CONTAINER_NAME:-}" ]; then
|
||||||
|
echo "Error: No Artifact --container was specified."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Create artifact archive
|
||||||
|
sudo mkdir -p "/var/lib/tripleo/artifacts/${CONTAINER_NAME}"
|
||||||
|
|
||||||
|
for FILE in ${FILES[@]}; do
|
||||||
|
check_file "$FILE"
|
||||||
|
sudo mv -v "$FILE" "/var/lib/tripleo/artifacts/${CONTAINER_NAME}/"
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ -n "${ENVIRONMENT_FILE:-}" ]; then
|
||||||
|
echo "Creating heat environment file: ${ENVIRONMENT_FILE}"
|
||||||
|
mkdir -p $(dirname "${ENVIRONMENT_FILE}")
|
||||||
|
/bin/python3 <<EOC
|
||||||
|
import os
|
||||||
|
import yaml
|
||||||
|
key = 'DeployArtifactFILEs'
|
||||||
|
urls = [
|
||||||
|
"/var/lib/tripleo/artifacts/${CONTAINER_NAME}/{}".format(i)
|
||||||
|
for i in files
|
||||||
|
]
|
||||||
|
content = {'parameter_defaults': {key: []}}
|
||||||
|
content['parameter_defaults'][key].extend(urls)
|
||||||
|
with open("${ENVIRONMENT_FILE}", 'w') as f:
|
||||||
|
yaml.dump(content, f, default_flow_style=False)
|
||||||
|
EOC
|
||||||
|
fi
|
||||||
@ -14,7 +14,7 @@
|
|||||||
# under the License.
|
# under the License.
|
||||||
|
|
||||||
#
|
#
|
||||||
# Script to upload directory tarball of puppet modules to a swift container.
|
# Script to upload directory tarball of puppet modules to a artifact container.
|
||||||
#
|
#
|
||||||
set -eu
|
set -eu
|
||||||
set -o pipefail
|
set -o pipefail
|
||||||
@ -34,14 +34,14 @@ function show_options {
|
|||||||
echo "Options:"
|
echo "Options:"
|
||||||
echo " -h, --help -- print this help."
|
echo " -h, --help -- print this help."
|
||||||
echo " -d <directory> -- Puppet modules directory. Required."
|
echo " -d <directory> -- Puppet modules directory. Required."
|
||||||
echo " -c <container> -- Swift container to use."
|
echo " -c <container> -- Artifact container to use."
|
||||||
echo " Default: overcloud-artifacts"
|
echo " Default: overcloud-artifacts"
|
||||||
echo " --environment <env> -- Generate this heat <env>"
|
echo " --environment <env> -- Generate this heat <env>"
|
||||||
echo " Default: $HOME/.tripleo/environments/puppet-modules-url.yaml"
|
echo " Default: $HOME/.tripleo/environments/puppet-modules-url.yaml"
|
||||||
echo " --seconds <seconds> -- Number of seconds for temp URL."
|
echo " --seconds <seconds> -- Number of seconds for temp URL."
|
||||||
echo " Default: 31536000 (one year)"
|
echo " Default: 31536000 (one year)"
|
||||||
echo
|
echo
|
||||||
echo "Upload a directory tarball of puppet modules to a swift container"
|
echo "Upload a directory tarball of puppet modules to a artifact container"
|
||||||
echo "and generate a heat environment file containing the required"
|
echo "and generate a heat environment file containing the required"
|
||||||
echo "DeployArtifactURLs parameter so that it is used for TripleO deployment."
|
echo "DeployArtifactURLs parameter so that it is used for TripleO deployment."
|
||||||
echo
|
echo
|
||||||
@ -79,7 +79,7 @@ done
|
|||||||
modules_directory=${MODULES_DIRECTORY%/}
|
modules_directory=${MODULES_DIRECTORY%/}
|
||||||
modules_directory=${modules_directory#/}
|
modules_directory=${modules_directory#/}
|
||||||
|
|
||||||
: ${CONTAINER_NAME:?No swift --container was specified}
|
: ${CONTAINER_NAME:?No artifact --container was specified}
|
||||||
|
|
||||||
echo "Creating tarball..."
|
echo "Creating tarball..."
|
||||||
PUPPET_MODULES_TMP=$(mktemp -d -t 'puppet-modules-XXXXXXX')
|
PUPPET_MODULES_TMP=$(mktemp -d -t 'puppet-modules-XXXXXXX')
|
||||||
@ -87,7 +87,7 @@ tar --transform "s|${modules_directory}|etc/puppet/modules|" \
|
|||||||
-czf "$PUPPET_MODULES_TMP/puppet-modules.tar.gz" $MODULES_DIRECTORY
|
-czf "$PUPPET_MODULES_TMP/puppet-modules.tar.gz" $MODULES_DIRECTORY
|
||||||
echo "Tarball created."
|
echo "Tarball created."
|
||||||
|
|
||||||
upload-swift-artifacts -f "$PUPPET_MODULES_TMP/puppet-modules.tar.gz" \
|
upload-artifacts -f "$PUPPET_MODULES_TMP/puppet-modules.tar.gz" \
|
||||||
-c "$CONTAINER_NAME" \
|
-c "$CONTAINER_NAME" \
|
||||||
--environment "$ENVIRONMENT_FILE" \
|
--environment "$ENVIRONMENT_FILE" \
|
||||||
--seconds "$SECONDS"
|
--seconds "$SECONDS"
|
||||||
|
|||||||
@ -1,173 +0,0 @@
|
|||||||
#!/bin/bash
|
|
||||||
#set -x
|
|
||||||
# Copyright 2015 Red Hat, Inc.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
||||||
# not use this file except in compliance with the License. You may obtain
|
|
||||||
# a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
||||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
||||||
# License for the specific language governing permissions and limitations
|
|
||||||
# under the License.
|
|
||||||
|
|
||||||
#
|
|
||||||
# Script to upload files to a Swift container for deployment via
|
|
||||||
# TripleO Heat Templates.
|
|
||||||
#
|
|
||||||
set -eu
|
|
||||||
set -o pipefail
|
|
||||||
SCRIPT_NAME=$(basename $0)
|
|
||||||
SCRIPT_HOME=$(cd $(dirname $0); pwd)
|
|
||||||
|
|
||||||
function show_options {
|
|
||||||
echo "Usage: $SCRIPT_NAME"
|
|
||||||
echo
|
|
||||||
echo "Options:"
|
|
||||||
echo " -h, --help -- print this help."
|
|
||||||
echo " -f <file> -- File(s) to upload."
|
|
||||||
echo " -c <container> -- Swift container to use."
|
|
||||||
echo " Default: overcloud-artifacts"
|
|
||||||
echo " --environment <env> -- Generate this heat <env>"
|
|
||||||
echo " Default: $HOME/.tripleo/environments/deployment-artifacts.yaml"
|
|
||||||
echo " --seconds <seconds> -- Number of seconds for temp URLs."
|
|
||||||
echo " Default: 31536000 (one year)"
|
|
||||||
echo
|
|
||||||
echo "Upload a set of files to Swift and generate a Heat environment"
|
|
||||||
echo "file containing the required DeployArtifactURLs parameter(s) "
|
|
||||||
echo "so that it is used with TripleO Heat Templates."
|
|
||||||
echo
|
|
||||||
exit $1
|
|
||||||
}
|
|
||||||
|
|
||||||
function check_file {
|
|
||||||
local FILE=$1
|
|
||||||
supported_files=(" RPM " "gzip compressed data")
|
|
||||||
array_length=${#supported_files[@]}
|
|
||||||
|
|
||||||
local test_type=`file $FILE`
|
|
||||||
|
|
||||||
local i=0
|
|
||||||
local matched=0
|
|
||||||
|
|
||||||
while [ $i -ne $array_length -a $matched -eq 0 ]; do
|
|
||||||
if [[ "$test_type" =~ ${supported_files[$i]} ]]; then
|
|
||||||
matched=1
|
|
||||||
fi
|
|
||||||
i=$((i+1))
|
|
||||||
done
|
|
||||||
|
|
||||||
if [ $matched -eq 0 ]; then
|
|
||||||
echo "Not a supported file type: $FILE"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
TEMP=`getopt -o he:f:c:s: -l help,environment:,file:,container:,seconds: -n $SCRIPT_NAME -- "$@"`
|
|
||||||
if [ $? != 0 ]; then
|
|
||||||
echo "Terminating..." >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Note the quotes around `$TEMP': they are essential!
|
|
||||||
eval set -- "$TEMP"
|
|
||||||
|
|
||||||
ENVIRONMENT_FILE="$HOME/.tripleo/environments/deployment-artifacts.yaml"
|
|
||||||
FILES=
|
|
||||||
CONTAINER_NAME=overcloud-artifacts
|
|
||||||
SECONDS=31536000
|
|
||||||
|
|
||||||
while true ; do
|
|
||||||
case "$1" in
|
|
||||||
-h|--help) show_options 0 >&2;;
|
|
||||||
-e|--environment) ENVIRONMENT_FILE=$2 ; shift 2;;
|
|
||||||
-f|--file) FILES+=" $2" ; shift 2;;
|
|
||||||
-c|--container) CONTAINER_NAME=$2 ; shift 2;;
|
|
||||||
-s|--seconds) SECONDS=$2 ; shift 2;;
|
|
||||||
--) shift ; break;;
|
|
||||||
*) echo "Error: unsupported option $1." ; exit 1;;
|
|
||||||
esac
|
|
||||||
done
|
|
||||||
|
|
||||||
if [ -z "${FILES:-}" ]; then
|
|
||||||
echo "Error: No files were specified."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -z "${CONTAINER_NAME:-}" ]; then
|
|
||||||
echo "Error: No Swift --container was specified."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
function create_heat_env {
|
|
||||||
local heat_env_file=$1
|
|
||||||
mkdir -p $(dirname "$heat_env_file")
|
|
||||||
cat > "$heat_env_file" <<-EOF_CAT
|
|
||||||
# Heat environment to deploy artifacts via Swift Temp URL(s)
|
|
||||||
parameter_defaults:
|
|
||||||
DeployArtifactURLs:
|
|
||||||
EOF_CAT
|
|
||||||
}
|
|
||||||
|
|
||||||
# create the container (doing it more than once won't hurt...) and grab the
|
|
||||||
# account
|
|
||||||
SWIFT_ACCOUNT=$(openstack container create $CONTAINER_NAME -f value -c account)
|
|
||||||
|
|
||||||
# This works with newer openstackclient which displays the endpoints in list
|
|
||||||
SWIFT_INTERNAL_URL=$(openstack endpoint list | grep swift | grep internal | sed -e "s|.*\(http.*\)://\([^/]*\)/.*|\1://\2|" || true)
|
|
||||||
|
|
||||||
if [ -z "$SWIFT_INTERNAL_URL" ]; then
|
|
||||||
# fallback to old openstack client 'show' which displays all the URLs
|
|
||||||
SWIFT_INTERNAL_URL=$(openstack endpoint show swift | grep internalurl | sed -e "s|.*\(http.*\)://\([^/]*\)/.*|\1://\2|")
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Does the Temp-URL-Key exist on this container?
|
|
||||||
# jq will output "null" if it doesn't
|
|
||||||
# If not set it...
|
|
||||||
SWIFT_TEMP_URL_KEY=$(openstack container show $CONTAINER_NAME -f json 2>/dev/null | jq -r '.properties."Temp-Url-Key"' || true)
|
|
||||||
if [ "$SWIFT_TEMP_URL_KEY" == "null" ]; then
|
|
||||||
echo "Creating new Swift Temp-Url-Key for container: $CONTAINER_NAME"
|
|
||||||
SWIFT_TEMP_URL_KEY=$(uuidgen | sha1sum | awk '{print $1}')
|
|
||||||
openstack container set --property "Temp-Url-Key=$SWIFT_TEMP_URL_KEY" "${CONTAINER_NAME}"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -n "${ENVIRONMENT_FILE:-}" ]; then
|
|
||||||
echo "Creating heat environment file: $ENVIRONMENT_FILE"
|
|
||||||
create_heat_env "$ENVIRONMENT_FILE"
|
|
||||||
else
|
|
||||||
echo "No environment file specified... skipping creation of Heat environment."
|
|
||||||
fi
|
|
||||||
|
|
||||||
function get_tempurl {
|
|
||||||
# https://docs.openstack.org/swift/latest/api/temporary_url_middleware.html#hmac-sha1-signature-for-temporary-urls
|
|
||||||
local FILE="$1"
|
|
||||||
local SWIFT_METHOD='GET'
|
|
||||||
local SWIFT_PATH="/v1/${SWIFT_ACCOUNT}/${CONTAINER_NAME}/$(basename $FILE)"
|
|
||||||
local SWIFT_EXPIRES=$(( $(date '+%s') + $SECONDS ))
|
|
||||||
local SWIFT_SIG=$(printf '%s\n%s\n%s' $SWIFT_METHOD $SWIFT_EXPIRES $SWIFT_PATH | openssl sha1 -hmac $SWIFT_TEMP_URL_KEY | sed 's/^.* //')
|
|
||||||
echo -n "${SWIFT_PATH}?temp_url_sig=${SWIFT_SIG}&temp_url_expires=${SWIFT_EXPIRES}"
|
|
||||||
}
|
|
||||||
|
|
||||||
function upload_file {
|
|
||||||
|
|
||||||
local FILE="$1"
|
|
||||||
echo "Uploading file to swift: $1"
|
|
||||||
pushd $(dirname $FILE) &>/dev/null
|
|
||||||
openstack object create "$CONTAINER_NAME" $(basename $FILE)
|
|
||||||
popd &>/dev/null
|
|
||||||
echo "Upload complete."
|
|
||||||
|
|
||||||
local URL=$(get_tempurl "$FILE")
|
|
||||||
echo " - '${SWIFT_INTERNAL_URL}$URL'" >> $ENVIRONMENT_FILE
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
for FILE in ${FILES[@]}; do
|
|
||||||
check_file "$FILE"
|
|
||||||
upload_file "$FILE"
|
|
||||||
done
|
|
||||||
1
scripts/upload-swift-artifacts
Symbolic link
1
scripts/upload-swift-artifacts
Symbolic link
@ -0,0 +1 @@
|
|||||||
|
upload-artifacts
|
||||||
Loading…
Reference in New Issue
Block a user