zuul
/
nodepool
Code Issues Proposed changes

Use optional upload script for uploading an image 

The script can be used to by-pass the drivers upload mehtod to use a
faster upload method or doing extra stepa before or after the actual
uplaod.

Change-Id: I148b853a23b1035e95def97d5b80fa12662b8251
This commit is contained in:
Benedikt Loeffler 2021-07-14 12:27:52 +02:00
parent f399292401
commit d6d3336699
13 changed files with 244 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
doc/source/aws.rst
View File

@ -151,6 +151,23 @@ Selecting the ``aws`` driver adds the following options to the
The number of times to retry launching a node before considering
the request failed.
.. attr:: upload-script
:type: string
:default: None
Filename of an optional script that can be called to upload an image. This
can be used to by-pass the drivers upload mehtod and do the upload by a
scrpt. The script will be called as follows and should have the external
image ID in the output:
``<SCRIPT> <IMAGE_NAME> <PATH_TO_IMAGE_FILE> <PROVIDER_NAME> <CLOUD_NAME>
<BUILD_ID> <UPLOAD_ID>``
If the script returns with result code 0 it is treated as successful and
the external image ID will parsed from the output with the regex ``Image
ID: ([\w-]+)``. Otherwise or when the regex did not match the upload is
treated as failed.
.. attr:: post-upload-hook
:type: string
:default: None

17
doc/source/azure.rst
View File

@ -195,6 +195,23 @@ section of the configuration.
The number of times to retry launching a server before
considering the request failed.
.. attr:: upload-script
:type: string
:default: None
Filename of an optional script that can be called to upload an image. This
can be used to by-pass the drivers upload mehtod and do the upload by a
scrpt. The script will be called as follows and should have the external
image ID in the output:
``<SCRIPT> <IMAGE_NAME> <PATH_TO_IMAGE_FILE> <PROVIDER_NAME> <CLOUD_NAME>
<BUILD_ID> <UPLOAD_ID>``
If the script returns with result code 0 it is treated as successful and
the external image ID will parsed from the output with the regex ``Image
ID: ([\w-]+)``. Otherwise or when the regex did not match the upload is
treated as failed.
.. attr:: post-upload-hook
:type: string
:default: None

17
doc/source/ibmvpc.rst
View File

@ -238,6 +238,23 @@ section of the configuration.
The number of times to retry launching a server before
considering the request failed.
.. attr:: upload-script
:type: string
:default: None
Filename of an optional script that can be called to upload an image. This
can be used to by-pass the drivers upload mehtod and do the upload by a
scrpt. The script will be called as follows and should have the external
image ID in the output:
``<SCRIPT> <IMAGE_NAME> <PATH_TO_IMAGE_FILE> <PROVIDER_NAME> <CLOUD_NAME>
<BUILD_ID> <UPLOAD_ID>``
If the script returns with result code 0 it is treated as successful and
the external image ID will parsed from the output with the regex ``Image
ID: ([\w-]+)``. Otherwise or when the regex did not match the upload is
treated as failed.
.. attr:: post-upload-hook
:type: string
:default: None

18
doc/source/openstack.rst
View File

@ -46,6 +46,7 @@ Selecting the OpenStack driver adds the following options to the
launch-retries: 3
image-name-format: '{image_name}-{timestamp}'
hostname-format: '{label.name}-{provider.name}-{node.id}'
upload-script: /usr/bin/custom-upload-script
post-upload-hook: /usr/bin/custom-hook
diskimages:
- name: trusty
@ -168,6 +169,23 @@ Selecting the OpenStack driver adds the following options to the
Format for image names that are uploaded to providers.
.. attr:: upload-script
:type: string
:default: None
Filename of an optional script that can be called to upload an image. This
can be used to by-pass the drivers upload mehtod and do the upload by a
scrpt. The script will be called as follows and should have the external
image ID in the output:
``<SCRIPT> <IMAGE_NAME> <PATH_TO_IMAGE_FILE> <PROVIDER_NAME> <CLOUD_NAME>
<BUILD_ID> <UPLOAD_ID>``
If the script returns with result code 0 it is treated as successful and
the external image ID will parsed from the output with the regex ``Image
ID: ([\w-]+)``. Otherwise or when the regex did not match the upload is
treated as failed.
.. attr:: post-upload-hook
:type: string
:default: None

80
nodepool/builder.py
View File

@ -1127,22 +1127,70 @@ class UploadWorker(BaseWorker):
meta['nodepool_build_id'] = build_id
meta['nodepool_upload_id'] = upload_id
try:
external_id = manager.uploadImage(
provider_image,
ext_image_name, filename,
image_type=image.extension,
meta=meta,
md5=image.md5,
sha256=image.sha256,
)
except Exception:
self.log.exception(
"Failed to upload build %s of image %s to provider %s" %
(build_id, image_name, provider.name))
data = zk.ImageUpload()
data.state = zk.FAILED
return data
if provider.upload_script:
try:
cmd = [
provider.upload_script,
ext_image_name,
filename,
provider.name,
provider.provider["cloud"],
build_id,
upload_id,
]
self.log.info(
'Uploading %s to %s using upload script: %s',
image_name,
provider.provider["cloud"],
' '.join(cmd))
p = subprocess.run(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, check=True)
upload_script_output = p.stdout.decode()
# parse image id from script output
ids = re.findall(r"Image ID: ([\w-]+)", upload_script_output)
if not ids:
raise Exception(
'Could not parse image id from upload script output')
external_id = ids[0]
except Exception as e:
if isinstance(e, subprocess.CalledProcessError):
self.log.error(
'Upload script failed to upload %s with exit code'
'%s\nstdout:\n%s\nstderr:\n%s',
image_name,
e.returncode,
e.stdout.decode(),
e.stderr.decode())
else:
self.log.exception(
'Unknown exception during upload script')
data = zk.ImageUpload()
data.state = zk.FAILED
return data
self.log.info(
'Upload script successfully uploaded %s\n'
'stdout:\n%s\nstderr:\n%s',
image_name, p.stdout.decode(), p.stderr.decode())
else:
try:
external_id = manager.uploadImage(
provider_image,
ext_image_name, filename,
image_type=image.extension,
meta=meta,
md5=image.md5,
sha256=image.sha256,
)
except Exception:
self.log.exception(
"Failed to upload build %s of image %s to provider %s" %
(build_id, image_name, provider.name))
data = zk.ImageUpload()
data.state = zk.FAILED
return data
if provider.post_upload_hook:
try:

1
nodepool/driver/aws/config.py
View File

@ -294,6 +294,7 @@ class AwsProviderConfig(ProviderConfig):
self.object_storage = self.provider.get('object-storage')
self.image_type = self.provider.get('image-format', 'raw')
self.image_name_format = '{image_name}-{timestamp}'
self.upload_script = self.provider.get('upload-script')
self.post_upload_hook = self.provider.get('post-upload-hook')
self.max_servers = self.provider.get('max-servers', math.inf)
self.max_cores = self.provider.get('max-cores', math.inf)

1
nodepool/driver/azure/config.py
View File

@ -264,6 +264,7 @@ class AzureProviderConfig(ProviderConfig):
def load(self, config):
self.image_type = 'vhd'
self.image_name_format = '{image_name}-{timestamp}'
self.upload_script = self.provider.get('upload-script')
self.post_upload_hook = self.provider.get('post-upload-hook')
self.rate = self.provider.get('rate', 1)

1
nodepool/driver/ibmvpc/config.py
View File

@ -241,6 +241,7 @@ class IBMVPCProviderConfig(ProviderConfig):
def load(self, config):
self.image_type = self.provider.get('image-format', 'qcow2')
self.image_name_format = 'npimage{upload_id}'
self.upload_script = self.provider.get('upload-script')
self.post_upload_hook = self.provider.get('post-upload-hook')
self.rate = self.provider.get('rate', 1)

3
nodepool/driver/openstack/config.py
View File

@ -175,6 +175,7 @@ class OpenStackProviderConfig(ProviderConfig):
self.cloud_images = {}
self.hostname_format = None
self.image_name_format = None
self.upload_script = None
self.post_upload_hook = None
super().__init__(provider)
@ -217,6 +218,7 @@ class OpenStackProviderConfig(ProviderConfig):
'image-name-format',
'{image_name}-{timestamp}'
)
self.upload_script = self.provider.get('upload-script')
self.post_upload_hook = self.provider.get('post-upload-hook')
default_port_mapping = {
@ -347,6 +349,7 @@ class OpenStackProviderConfig(ProviderConfig):
'pools': [pool],
'diskimages': [provider_diskimage],
'cloud-images': [provider_cloud_images],
'upload-script': str,
'post-upload-hook': str,
})
return v.Schema(schema)

1
nodepool/tests/fixtures/config_validate/good.yaml vendored
View File

@ -80,6 +80,7 @@ providers:
boot-timeout: 120
rate: 0.001
port-cleanup-interval: 0
upload-script: /usr/bin/upload-script
post-upload-hook: /usr/bin/upload-hook
diskimages:
- name: trusty

50
nodepool/tests/fixtures/node_upload_script.yaml vendored Normal file
View File

@ -0,0 +1,50 @@
elements-dir: .
images-dir: '{images_dir}'
build-log-dir: '{build_log_dir}'
zookeeper-servers:
- host: {zookeeper_host}
port: {zookeeper_port}
chroot: {zookeeper_chroot}
zookeeper-tls:
ca: {zookeeper_ca}
cert: {zookeeper_cert}
key: {zookeeper_key}
labels:
- name: fake-label
min-ready: 0
providers:
- name: fake-provider
cloud: fake
driver: fake
region-name: fake-region
rate: 0.0001
upload-script: nodepool/tests/upload-script
diskimages:
- name: fake-image
pools:
- name: main
max-servers: 96
availability-zones:
- az1
labels:
- name: fake-label
diskimage: fake-image
min-ram: 8192
flavor-name: 'Fake'
diskimages:
- name: fake-image
elements:
- fedora
- vm
release: 21
dib-cmd: nodepool/tests/fake-image-create
env-vars:
TMPDIR: /opt/dib_tmp
DIB_IMAGE_CACHE: /opt/dib_cache
DIB_CLOUD_IMAGES: http://download.fedoraproject.org/pub/fedora/linux/releases/test/21-Beta/Cloud/Images/x86_64/
BASE_IMAGE_FILE: Fedora-Cloud-Base-20141029-21_Beta.x86_64.qcow2

11
nodepool/tests/unit/test_builder.py
View File

@ -584,6 +584,17 @@ class TestNodePoolBuilder(tests.DBTestCase):
self.waitForUploadRecordDeletion(image.provider_name, image.image_name,
image.build_id, image.id)
def test_upload_script(self):
configfile = self.setup_config('node_upload_script.yaml')
self.useBuilder(configfile)
self.waitForImage('fake-provider', 'fake-image')
newest_builds = self.zk.getMostRecentBuilds(1, 'fake-image',
state=zk.READY)
uploads = self.zk.getUploads('fake-image', newest_builds[0].id,
'fake-provider', states=[zk.READY])
self.assertTrue(uploads[0].external_id, 'fake-image-12345-abcdef')
def test_post_upload_hook(self):
configfile = self.setup_config('node_upload_hook.yaml')
bldr = self.useBuilder(configfile)

43
nodepool/tests/upload-script Executable file
View File

@ -0,0 +1,43 @@
#!/bin/bash
set -euo pipefail
if [ $# -ne 6 ]
then
echo "Usage: $0 <IMAGENAME> <SOURCEIMAGE> <PROVIDER> <CLOUD> <BUILDID> <UPLOADID>"
exit 1
fi
IMAGENAME="${1}"
SOURCEIMAGE="${2}"
PROVIDER="${3}"
CLOUD="${4}"
BUILDID="${5}"
UPLOADID="${6}"
echo "Uploading $IMAGENAME to $CLOUD"
# For a better performace the image is directly uploaded to ceph instead of using the glance api for uploading the image
#
# Create a new image UUID
# UUID=$(uuidgen)
#
# Create an empty image
# glance image-create --disk-format "${SOURCEIMAGE##*.}" --container-format bare --id "${UUID}" --name "${IMAGENAME}" --visibility "private" --property "nodepool_build_id=${BUILDID}" --property "nodepool_upload_id=${UPLOADID}" <&-
#
# Import the image directly to ceph
# rbd import "${SOURCEIMAGE}" "${POOL}/${UUID}"
#
# Create a snap from the imported image
# rbd -p "${POOL}" snap create "${UUID}@snap"
# rbd -p "${POOL}" snap protect "${UUID}@snap"
#
# Add the ceph snap to the "empty" image
# glance location-add --url "rbd://${FSID}/${POOL}/${UUID}/snap" "${UUID}"
#
# Return the external image ID for the new created image
# echo "Image ID: ${UUID}"
# Mandatory: Return the external image ID
echo "Image ID: fake-image-12345-abcdef"