Use optional upload script for uploading an image
The script can be used to by-pass the drivers upload mehtod to use a faster upload method or doing extra stepa before or after the actual uplaod. Change-Id: I148b853a23b1035e95def97d5b80fa12662b8251
This commit is contained in:
parent
f399292401
commit
d6d3336699
|
@ -151,6 +151,23 @@ Selecting the ``aws`` driver adds the following options to the
|
||||||
The number of times to retry launching a node before considering
|
The number of times to retry launching a node before considering
|
||||||
the request failed.
|
the request failed.
|
||||||
|
|
||||||
|
.. attr:: upload-script
|
||||||
|
:type: string
|
||||||
|
:default: None
|
||||||
|
|
||||||
|
Filename of an optional script that can be called to upload an image. This
|
||||||
|
can be used to by-pass the drivers upload mehtod and do the upload by a
|
||||||
|
scrpt. The script will be called as follows and should have the external
|
||||||
|
image ID in the output:
|
||||||
|
|
||||||
|
``<SCRIPT> <IMAGE_NAME> <PATH_TO_IMAGE_FILE> <PROVIDER_NAME> <CLOUD_NAME>
|
||||||
|
<BUILD_ID> <UPLOAD_ID>``
|
||||||
|
|
||||||
|
If the script returns with result code 0 it is treated as successful and
|
||||||
|
the external image ID will parsed from the output with the regex ``Image
|
||||||
|
ID: ([\w-]+)``. Otherwise or when the regex did not match the upload is
|
||||||
|
treated as failed.
|
||||||
|
|
||||||
.. attr:: post-upload-hook
|
.. attr:: post-upload-hook
|
||||||
:type: string
|
:type: string
|
||||||
:default: None
|
:default: None
|
||||||
|
|
|
@ -195,6 +195,23 @@ section of the configuration.
|
||||||
The number of times to retry launching a server before
|
The number of times to retry launching a server before
|
||||||
considering the request failed.
|
considering the request failed.
|
||||||
|
|
||||||
|
.. attr:: upload-script
|
||||||
|
:type: string
|
||||||
|
:default: None
|
||||||
|
|
||||||
|
Filename of an optional script that can be called to upload an image. This
|
||||||
|
can be used to by-pass the drivers upload mehtod and do the upload by a
|
||||||
|
scrpt. The script will be called as follows and should have the external
|
||||||
|
image ID in the output:
|
||||||
|
|
||||||
|
``<SCRIPT> <IMAGE_NAME> <PATH_TO_IMAGE_FILE> <PROVIDER_NAME> <CLOUD_NAME>
|
||||||
|
<BUILD_ID> <UPLOAD_ID>``
|
||||||
|
|
||||||
|
If the script returns with result code 0 it is treated as successful and
|
||||||
|
the external image ID will parsed from the output with the regex ``Image
|
||||||
|
ID: ([\w-]+)``. Otherwise or when the regex did not match the upload is
|
||||||
|
treated as failed.
|
||||||
|
|
||||||
.. attr:: post-upload-hook
|
.. attr:: post-upload-hook
|
||||||
:type: string
|
:type: string
|
||||||
:default: None
|
:default: None
|
||||||
|
|
|
@ -238,6 +238,23 @@ section of the configuration.
|
||||||
The number of times to retry launching a server before
|
The number of times to retry launching a server before
|
||||||
considering the request failed.
|
considering the request failed.
|
||||||
|
|
||||||
|
.. attr:: upload-script
|
||||||
|
:type: string
|
||||||
|
:default: None
|
||||||
|
|
||||||
|
Filename of an optional script that can be called to upload an image. This
|
||||||
|
can be used to by-pass the drivers upload mehtod and do the upload by a
|
||||||
|
scrpt. The script will be called as follows and should have the external
|
||||||
|
image ID in the output:
|
||||||
|
|
||||||
|
``<SCRIPT> <IMAGE_NAME> <PATH_TO_IMAGE_FILE> <PROVIDER_NAME> <CLOUD_NAME>
|
||||||
|
<BUILD_ID> <UPLOAD_ID>``
|
||||||
|
|
||||||
|
If the script returns with result code 0 it is treated as successful and
|
||||||
|
the external image ID will parsed from the output with the regex ``Image
|
||||||
|
ID: ([\w-]+)``. Otherwise or when the regex did not match the upload is
|
||||||
|
treated as failed.
|
||||||
|
|
||||||
.. attr:: post-upload-hook
|
.. attr:: post-upload-hook
|
||||||
:type: string
|
:type: string
|
||||||
:default: None
|
:default: None
|
||||||
|
|
|
@ -46,6 +46,7 @@ Selecting the OpenStack driver adds the following options to the
|
||||||
launch-retries: 3
|
launch-retries: 3
|
||||||
image-name-format: '{image_name}-{timestamp}'
|
image-name-format: '{image_name}-{timestamp}'
|
||||||
hostname-format: '{label.name}-{provider.name}-{node.id}'
|
hostname-format: '{label.name}-{provider.name}-{node.id}'
|
||||||
|
upload-script: /usr/bin/custom-upload-script
|
||||||
post-upload-hook: /usr/bin/custom-hook
|
post-upload-hook: /usr/bin/custom-hook
|
||||||
diskimages:
|
diskimages:
|
||||||
- name: trusty
|
- name: trusty
|
||||||
|
@ -168,6 +169,23 @@ Selecting the OpenStack driver adds the following options to the
|
||||||
|
|
||||||
Format for image names that are uploaded to providers.
|
Format for image names that are uploaded to providers.
|
||||||
|
|
||||||
|
.. attr:: upload-script
|
||||||
|
:type: string
|
||||||
|
:default: None
|
||||||
|
|
||||||
|
Filename of an optional script that can be called to upload an image. This
|
||||||
|
can be used to by-pass the drivers upload mehtod and do the upload by a
|
||||||
|
scrpt. The script will be called as follows and should have the external
|
||||||
|
image ID in the output:
|
||||||
|
|
||||||
|
``<SCRIPT> <IMAGE_NAME> <PATH_TO_IMAGE_FILE> <PROVIDER_NAME> <CLOUD_NAME>
|
||||||
|
<BUILD_ID> <UPLOAD_ID>``
|
||||||
|
|
||||||
|
If the script returns with result code 0 it is treated as successful and
|
||||||
|
the external image ID will parsed from the output with the regex ``Image
|
||||||
|
ID: ([\w-]+)``. Otherwise or when the regex did not match the upload is
|
||||||
|
treated as failed.
|
||||||
|
|
||||||
.. attr:: post-upload-hook
|
.. attr:: post-upload-hook
|
||||||
:type: string
|
:type: string
|
||||||
:default: None
|
:default: None
|
||||||
|
|
|
@ -1127,22 +1127,70 @@ class UploadWorker(BaseWorker):
|
||||||
meta['nodepool_build_id'] = build_id
|
meta['nodepool_build_id'] = build_id
|
||||||
meta['nodepool_upload_id'] = upload_id
|
meta['nodepool_upload_id'] = upload_id
|
||||||
|
|
||||||
try:
|
if provider.upload_script:
|
||||||
external_id = manager.uploadImage(
|
try:
|
||||||
provider_image,
|
cmd = [
|
||||||
ext_image_name, filename,
|
provider.upload_script,
|
||||||
image_type=image.extension,
|
ext_image_name,
|
||||||
meta=meta,
|
filename,
|
||||||
md5=image.md5,
|
provider.name,
|
||||||
sha256=image.sha256,
|
provider.provider["cloud"],
|
||||||
)
|
build_id,
|
||||||
except Exception:
|
upload_id,
|
||||||
self.log.exception(
|
]
|
||||||
"Failed to upload build %s of image %s to provider %s" %
|
self.log.info(
|
||||||
(build_id, image_name, provider.name))
|
'Uploading %s to %s using upload script: %s',
|
||||||
data = zk.ImageUpload()
|
image_name,
|
||||||
data.state = zk.FAILED
|
provider.provider["cloud"],
|
||||||
return data
|
' '.join(cmd))
|
||||||
|
p = subprocess.run(cmd, stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE, check=True)
|
||||||
|
upload_script_output = p.stdout.decode()
|
||||||
|
|
||||||
|
# parse image id from script output
|
||||||
|
ids = re.findall(r"Image ID: ([\w-]+)", upload_script_output)
|
||||||
|
if not ids:
|
||||||
|
raise Exception(
|
||||||
|
'Could not parse image id from upload script output')
|
||||||
|
external_id = ids[0]
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
if isinstance(e, subprocess.CalledProcessError):
|
||||||
|
self.log.error(
|
||||||
|
'Upload script failed to upload %s with exit code'
|
||||||
|
'%s\nstdout:\n%s\nstderr:\n%s',
|
||||||
|
image_name,
|
||||||
|
e.returncode,
|
||||||
|
e.stdout.decode(),
|
||||||
|
e.stderr.decode())
|
||||||
|
else:
|
||||||
|
self.log.exception(
|
||||||
|
'Unknown exception during upload script')
|
||||||
|
data = zk.ImageUpload()
|
||||||
|
data.state = zk.FAILED
|
||||||
|
return data
|
||||||
|
|
||||||
|
self.log.info(
|
||||||
|
'Upload script successfully uploaded %s\n'
|
||||||
|
'stdout:\n%s\nstderr:\n%s',
|
||||||
|
image_name, p.stdout.decode(), p.stderr.decode())
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
external_id = manager.uploadImage(
|
||||||
|
provider_image,
|
||||||
|
ext_image_name, filename,
|
||||||
|
image_type=image.extension,
|
||||||
|
meta=meta,
|
||||||
|
md5=image.md5,
|
||||||
|
sha256=image.sha256,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
self.log.exception(
|
||||||
|
"Failed to upload build %s of image %s to provider %s" %
|
||||||
|
(build_id, image_name, provider.name))
|
||||||
|
data = zk.ImageUpload()
|
||||||
|
data.state = zk.FAILED
|
||||||
|
return data
|
||||||
|
|
||||||
if provider.post_upload_hook:
|
if provider.post_upload_hook:
|
||||||
try:
|
try:
|
||||||
|
|
|
@ -294,6 +294,7 @@ class AwsProviderConfig(ProviderConfig):
|
||||||
self.object_storage = self.provider.get('object-storage')
|
self.object_storage = self.provider.get('object-storage')
|
||||||
self.image_type = self.provider.get('image-format', 'raw')
|
self.image_type = self.provider.get('image-format', 'raw')
|
||||||
self.image_name_format = '{image_name}-{timestamp}'
|
self.image_name_format = '{image_name}-{timestamp}'
|
||||||
|
self.upload_script = self.provider.get('upload-script')
|
||||||
self.post_upload_hook = self.provider.get('post-upload-hook')
|
self.post_upload_hook = self.provider.get('post-upload-hook')
|
||||||
self.max_servers = self.provider.get('max-servers', math.inf)
|
self.max_servers = self.provider.get('max-servers', math.inf)
|
||||||
self.max_cores = self.provider.get('max-cores', math.inf)
|
self.max_cores = self.provider.get('max-cores', math.inf)
|
||||||
|
|
|
@ -264,6 +264,7 @@ class AzureProviderConfig(ProviderConfig):
|
||||||
def load(self, config):
|
def load(self, config):
|
||||||
self.image_type = 'vhd'
|
self.image_type = 'vhd'
|
||||||
self.image_name_format = '{image_name}-{timestamp}'
|
self.image_name_format = '{image_name}-{timestamp}'
|
||||||
|
self.upload_script = self.provider.get('upload-script')
|
||||||
self.post_upload_hook = self.provider.get('post-upload-hook')
|
self.post_upload_hook = self.provider.get('post-upload-hook')
|
||||||
|
|
||||||
self.rate = self.provider.get('rate', 1)
|
self.rate = self.provider.get('rate', 1)
|
||||||
|
|
|
@ -241,6 +241,7 @@ class IBMVPCProviderConfig(ProviderConfig):
|
||||||
def load(self, config):
|
def load(self, config):
|
||||||
self.image_type = self.provider.get('image-format', 'qcow2')
|
self.image_type = self.provider.get('image-format', 'qcow2')
|
||||||
self.image_name_format = 'npimage{upload_id}'
|
self.image_name_format = 'npimage{upload_id}'
|
||||||
|
self.upload_script = self.provider.get('upload-script')
|
||||||
self.post_upload_hook = self.provider.get('post-upload-hook')
|
self.post_upload_hook = self.provider.get('post-upload-hook')
|
||||||
|
|
||||||
self.rate = self.provider.get('rate', 1)
|
self.rate = self.provider.get('rate', 1)
|
||||||
|
|
|
@ -175,6 +175,7 @@ class OpenStackProviderConfig(ProviderConfig):
|
||||||
self.cloud_images = {}
|
self.cloud_images = {}
|
||||||
self.hostname_format = None
|
self.hostname_format = None
|
||||||
self.image_name_format = None
|
self.image_name_format = None
|
||||||
|
self.upload_script = None
|
||||||
self.post_upload_hook = None
|
self.post_upload_hook = None
|
||||||
super().__init__(provider)
|
super().__init__(provider)
|
||||||
|
|
||||||
|
@ -217,6 +218,7 @@ class OpenStackProviderConfig(ProviderConfig):
|
||||||
'image-name-format',
|
'image-name-format',
|
||||||
'{image_name}-{timestamp}'
|
'{image_name}-{timestamp}'
|
||||||
)
|
)
|
||||||
|
self.upload_script = self.provider.get('upload-script')
|
||||||
self.post_upload_hook = self.provider.get('post-upload-hook')
|
self.post_upload_hook = self.provider.get('post-upload-hook')
|
||||||
|
|
||||||
default_port_mapping = {
|
default_port_mapping = {
|
||||||
|
@ -347,6 +349,7 @@ class OpenStackProviderConfig(ProviderConfig):
|
||||||
'pools': [pool],
|
'pools': [pool],
|
||||||
'diskimages': [provider_diskimage],
|
'diskimages': [provider_diskimage],
|
||||||
'cloud-images': [provider_cloud_images],
|
'cloud-images': [provider_cloud_images],
|
||||||
|
'upload-script': str,
|
||||||
'post-upload-hook': str,
|
'post-upload-hook': str,
|
||||||
})
|
})
|
||||||
return v.Schema(schema)
|
return v.Schema(schema)
|
||||||
|
|
|
@ -80,6 +80,7 @@ providers:
|
||||||
boot-timeout: 120
|
boot-timeout: 120
|
||||||
rate: 0.001
|
rate: 0.001
|
||||||
port-cleanup-interval: 0
|
port-cleanup-interval: 0
|
||||||
|
upload-script: /usr/bin/upload-script
|
||||||
post-upload-hook: /usr/bin/upload-hook
|
post-upload-hook: /usr/bin/upload-hook
|
||||||
diskimages:
|
diskimages:
|
||||||
- name: trusty
|
- name: trusty
|
||||||
|
|
|
@ -0,0 +1,50 @@
|
||||||
|
elements-dir: .
|
||||||
|
images-dir: '{images_dir}'
|
||||||
|
build-log-dir: '{build_log_dir}'
|
||||||
|
|
||||||
|
zookeeper-servers:
|
||||||
|
- host: {zookeeper_host}
|
||||||
|
port: {zookeeper_port}
|
||||||
|
chroot: {zookeeper_chroot}
|
||||||
|
|
||||||
|
zookeeper-tls:
|
||||||
|
ca: {zookeeper_ca}
|
||||||
|
cert: {zookeeper_cert}
|
||||||
|
key: {zookeeper_key}
|
||||||
|
|
||||||
|
labels:
|
||||||
|
- name: fake-label
|
||||||
|
min-ready: 0
|
||||||
|
|
||||||
|
providers:
|
||||||
|
- name: fake-provider
|
||||||
|
cloud: fake
|
||||||
|
driver: fake
|
||||||
|
region-name: fake-region
|
||||||
|
rate: 0.0001
|
||||||
|
upload-script: nodepool/tests/upload-script
|
||||||
|
diskimages:
|
||||||
|
- name: fake-image
|
||||||
|
pools:
|
||||||
|
- name: main
|
||||||
|
max-servers: 96
|
||||||
|
availability-zones:
|
||||||
|
- az1
|
||||||
|
labels:
|
||||||
|
- name: fake-label
|
||||||
|
diskimage: fake-image
|
||||||
|
min-ram: 8192
|
||||||
|
flavor-name: 'Fake'
|
||||||
|
|
||||||
|
diskimages:
|
||||||
|
- name: fake-image
|
||||||
|
elements:
|
||||||
|
- fedora
|
||||||
|
- vm
|
||||||
|
release: 21
|
||||||
|
dib-cmd: nodepool/tests/fake-image-create
|
||||||
|
env-vars:
|
||||||
|
TMPDIR: /opt/dib_tmp
|
||||||
|
DIB_IMAGE_CACHE: /opt/dib_cache
|
||||||
|
DIB_CLOUD_IMAGES: http://download.fedoraproject.org/pub/fedora/linux/releases/test/21-Beta/Cloud/Images/x86_64/
|
||||||
|
BASE_IMAGE_FILE: Fedora-Cloud-Base-20141029-21_Beta.x86_64.qcow2
|
|
@ -584,6 +584,17 @@ class TestNodePoolBuilder(tests.DBTestCase):
|
||||||
self.waitForUploadRecordDeletion(image.provider_name, image.image_name,
|
self.waitForUploadRecordDeletion(image.provider_name, image.image_name,
|
||||||
image.build_id, image.id)
|
image.build_id, image.id)
|
||||||
|
|
||||||
|
def test_upload_script(self):
|
||||||
|
configfile = self.setup_config('node_upload_script.yaml')
|
||||||
|
self.useBuilder(configfile)
|
||||||
|
self.waitForImage('fake-provider', 'fake-image')
|
||||||
|
|
||||||
|
newest_builds = self.zk.getMostRecentBuilds(1, 'fake-image',
|
||||||
|
state=zk.READY)
|
||||||
|
uploads = self.zk.getUploads('fake-image', newest_builds[0].id,
|
||||||
|
'fake-provider', states=[zk.READY])
|
||||||
|
self.assertTrue(uploads[0].external_id, 'fake-image-12345-abcdef')
|
||||||
|
|
||||||
def test_post_upload_hook(self):
|
def test_post_upload_hook(self):
|
||||||
configfile = self.setup_config('node_upload_hook.yaml')
|
configfile = self.setup_config('node_upload_hook.yaml')
|
||||||
bldr = self.useBuilder(configfile)
|
bldr = self.useBuilder(configfile)
|
||||||
|
|
|
@ -0,0 +1,43 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
if [ $# -ne 6 ]
|
||||||
|
then
|
||||||
|
echo "Usage: $0 <IMAGENAME> <SOURCEIMAGE> <PROVIDER> <CLOUD> <BUILDID> <UPLOADID>"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
IMAGENAME="${1}"
|
||||||
|
SOURCEIMAGE="${2}"
|
||||||
|
PROVIDER="${3}"
|
||||||
|
CLOUD="${4}"
|
||||||
|
BUILDID="${5}"
|
||||||
|
UPLOADID="${6}"
|
||||||
|
|
||||||
|
echo "Uploading $IMAGENAME to $CLOUD"
|
||||||
|
|
||||||
|
# For a better performace the image is directly uploaded to ceph instead of using the glance api for uploading the image
|
||||||
|
#
|
||||||
|
# Create a new image UUID
|
||||||
|
# UUID=$(uuidgen)
|
||||||
|
#
|
||||||
|
# Create an empty image
|
||||||
|
# glance image-create --disk-format "${SOURCEIMAGE##*.}" --container-format bare --id "${UUID}" --name "${IMAGENAME}" --visibility "private" --property "nodepool_build_id=${BUILDID}" --property "nodepool_upload_id=${UPLOADID}" <&-
|
||||||
|
#
|
||||||
|
# Import the image directly to ceph
|
||||||
|
# rbd import "${SOURCEIMAGE}" "${POOL}/${UUID}"
|
||||||
|
#
|
||||||
|
# Create a snap from the imported image
|
||||||
|
# rbd -p "${POOL}" snap create "${UUID}@snap"
|
||||||
|
# rbd -p "${POOL}" snap protect "${UUID}@snap"
|
||||||
|
#
|
||||||
|
# Add the ceph snap to the "empty" image
|
||||||
|
# glance location-add --url "rbd://${FSID}/${POOL}/${UUID}/snap" "${UUID}"
|
||||||
|
#
|
||||||
|
# Return the external image ID for the new created image
|
||||||
|
# echo "Image ID: ${UUID}"
|
||||||
|
|
||||||
|
|
||||||
|
# Mandatory: Return the external image ID
|
||||||
|
echo "Image ID: fake-image-12345-abcdef"
|
Loading…
Reference in New Issue