Launcher: fix image decompression

We did not correctly check the image filename extensions when
deciding whether to decompress an image.  Correct that by downloading
to a path with the extension (since zstd will want to see a file
with a .zst extension and then remove it), and then check the extension
of the downloaded path.

Update the tests to exercise this.

Add zstd to bindep so it's installed for the tests and expressed as
a requirement for Zuul in general.  Remove the launcher-specific addition
of zstd to the container image for simplicity.

Change-Id: Ie6ba9c54ed0246157fb0d1ea820b758872e60d98

Dockerfile

@@ -124,11 +124,6 @@ FROM zuul as zuul-fingergw
 CMD ["/usr/local/bin/zuul-fingergw", "-f"]
 
 FROM zuul as zuul-launcher
-# Install zstd to decompress images
-RUN apt-get update \
-  && apt-get install -y zstd \
-  && apt-get clean \
-  && rm -rf /var/lib/apt/lists/*
 CMD ["/usr/local/bin/zuul-launcher", "-f"]
 
 FROM zuul as zuul-merger

bindep.txt

@@ -67,4 +67,4 @@ openafs-client [platform:debian]
 krb5-user [platform:debian]
 setpriv [platform:ubuntu-bionic]
 util-linux [platform:apt platform:rpm platform:apk !platform:ubuntu-bionic]
-
+zstd

tests/unit/test_launcher.py

@@ -41,13 +41,18 @@ from tests.base import (
 class ImageMocksFixture(ResponsesFixture):
     def __init__(self):
         super().__init__()
-        raw_body = "test raw image"
+        raw_body = 'test raw image'
+        zst_body = b'(\xb5/\xfd\x04Xy\x00\x00test raw image\n\xde\x9d\x9c\xfb'
         qcow2_body = "test qcow2 image"
         self.requests_mock.add_passthru("http://localhost")
         self.requests_mock.add(
             responses.GET,
             'http://example.com/image.raw',
             body=raw_body)
+        self.requests_mock.add(
+            responses.GET,
+            'http://example.com/image.raw.zst',
+            body=zst_body)
         self.requests_mock.add(
             responses.GET,
             'http://example.com/image.qcow2',
@@ -56,6 +61,10 @@ class ImageMocksFixture(ResponsesFixture):
             responses.HEAD,
             'http://example.com/image.raw',
             headers={'content-length': str(len(raw_body))})
+        self.requests_mock.add(
+            responses.HEAD,
+            'http://example.com/image.raw.zst',
+            headers={'content-length': str(len(zst_body))})
         self.requests_mock.add(
             responses.HEAD,
             'http://example.com/image.qcow2',
@@ -70,7 +79,7 @@ class LauncherBaseTestCase(ZuulTestCase):
             'artifacts': [
                 {
                     'name': 'raw image',
-                    'url': 'http://example.com/image.raw',
+                    'url': 'http://example.com/image.raw.zst',
                     'metadata': {
                         'type': 'zuul_image',
                         'image_name': 'debian-local',
@@ -99,7 +108,7 @@ class LauncherBaseTestCase(ZuulTestCase):
             'artifacts': [
                 {
                     'name': 'raw image',
-                    'url': 'http://example.com/image.raw',
+                    'url': 'http://example.com/image.raw.zst',
                     'metadata': {
                         'type': 'zuul_image',
                         'image_name': 'ubuntu-local',
@@ -402,7 +411,7 @@ class TestLauncher(LauncherBaseTestCase):
                 name=image.name,
                 canonical_name=image.canonical_name,
                 project_canonical_name=image.project_canonical_name,
-                url='http://example.com/image.raw',
+                url='http://example.com/image.raw.zst',
                 timestamp=time.time(),
             )
             with iba.locked(self.zk_client):

zuul/launcher/server.py

@@ -1683,7 +1683,9 @@ class Launcher:
                     f.write(chunk)
 
     def downloadArtifact(self, image_build_artifact):
+        ext = image_build_artifact.url.split('.')[-1]
         path = os.path.join(self.temp_dir, image_build_artifact.uuid)
+        path = f'{path}.{ext}'
         self.log.info("Downloading artifact %s into %s",
                       image_build_artifact, path)
         futures = []