Add pause option for image uploads

We need to ability to pause uploads of our images to a providers. For
example, we've build a good image, but it includes a regression for a
specific provider.

By setting the pause option of the image, we are able to delete the
bad image from the provider and prevent nodepool-builder from
re-uploading it.

Change-Id: Ib03852ea378fcdc985825b2e477036b06c53ea57
Signed-off-by: Paul Belanger <pabelanger@redhat.com>

doc/source/configuration.rst

@@ -466,6 +466,7 @@ Example::
 
   images:
     - name: precise
+      pause: False
       min-ram: 8192
       name-filter: 'something to match'
       username: jenkins
@@ -494,6 +495,10 @@ Example::
     `name-filter` to ``Performance`` will ensure the chosen flavor also
     contains this string as well as meeting `min-ram` requirements).
 
+  ``pause`` (bool)
+    When set to True, nodepool-builder will not upload the image to the
+    provider.
+
   ``username``
     Nodepool expects that user to exist after running the script indicated by
     ``setup``. Default ``jenkins``

nodepool/builder.py

@@ -789,7 +789,6 @@ class UploadWorker(BaseWorker):
                       (build_id, filename, provider.name))
 
         manager = self._config.provider_managers[provider.name]
-
         provider_image = provider.images.get(image_name)
         if provider_image is None:
             raise exceptions.BuilderInvalidCommandError(
@@ -863,6 +862,10 @@ class UploadWorker(BaseWorker):
         if image.name not in self._config.images_in_use:
             return
 
+        # Check if image uploads are paused.
+        if provider.images.get(image.name).pause:
+            return
+
         # Search for the most recent 'ready' image build
         builds = self._zk.getMostRecentBuilds(1, image.name,
                                               zk.READY)

nodepool/config.py

@@ -226,6 +226,7 @@ def loadConfig(config_path):
             i.name_filter = image.get('name-filter', None)
             i.username = image.get('username', 'jenkins')
             i.user_home = image.get('user-home', '/home/jenkins')
+            i.pause = bool(image.get('pause', False))
             i.private_key = image.get('private-key',
                                       '/var/lib/jenkins/.ssh/id_rsa')
             i.config_drive = image.get('config-drive', None)

nodepool/tests/fixtures/node_image_upload_pause.yaml

@@ -0,0 +1,80 @@
+script-dir: .
+elements-dir: .
+images-dir: '{images_dir}'
+
+cron:
+  check: '*/15 * * * *'
+  cleanup: '*/1 * * * *'
+
+zmq-publishers:
+  - tcp://localhost:8881
+
+gearman-servers:
+  - host: localhost
+    port: {gearman_port}
+
+zookeeper-servers:
+  - host: {zookeeper_host}
+    port: {zookeeper_port}
+    chroot: {zookeeper_chroot}
+
+labels:
+  - name: fake-label
+    image: fake-image
+    min-ready: 1
+    providers:
+      - name: fake-provider
+  - name: fake-label2
+    image: fake-image2
+    min-ready: 1
+    providers:
+      - name: fake-provider
+
+providers:
+  - name: fake-provider
+    region-name: fake-region
+    keypair: 'if-present-use-this-keypair'
+    username: 'fake'
+    password: 'fake'
+    auth-url: 'fake'
+    project-id: 'fake'
+    max-servers: 96
+    pool: 'fake'
+    networks:
+      - net-id: 'some-uuid'
+    rate: 0.0001
+    images:
+      - name: fake-image
+        pause: True
+        min-ram: 8192
+        name-filter: 'Fake'
+        meta:
+          key: value
+          key2: value
+      - name: fake-image2
+        min-ram: 8192
+
+targets:
+  - name: fake-target
+
+diskimages:
+  - name: fake-image
+    elements:
+      - fedora
+      - vm
+    release: 21
+    env-vars:
+      TMPDIR: /opt/dib_tmp
+      DIB_IMAGE_CACHE: /opt/dib_cache
+      DIB_CLOUD_IMAGES: http://download.fedoraproject.org/pub/fedora/linux/releases/test/21-Beta/Cloud/Images/x86_64/
+      BASE_IMAGE_FILE: Fedora-Cloud-Base-20141029-21_Beta.x86_64.qcow2
+  - name: fake-image2
+    elements:
+      - fedora
+      - vm
+    release: 21
+    env-vars:
+      TMPDIR: /opt/dib_tmp
+      DIB_IMAGE_CACHE: /opt/dib_cache
+      DIB_CLOUD_IMAGES: http://download.fedoraproject.org/pub/fedora/linux/releases/test/21-Beta/Cloud/Images/x86_64/
+      BASE_IMAGE_FILE: Fedora-Cloud-Base-20141029-21_Beta.x86_64.qcow2

nodepool/tests/test_commands.py

@@ -154,6 +154,19 @@ class TestNodepoolCMD(tests.DBTestCase):
         self.assert_listed(configfile, ['dib-image-list'], 1, 'fake-image', 0)
         self.assert_listed(configfile, ['dib-image-list'], 1, 'fake-image2', 1)
 
+    def test_dib_image_upload_pause(self):
+        configfile = self.setup_config('node_image_upload_pause.yaml')
+        self._useBuilder(configfile)
+        pool = self.useNodepool(configfile, watermark_sleep=1)
+        pool.start()
+        self.waitForNodes(pool)
+        # Make sure diskimages were built.
+        self.assert_listed(configfile, ['dib-image-list'], 1, 'fake-image', 1)
+        self.assert_listed(configfile, ['dib-image-list'], 1, 'fake-image2', 1)
+        # fake-image will be missing, since it is paused.
+        self.assert_listed(configfile, ['image-list'], 3, 'fake-image', 0)
+        self.assert_listed(configfile, ['image-list'], 3, 'fake-image2', 1)
+
     def test_dib_image_delete(self):
         configfile = self.setup_config('node.yaml')
         pool = self.useNodepool(configfile, watermark_sleep=1)