Merge "AWS: only import from same s3 region"

tests/fixtures/layouts/aws/nodepool-image-snapshot-region.yaml

@@ -0,0 +1,90 @@
+- pipeline:
+    name: check
+    manager: independent
+    trigger:
+      gerrit:
+        - event: patchset-created
+    success:
+      gerrit:
+        Verified: 1
+    failure:
+      gerrit:
+        Verified: -1
+
+- pipeline:
+    name: image
+    manager: independent
+    trigger:
+      zuul:
+        - event: image-build
+    success:
+      zuul:
+        image-built: true
+        image-validated: true
+
+- job:
+    name: base
+    parent: null
+    run: playbooks/base.yaml
+    nodeset:
+      nodes:
+        - label: ubuntu-xenial
+          name: controller
+
+- job:
+    name: build-debian-local-image
+    image-build-name: debian-local
+
+- project:
+    name: org/common-config
+    image:
+      jobs:
+        - build-debian-local-image
+
+- image:
+    name: debian-local
+    type: zuul
+
+- flavor:
+    name: normal
+
+- label:
+    name: debian-local-normal
+    image: debian-local
+    flavor: normal
+
+- section:
+    name: aws-base
+    abstract: true
+    connection: aws
+    boot-timeout: 120
+    launch-timeout: 600
+
+- section:
+    name: aws-us-east-1
+    parent: aws-base
+    region: us-east-1
+    object-storage:
+      bucket-name: zuul
+    flavors:
+      - name: normal
+        instance-type: t3.medium
+    images:
+      - name: debian-local
+
+- provider:
+    name: aws-us-east-1-main
+    section: aws-us-east-1
+    labels:
+      - name: debian-local-normal
+        key-name: zuul
+
+# This provider is only here to store the original image upload.
+- section:
+    name: aws-us-west-1
+    parent: aws-base
+    region: us-west-1
+
+- provider:
+    name: aws-us-west-1-main
+    section: aws-us-west-1

tests/unit/test_aws_driver.py

@@ -87,6 +87,24 @@ class TestAwsDriver(BaseCloudDriverTest):
             ]
         }
     }
+    s3_region_debian_return_data = {
+        'zuul': {
+            'artifacts': [
+                {
+                    'name': 'raw image',
+                    'url': 's3://zuulwest/image.raw',
+                    'metadata': {
+                        'type': 'zuul_image',
+                        'image_name': 'debian-local',
+                        'format': 'raw',
+                        'sha256': ('59984dd82f51edb3777b969739a92780'
+                                   'a520bb314b8d64b294d5de976bd8efb9'),
+                        'md5sum': '262278e1632567a907e4604e9edd2e83',
+                    }
+                },
+            ]
+        }
+    }
 
     def setUp(self):
         self.initTestConfig()
@@ -109,6 +127,9 @@ class TestAwsDriver(BaseCloudDriverTest):
         self.s3_client = boto3.client('s3', region_name='us-east-1')
         self.iam = boto3.resource('iam', region_name='us-east-1')
         self.s3.create_bucket(Bucket='zuul')
+        location = {'LocationConstraint': 'us-west-1'}
+        self.s3.create_bucket(Bucket="zuulwest",
+                              CreateBucketConfiguration=location)
 
         # A list of args to method calls for validation
         self.run_instances_calls = []
@@ -367,6 +388,21 @@ class TestAwsDriver(BaseCloudDriverTest):
                           Key='image.raw')
         self._test_diskimage()
 
+    @simple_layout('layouts/aws/nodepool-image-snapshot-region.yaml',
+                   enable_nodepool=True)
+    @return_data(
+        'build-debian-local-image',
+        'refs/heads/master',
+        s3_region_debian_return_data,
+    )
+    def test_aws_diskimage_s3_region_download(self):
+        # The image in a bucket in a different region should be
+        # downloaded without using a direct import.
+        bucket = self.s3.Bucket('zuulwest')
+        bucket.put_object(Body=b'hi',
+                          Key='image.raw')
+        self._test_diskimage()
+
     @simple_layout('layouts/nodepool-multi-provider.yaml',
                    enable_nodepool=True)
     def test_aws_resource_cleanup(self):

zuul/driver/aws/awsendpoint.py

@@ -730,6 +730,11 @@ class AwsProviderEndpoint(BaseProviderEndpoint):
                 iops=label.iops))
         return quota
 
+    def _getBucketRegion(self, bucket_name):
+        data = self.s3_client.get_bucket_location(Bucket=bucket_name)
+        # None means us-east-1 for s3 buckets
+        return data['LocationConstraint'] or 'us-east-1'
+
     def downloadUrl(self, url, path):
         if not url.startswith('s3://'):
             return None
@@ -737,6 +742,8 @@ class AwsProviderEndpoint(BaseProviderEndpoint):
         url_parts = urllib.parse.urlparse(url)
         bucket_name = url_parts.netloc
         object_filename = url_parts.path.lstrip('/')
+
+        self.log.debug("Downloading %s to %s", url, path)
         self.s3_client.download_file(bucket_name, object_filename, path)
         return path
 
@@ -761,6 +768,9 @@ class AwsProviderEndpoint(BaseProviderEndpoint):
         bucket_name = url_parts.netloc
         object_filename = url_parts.path.lstrip('/')
 
+        if self._getBucketRegion(bucket_name) != self.region:
+            return None
+
         return AwsImageImportJob(
             self,
             provider_image, image_name,