Browse Source

Host some patched jars in common artifacts

Host these jars, and also provide tooling to recreate them.

These are needed for proper S3-as-a-datasource capability.

Change-Id: Ied75211044b8f51623d87d36aa0588632645eea3
(cherry picked from commit d4e774245b)
Jeremy Freudberg 10 months ago
parent
commit
abacf8f2d4

BIN
common-artifacts/hadoop-aws-2.7.1.jar View File


BIN
common-artifacts/hadoop-aws-2.7.3.jar View File


BIN
common-artifacts/hadoop-aws-2.7.5.jar View File


+ 17
- 0
tools/generate-patched-hadoop-aws.sh View File

@@ -0,0 +1,17 @@
1
+#!/bin/bash
2
+
3
+set -e
4
+
5
+# First, install jdk, maven, git, etc.
6
+
7
+for version in 2.7.1 2.7.3 2.7.5; do
8
+    git clone git://git.apache.org/hadoop.git -b branch-$version --depth 1
9
+    cd hadoop/hadoop-tools/hadoop-aws/
10
+    git apply ../../../path-style-access.patch
11
+    mvn package -Dhadoop.version=$version
12
+    mv target/hadoop-aws-$version.jar ../../../
13
+    cd ../../../
14
+    rm -rf hadoop
15
+done
16
+
17
+ls hadoop-aws-*.jar

+ 48
- 0
tools/path-style-access.patch View File

@@ -0,0 +1,48 @@
1
+diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
2
+index 3486dfb..ad19d00 100644
3
+--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
4
++++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
5
+@@ -35,6 +35,7 @@
6
+ 
7
+   //use a custom endpoint?
8
+   public static final String ENDPOINT = "fs.s3a.endpoint";
9
++  public static final String PATH_STYLE_ACCESS = "fs.s3a.path.style.access";
10
+   //connect to s3 through a proxy server?
11
+   public static final String PROXY_HOST = "fs.s3a.proxy.host";
12
+   public static final String PROXY_PORT = "fs.s3a.proxy.port";
13
+diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
14
+index 91a606c..17b272a 100644
15
+--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
16
++++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
17
+@@ -40,6 +40,7 @@
18
+ 
19
+ import com.amazonaws.auth.InstanceProfileCredentialsProvider;
20
+ import com.amazonaws.services.s3.AmazonS3Client;
21
++import com.amazonaws.services.s3.S3ClientOptions;
22
+ import com.amazonaws.services.s3.model.CannedAccessControlList;
23
+ import com.amazonaws.services.s3.model.DeleteObjectsRequest;
24
+ import com.amazonaws.services.s3.model.ListObjectsRequest;
25
+@@ -244,6 +245,8 @@ public void initialize(URI name, Configuration conf) throws IOException {
26
+       }
27
+     }
28
+ 
29
++    enablePathStyleAccessIfRequired(conf);
30
++
31
+     maxKeys = conf.getInt(MAX_PAGING_KEYS, DEFAULT_MAX_PAGING_KEYS);
32
+     partSize = conf.getLong(MULTIPART_SIZE, DEFAULT_MULTIPART_SIZE);
33
+     multiPartThreshold = conf.getInt(MIN_MULTIPART_THRESHOLD,
34
+@@ -314,6 +317,14 @@ public void initialize(URI name, Configuration conf) throws IOException {
35
+     setConf(conf);
36
+   }
37
+ 
38
++  private void enablePathStyleAccessIfRequired(Configuration conf) {
39
++    final boolean pathStyleAccess = conf.getBoolean(PATH_STYLE_ACCESS, false);
40
++    if (pathStyleAccess) {
41
++      LOG.debug("Enabling path style access!");
42
++      s3.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true));
43
++    }
44
++  }
45
++
46
+   /**
47
+    * Return the protocol scheme for the FileSystem.
48
+    *

Loading…
Cancel
Save