Fix

apache · Dec 29, 2024 · 7f0a99b · 7f0a99b
1 parent 50a4d15
commit 7f0a99b
Show file tree

Hide file tree

Showing 6 changed files with 20 additions and 7 deletions.
diff --git a/bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/OSSTokenProvider.java b/bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/OSSTokenProvider.java
@@ -156,7 +156,7 @@ private String createPolicy(Set<String> readLocations, Set<String> writeLocation
                   key ->
                       Statement.builder()
                           .effect(Effect.ALLOW)
-                          .addAction("oss:ListBucket")
+                          .addAction("oss:ListObjects")
                           .addResource(key)
                           .condition(getCondition(uri)));
               // GetBucketLocation

diff --git a/bundles/aws/src/main/java/org/apache/gravitino/s3/fs/S3FileSystemProvider.java b/bundles/aws/src/main/java/org/apache/gravitino/s3/fs/S3FileSystemProvider.java
@@ -62,14 +62,15 @@ public FileSystem getFileSystem(Path path, Map<String, String> config) throws IO
     Map<String, String> hadoopConfMap =
         FileSystemUtils.toHadoopConfigMap(config, GRAVITINO_KEY_TO_S3_HADOOP_KEY);
 
-    if (!hadoopConfMap.containsKey(S3_CREDENTIAL_KEY)
-        && config.containsKey(
-            GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_SERVER_URI_KEY)) {
-      hadoopConfMap.put(
-          Constants.AWS_CREDENTIALS_PROVIDER, S3SessionCredentialProvider.class.getCanonicalName());
+    hadoopConfMap.forEach(configuration::set);
+    if (!hadoopConfMap.containsKey(S3_CREDENTIAL_KEY)) {
+      configuration.set(S3_CREDENTIAL_KEY, S3_SIMPLE_CREDENTIAL);
     }
 
-    hadoopConfMap.forEach(configuration::set);
+    if (config.containsKey(GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_SERVER_URI_KEY)) {
+      configuration.set(
+          Constants.AWS_CREDENTIALS_PROVIDER, S3SessionCredentialProvider.class.getCanonicalName());
+    }
 
     // Hadoop-aws 2 does not support IAMInstanceCredentialsProvider
     checkAndSetCredentialProvider(configuration);

diff --git a/bundles/gcp/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java b/bundles/gcp/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java
@@ -18,11 +18,13 @@
  */
 package org.apache.gravitino.gcs.fs;
 
+import com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem;
 import com.google.cloud.hadoop.util.AccessTokenProvider;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableMap;
 import java.io.IOException;
 import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider;
 import org.apache.gravitino.catalog.hadoop.fs.FileSystemUtils;
 import org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystemConfiguration;
@@ -55,6 +57,11 @@ public FileSystem getFileSystem(Path path, Map<String, String> config) throws IO
             "fs.gs.auth.access.token.provider.impl", GCSCredentialProvider.class.getName());
       }
     }
+
+    if (StringUtils.isBlank(configuration.get("fs.gs.impl"))) {
+      configuration.set("fs.gs.impl", GoogleHadoopFileSystem.class.getName());
+    }
+
     return FileSystem.newInstance(path.toUri(), configuration);
   }
 

diff --git a/...avitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSCredentialIT.java b/...avitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSCredentialIT.java
@@ -92,6 +92,7 @@ public void startUp() throws Exception {
     conf.set("fs.gvfs.impl.disable.cache", "true");
     conf.set("fs.gravitino.server.uri", serverUri);
     conf.set("fs.gravitino.client.metalake", metalakeName);
+    conf.set("fs.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem");
 
     // Pass this configuration to the real file system
     conf.set(GCSProperties.GCS_SERVICE_ACCOUNT_JSON_PATH, SERVICE_ACCOUNT_FILE);

diff --git a/.../apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java b/.../apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java
@@ -88,6 +88,7 @@ public void startUp() throws Exception {
     conf.set("fs.gvfs.impl.disable.cache", "true");
     conf.set("fs.gravitino.server.uri", serverUri);
     conf.set("fs.gravitino.client.metalake", metalakeName);
+    conf.set("fs.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem");
 
     // Pass this configuration to the real file system
     conf.set(GCSProperties.GCS_SERVICE_ACCOUNT_JSON_PATH, SERVICE_ACCOUNT_FILE);

diff --git a/...ravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemS3CredentialIT.java b/...ravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemS3CredentialIT.java
@@ -50,6 +50,7 @@ public class GravitinoVirtualFileSystemS3CredentialIT extends GravitinoVirtualFi
   public static final String BUCKET_NAME = System.getenv("S3_STS_BUCKET_NAME");
   public static final String S3_ACCESS_KEY = System.getenv("S3_STS_ACCESS_KEY_ID");
   public static final String S3_SECRET_KEY = System.getenv("S3_STS_SECRET_ACCESS_KEY");
+  public static final String S3_ENDPOINT = System.getenv("S3_STS_ENDPOINT");
   public static final String S3_REGION = System.getenv("S3_STS_REGION");
   public static final String S3_ROLE_ARN = System.getenv("S3_STS_ROLE_ARN");
 
@@ -82,6 +83,7 @@ public void startUp() throws Exception {
     Map<String, String> properties = Maps.newHashMap();
     properties.put(S3Properties.GRAVITINO_S3_ACCESS_KEY_ID, S3_ACCESS_KEY);
     properties.put(S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY, S3_SECRET_KEY);
+    properties.put(S3Properties.GRAVITINO_S3_ENDPOINT, S3_ENDPOINT);
     properties.put(
         "gravitino.bypass.fs.s3a.aws.credentials.provider",
         "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider");
@@ -109,6 +111,7 @@ public void startUp() throws Exception {
     // Pass this configuration to the real file system
     conf.set(S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY, S3_SECRET_KEY);
     conf.set(S3Properties.GRAVITINO_S3_ACCESS_KEY_ID, S3_ACCESS_KEY);
+    conf.set(S3Properties.GRAVITINO_S3_ENDPOINT, S3_ENDPOINT);
     conf.set(S3Properties.GRAVITINO_S3_REGION, S3_REGION);
     conf.set(S3Properties.GRAVITINO_S3_ROLE_ARN, S3_ROLE_ARN);
   }