From 528d0eacac536569ff9109f70c4ca3af0284bccd Mon Sep 17 00:00:00 2001 From: FANNG Date: Mon, 30 Dec 2024 14:08:26 +0800 Subject: [PATCH] [#6024] feat(iceberg): refactor Iceberg credential code to reuse credential component in Gravitino server (#6021) ### What changes were proposed in this pull request? 1. reuse `CatalogCredentialManager` to manage the credentials in Iceberg catalog. 2. use `CatalogWrapperForREST` to manage some REST specific operations like credential vending. 3. depracate `catalog-provider-type` , use `catalog-providers` instead. ### Why are the changes needed? Fix: #6024 ### Does this PR introduce _any_ user-facing change? yes, depracate `catalog-provider-type` , use `catalog-providers`, do some compatibility work. ### How was this patch tested? run with s3 token --- .../credential/CredentialConstants.java | 2 +- .../test/FilesetCatalogCredentialIT.java | 3 - .../credential/CatalogCredentialManager.java | 16 ++ .../credential/CredentialProviderManager.java | 73 -------- .../gravitino/credential/CredentialUtils.java | 28 +--- .../credential/config/CredentialConfig.java | 10 ++ .../iceberg-rest-server/rewrite_config.py | 3 +- docs/iceberg-rest-service.md | 99 +++++------ .../iceberg/common/IcebergConfig.java | 6 +- .../common/ops/IcebergCatalogWrapper.java | 28 +--- .../service/CatalogWrapperForREST.java | 157 ++++++++++++++++++ .../service/IcebergCatalogWrapperManager.java | 48 ++---- .../IcebergTableOperationExecutor.java | 4 +- .../service/rest/IcebergTableOperations.java | 77 +-------- .../api/event/IcebergRequestContext.java | 25 ++- .../test/IcebergRESTADLSTokenIT.java | 2 +- .../test/IcebergRESTAzureAccountKeyIT.java | 2 +- .../integration/test/IcebergRESTGCSIT.java | 2 +- .../integration/test/IcebergRESTOSSIT.java | 2 +- .../test/IcebergRESTOSSSecretIT.java | 2 +- .../integration/test/IcebergRESTS3IT.java | 2 +- ...tIcebergCatalogWrapperManagerForREST.java} | 2 +- ...orTest.java => CatalogWrapperForTest.java} | 8 +- .../IcebergCatalogWrapperManagerForTest.java | 7 +- .../service/rest/IcebergRestTestUtil.java | 3 +- .../rest/MockIcebergTableOperations.java | 4 +- 26 files changed, 318 insertions(+), 297 deletions(-) delete mode 100644 core/src/main/java/org/apache/gravitino/credential/CredentialProviderManager.java create mode 100644 iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/CatalogWrapperForREST.java rename iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/{TestIcebergCatalogWrapperManager.java => TestIcebergCatalogWrapperManagerForREST.java} (98%) rename iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/{IcebergCatalogWrapperForTest.java => CatalogWrapperForTest.java} (89%) diff --git a/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java index 7d552deb6bf..a6e0d54bfad 100644 --- a/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java +++ b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java @@ -20,7 +20,7 @@ package org.apache.gravitino.credential; public class CredentialConstants { - public static final String CREDENTIAL_PROVIDER_TYPE = "credential-provider-type"; + @Deprecated public static final String CREDENTIAL_PROVIDER_TYPE = "credential-provider-type"; public static final String CREDENTIAL_PROVIDERS = "credential-providers"; public static final String CREDENTIAL_CACHE_EXPIRE_RATIO = "credential-cache-expire-ratio"; public static final String CREDENTIAL_CACHE_MAX_SIZE = "credential-cache-max-size"; diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/FilesetCatalogCredentialIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/FilesetCatalogCredentialIT.java index 94239fef28f..3dc3ad82ae4 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/FilesetCatalogCredentialIT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/FilesetCatalogCredentialIT.java @@ -86,9 +86,6 @@ public void startUp() throws Exception { S3TokenCredential.S3_TOKEN_CREDENTIAL_TYPE + "," + S3SecretKeyCredential.S3_SECRET_KEY_CREDENTIAL_TYPE); - properties.put( - CredentialConstants.CREDENTIAL_PROVIDER_TYPE, - S3SecretKeyCredential.S3_SECRET_KEY_CREDENTIAL_TYPE); properties.put(S3Properties.GRAVITINO_S3_ACCESS_KEY_ID, S3_ACCESS_KEY); properties.put(S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY, S3_SECRET_KEY); properties.put(S3Properties.GRAVITINO_S3_ENDPOINT, "s3.ap-southeast-2.amazonaws.com"); diff --git a/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialManager.java b/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialManager.java index 0e407a399b4..7fbead57a5e 100644 --- a/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialManager.java +++ b/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialManager.java @@ -51,6 +51,17 @@ public Credential getCredential(String credentialType, CredentialContext context return credentialCache.getCredential(credentialCacheKey, cacheKey -> doGetCredential(cacheKey)); } + // Get credential with only one credential provider. + public Credential getCredential(CredentialContext context) { + if (credentialProviders.size() == 0) { + throw new IllegalArgumentException("There are no credential provider for the catalog."); + } else if (credentialProviders.size() > 1) { + throw new UnsupportedOperationException( + "There are multiple credential providers for the catalog."); + } + return getCredential(credentialProviders.keySet().iterator().next(), context); + } + @Override public void close() { credentialProviders @@ -67,6 +78,11 @@ public void close() { e); } }); + try { + credentialCache.close(); + } catch (IOException e) { + LOG.warn("Close credential cache failed, catalog: {}", catalogName, e); + } } private Credential doGetCredential(CredentialCacheKey credentialCacheKey) { diff --git a/core/src/main/java/org/apache/gravitino/credential/CredentialProviderManager.java b/core/src/main/java/org/apache/gravitino/credential/CredentialProviderManager.java deleted file mode 100644 index b583bedcfdf..00000000000 --- a/core/src/main/java/org/apache/gravitino/credential/CredentialProviderManager.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.gravitino.credential; - -import com.google.common.base.Preconditions; -import java.io.IOException; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import javax.annotation.Nullable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class CredentialProviderManager { - - private static final Logger LOG = LoggerFactory.getLogger(CredentialProviderManager.class); - private Map credentialProviders; - - public CredentialProviderManager() { - this.credentialProviders = new ConcurrentHashMap<>(); - } - - public void registerCredentialProvider( - String catalogName, CredentialProvider credentialProvider) { - CredentialProvider current = credentialProviders.putIfAbsent(catalogName, credentialProvider); - Preconditions.checkState( - !credentialProvider.equals(current), - String.format( - "Should not register multiple times to CredentialProviderManager, catalog: %s, " - + "credential provider: %s", - catalogName, credentialProvider.credentialType())); - LOG.info( - "Register catalog:%s credential provider:%s to CredentialProviderManager", - catalogName, credentialProvider.credentialType()); - } - - public void unregisterCredentialProvider(String catalogName) { - CredentialProvider credentialProvider = credentialProviders.remove(catalogName); - // Not all catalog has credential provider - if (credentialProvider != null) { - LOG.info( - "Unregister catalog:{} credential provider:{} to CredentialProviderManager", - catalogName, - credentialProvider.credentialType()); - try { - credentialProvider.close(); - } catch (IOException e) { - LOG.warn("Close credential provider failed", e); - } - } - } - - @Nullable - public CredentialProvider getCredentialProvider(String catalogName) { - return credentialProviders.get(catalogName); - } -} diff --git a/core/src/main/java/org/apache/gravitino/credential/CredentialUtils.java b/core/src/main/java/org/apache/gravitino/credential/CredentialUtils.java index 9a202ec9747..9d2ea43e664 100644 --- a/core/src/main/java/org/apache/gravitino/credential/CredentialUtils.java +++ b/core/src/main/java/org/apache/gravitino/credential/CredentialUtils.java @@ -19,30 +19,20 @@ package org.apache.gravitino.credential; -import com.google.common.base.Splitter; -import com.google.common.collect.ImmutableSet; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Supplier; import java.util.stream.Collectors; -import org.apache.gravitino.utils.PrincipalUtils; +import org.apache.gravitino.credential.config.CredentialConfig; public class CredentialUtils { - private static final Splitter splitter = Splitter.on(","); - - public static Credential vendCredential(CredentialProvider credentialProvider, String[] path) { - PathBasedCredentialContext pathBasedCredentialContext = - new PathBasedCredentialContext( - PrincipalUtils.getCurrentUserName(), ImmutableSet.copyOf(path), Collections.emptySet()); - return credentialProvider.getCredential(pathBasedCredentialContext); - } - public static Map loadCredentialProviders( Map catalogProperties) { - Set credentialProviders = - CredentialUtils.getCredentialProvidersByOrder(() -> catalogProperties); + CredentialConfig credentialConfig = new CredentialConfig(catalogProperties); + List credentialProviders = credentialConfig.get(CredentialConfig.CREDENTIAL_PROVIDERS); return credentialProviders.stream() .collect( @@ -80,14 +70,8 @@ private static Set getCredentialProvidersFromProperties(Map> CREDENTIAL_PROVIDERS = + new ConfigBuilder(CredentialConstants.CREDENTIAL_PROVIDERS) + .doc("Credential providers, separated by comma.") + .version(ConfigConstants.VERSION_0_8_0) + .stringConf() + .toSequence() + .createWithDefault(Collections.emptyList()); + public static final ConfigEntry CREDENTIAL_CACHE_EXPIRE_RATIO = new ConfigBuilder(CredentialConstants.CREDENTIAL_CACHE_EXPIRE_RATIO) .doc( diff --git a/dev/docker/iceberg-rest-server/rewrite_config.py b/dev/docker/iceberg-rest-server/rewrite_config.py index 624c67750ca..d607eb6ab42 100755 --- a/dev/docker/iceberg-rest-server/rewrite_config.py +++ b/dev/docker/iceberg-rest-server/rewrite_config.py @@ -22,7 +22,8 @@ "GRAVITINO_IO_IMPL" : "io-impl", "GRAVITINO_URI" : "uri", "GRAVITINO_WAREHOUSE" : "warehouse", - "GRAVITINO_CREDENTIAL_PROVIDER_TYPE" : "credential-provider-type", + "GRAVITINO_CREDENTIAL_PROVIDER_TYPE" : "credential-providers", + "GRAVITINO_CREDENTIAL_PROVIDERS" : "credential-providers", "GRAVITINO_GCS_CREDENTIAL_FILE_PATH" : "gcs-credential-file-path", "GRAVITINO_S3_ACCESS_KEY" : "s3-access-key-id", "GRAVITINO_S3_SECRET_KEY" : "s3-secret-access-key", diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md index f31aa13685a..3c2f27a3d1c 100644 --- a/docs/iceberg-rest-service.md +++ b/docs/iceberg-rest-service.md @@ -106,22 +106,23 @@ The detailed configuration items are as follows: Gravitino Iceberg REST service supports using static S3 secret key or generating temporary token to access S3 data. -| Configuration item | Description | Default value | Required | Since Version | -|----------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|----------------------------------------------------|------------------| -| `gravitino.iceberg-rest.io-impl` | The IO implementation for `FileIO` in Iceberg, use `org.apache.iceberg.aws.s3.S3FileIO` for S3. | (none) | No | 0.6.0-incubating | -| `gravitino.iceberg-rest.credential-provider-type` | Supports `s3-token` and `s3-secret-key` for S3. `s3-token` generates a temporary token according to the query data path while `s3-secret-key` using the s3 secret access key to access S3 data. | (none) | No | 0.7.0-incubating | -| `gravitino.iceberg-rest.s3-access-key-id` | The static access key ID used to access S3 data. | (none) | No | 0.6.0-incubating | -| `gravitino.iceberg-rest.s3-secret-access-key` | The static secret access key used to access S3 data. | (none) | No | 0.6.0-incubating | -| `gravitino.iceberg-rest.s3-endpoint` | An alternative endpoint of the S3 service, This could be used for S3FileIO with any s3-compatible object storage service that has a different endpoint, or access a private S3 endpoint in a virtual private cloud. | (none) | No | 0.6.0-incubating | -| `gravitino.iceberg-rest.s3-region` | The region of the S3 service, like `us-west-2`. | (none) | No | 0.6.0-incubating | -| `gravitino.iceberg-rest.s3-role-arn` | The ARN of the role to access the S3 data. | (none) | Yes, when `credential-provider-type` is `s3-token` | 0.7.0-incubating | -| `gravitino.iceberg-rest.s3-external-id` | The S3 external id to generate token, only used when `credential-provider-type` is `s3-token`. | (none) | No | 0.7.0-incubating | -| `gravitino.iceberg-rest.s3-token-expire-in-secs` | The S3 session token expire time in secs, it couldn't exceed the max session time of the assumed role, only used when `credential-provider-type` is `s3-token`. | 3600 | No | 0.7.0-incubating | -| `gravitino.iceberg-rest.s3-token-service-endpoint` | An alternative endpoint of the S3 token service, This could be used with s3-compatible object storage service like MINIO that has a different STS endpoint. | (none) | No | 0.8.0-incubating | +| Configuration item | Description | Default value | Required | Since Version | +|----------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|------------------------------------------------|------------------| +| `gravitino.iceberg-rest.io-impl` | The IO implementation for `FileIO` in Iceberg, use `org.apache.iceberg.aws.s3.S3FileIO` for S3. | (none) | No | 0.6.0-incubating | +| `gravitino.iceberg-rest.credential-provider-type` | Deprecated, please use `gravitino.iceberg-rest.credential-providers` instead. | (none) | No | 0.7.0-incubating | +| `gravitino.iceberg-rest.credential-providers` | Supports `s3-token` and `s3-secret-key` for S3. `s3-token` generates a temporary token according to the query data path while `s3-secret-key` using the s3 secret access key to access S3 data. | (none) | No | 0.7.0-incubating | +| `gravitino.iceberg-rest.s3-access-key-id` | The static access key ID used to access S3 data. | (none) | No | 0.6.0-incubating | +| `gravitino.iceberg-rest.s3-secret-access-key` | The static secret access key used to access S3 data. | (none) | No | 0.6.0-incubating | +| `gravitino.iceberg-rest.s3-endpoint` | An alternative endpoint of the S3 service, This could be used for S3FileIO with any s3-compatible object storage service that has a different endpoint, or access a private S3 endpoint in a virtual private cloud. | (none) | No | 0.6.0-incubating | +| `gravitino.iceberg-rest.s3-region` | The region of the S3 service, like `us-west-2`. | (none) | No | 0.6.0-incubating | +| `gravitino.iceberg-rest.s3-role-arn` | The ARN of the role to access the S3 data. | (none) | Yes, when `credential-providers` is `s3-token` | 0.7.0-incubating | +| `gravitino.iceberg-rest.s3-external-id` | The S3 external id to generate token, only used when `credential-providers` is `s3-token`. | (none) | No | 0.7.0-incubating | +| `gravitino.iceberg-rest.s3-token-expire-in-secs` | The S3 session token expire time in secs, it couldn't exceed the max session time of the assumed role, only used when `credential-providers` is `s3-token`. | 3600 | No | 0.7.0-incubating | +| `gravitino.iceberg-rest.s3-token-service-endpoint` | An alternative endpoint of the S3 token service, This could be used with s3-compatible object storage service like MINIO that has a different STS endpoint. | (none) | No | 0.8.0-incubating | For other Iceberg s3 properties not managed by Gravitino like `s3.sse.type`, you could config it directly by `gravitino.iceberg-rest.s3.sse.type`. -If you set `credential-provider-type` explicitly, please downloading [Gravitino AWS bundle jar](https://mvnrepository.com/artifact/org.apache.gravitino/aws-bundle), and place it to the classpath of Iceberg REST server. +If you set `credential-providers` explicitly, please downloading [Gravitino AWS bundle jar](https://mvnrepository.com/artifact/org.apache.gravitino/aws-bundle), and place it to the classpath of Iceberg REST server. :::info To configure the JDBC catalog backend, set the `gravitino.iceberg-rest.warehouse` parameter to `s3://{bucket_name}/${prefix_name}`. For the Hive catalog backend, set `gravitino.iceberg-rest.warehouse` to `s3a://{bucket_name}/${prefix_name}`. Additionally, download the [Iceberg AWS bundle](https://mvnrepository.com/artifact/org.apache.iceberg/iceberg-aws-bundle) and place it in the classpath of Iceberg REST server. @@ -134,18 +135,19 @@ Gravitino Iceberg REST service supports using static access-key-id and secret-ac | Configuration item | Description | Default value | Required | Since Version | |---------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------|------------------------------------------------------|------------------| | `gravitino.iceberg-rest.io-impl` | The IO implementation for `FileIO` in Iceberg, use `org.apache.iceberg.aliyun.oss.OSSFileIO` for OSS. | (none) | No | 0.6.0-incubating | -| `gravitino.iceberg-rest.credential-provider-type` | Supports `oss-token` and `oss-secret-key` for OSS. `oss-token` generates a temporary token according to the query data path while `oss-secret-key` using the oss secret access key to access S3 data. | (none) | No | 0.7.0-incubating | +| `gravitino.iceberg-rest.credential-provider-type` | Deprecated, please use `gravitino.iceberg-rest.credential-providers` instead. | (none) | No | 0.7.0-incubating | +| `gravitino.iceberg-rest.credential-providers` | Supports `oss-token` and `oss-secret-key` for OSS. `oss-token` generates a temporary token according to the query data path while `oss-secret-key` using the oss secret access key to access S3 data. | (none) | No | 0.7.0-incubating | | `gravitino.iceberg-rest.oss-access-key-id` | The static access key ID used to access OSS data. | (none) | No | 0.7.0-incubating | | `gravitino.iceberg-rest.oss-secret-access-key` | The static secret access key used to access OSS data. | (none) | No | 0.7.0-incubating | | `gravitino.iceberg-rest.oss-endpoint` | The endpoint of Aliyun OSS service. | (none) | No | 0.7.0-incubating | -| `gravitino.iceberg-rest.oss-region` | The region of the OSS service, like `oss-cn-hangzhou`, only used when `credential-provider-type` is `oss-token`. | (none) | No | 0.8.0-incubating | -| `gravitino.iceberg-rest.oss-role-arn` | The ARN of the role to access the OSS data, only used when `credential-provider-type` is `oss-token`. | (none) | Yes, when `credential-provider-type` is `oss-token`. | 0.8.0-incubating | -| `gravitino.iceberg-rest.oss-external-id` | The OSS external id to generate token, only used when `credential-provider-type` is `oss-token`. | (none) | No | 0.8.0-incubating | -| `gravitino.iceberg-rest.oss-token-expire-in-secs` | The OSS security token expire time in secs, only used when `credential-provider-type` is `oss-token`. | 3600 | No | 0.8.0-incubating | +| `gravitino.iceberg-rest.oss-region` | The region of the OSS service, like `oss-cn-hangzhou`, only used when `credential-providers` is `oss-token`. | (none) | No | 0.8.0-incubating | +| `gravitino.iceberg-rest.oss-role-arn` | The ARN of the role to access the OSS data, only used when `credential-providers` is `oss-token`. | (none) | Yes, when `credential-provider-type` is `oss-token`. | 0.8.0-incubating | +| `gravitino.iceberg-rest.oss-external-id` | The OSS external id to generate token, only used when `credential-providers` is `oss-token`. | (none) | No | 0.8.0-incubating | +| `gravitino.iceberg-rest.oss-token-expire-in-secs` | The OSS security token expire time in secs, only used when `credential-providers` is `oss-token`. | 3600 | No | 0.8.0-incubating | For other Iceberg OSS properties not managed by Gravitino like `client.security-token`, you could config it directly by `gravitino.iceberg-rest.client.security-token`. -If you set `credential-provider-type` explicitly, please downloading [Gravitino Aliyun bundle jar](https://mvnrepository.com/artifact/org.apache.gravitino/aliyun-bundle), and place it to the classpath of Iceberg REST server. +If you set `credential-providers` explicitly, please downloading [Gravitino Aliyun bundle jar](https://mvnrepository.com/artifact/org.apache.gravitino/aliyun-bundle), and place it to the classpath of Iceberg REST server. :::info Please set the `gravitino.iceberg-rest.warehouse` parameter to `oss://{bucket_name}/${prefix_name}`. Additionally, download the [Aliyun OSS SDK](https://gosspublic.alicdn.com/sdks/java/aliyun_java_sdk_3.10.2.zip) and copy `aliyun-sdk-oss-3.10.2.jar`, `hamcrest-core-1.1.jar`, `jdom2-2.0.6.jar` in the classpath of Iceberg REST server, `iceberg-rest-server/libs` for the auxiliary server, `libs` for the standalone server. @@ -158,12 +160,13 @@ Supports using static GCS credential file or generating GCS token to access GCS | Configuration item | Description | Default value | Required | Since Version | |---------------------------------------------------|----------------------------------------------------------------------------------------------------|---------------|----------|------------------| | `gravitino.iceberg-rest.io-impl` | The io implementation for `FileIO` in Iceberg, use `org.apache.iceberg.gcp.gcs.GCSFileIO` for GCS. | (none) | No | 0.6.0-incubating | -| `gravitino.iceberg-rest.credential-provider-type` | Supports `gcs-token`, generates a temporary token according to the query data path. | (none) | No | 0.7.0-incubating | -| `gravitino.iceberg-rest.gcs-credential-file-path` | The location of GCS credential file, only used when `credential-provider-type` is `gcs-token`. | (none) | No | 0.7.0-incubating | +| `gravitino.iceberg-rest.credential-provider-type` | Deprecated, please use `gravitino.iceberg-rest.credential-providers` instead. | (none) | No | 0.7.0-incubating | +| `gravitino.iceberg-rest.credential-providers` | Supports `gcs-token`, generates a temporary token according to the query data path. | (none) | No | 0.7.0-incubating | +| `gravitino.iceberg-rest.gcs-credential-file-path` | The location of GCS credential file, only used when `credential-providers` is `gcs-token`. | (none) | No | 0.7.0-incubating | For other Iceberg GCS properties not managed by Gravitino like `gcs.project-id`, you could config it directly by `gravitino.iceberg-rest.gcs.project-id`. -If you set `credential-provider-type` explicitly, please downloading [Gravitino GCP bundle jar](https://mvnrepository.com/artifact/org.apache.gravitino/gcp-bundle), and place it to the classpath of Iceberg REST server. +If you set `credential-providers` explicitly, please downloading [Gravitino GCP bundle jar](https://mvnrepository.com/artifact/org.apache.gravitino/gcp-bundle), and place it to the classpath of Iceberg REST server. Please make sure the credential file is accessible by Gravitino, like using `export GOOGLE_APPLICATION_CREDENTIALS=/xx/application_default_credentials.json` before Gravitino Iceberg REST server is started. @@ -178,17 +181,18 @@ Gravitino Iceberg REST service supports generating SAS token to access ADLS data | Configuration item | Description | Default value | Required | Since Version | |-----------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|----------|------------------| | `gravitino.iceberg-rest.io-impl` | The IO implementation for `FileIO` in Iceberg, use `org.apache.iceberg.azure.adlsv2.ADLSFileIO` for ADLS. | (none) | Yes | 0.8.0-incubating | -| `gravitino.iceberg-rest.credential-provider-type` | Supports `adls-token` and `azure-account-key`. `adls-token` generates a temporary token according to the query data path while `azure-account-key` uses a storage account key to access ADLS data. | (none) | Yes | 0.8.0-incubating | +| `gravitino.iceberg-rest.credential-provider-type` | Deprecated, please use `gravitino.iceberg-rest.credential-providers` instead. | (none) | No | 0.7.0-incubating | +| `gravitino.iceberg-rest.credential-providers` | Supports `adls-token` and `azure-account-key`. `adls-token` generates a temporary token according to the query data path while `azure-account-key` uses a storage account key to access ADLS data. | (none) | Yes | 0.8.0-incubating | | `gravitino.iceberg-rest.azure-storage-account-name` | The static storage account name used to access ADLS data. | (none) | Yes | 0.8.0-incubating | | `gravitino.iceberg-rest.azure-storage-account-key` | The static storage account key used to access ADLS data. | (none) | Yes | 0.8.0-incubating | -| `gravitino.iceberg-rest.azure-tenant-id` | Azure Active Directory (AAD) tenant ID, only used when `credential-provider-type` is `adls-token`. | (none) | Yes | 0.8.0-incubating | -| `gravitino.iceberg-rest.azure-client-id` | Azure Active Directory (AAD) client ID used for authentication, only used when `credential-provider-type` is `adls-token`. | (none) | Yes | 0.8.0-incubating | -| `gravitino.iceberg-rest.azure-client-secret` | Azure Active Directory (AAD) client secret used for authentication, only used when `credential-provider-type` is `adls-token`. | (none) | Yes | 0.8.0-incubating | -| `gravitino.iceberg-rest.adls-token-expire-in-secs` | The ADLS SAS token expire time in secs, only used when `credential-provider-type` is `adls-token`. | 3600 | No | 0.8.0-incubating | +| `gravitino.iceberg-rest.azure-tenant-id` | Azure Active Directory (AAD) tenant ID, only used when `credential-providers` is `adls-token`. | (none) | Yes | 0.8.0-incubating | +| `gravitino.iceberg-rest.azure-client-id` | Azure Active Directory (AAD) client ID used for authentication, only used when `credential-providers` is `adls-token`. | (none) | Yes | 0.8.0-incubating | +| `gravitino.iceberg-rest.azure-client-secret` | Azure Active Directory (AAD) client secret used for authentication, only used when `credential-providers` is `adls-token`. | (none) | Yes | 0.8.0-incubating | +| `gravitino.iceberg-rest.adls-token-expire-in-secs` | The ADLS SAS token expire time in secs, only used when `credential-providers` is `adls-token`. | 3600 | No | 0.8.0-incubating | For other Iceberg ADLS properties not managed by Gravitino like `adls.read.block-size-bytes`, you could config it directly by `gravitino.iceberg-rest.adls.read.block-size-bytes`. -If you set `credential-provider-type` explicitly, please downloading [Gravitino Azure bundle jar](https://mvnrepository.com/artifact/org.apache.gravitino/azure-bundle), and place it to the classpath of Iceberg REST server. +If you set `credential-providers` explicitly, please downloading [Gravitino Azure bundle jar](https://mvnrepository.com/artifact/org.apache.gravitino/azure-bundle), and place it to the classpath of Iceberg REST server. :::info Please set `gravitino.iceberg-rest.warehouse` to `abfs[s]://{container-name}@{storage-account-name}.dfs.core.windows.net/{path}`, and download the [Iceberg Azure bundle](https://mvnrepository.com/artifact/org.apache.iceberg/iceberg-azure-bundle) and place it in the classpath of Iceberg REST server. @@ -415,7 +419,7 @@ For example, we can configure Spark catalog options to use Gravitino Iceberg RES --conf spark.sql.catalog.rest.uri=http://127.0.0.1:9001/iceberg/ ``` -You may need to adjust the Iceberg Spark runtime jar file name according to the real version number in your environment. If you want to access the data stored in cloud, you need to download corresponding jars (please refer to the cloud storage part) and place it in the classpath of Spark. If you want to enable credential vending, please set `credential-provider-type` to a proper value in the server side, set `spark.sql.catalog.rest.header.X-Iceberg-Access-Delegation` = `vended-credentials` in the client side. +You may need to adjust the Iceberg Spark runtime jar file name according to the real version number in your environment. If you want to access the data stored in cloud, you need to download corresponding jars (please refer to the cloud storage part) and place it in the classpath of Spark. If you want to enable credential vending, please set `credential-providers` to a proper value in the server side, set `spark.sql.catalog.rest.header.X-Iceberg-Access-Delegation` = `vended-credentials` in the client side. For other storages not managed by Gravitino, the properties wouldn't transfer from the server to client automatically, if you want to pass custom properties to initialize `FileIO`, you could add it by `spark.sql.catalog.${iceberg_catalog_name}.${configuration_key}` = `{property_value}`. @@ -441,24 +445,25 @@ docker run -d -p 9001:9001 apache/gravitino-iceberg-rest:0.7.0-incubating Gravitino Iceberg REST server in docker image could access local storage by default, you could set the following environment variables if the storage is cloud/remote storage like S3, please refer to [storage section](#storage) for more details. -| Environment variables | Configuration items | Since version | -|-----------------------------------------|-----------------------------------------------------|-------------------| -| `GRAVITINO_IO_IMPL` | `gravitino.iceberg-rest.io-impl` | 0.7.0-incubating | -| `GRAVITINO_URI` | `gravitino.iceberg-rest.uri` | 0.7.0-incubating | -| `GRAVITINO_WAREHOUSE` | `gravitino.iceberg-rest.warehouse` | 0.7.0-incubating | -| `GRAVITINO_CREDENTIAL_PROVIDER_TYPE` | `gravitino.iceberg-rest.credential-provider-type` | 0.7.0-incubating | -| `GRAVITINO_GCS_CREDENTIAL_FILE_PATH` | `gravitino.iceberg-rest.gcs-credential-file-path` | 0.7.0-incubating | -| `GRAVITINO_S3_ACCESS_KEY` | `gravitino.iceberg-rest.s3-access-key-id` | 0.7.0-incubating | -| `GRAVITINO_S3_SECRET_KEY` | `gravitino.iceberg-rest.s3-secret-access-key` | 0.7.0-incubating | -| `GRAVITINO_S3_REGION` | `gravitino.iceberg-rest.s3-region` | 0.7.0-incubating | -| `GRAVITINO_S3_ROLE_ARN` | `gravitino.iceberg-rest.s3-role-arn` | 0.7.0-incubating | -| `GRAVITINO_S3_EXTERNAL_ID` | `gravitino.iceberg-rest.s3-external-id` | 0.7.0-incubating | -| `GRAVITINO_S3_TOKEN_SERVICE_ENDPOINT` | `gravitino.iceberg-rest.s3-token-service-endpoint` | 0.8.0-incubating | -| `GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME` | `gravitino.iceberg-rest.azure-storage-account-name` | 0.8.0-incubating | -| `GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY` | `gravitino.iceberg-rest.azure-storage-account-key` | 0.8.0-incubating | -| `GRAVITINO_AZURE_TENANT_ID` | `gravitino.iceberg-rest.azure-tenant-id` | 0.8.0-incubating | -| `GRAVITINO_AZURE_CLIENT_ID` | `gravitino.iceberg-rest.azure-client-id` | 0.8.0-incubating | -| `GRAVITINO_AZURE_CLIENT_SECRET` | `gravitino.iceberg-rest.azure-client-secret` | 0.8.0-incubating | +| Environment variables | Configuration items | Since version | +|----------------------------------------|-----------------------------------------------------|------------------| +| `GRAVITINO_IO_IMPL` | `gravitino.iceberg-rest.io-impl` | 0.7.0-incubating | +| `GRAVITINO_URI` | `gravitino.iceberg-rest.uri` | 0.7.0-incubating | +| `GRAVITINO_WAREHOUSE` | `gravitino.iceberg-rest.warehouse` | 0.7.0-incubating | +| `GRAVITINO_CREDENTIAL_PROVIDER_TYPE` | `gravitino.iceberg-rest.credential-providers` | 0.8.0-incubating | +| `GRAVITINO_CREDENTIAL_PROVIDERS` | `gravitino.iceberg-rest.credential-providers` | 0.8.0-incubating | +| `GRAVITINO_GCS_CREDENTIAL_FILE_PATH` | `gravitino.iceberg-rest.gcs-credential-file-path` | 0.7.0-incubating | +| `GRAVITINO_S3_ACCESS_KEY` | `gravitino.iceberg-rest.s3-access-key-id` | 0.7.0-incubating | +| `GRAVITINO_S3_SECRET_KEY` | `gravitino.iceberg-rest.s3-secret-access-key` | 0.7.0-incubating | +| `GRAVITINO_S3_REGION` | `gravitino.iceberg-rest.s3-region` | 0.7.0-incubating | +| `GRAVITINO_S3_ROLE_ARN` | `gravitino.iceberg-rest.s3-role-arn` | 0.7.0-incubating | +| `GRAVITINO_S3_EXTERNAL_ID` | `gravitino.iceberg-rest.s3-external-id` | 0.7.0-incubating | +| `GRAVITINO_S3_TOKEN_SERVICE_ENDPOINT` | `gravitino.iceberg-rest.s3-token-service-endpoint` | 0.8.0-incubating | +| `GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME` | `gravitino.iceberg-rest.azure-storage-account-name` | 0.8.0-incubating | +| `GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY` | `gravitino.iceberg-rest.azure-storage-account-key` | 0.8.0-incubating | +| `GRAVITINO_AZURE_TENANT_ID` | `gravitino.iceberg-rest.azure-tenant-id` | 0.8.0-incubating | +| `GRAVITINO_AZURE_CLIENT_ID` | `gravitino.iceberg-rest.azure-client-id` | 0.8.0-incubating | +| `GRAVITINO_AZURE_CLIENT_SECRET` | `gravitino.iceberg-rest.azure-client-secret` | 0.8.0-incubating | Or build it manually to add custom configuration or logics: diff --git a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java index 60a7491b854..638d0c6d311 100644 --- a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java +++ b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/IcebergConfig.java @@ -239,9 +239,13 @@ public class IcebergConfig extends Config implements OverwriteDefaultConfig { .toSequence() .createWithDefault(Collections.emptyList()); + @Deprecated public static final ConfigEntry CREDENTIAL_PROVIDER_TYPE = new ConfigBuilder(CredentialConstants.CREDENTIAL_PROVIDER_TYPE) - .doc("The credential provider type for Iceberg") + .doc( + String.format( + "Deprecated, please use %s instead, The credential provider type for Iceberg", + CredentialConstants.CREDENTIAL_PROVIDERS)) .version(ConfigConstants.VERSION_0_7_0) .stringConf() .create(); diff --git a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java index 0ed62b26f7f..d444c55a750 100644 --- a/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java +++ b/iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/ops/IcebergCatalogWrapper.java @@ -19,23 +19,19 @@ package org.apache.gravitino.iceberg.common.ops; import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableSet; import java.sql.Driver; import java.sql.DriverManager; import java.util.Locale; import java.util.Map; import java.util.Optional; -import java.util.Set; import java.util.function.Supplier; import lombok.Getter; import lombok.Setter; import org.apache.commons.lang3.StringUtils; -import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; import org.apache.gravitino.iceberg.common.IcebergCatalogBackend; import org.apache.gravitino.iceberg.common.IcebergConfig; import org.apache.gravitino.iceberg.common.utils.IcebergCatalogUtil; import org.apache.gravitino.utils.IsolatedClassLoader; -import org.apache.gravitino.utils.MapUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.iceberg.Transaction; @@ -62,6 +58,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * A wrapper for Iceberg catalog backend, provides the common interface for Iceberg REST server and + * Gravitino Iceberg catalog. + */ public class IcebergCatalogWrapper implements AutoCloseable { public static final Logger LOG = LoggerFactory.getLogger(IcebergCatalogWrapper.class); @@ -70,14 +70,7 @@ public class IcebergCatalogWrapper implements AutoCloseable { private SupportsNamespaces asNamespaceCatalog; private final IcebergCatalogBackend catalogBackend; private String catalogUri = null; - private Map catalogConfigToClients; private Map catalogPropertiesMap; - private static final Set catalogPropertiesToClientKeys = - ImmutableSet.of( - IcebergConstants.IO_IMPL, - IcebergConstants.AWS_S3_REGION, - IcebergConstants.ICEBERG_S3_ENDPOINT, - IcebergConstants.ICEBERG_OSS_ENDPOINT); public IcebergCatalogWrapper(IcebergConfig icebergConfig) { this.catalogBackend = @@ -97,10 +90,6 @@ public IcebergCatalogWrapper(IcebergConfig icebergConfig) { if (catalog instanceof SupportsNamespaces) { this.asNamespaceCatalog = (SupportsNamespaces) catalog; } - this.catalogConfigToClients = - MapUtils.getFilteredMap( - icebergConfig.getIcebergCatalogProperties(), - key -> catalogPropertiesToClientKeys.contains(key)); this.catalogPropertiesMap = icebergConfig.getIcebergCatalogProperties(); } @@ -307,14 +296,7 @@ private void closePostgreSQLCatalogResource() { // Some io and security configuration should pass to Iceberg REST client private LoadTableResponse injectTableConfig(Supplier supplier) { LoadTableResponse loadTableResponse = supplier.get(); - return LoadTableResponse.builder() - .withTableMetadata(loadTableResponse.tableMetadata()) - .addAllConfig(getCatalogConfigToClient()) - .build(); - } - - private Map getCatalogConfigToClient() { - return catalogConfigToClients; + return LoadTableResponse.builder().withTableMetadata(loadTableResponse.tableMetadata()).build(); } @Getter diff --git a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/CatalogWrapperForREST.java b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/CatalogWrapperForREST.java new file mode 100644 index 00000000000..8ae7bd66ddc --- /dev/null +++ b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/CatalogWrapperForREST.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.iceberg.service; + +import com.google.common.collect.ImmutableSet; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants; +import org.apache.gravitino.credential.CatalogCredentialManager; +import org.apache.gravitino.credential.Credential; +import org.apache.gravitino.credential.CredentialConstants; +import org.apache.gravitino.credential.CredentialPropertyUtils; +import org.apache.gravitino.credential.PathBasedCredentialContext; +import org.apache.gravitino.iceberg.common.IcebergConfig; +import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper; +import org.apache.gravitino.utils.MapUtils; +import org.apache.gravitino.utils.PrincipalUtils; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableProperties; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.ServiceUnavailableException; +import org.apache.iceberg.rest.requests.CreateTableRequest; +import org.apache.iceberg.rest.responses.LoadTableResponse; + +/** Process Iceberg REST specific operations, like credential vending. */ +public class CatalogWrapperForREST extends IcebergCatalogWrapper { + + private final CatalogCredentialManager catalogCredentialManager; + + private final Map catalogConfigToClients; + + private static final Set catalogPropertiesToClientKeys = + ImmutableSet.of( + IcebergConstants.IO_IMPL, + IcebergConstants.AWS_S3_REGION, + IcebergConstants.ICEBERG_S3_ENDPOINT, + IcebergConstants.ICEBERG_OSS_ENDPOINT); + + public CatalogWrapperForREST(String catalogName, IcebergConfig config) { + super(config); + this.catalogConfigToClients = + MapUtils.getFilteredMap( + config.getIcebergCatalogProperties(), + key -> catalogPropertiesToClientKeys.contains(key)); + // To be compatible with old properties + Map catalogProperties = checkForCompatibility(config.getAllConfig()); + this.catalogCredentialManager = new CatalogCredentialManager(catalogName, catalogProperties); + } + + public LoadTableResponse createTable( + Namespace namespace, CreateTableRequest request, boolean requestCredential) { + LoadTableResponse loadTableResponse = super.createTable(namespace, request); + if (requestCredential) { + return injectCredentialConfig( + TableIdentifier.of(namespace, request.name()), loadTableResponse); + } + return loadTableResponse; + } + + public LoadTableResponse loadTable(TableIdentifier identifier, boolean requestCredential) { + LoadTableResponse loadTableResponse = super.loadTable(identifier); + if (requestCredential) { + return injectCredentialConfig(identifier, loadTableResponse); + } + return loadTableResponse; + } + + @Override + public void close() { + if (catalogCredentialManager != null) { + catalogCredentialManager.close(); + } + } + + private Map getCatalogConfigToClient() { + return catalogConfigToClients; + } + + private LoadTableResponse injectCredentialConfig( + TableIdentifier tableIdentifier, LoadTableResponse loadTableResponse) { + TableMetadata tableMetadata = loadTableResponse.tableMetadata(); + String[] path = + Stream.of( + tableMetadata.location(), + tableMetadata.property(TableProperties.WRITE_DATA_LOCATION, ""), + tableMetadata.property(TableProperties.WRITE_METADATA_LOCATION, "")) + .filter(StringUtils::isNotBlank) + .toArray(String[]::new); + + PathBasedCredentialContext context = + new PathBasedCredentialContext( + PrincipalUtils.getCurrentUserName(), ImmutableSet.copyOf(path), Collections.emptySet()); + Credential credential = catalogCredentialManager.getCredential(context); + if (credential == null) { + throw new ServiceUnavailableException("Couldn't generate credential, %s", context); + } + + LOG.info( + "Generate credential: {} for Iceberg table: {}", + credential.credentialType(), + tableIdentifier); + + Map credentialConfig = CredentialPropertyUtils.toIcebergProperties(credential); + return LoadTableResponse.builder() + .withTableMetadata(loadTableResponse.tableMetadata()) + .addAllConfig(loadTableResponse.config()) + .addAllConfig(getCatalogConfigToClient()) + .addAllConfig(credentialConfig) + .build(); + } + + @SuppressWarnings("deprecation") + private Map checkForCompatibility(Map properties) { + HashMap normalizedProperties = new HashMap<>(properties); + String credentialProviderType = properties.get(CredentialConstants.CREDENTIAL_PROVIDER_TYPE); + String credentialProviders = properties.get(CredentialConstants.CREDENTIAL_PROVIDERS); + if (StringUtils.isNotBlank(credentialProviders) + && StringUtils.isNotBlank(credentialProviderType)) { + throw new IllegalArgumentException( + String.format( + "Should not set both %s and %s", + CredentialConstants.CREDENTIAL_PROVIDER_TYPE, + CredentialConstants.CREDENTIAL_PROVIDERS)); + } + + if (StringUtils.isNotBlank(credentialProviderType)) { + LOG.warn( + "%s is deprecated, please use %s instead.", + CredentialConstants.CREDENTIAL_PROVIDER_TYPE, CredentialConstants.CREDENTIAL_PROVIDERS); + normalizedProperties.put(CredentialConstants.CREDENTIAL_PROVIDERS, credentialProviderType); + } + + return normalizedProperties; + } +} diff --git a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/IcebergCatalogWrapperManager.java b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/IcebergCatalogWrapperManager.java index 6e25ceec427..7b3e18109fa 100644 --- a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/IcebergCatalogWrapperManager.java +++ b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/IcebergCatalogWrapperManager.java @@ -27,10 +27,6 @@ import java.util.Optional; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import org.apache.commons.lang3.StringUtils; -import org.apache.gravitino.credential.CredentialProvider; -import org.apache.gravitino.credential.CredentialProviderFactory; -import org.apache.gravitino.credential.CredentialProviderManager; import org.apache.gravitino.iceberg.common.IcebergConfig; import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper; import org.apache.gravitino.iceberg.service.provider.IcebergConfigProvider; @@ -38,17 +34,15 @@ import org.slf4j.LoggerFactory; public class IcebergCatalogWrapperManager implements AutoCloseable { + public static final Logger LOG = LoggerFactory.getLogger(IcebergCatalogWrapperManager.class); - private final Cache icebergCatalogWrapperCache; + private final Cache icebergCatalogWrapperCache; private final IcebergConfigProvider configProvider; - private CredentialProviderManager credentialProviderManager; - public IcebergCatalogWrapperManager( Map properties, IcebergConfigProvider configProvider) { - this.credentialProviderManager = new CredentialProviderManager(); this.configProvider = configProvider; this.icebergCatalogWrapperCache = Caffeine.newBuilder() @@ -61,7 +55,6 @@ public IcebergCatalogWrapperManager( String catalogName = (String) k; LOG.info("Remove IcebergCatalogWrapper cache {}.", catalogName); closeIcebergCatalogWrapper((IcebergCatalogWrapper) v); - credentialProviderManager.unregisterCredentialProvider(catalogName); }) .scheduler( Scheduler.forScheduledExecutorService( @@ -79,44 +72,33 @@ public IcebergCatalogWrapperManager( * ([^/]*\/), end with / * @return the instance of IcebergCatalogWrapper. */ - public IcebergCatalogWrapper getOps(String rawPrefix) { + public CatalogWrapperForREST getOps(String rawPrefix) { String catalogName = IcebergRestUtils.getCatalogName(rawPrefix); return getCatalogWrapper(catalogName); } - public IcebergCatalogWrapper getCatalogWrapper(String catalogName) { - IcebergCatalogWrapper catalogWrapper = + public CatalogWrapperForREST getCatalogWrapper(String catalogName) { + CatalogWrapperForREST catalogWrapperForREST = icebergCatalogWrapperCache.get(catalogName, k -> createCatalogWrapper(catalogName)); // Reload conf to reset UserGroupInformation or icebergTableOps will always use // Simple auth. - catalogWrapper.reloadHadoopConf(); - return catalogWrapper; - } - - public CredentialProvider getCredentialProvider(String catalogName) { - return credentialProviderManager.getCredentialProvider(catalogName); + catalogWrapperForREST.reloadHadoopConf(); + return catalogWrapperForREST; } - @VisibleForTesting - protected IcebergCatalogWrapper createIcebergCatalogWrapper(IcebergConfig icebergConfig) { - return new IcebergCatalogWrapper(icebergConfig); - } - - private IcebergCatalogWrapper createCatalogWrapper(String catalogName) { + private CatalogWrapperForREST createCatalogWrapper(String catalogName) { Optional icebergConfig = configProvider.getIcebergCatalogConfig(catalogName); if (!icebergConfig.isPresent()) { throw new RuntimeException("Couldn't find Iceberg configuration for " + catalogName); } + return createCatalogWrapper(catalogName, icebergConfig.get()); + } - IcebergConfig config = icebergConfig.get(); - String credentialProviderType = config.get(IcebergConfig.CREDENTIAL_PROVIDER_TYPE); - if (StringUtils.isNotBlank(credentialProviderType)) { - CredentialProvider credentialProvider = - CredentialProviderFactory.create(credentialProviderType, config.getAllConfig()); - credentialProviderManager.registerCredentialProvider(catalogName, credentialProvider); - } - - return createIcebergCatalogWrapper(icebergConfig.get()); + // Overriding this method to create a new CatalogWrapperForREST for test; + @VisibleForTesting + protected CatalogWrapperForREST createCatalogWrapper( + String catalogName, IcebergConfig icebergConfig) { + return new CatalogWrapperForREST(catalogName, icebergConfig); } private void closeIcebergCatalogWrapper(IcebergCatalogWrapper catalogWrapper) { diff --git a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/dispatcher/IcebergTableOperationExecutor.java b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/dispatcher/IcebergTableOperationExecutor.java index e6385bfdc6f..31e94ab9f60 100644 --- a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/dispatcher/IcebergTableOperationExecutor.java +++ b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/dispatcher/IcebergTableOperationExecutor.java @@ -42,7 +42,7 @@ public LoadTableResponse createTable( IcebergRequestContext context, Namespace namespace, CreateTableRequest createTableRequest) { return icebergCatalogWrapperManager .getCatalogWrapper(context.catalogName()) - .createTable(namespace, createTableRequest); + .createTable(namespace, createTableRequest, context.requestCredentialVending()); } @Override @@ -74,7 +74,7 @@ public LoadTableResponse loadTable( IcebergRequestContext context, TableIdentifier tableIdentifier) { return icebergCatalogWrapperManager .getCatalogWrapper(context.catalogName()) - .loadTable(tableIdentifier); + .loadTable(tableIdentifier, context.requestCredentialVending()); } @Override diff --git a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableOperations.java b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableOperations.java index 12f9c5055bb..96fc98921f3 100644 --- a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableOperations.java +++ b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/iceberg/service/rest/IcebergTableOperations.java @@ -23,8 +23,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; -import java.util.Map; -import java.util.stream.Stream; import javax.inject.Inject; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.Consumes; @@ -33,7 +31,6 @@ import javax.ws.rs.GET; import javax.ws.rs.HEAD; import javax.ws.rs.HeaderParam; -import javax.ws.rs.NotSupportedException; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; @@ -43,23 +40,14 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import org.apache.commons.lang3.StringUtils; -import org.apache.gravitino.credential.Credential; -import org.apache.gravitino.credential.CredentialConstants; -import org.apache.gravitino.credential.CredentialPropertyUtils; -import org.apache.gravitino.credential.CredentialProvider; -import org.apache.gravitino.credential.CredentialUtils; -import org.apache.gravitino.iceberg.service.IcebergCatalogWrapperManager; import org.apache.gravitino.iceberg.service.IcebergObjectMapper; import org.apache.gravitino.iceberg.service.IcebergRestUtils; import org.apache.gravitino.iceberg.service.dispatcher.IcebergTableOperationDispatcher; import org.apache.gravitino.iceberg.service.metrics.IcebergMetricsManager; import org.apache.gravitino.listener.api.event.IcebergRequestContext; import org.apache.gravitino.metrics.MetricNames; -import org.apache.iceberg.TableMetadata; -import org.apache.iceberg.TableProperties; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.exceptions.ServiceUnavailableException; import org.apache.iceberg.rest.RESTUtil; import org.apache.iceberg.rest.requests.CreateTableRequest; import org.apache.iceberg.rest.requests.ReportMetricsRequest; @@ -79,7 +67,6 @@ public class IcebergTableOperations { @VisibleForTesting public static final String X_ICEBERG_ACCESS_DELEGATION = "X-Iceberg-Access-Delegation"; - private IcebergCatalogWrapperManager icebergCatalogWrapperManager; private IcebergMetricsManager icebergMetricsManager; private ObjectMapper icebergObjectMapper; @@ -89,10 +76,8 @@ public class IcebergTableOperations { @Inject public IcebergTableOperations( - IcebergCatalogWrapperManager icebergCatalogWrapperManager, IcebergMetricsManager icebergMetricsManager, IcebergTableOperationDispatcher tableOperationDispatcher) { - this.icebergCatalogWrapperManager = icebergCatalogWrapperManager; this.icebergMetricsManager = icebergMetricsManager; this.tableOperationDispatcher = tableOperationDispatcher; this.icebergObjectMapper = IcebergObjectMapper.getInstance(); @@ -132,18 +117,11 @@ public Response createTable( createTableRequest, accessDelegation, isCredentialVending); - IcebergRequestContext context = new IcebergRequestContext(httpServletRequest(), catalogName); + IcebergRequestContext context = + new IcebergRequestContext(httpServletRequest(), catalogName, isCredentialVending); LoadTableResponse loadTableResponse = tableOperationDispatcher.createTable(context, icebergNS, createTableRequest); - if (isCredentialVending) { - return IcebergRestUtils.ok( - injectCredentialConfig( - catalogName, - TableIdentifier.of(icebergNS, createTableRequest.name()), - loadTableResponse)); - } else { - return IcebergRestUtils.ok(loadTableResponse); - } + return IcebergRestUtils.ok(loadTableResponse); } @POST @@ -221,15 +199,11 @@ public Response loadTable( isCredentialVending); // todo support snapshots TableIdentifier tableIdentifier = TableIdentifier.of(icebergNS, table); - IcebergRequestContext context = new IcebergRequestContext(httpServletRequest(), catalogName); + IcebergRequestContext context = + new IcebergRequestContext(httpServletRequest(), catalogName, isCredentialVending); LoadTableResponse loadTableResponse = tableOperationDispatcher.loadTable(context, tableIdentifier); - if (isCredentialVending) { - return IcebergRestUtils.ok( - injectCredentialConfig(catalogName, tableIdentifier, loadTableResponse)); - } else { - return IcebergRestUtils.ok(loadTableResponse); - } + return IcebergRestUtils.ok(loadTableResponse); } @HEAD @@ -287,45 +261,6 @@ private String SerializeUpdateTableRequest(UpdateTableRequest updateTableRequest } } - private LoadTableResponse injectCredentialConfig( - String catalogName, TableIdentifier tableIdentifier, LoadTableResponse loadTableResponse) { - CredentialProvider credentialProvider = - icebergCatalogWrapperManager.getCredentialProvider(catalogName); - if (credentialProvider == null) { - throw new NotSupportedException( - "Doesn't support credential vending, please add " - + CredentialConstants.CREDENTIAL_PROVIDER_TYPE - + " to the catalog configurations"); - } - - TableMetadata tableMetadata = loadTableResponse.tableMetadata(); - String[] path = - Stream.of( - tableMetadata.location(), - tableMetadata.property(TableProperties.WRITE_DATA_LOCATION, ""), - tableMetadata.property(TableProperties.WRITE_METADATA_LOCATION, "")) - .filter(StringUtils::isNotBlank) - .toArray(String[]::new); - - Credential credential = CredentialUtils.vendCredential(credentialProvider, path); - if (credential == null) { - throw new ServiceUnavailableException( - "Couldn't generate credential for %s", credentialProvider.credentialType()); - } - - LOG.info( - "Generate credential: {} for Iceberg table: {}", - credential.credentialType(), - tableIdentifier); - - Map credentialConfig = CredentialPropertyUtils.toIcebergProperties(credential); - return LoadTableResponse.builder() - .withTableMetadata(loadTableResponse.tableMetadata()) - .addAllConfig(loadTableResponse.config()) - .addAllConfig(credentialConfig) - .build(); - } - private boolean isCredentialVending(String accessDelegation) { if (StringUtils.isBlank(accessDelegation)) { return false; diff --git a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/listener/api/event/IcebergRequestContext.java b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/listener/api/event/IcebergRequestContext.java index c0849a117aa..c46fcdfd097 100644 --- a/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/listener/api/event/IcebergRequestContext.java +++ b/iceberg/iceberg-rest-server/src/main/java/org/apache/gravitino/listener/api/event/IcebergRequestContext.java @@ -33,19 +33,33 @@ public class IcebergRequestContext { private final String userName; private final String remoteHostName; private final Map httpHeaders; + private final boolean requestCredentialVending; /** - * Constructs a new {@code IcebergRequestContext} with specified HTTP request and catalog name. + * Constructs a new {@code IcebergRequestContext} instance. * * @param httpRequest The HttpServletRequest object containing request details. * @param catalogName The name of the catalog to be accessed in the request. */ public IcebergRequestContext(HttpServletRequest httpRequest, String catalogName) { + this(httpRequest, catalogName, false); + } + + /** + * Constructs a new {@code IcebergRequestContext} instance. + * + * @param httpRequest The HttpServletRequest object containing request details. + * @param catalogName The name of the catalog to be accessed in the request. + * @param requestCredentialVending Whether the request is for credential vending. + */ + public IcebergRequestContext( + HttpServletRequest httpRequest, String catalogName, boolean requestCredentialVending) { this.httpServletRequest = httpRequest; this.remoteHostName = httpRequest.getRemoteHost(); this.httpHeaders = IcebergRestUtils.getHttpHeaders(httpRequest); this.catalogName = catalogName; this.userName = PrincipalUtils.getCurrentUserName(); + this.requestCredentialVending = requestCredentialVending; } /** @@ -84,6 +98,15 @@ public Map httpHeaders() { return httpHeaders; } + /** + * Checks if the request is for credential vending. + * + * @return true if the request is for credential vending, false otherwise. + */ + public boolean requestCredentialVending() { + return requestCredentialVending; + } + /** * Retrieves the HttpServletRequest object. This method is deprecated and should be used * cautiously. diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java index b663251e0e6..52ccb876df9 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java @@ -92,7 +92,7 @@ private Map getADLSConfig() { Map configMap = new HashMap(); configMap.put( - IcebergConfig.ICEBERG_CONFIG_PREFIX + CredentialConstants.CREDENTIAL_PROVIDER_TYPE, + IcebergConfig.ICEBERG_CONFIG_PREFIX + CredentialConstants.CREDENTIAL_PROVIDERS, ADLSTokenCredential.ADLS_TOKEN_CREDENTIAL_TYPE); configMap.put( IcebergConfig.ICEBERG_CONFIG_PREFIX + AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME, diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java index 695b72ed4b3..f999f84f58d 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java @@ -82,7 +82,7 @@ private Map getADLSConfig() { Map configMap = new HashMap(); configMap.put( - IcebergConfig.ICEBERG_CONFIG_PREFIX + CredentialConstants.CREDENTIAL_PROVIDER_TYPE, + IcebergConfig.ICEBERG_CONFIG_PREFIX + CredentialConstants.CREDENTIAL_PROVIDERS, AzureAccountKeyCredential.AZURE_ACCOUNT_KEY_CREDENTIAL_TYPE); configMap.put( IcebergConfig.ICEBERG_CONFIG_PREFIX + AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME, diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSIT.java index 11ee27bf449..523d8773748 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSIT.java @@ -73,7 +73,7 @@ private Map getGCSConfig() { Map configMap = new HashMap(); configMap.put( - IcebergConfig.ICEBERG_CONFIG_PREFIX + CredentialConstants.CREDENTIAL_PROVIDER_TYPE, + IcebergConfig.ICEBERG_CONFIG_PREFIX + CredentialConstants.CREDENTIAL_PROVIDERS, GCSTokenCredential.GCS_TOKEN_CREDENTIAL_TYPE); configMap.put( IcebergConfig.ICEBERG_CONFIG_PREFIX diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java index af70253d84f..4c4b4a953bc 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java @@ -86,7 +86,7 @@ private Map getOSSConfig() { Map configMap = new HashMap(); configMap.put( - IcebergConfig.ICEBERG_CONFIG_PREFIX + CredentialConstants.CREDENTIAL_PROVIDER_TYPE, + IcebergConfig.ICEBERG_CONFIG_PREFIX + CredentialConstants.CREDENTIAL_PROVIDERS, OSSTokenCredential.OSS_TOKEN_CREDENTIAL_TYPE); configMap.put(IcebergConfig.ICEBERG_CONFIG_PREFIX + OSSProperties.GRAVITINO_OSS_REGION, region); configMap.put( diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSSecretIT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSSecretIT.java index cd5c99c46d5..0be69cbe3d7 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSSecretIT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSSecretIT.java @@ -79,7 +79,7 @@ private Map getOSSConfig() { Map configMap = new HashMap(); configMap.put( - IcebergConfig.ICEBERG_CONFIG_PREFIX + CredentialConstants.CREDENTIAL_PROVIDER_TYPE, + IcebergConfig.ICEBERG_CONFIG_PREFIX + CredentialConstants.CREDENTIAL_PROVIDERS, OSSSecretKeyCredential.OSS_SECRET_KEY_CREDENTIAL_TYPE); configMap.put( IcebergConfig.ICEBERG_CONFIG_PREFIX + OSSProperties.GRAVITINO_OSS_ENDPOINT, endpoint); diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3IT.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3IT.java index 7e16273245d..e906018f525 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3IT.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3IT.java @@ -87,7 +87,7 @@ private Map getS3Config() { Map configMap = new HashMap(); configMap.put( - IcebergConfig.ICEBERG_CONFIG_PREFIX + CredentialConstants.CREDENTIAL_PROVIDER_TYPE, + IcebergConfig.ICEBERG_CONFIG_PREFIX + CredentialConstants.CREDENTIAL_PROVIDERS, S3TokenCredential.S3_TOKEN_CREDENTIAL_TYPE); configMap.put(IcebergConfig.ICEBERG_CONFIG_PREFIX + S3Properties.GRAVITINO_S3_REGION, region); configMap.put( diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/TestIcebergCatalogWrapperManager.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/TestIcebergCatalogWrapperManagerForREST.java similarity index 98% rename from iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/TestIcebergCatalogWrapperManager.java rename to iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/TestIcebergCatalogWrapperManagerForREST.java index 85a7fdc04e9..fad31e816d4 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/TestIcebergCatalogWrapperManager.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/TestIcebergCatalogWrapperManagerForREST.java @@ -28,7 +28,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -public class TestIcebergCatalogWrapperManager { +public class TestIcebergCatalogWrapperManagerForREST { private static final String DEFAULT_CATALOG = "memory"; diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergCatalogWrapperForTest.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/CatalogWrapperForTest.java similarity index 89% rename from iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergCatalogWrapperForTest.java rename to iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/CatalogWrapperForTest.java index f6326dd229e..423b52d577d 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergCatalogWrapperForTest.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/CatalogWrapperForTest.java @@ -19,7 +19,7 @@ package org.apache.gravitino.iceberg.service.rest; import org.apache.gravitino.iceberg.common.IcebergConfig; -import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper; +import org.apache.gravitino.iceberg.service.CatalogWrapperForREST; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.TableMetadata; @@ -32,9 +32,9 @@ import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; // Used to override registerTable -public class IcebergCatalogWrapperForTest extends IcebergCatalogWrapper { - public IcebergCatalogWrapperForTest(IcebergConfig icebergConfig) { - super(icebergConfig); +public class CatalogWrapperForTest extends CatalogWrapperForREST { + public CatalogWrapperForTest(String catalogName, IcebergConfig icebergConfig) { + super(catalogName, icebergConfig); } @Override diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergCatalogWrapperManagerForTest.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergCatalogWrapperManagerForTest.java index 361b086d987..445b9f7451d 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergCatalogWrapperManagerForTest.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergCatalogWrapperManagerForTest.java @@ -21,7 +21,7 @@ import java.util.Map; import org.apache.gravitino.iceberg.common.IcebergConfig; -import org.apache.gravitino.iceberg.common.ops.IcebergCatalogWrapper; +import org.apache.gravitino.iceberg.service.CatalogWrapperForREST; import org.apache.gravitino.iceberg.service.IcebergCatalogWrapperManager; import org.apache.gravitino.iceberg.service.provider.IcebergConfigProvider; @@ -33,7 +33,8 @@ public IcebergCatalogWrapperManagerForTest( } @Override - public IcebergCatalogWrapper createIcebergCatalogWrapper(IcebergConfig icebergConfig) { - return new IcebergCatalogWrapperForTest(icebergConfig); + public CatalogWrapperForREST createCatalogWrapper( + String catalogName, IcebergConfig icebergConfig) { + return new CatalogWrapperForTest(catalogName, icebergConfig); } } diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergRestTestUtil.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergRestTestUtil.java index 19309dc05ad..01c063f49c3 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergRestTestUtil.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/IcebergRestTestUtil.java @@ -104,8 +104,7 @@ public static ResourceConfig getIcebergResourceConfig( StaticIcebergConfigProvider.class.getName()); catalogConf.put(String.format("%s.catalog-backend-name", catalogConfigPrefix), PREFIX); catalogConf.put( - CredentialConstants.CREDENTIAL_PROVIDER_TYPE, - DummyCredentialProvider.DUMMY_CREDENTIAL_TYPE); + CredentialConstants.CREDENTIAL_PROVIDERS, DummyCredentialProvider.DUMMY_CREDENTIAL_TYPE); IcebergConfigProvider configProvider = IcebergConfigProviderFactory.create(catalogConf); configProvider.initialize(catalogConf); // used to override register table interface diff --git a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/MockIcebergTableOperations.java b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/MockIcebergTableOperations.java index 9b9bd930634..a6d9539c95b 100644 --- a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/MockIcebergTableOperations.java +++ b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/service/rest/MockIcebergTableOperations.java @@ -21,7 +21,6 @@ import javax.inject.Inject; import javax.servlet.http.HttpServletRequest; -import org.apache.gravitino.iceberg.service.IcebergCatalogWrapperManager; import org.apache.gravitino.iceberg.service.dispatcher.IcebergTableOperationDispatcher; import org.apache.gravitino.iceberg.service.metrics.IcebergMetricsManager; @@ -29,10 +28,9 @@ public class MockIcebergTableOperations extends IcebergTableOperations { @Inject public MockIcebergTableOperations( - IcebergCatalogWrapperManager icebergCatalogWrapperManager, IcebergMetricsManager icebergMetricsManager, IcebergTableOperationDispatcher tableOperationDispatcher) { - super(icebergCatalogWrapperManager, icebergMetricsManager, tableOperationDispatcher); + super(icebergMetricsManager, tableOperationDispatcher); } // HTTP request is null in Jersey test, create a mock request