Skip to content

Commit

Permalink
Merge branch 'main' of github.com:datastrato/graviton into 5472
Browse files Browse the repository at this point in the history
  • Loading branch information
yuqi1129 committed Jan 7, 2025
2 parents 44014d9 + 6ad3d3b commit f4968bd
Show file tree
Hide file tree
Showing 158 changed files with 8,904 additions and 897 deletions.
40 changes: 11 additions & 29 deletions .github/workflows/auto-cherry-pick.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,60 +7,42 @@ on:
types: ["closed"]

jobs:
cherry_pick_branch_0_5:
runs-on: ubuntu-latest
name: Cherry pick into branch_0.5
if: ${{ contains(github.event.pull_request.labels.*.name, 'branch-0.5') && github.event.pull_request.merged == true }}
steps:
- name: Checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Cherry pick into branch-0.5
uses: carloscastrojumo/[email protected]
with:
branch: branch-0.5
labels: |
cherry-pick
reviewers: |
jerryshao
cherry_pick_branch_0_6:
cherry_pick_branch_0_7:
runs-on: ubuntu-latest
name: Cherry pick into branch_0.6
if: ${{ contains(github.event.pull_request.labels.*.name, 'branch-0.6') && github.event.pull_request.merged == true }}
name: Cherry pick into branch_0.7
if: ${{ contains(github.event.pull_request.labels.*.name, 'branch-0.7') && github.event.pull_request.merged == true }}
steps:
- name: Checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Cherry pick into branch-0.6
- name: Cherry pick into branch-0.7
uses: carloscastrojumo/[email protected]
with:
branch: branch-0.6
branch: branch-0.7
labels: |
cherry-pick
reviewers: |
jerryshao
cherry_pick_branch_0_7:
cherry_pick_branch_0_8:
runs-on: ubuntu-latest
name: Cherry pick into branch_0.7
if: ${{ contains(github.event.pull_request.labels.*.name, 'branch-0.7') && github.event.pull_request.merged == true }}
name: Cherry pick into branch_0.8
if: ${{ contains(github.event.pull_request.labels.*.name, 'branch-0.8') && github.event.pull_request.merged == true }}
steps:
- name: Checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Cherry pick into branch-0.7
- name: Cherry pick into branch-0.8
uses: carloscastrojumo/[email protected]
with:
branch: branch-0.7
branch: branch-0.8
labels: |
cherry-pick
reviewers: |
jerryshao
FANNG1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
1 change: 1 addition & 0 deletions .github/workflows/backend-integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ jobs:
- meta/**
- scripts/**
- server/**
- bundles/**
- server-common/**
- build.gradle.kts
- gradle.properties
Expand Down
89 changes: 89 additions & 0 deletions .github/workflows/gvfs-fuse-build-test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
name: Build gvfs-fuse and testing

# Controls when the workflow will run
on:
push:
branches: [ "main", "branch-*" ]
pull_request:
branches: [ "main", "branch-*" ]
workflow_dispatch:

concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true

jobs:
changes:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: dorny/paths-filter@v2
id: filter
with:
filters: |
source_changes:
- .github/**
- api/**
- bin/**
- catalogs/hadoop/**
- clients/filesystem-fuse/**
- common/**
- conf/**
- core/**
- dev/**
- gradle/**
- meta/**
- scripts/**
- server/**
- server-common/**
- build.gradle.kts
- gradle.properties
- gradlew
- setting.gradle.kts
outputs:
source_changes: ${{ steps.filter.outputs.source_changes }}

# Build for AMD64 architecture
Gvfs-Build:
needs: changes
if: needs.changes.outputs.source_changes == 'true'
runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
matrix:
architecture: [linux/amd64]
java-version: [ 17 ]
env:
PLATFORM: ${{ matrix.architecture }}
steps:
- uses: actions/checkout@v3

- uses: actions/setup-java@v4
with:
java-version: ${{ matrix.java-version }}
distribution: 'temurin'
cache: 'gradle'

- name: Set up QEMU
uses: docker/setup-qemu-action@v2

- name: Check required command
run: |
dev/ci/check_commands.sh
- name: Build and test Gravitino
run: |
./gradlew :clients:filesystem-fuse:build -PenableFuse=true
- name: Free up disk space
run: |
dev/ci/util_free_space.sh
- name: Upload tests reports
uses: actions/upload-artifact@v3
if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
with:
name: Gvfs-fuse integrate-test-reports-${{ matrix.java-version }}
path: |
clients/filesystem-fuse/build/test/log/*.log
4 changes: 4 additions & 0 deletions api/src/main/java/org/apache/gravitino/MetadataObjects.java
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import java.util.Collections;
import java.util.List;
import javax.annotation.Nullable;
import org.apache.commons.lang3.StringUtils;
Expand Down Expand Up @@ -151,6 +152,9 @@ public static MetadataObject parse(String fullName, MetadataObject.Type type) {
StringUtils.isNotBlank(fullName), "Metadata object full name cannot be blank");

List<String> parts = DOT_SPLITTER.splitToList(fullName);
if (type == MetadataObject.Type.ROLE) {
return MetadataObjects.of(Collections.singletonList(fullName), MetadataObject.Type.ROLE);
}

return MetadataObjects.of(parts, type);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ public interface SupportsCredentials {
* org.apache.gravitino.file.Fileset}, {@link org.apache.gravitino.rel.Table}. There will be
* at most one credential for one credential type.
*/
Credential[] getCredentials() throws NoSuchCredentialException;
Credential[] getCredentials();

/**
* Retrieves an {@link Credential} object based on the specified credential type.
Expand Down
15 changes: 15 additions & 0 deletions api/src/test/java/org/apache/gravitino/TestMetadataObjects.java
Original file line number Diff line number Diff line change
Expand Up @@ -84,4 +84,19 @@ public void testColumnObject() {
MetadataObjects.of(
Lists.newArrayList("catalog", "schema", "table"), MetadataObject.Type.COLUMN));
}

@Test
public void testRoleObject() {
MetadataObject roleObject = MetadataObjects.of(null, "role.test", MetadataObject.Type.ROLE);
Assertions.assertEquals("role.test", roleObject.fullName());

MetadataObject roleObject1 = MetadataObjects.of(null, "role", MetadataObject.Type.ROLE);
Assertions.assertEquals("role", roleObject1.fullName());

MetadataObject roleObject2 = MetadataObjects.parse("role.test", MetadataObject.Type.ROLE);
Assertions.assertEquals("role.test", roleObject2.fullName());

MetadataObject roleObject3 = MetadataObjects.parse("role", MetadataObject.Type.ROLE);
Assertions.assertEquals("role", roleObject3.fullName());
}
}
3 changes: 3 additions & 0 deletions build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -501,6 +501,9 @@ subprojects {
exclude("test/**")
}
}
tasks.named("compileJava").configure {
dependsOn("spotlessCheck")
}
}

tasks.rat {
Expand Down
1 change: 1 addition & 0 deletions bundles/aws-bundle/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ tasks.withType(ShadowJar::class.java) {
relocate("org.apache.commons.lang3", "org.apache.gravitino.aws.shaded.org.apache.commons.lang3")
relocate("com.google.common", "org.apache.gravitino.aws.shaded.com.google.common")
relocate("com.fasterxml.jackson", "org.apache.gravitino.aws.shaded.com.fasterxml.jackson")
mergeServiceFiles()
}

tasks.jar {
Expand Down
1 change: 1 addition & 0 deletions bundles/azure-bundle/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ tasks.withType(ShadowJar::class.java) {
relocate("com.fasterxml", "org.apache.gravitino.azure.shaded.com.fasterxml")
relocate("com.google.common", "org.apache.gravitino.azure.shaded.com.google.common")
relocate("org.eclipse.jetty", "org.apache.gravitino.azure.shaded.org.eclipse.jetty")
mergeServiceFiles()
}

tasks.jar {
Expand Down
1 change: 1 addition & 0 deletions bundles/gcp-bundle/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ tasks.withType(ShadowJar::class.java) {
relocate("com.google.common", "org.apache.gravitino.gcp.shaded.com.google.common")
relocate("com.fasterxml", "org.apache.gravitino.gcp.shaded.com.fasterxml")
relocate("org.eclipse.jetty", "org.apache.gravitino.gcp.shaded.org.eclipse.jetty")
mergeServiceFiles()
}

tasks.jar {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,13 @@ private CredentialAccessBoundary getAccessBoundary(
CredentialAccessBoundary.newBuilder();
readBuckets.forEach(
bucket -> {
// Hadoop GCS connector needs to get bucket info
AccessBoundaryRule bucketInfoRule =
AccessBoundaryRule.newBuilder()
.setAvailableResource(toGCSBucketResource(bucket))
.setAvailablePermissions(Arrays.asList("inRole:roles/storage.legacyBucketReader"))
.build();
credentialAccessBoundaryBuilder.addRule(bucketInfoRule);
List<String> readConditions = readExpressions.get(bucket);
AccessBoundaryRule rule =
getAccessBoundaryRule(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,48 +21,74 @@

/* User friendly error messages. */
public class ErrorMessages {
public static final String UNSUPPORTED_COMMAND = "Unsupported or unknown command.";
public static final String UNKNOWN_ENTITY = "Unknown entity.";
public static final String TOO_MANY_ARGUMENTS = "Too many arguments.";
public static final String UNKNOWN_METALAKE = "Unknown metalake name.";
public static final String UNKNOWN_CATALOG = "Unknown catalog name.";
public static final String UNKNOWN_SCHEMA = "Unknown schema name.";
public static final String UNKNOWN_TABLE = "Unknown table name.";
public static final String UNKNOWN_MODEL = "Unknown model name.";
public static final String MALFORMED_NAME = "Malformed entity name.";
public static final String MISSING_NAME = "Missing --name option.";
public static final String MISSING_METALAKE = "Missing --metalake option.";
public static final String MISSING_GROUP = "Missing --group option.";
public static final String MISSING_USER = "Missing --user option.";
public static final String MISSING_ROLE = "Missing --role option.";
public static final String MISSING_TAG = "Missing --tag option.";
public static final String METALAKE_EXISTS = "Metalake already exists.";
public static final String CATALOG_EXISTS = "Catalog already exists.";
public static final String SCHEMA_EXISTS = "Schema already exists.";
public static final String UNKNOWN_USER = "Unknown user.";
public static final String USER_EXISTS = "User already exists.";
public static final String UNKNOWN_GROUP = "Unknown group.";
public static final String GROUP_EXISTS = "Group already exists.";
public static final String UNKNOWN_TAG = "Unknown tag.";
public static final String MULTIPLE_TAG_COMMAND_ERROR =
"Error: The current command only supports one --tag option.";
public static final String TAG_EXISTS = "Tag already exists.";
public static final String UNKNOWN_COLUMN = "Unknown column.";
public static final String COLUMN_EXISTS = "Column already exists.";
public static final String UNKNOWN_TOPIC = "Unknown topic.";
public static final String TOPIC_EXISTS = "Topic already exists.";
public static final String UNKNOWN_FILESET = "Unknown fileset.";
public static final String FILESET_EXISTS = "Fileset already exists.";
public static final String TAG_EMPTY = "Error: Must configure --tag option.";
public static final String UNKNOWN_ROLE = "Unknown role.";
public static final String GROUP_EXISTS = "Group already exists.";
public static final String METALAKE_EXISTS = "Metalake already exists.";
public static final String MODEL_EXISTS = "Model already exists.";
public static final String ROLE_EXISTS = "Role already exists.";
public static final String SCHEMA_EXISTS = "Schema already exists.";
public static final String TABLE_EXISTS = "Table already exists.";
public static final String MODEL_EXISTS = "Model already exists.";
public static final String INVALID_SET_COMMAND =
"Unsupported combination of options either use --name, --user, --group or --property and --value.";
public static final String INVALID_REMOVE_COMMAND =
"Unsupported combination of options either use --name or --property.";
public static final String TAG_EXISTS = "Tag already exists.";
public static final String TOPIC_EXISTS = "Topic already exists.";
public static final String USER_EXISTS = "User already exists.";

public static final String ENTITY_IN_USE = " in use, please disable it first.";

public static final String INVALID_ENABLE_DISABLE =
"Unable to us --enable and --disable at the same time";
public static final String INVALID_OWNER_COMMAND =
"Unsupported combination of options either use --user or --group.";
public static final String INVALID_REMOVE_COMMAND =
"Unsupported combination of options either use --name or --property.";
public static final String INVALID_SET_COMMAND =
"Unsupported combination of options either use --name, --user, --group or --property and --value.";

public static final String HELP_FAILED = "Failed to load help message: ";

public static final String MALFORMED_NAME = "Malformed entity name.";
public static final String MISSING_COLUMN_FILE = "Missing --columnfile option.";
public static final String MISSING_DATATYPE = "Missing --datatype option.";
public static final String MISSING_ENTITIES = "Missing required entity names: ";

public static final String MISSING_GROUP = "Missing --group option.";
public static final String MISSING_METALAKE = "Missing --metalake option.";
public static final String MISSING_NAME = "Missing --name option.";
public static final String MISSING_PRIVILEGES = "Missing --privilege option.";
public static final String MISSING_PROPERTY = "Missing --property option.";
public static final String MISSING_PROPERTY_AND_VALUE = "Missing --property and --value options.";
public static final String MISSING_ROLE = "Missing --role option.";
public static final String MISSING_TAG = "Missing --tag option.";
public static final String MISSING_URI = "Missing --uri option.";
public static final String MISSING_USER = "Missing --user option.";
public static final String MISSING_VALUE = "Missing --value option.";

public static final String MULTIPLE_ROLE_COMMAND_ERROR =
"This command only supports one --role option.";
public static final String MULTIPLE_TAG_COMMAND_ERROR =
"This command only supports one --tag option.";
public static final String MISSING_PROVIDER = "Missing --provider option.";

public static final String REGISTER_FAILED = "Failed to register model: ";

public static final String UNKNOWN_CATALOG = "Unknown catalog name.";
public static final String UNKNOWN_COLUMN = "Unknown column name.";
public static final String UNKNOWN_ENTITY = "Unknown entity.";
public static final String UNKNOWN_FILESET = "Unknown fileset name.";
public static final String UNKNOWN_GROUP = "Unknown group.";
public static final String UNKNOWN_METALAKE = "Unknown metalake name.";
public static final String UNKNOWN_MODEL = "Unknown model name.";
public static final String UNKNOWN_PRIVILEGE = "Unknown privilege";
public static final String UNKNOWN_ROLE = "Unknown role.";
public static final String UNKNOWN_SCHEMA = "Unknown schema name.";
public static final String UNKNOWN_TABLE = "Unknown table name.";
public static final String UNKNOWN_TAG = "Unknown tag.";
public static final String UNKNOWN_TOPIC = "Unknown topic name.";
public static final String UNKNOWN_USER = "Unknown user.";

public static final String PARSE_ERROR = "Error parsing command line: ";
public static final String TOO_MANY_ARGUMENTS = "Too many arguments.";
public static final String UNSUPPORTED_ACTION = "Entity doesn't support this action.";
public static final String UNSUPPORTED_COMMAND = "Unsupported or unknown command.";
}
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ public String getMetalakeName() {
}

System.err.println(ErrorMessages.MISSING_METALAKE);
Main.exit(-1);

return null;
}
Expand Down
Loading

0 comments on commit f4968bd

Please sign in to comment.