Merge pull request #1 from baisui1981/tis-release-0.10.1

integer hudi with TIS
qlangtech · Mar 10, 2022 · 8541805 · 8541805
2 parents 84fb390 + 4f43977
commit 8541805
Show file tree

Hide file tree

Showing 9 changed files with 753 additions and 712 deletions.
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/IExtraHadoopFileSystemGetter.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/IExtraHadoopFileSystemGetter.java
@@ -0,0 +1,13 @@
+package org.apache.hudi.common.fs;
+
+import org.apache.hadoop.fs.FileSystem;
+
+/**
+ * @author: 百岁（[email protected]）
+ * @create: 2022-03-02 10:46
+ **/
+public interface IExtraHadoopFileSystemGetter {
+    String HUDI_FILESYSTEM_NAME = "hudi_hdfs";
+
+    FileSystem getHadoopFileSystem(String path);
+}
diff --git a/hudi-flink/pom.xml b/hudi-flink/pom.xml
@@ -25,7 +25,7 @@
   </parent>
   <modelVersion>4.0.0</modelVersion>
 
-  <artifactId>hudi-flink_${scala.binary.version}</artifactId>
+  <artifactId>tis-hudi-flink_${scala.binary.version}</artifactId>
   <packaging>jar</packaging>
 
   <properties>

diff --git a/hudi-kafka-connect/pom.xml b/hudi-kafka-connect/pom.xml
@@ -142,7 +142,7 @@
         <!-- NOTE: This is temp (SchemaProvide dep) until PR3162 lands -->
         <dependency>
             <groupId>org.apache.hudi</groupId>
-            <artifactId>hudi-flink_${scala.binary.version}</artifactId>
+            <artifactId>tis-hudi-flink_${scala.binary.version}</artifactId>
             <version>${project.version}</version>
         </dependency>
         <dependency>
@@ -174,7 +174,7 @@
             <groupId>log4j</groupId>
             <artifactId>log4j</artifactId>
         </dependency>
-        
+
         <!-- Fasterxml -->
         <dependency>
             <groupId>com.fasterxml.jackson.module</groupId>

diff --git a/packaging/hudi-flink-bundle/package.sh b/packaging/hudi-flink-bundle/package.sh
@@ -0,0 +1 @@
+mvn install -DskipTests -Pflink-bundle-shade-hive2
diff --git a/packaging/hudi-flink-bundle/pom.xml b/packaging/hudi-flink-bundle/pom.xml
@@ -25,7 +25,7 @@
     <relativePath>../../pom.xml</relativePath>
   </parent>
   <modelVersion>4.0.0</modelVersion>
-  <artifactId>hudi-flink-bundle_${scala.binary.version}</artifactId>
+  <artifactId>tis-hudi-flink-bundle_${scala.binary.version}_hive_${hive.version}</artifactId>
   <packaging>jar</packaging>
 
   <properties>
@@ -76,7 +76,7 @@
                   <include>org.apache.hudi:hudi-common</include>
                   <include>org.apache.hudi:hudi-client-common</include>
                   <include>org.apache.hudi:hudi-flink-client</include>
-                  <include>org.apache.hudi:hudi-flink_${scala.binary.version}</include>
+                  <include>org.apache.hudi:tis-hudi-flink_${scala.binary.version}</include>
                   <include>org.apache.hudi:hudi-hive-sync</include>
                   <include>org.apache.hudi:hudi-sync-common</include>
                   <include>org.apache.hudi:hudi-hadoop-mr</include>
@@ -142,6 +142,9 @@
                   <include>org.datanucleus:datanucleus-core</include>
                   <include>org.datanucleus:datanucleus-api-jdo</include>
                   <include>org.apache.thrift:libfb303</include>
+                  <!-- baisui add for loading class org/apache/thrift/TBase-->
+                  <include>org.apache.thrift:libthrift</include>
+
                   <include>org.apache.orc:orc-core</include>
                   <include>org.apache.flink:flink-sql-connector-hive-2.3.6_${scala.binary.version}</include>
 
@@ -294,7 +297,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.hudi</groupId>
-      <artifactId>hudi-flink_${scala.binary.version}</artifactId>
+      <artifactId>tis-hudi-flink_${scala.binary.version}</artifactId>
       <version>${project.version}</version>
       <exclusions>
         <exclusion>
@@ -444,6 +447,7 @@
       <groupId>${hive.groupid}</groupId>
       <artifactId>hive-exec</artifactId>
       <version>${hive.version}</version>
+      <scope>${flink.bundle.hive.scope}</scope>
     </dependency>
     <dependency>
       <groupId>${hive.groupid}</groupId>
@@ -661,6 +665,9 @@
         <hive.version>2.3.1</hive.version>
         <flink.bundle.hive.scope>compile</flink.bundle.hive.scope>
       </properties>
+      <activation>
+        <activeByDefault>true</activeByDefault>
+      </activation>
       <dependencies>
         <dependency>
           <groupId>${hive.groupid}</groupId>

diff --git a/packaging/hudi-kafka-connect-bundle/pom.xml b/packaging/hudi-kafka-connect-bundle/pom.xml
@@ -83,7 +83,7 @@
                                     <include>org.apache.hudi:hudi-aws</include>
 
                                     <!-- NOTE: This is temp (SchemaProvide dep) until PR3162 lands -->
-                                    <include>org.apache.hudi:hudi-flink_${scala.binary.version}</include>
+                                    <include>org.apache.hudi:tis-hudi-flink_${scala.binary.version}</include>
                                     <include>org.apache.hudi:flink-core</include>
                                     <include>org.apache.hudi:hudi-flink-client</include>
                                     <include>org.apache.flink:flink-core</include>
@@ -115,7 +115,7 @@
                                     <include>org.objenesis:objenesis</include>
                                     <include>com.esotericsoftware:kryo-shaded</include>
                                     <include>com.esotericsoftware:minlog</include>
-                                    
+
                                     <include>org.apache.hbase:hbase-client</include>
                                     <include>org.apache.hbase:hbase-common</include>
                                     <include>org.apache.hbase:hbase-protocol</include>

diff --git a/packaging/hudi-utilities-bundle/pom.xml b/packaging/hudi-utilities-bundle/pom.xml
@@ -23,7 +23,7 @@
     <relativePath>../../pom.xml</relativePath>
   </parent>
   <modelVersion>4.0.0</modelVersion>
-  <artifactId>hudi-utilities-bundle_${scala.binary.version}</artifactId>
+  <artifactId>tis-hudi-utilities-bundle_${scala.binary.version}</artifactId>
   <packaging>jar</packaging>
 
   <properties>

diff --git a/pom.xml b/pom.xml
@@ -225,6 +225,7 @@
           </dependency>
         </dependencies>
         <configuration>
+          <skip>true</skip>
           <consoleOutput>true</consoleOutput>
           <encoding>UTF-8</encoding>
           <configLocation>style/checkstyle.xml</configLocation>
@@ -374,6 +375,7 @@
           <artifactId>apache-rat-plugin</artifactId>
           <version>${apache-rat-plugin.version}</version>
           <configuration>
+            <skip>true</skip>
             <excludeSubProjects>false</excludeSubProjects>
             <numUnapprovedLicenses>0</numUnapprovedLicenses>
             <licenses>
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		mvn install -DskipTests -Pflink-bundle-shade-hive2