diff --git a/java/adapter/orc/pom.xml b/java/adapter/orc/pom.xml
index 605b9871639ea..12ee1065ca463 100644
--- a/java/adapter/orc/pom.xml
+++ b/java/adapter/orc/pom.xml
@@ -131,11 +131,30 @@
     <build>
         <resources>
             <resource>
-                <directory>${arrow.cpp.build.dir}</directory>
+                <directory>${arrow.cpp.build.dir}/arrow_orc_jni</directory>
                 <includes>
                     <include>**/*arrow_orc_jni.*</include>
                 </includes>
             </resource>
         </resources>
+        <plugins>
+            <plugin>
+                <groupId>com.coderplus.maven.plugins</groupId>
+                <artifactId>copy-rename-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>copy-file</id>
+                        <phase>generate-sources</phase>
+                        <goals>
+                            <goal>copy</goal>
+                        </goals>
+                        <configuration>
+                            <sourceFile>${arrow.cpp.build.dir}</sourceFile>
+                            <destinationFile>${arrow.cpp.build.dir}/arrow_orc_jni</destinationFile>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
     </build>
 </project>
diff --git a/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcJniUtils.java b/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcJniUtils.java
index 2701c228709c2..9b599234bdf51 100644
--- a/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcJniUtils.java
+++ b/java/adapter/orc/src/main/java/org/apache/arrow/adapter/orc/OrcJniUtils.java
@@ -39,7 +39,7 @@ static void loadOrcAdapterLibraryFromJar()
     synchronized (OrcJniUtils.class) {
       if (!isLoaded) {
         final String libraryToLoad =
-            getNormalizedArch() + "/" + System.mapLibraryName(LIBRARY_NAME);
+            LIBRARY_NAME + "/" + getNormalizedArch() + "/" + System.mapLibraryName(LIBRARY_NAME);
         final File libraryFile =
             moveFileFromJarToTemp(System.getProperty("java.io.tmpdir"), libraryToLoad, LIBRARY_NAME);
         System.load(libraryFile.getAbsolutePath());
diff --git a/java/c/pom.xml b/java/c/pom.xml
index a999292979d56..2fd39a7ecbe46 100644
--- a/java/c/pom.xml
+++ b/java/c/pom.xml
@@ -76,12 +76,31 @@
     <build>
         <resources>
             <resource>
-                <directory>${arrow.c.jni.dist.dir}</directory>
+                <directory>${arrow.c.jni.dist.dir}/arrow_cdata_jni</directory>
                 <includes>
                     <include>**/*arrow_cdata_jni.*</include>
                 </includes>
             </resource>
         </resources>
+        <plugins>
+            <plugin>
+                <groupId>com.coderplus.maven.plugins</groupId>
+                <artifactId>copy-rename-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>copy-file</id>
+                        <phase>generate-sources</phase>
+                        <goals>
+                            <goal>copy</goal>
+                        </goals>
+                        <configuration>
+                            <sourceFile>${arrow.cpp.build.dir}</sourceFile>
+                            <destinationFile>${arrow.cpp.build.dir}/arrow_cdata_jni</destinationFile>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
     </build>
 
 </project>
diff --git a/java/c/src/main/java/org/apache/arrow/c/jni/JniLoader.java b/java/c/src/main/java/org/apache/arrow/c/jni/JniLoader.java
index e435461349257..ef9f432cf0036 100644
--- a/java/c/src/main/java/org/apache/arrow/c/jni/JniLoader.java
+++ b/java/c/src/main/java/org/apache/arrow/c/jni/JniLoader.java
@@ -80,7 +80,7 @@ private synchronized void loadRemaining() {
 
   private void load(String name) {
     final String libraryToLoad =
-        getNormalizedArch() + "/" + System.mapLibraryName(name);
+        name + "/" + getNormalizedArch() + "/" + System.mapLibraryName(name);
     try {
       File temp = File.createTempFile("jnilib-", ".tmp", new File(System.getProperty("java.io.tmpdir")));
       temp.deleteOnExit();
diff --git a/java/dataset/pom.xml b/java/dataset/pom.xml
index 7f4c93c7453bb..422b3ee923d88 100644
--- a/java/dataset/pom.xml
+++ b/java/dataset/pom.xml
@@ -157,7 +157,7 @@
     <build>
         <resources>
             <resource>
-                <directory>${arrow.cpp.build.dir}</directory>
+                <directory>${arrow.cpp.build.dir}/arrow_dataset_jni</directory>
                 <includes>
                     <include>**/*arrow_dataset_jni.*</include>
                 </includes>
@@ -165,6 +165,23 @@
         </resources>
 
         <plugins>
+            <plugin>
+                <groupId>com.coderplus.maven.plugins</groupId>
+                <artifactId>copy-rename-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>copy-file</id>
+                        <phase>generate-sources</phase>
+                        <goals>
+                            <goal>copy</goal>
+                        </goals>
+                        <configuration>
+                            <sourceFile>${arrow.cpp.build.dir}</sourceFile>
+                            <destinationFile>${arrow.cpp.build.dir}/arrow_dataset_jni</destinationFile>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
             <plugin>
                 <artifactId>maven-surefire-plugin</artifactId>
                 <configuration>
diff --git a/java/dataset/src/main/java/org/apache/arrow/dataset/jni/JniLoader.java b/java/dataset/src/main/java/org/apache/arrow/dataset/jni/JniLoader.java
index a3b31c73e8540..cf2f8fe29e8ba 100644
--- a/java/dataset/src/main/java/org/apache/arrow/dataset/jni/JniLoader.java
+++ b/java/dataset/src/main/java/org/apache/arrow/dataset/jni/JniLoader.java
@@ -79,7 +79,7 @@ private synchronized void loadRemaining() {
 
   private void load(String name) {
     final String libraryToLoad =
-        getNormalizedArch() + "/" + System.mapLibraryName(name);
+        name + "/" + getNormalizedArch() + "/" + System.mapLibraryName(name);
     try {
       File temp = File.createTempFile("jnilib-", ".tmp", new File(System.getProperty("java.io.tmpdir")));
       temp.deleteOnExit();
diff --git a/java/gandiva/pom.xml b/java/gandiva/pom.xml
index 330c156a0346b..46736078d55e2 100644
--- a/java/gandiva/pom.xml
+++ b/java/gandiva/pom.xml
@@ -114,7 +114,7 @@
     <build>
         <resources>
             <resource>
-                <directory>${arrow.cpp.build.dir}</directory>
+                <directory>${arrow.cpp.build.dir}/gandiva_jni</directory>
                 <includes>
                     <include>**/*gandiva_jni.*</include>
                 </includes>
@@ -129,6 +129,23 @@
             </extension>
         </extensions>
         <plugins>
+            <plugin>
+                <groupId>com.coderplus.maven.plugins</groupId>
+                <artifactId>copy-rename-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>copy-file</id>
+                        <phase>generate-sources</phase>
+                        <goals>
+                            <goal>copy</goal>
+                        </goals>
+                        <configuration>
+                            <sourceFile>${arrow.cpp.build.dir}</sourceFile>
+                            <destinationFile>${arrow.cpp.build.dir}/gandiva_jni</destinationFile>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
             <plugin>
                 <groupId>org.xolstice.maven.plugins</groupId>
                 <artifactId>protobuf-maven-plugin</artifactId>
diff --git a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniLoader.java b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniLoader.java
index 2528989f3784b..57748e9c8e1af 100644
--- a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniLoader.java
+++ b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/JniLoader.java
@@ -71,7 +71,7 @@ private static JniLoader setupInstance() throws GandivaException {
   private static void loadGandivaLibraryFromJar(final String tmpDir)
           throws IOException, GandivaException {
     final String libraryToLoad =
-        getNormalizedArch() + "/" + System.mapLibraryName(LIBRARY_NAME);
+        LIBRARY_NAME + "/" + getNormalizedArch() + "/" + System.mapLibraryName(LIBRARY_NAME);
     final File libraryFile = moveFileFromJarToTemp(tmpDir, libraryToLoad, LIBRARY_NAME);
     System.load(libraryFile.getAbsolutePath());
   }
diff --git a/java/pom.xml b/java/pom.xml
index 042488a5b949a..6d50fc160a665 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -605,6 +605,11 @@
           <artifactId>maven-site-plugin</artifactId>
           <version>3.7.1</version>
         </plugin>
+        <plugin>
+          <groupId>com.coderplus.maven.plugins</groupId>
+          <artifactId>copy-rename-maven-plugin</artifactId>
+          <version>1.0</version>
+        </plugin>
       </plugins>
     </pluginManagement>
   </build>