diff --git a/common/pom.xml b/common/pom.xml
index 5757351d9..540101d71 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -167,6 +167,24 @@ under the License.
         <groupId>net.alchim31.maven</groupId>
         <artifactId>scala-maven-plugin</artifactId>
       </plugin>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>build-helper-maven-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>add-shim-source</id>
+            <phase>generate-sources</phase>
+            <goals>
+              <goal>add-source</goal>
+            </goals>
+            <configuration>
+              <sources>
+                <source>src/main/${shims.source}</source>
+              </sources>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
     <resources>
       <resource>
diff --git a/common/src/main/scala/org/apache/comet/shims/ShimBatchReader.scala b/common/src/main/spark-3.x/org/apache/comet/shims/ShimBatchReader.scala
similarity index 100%
rename from common/src/main/scala/org/apache/comet/shims/ShimBatchReader.scala
rename to common/src/main/spark-3.x/org/apache/comet/shims/ShimBatchReader.scala
diff --git a/common/src/main/scala/org/apache/comet/shims/ShimFileFormat.scala b/common/src/main/spark-3.x/org/apache/comet/shims/ShimFileFormat.scala
similarity index 100%
rename from common/src/main/scala/org/apache/comet/shims/ShimFileFormat.scala
rename to common/src/main/spark-3.x/org/apache/comet/shims/ShimFileFormat.scala
diff --git a/common/src/main/scala/org/apache/comet/shims/ShimResolveDefaultColumns.scala b/common/src/main/spark-3.x/org/apache/comet/shims/ShimResolveDefaultColumns.scala
similarity index 100%
rename from common/src/main/scala/org/apache/comet/shims/ShimResolveDefaultColumns.scala
rename to common/src/main/spark-3.x/org/apache/comet/shims/ShimResolveDefaultColumns.scala
diff --git a/pom.xml b/pom.xml
index dd0ccc6e2..6d28c8168 100644
--- a/pom.xml
+++ b/pom.xml
@@ -88,6 +88,7 @@ under the License.
     <argLine>-ea -Xmx4g -Xss4m ${extraJavaTestArgs}</argLine>
     <additional.3_3.test.source>spark-3.3-plus</additional.3_3.test.source>
     <additional.3_4.test.source>spark-3.4</additional.3_4.test.source>
+    <shims.source>spark-3.x</shims.source>
   </properties>
 
   <dependencyManagement>
diff --git a/spark/pom.xml b/spark/pom.xml
index e0be7c656..66ff82909 100644
--- a/spark/pom.xml
+++ b/spark/pom.xml
@@ -250,6 +250,18 @@ under the License.
               </sources>
             </configuration>
           </execution>
+          <execution>
+            <id>add-shim-source</id>
+            <phase>generate-sources</phase>
+            <goals>
+              <goal>add-source</goal>
+            </goals>
+            <configuration>
+              <sources>
+                <source>src/main/${shims.source}</source>
+              </sources>
+            </configuration>
+          </execution>
         </executions>
       </plugin>
     </plugins>
diff --git a/spark/src/main/scala/org/apache/comet/shims/ShimCometBatchScanExec.scala b/spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometBatchScanExec.scala
similarity index 100%
rename from spark/src/main/scala/org/apache/comet/shims/ShimCometBatchScanExec.scala
rename to spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometBatchScanExec.scala
diff --git a/spark/src/main/scala/org/apache/comet/shims/ShimCometBroadcastExchangeExec.scala b/spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometBroadcastExchangeExec.scala
similarity index 100%
rename from spark/src/main/scala/org/apache/comet/shims/ShimCometBroadcastExchangeExec.scala
rename to spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometBroadcastExchangeExec.scala
diff --git a/spark/src/main/scala/org/apache/comet/shims/ShimCometBroadcastHashJoinExec.scala b/spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometBroadcastHashJoinExec.scala
similarity index 100%
rename from spark/src/main/scala/org/apache/comet/shims/ShimCometBroadcastHashJoinExec.scala
rename to spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometBroadcastHashJoinExec.scala
diff --git a/spark/src/main/scala/org/apache/comet/shims/ShimCometScanExec.scala b/spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometScanExec.scala
similarity index 100%
rename from spark/src/main/scala/org/apache/comet/shims/ShimCometScanExec.scala
rename to spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometScanExec.scala
diff --git a/spark/src/main/scala/org/apache/comet/shims/ShimCometShuffleExchangeExec.scala b/spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometShuffleExchangeExec.scala
similarity index 100%
rename from spark/src/main/scala/org/apache/comet/shims/ShimCometShuffleExchangeExec.scala
rename to spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometShuffleExchangeExec.scala
diff --git a/spark/src/main/scala/org/apache/comet/shims/ShimCometSparkSessionExtensions.scala b/spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometSparkSessionExtensions.scala
similarity index 100%
rename from spark/src/main/scala/org/apache/comet/shims/ShimCometSparkSessionExtensions.scala
rename to spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometSparkSessionExtensions.scala
diff --git a/spark/src/main/scala/org/apache/comet/shims/ShimCometTakeOrderedAndProjectExec.scala b/spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometTakeOrderedAndProjectExec.scala
similarity index 100%
rename from spark/src/main/scala/org/apache/comet/shims/ShimCometTakeOrderedAndProjectExec.scala
rename to spark/src/main/spark-3.x/org/apache/comet/shims/ShimCometTakeOrderedAndProjectExec.scala
diff --git a/spark/src/main/scala/org/apache/comet/shims/ShimQueryPlanSerde.scala b/spark/src/main/spark-3.x/org/apache/comet/shims/ShimQueryPlanSerde.scala
similarity index 100%
rename from spark/src/main/scala/org/apache/comet/shims/ShimQueryPlanSerde.scala
rename to spark/src/main/spark-3.x/org/apache/comet/shims/ShimQueryPlanSerde.scala
diff --git a/spark/src/main/scala/org/apache/comet/shims/ShimSQLConf.scala b/spark/src/main/spark-3.x/org/apache/comet/shims/ShimSQLConf.scala
similarity index 100%
rename from spark/src/main/scala/org/apache/comet/shims/ShimSQLConf.scala
rename to spark/src/main/spark-3.x/org/apache/comet/shims/ShimSQLConf.scala