Support multi-release jars like one.util:streamex (#67)

Support Multi-Release JARs (https://openjdk.java.net/jeps/238) like `one.util:streamex`.

src/main/groovy/com/palantir/gradle/shadowjar/ShadowJarConfigurationTask.java

@@ -16,15 +16,24 @@
 
 package com.palantir.gradle.shadowjar;
 
+import com.github.jengelman.gradle.plugins.shadow.relocation.CacheableRelocator;
+import com.github.jengelman.gradle.plugins.shadow.relocation.RelocateClassContext;
+import com.github.jengelman.gradle.plugins.shadow.relocation.RelocatePathContext;
 import com.github.jengelman.gradle.plugins.shadow.relocation.SimpleRelocator;
 import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar;
+import com.github.jengelman.gradle.plugins.shadow.transformers.ManifestAppenderTransformer;
+import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
 import java.util.Set;
 import java.util.jar.JarFile;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import java.util.zip.ZipEntry;
 import org.gradle.api.DefaultTask;
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.artifacts.ResolvedDependency;
@@ -35,11 +44,20 @@
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.Internal;
 import org.gradle.api.tasks.TaskAction;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 // Originally taken from https://github.com/johnrengelman/shadow/blob/d4e649d7dd014bfdd9575bfec92d7e74c3cf1aca/
 // src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ConfigureShadowRelocation.groovy
 public abstract class ShadowJarConfigurationTask extends DefaultTask {
+
+    private static final Logger log = LoggerFactory.getLogger(ShadowJarConfigurationTask.class);
+
     private static final String CLASS_SUFFIX = ".class";
+
+    // Multi-Release JAR Files are defined in https://openjdk.java.net/jeps/238
+    private static final Pattern MULTIRELEASE_JAR_PREFIX = Pattern.compile("^META-INF/versions/\\d+/");
+
     private final Property<ShadowJar> shadowJarProperty =
             getProject().getObjects().property(ShadowJar.class);
 
@@ -69,17 +87,21 @@ public final SetProperty<ResolvedDependency> getAcceptedDependencies() {
 
     @TaskAction
     public final void run() {
-        ShadowJar shadowJar = shadowJarProperty.get();
+        ShadowJar shadowJarTask = shadowJarProperty.get();
 
-        shadowJar.getDependencyFilter().include(acceptedDependencies.get()::contains);
+        shadowJarTask.getDependencyFilter().include(acceptedDependencies.get()::contains);
 
-        FileCollection jars = shadowJar.getDependencyFilter().resolve(getConfigurations());
+        FileCollection jars = shadowJarTask.getDependencyFilter().resolve(getConfigurations());
 
         Set<String> pathsInJars = jars.getFiles().stream()
                 .flatMap(jar -> {
                     try (JarFile jarFile = new JarFile(jar)) {
                         return Collections.list(jarFile.entries()).stream()
-                                .map(path -> path.getName())
+                                .filter(entry -> !entry.isDirectory())
+                                .map(ZipEntry::getName)
+                                .peek(path -> log.debug("Jar '{}' contains entry '{}'", jar.getName(), path))
+                                .peek(path -> Preconditions.checkState(
+                                        !path.startsWith("/"), "Unexpected absolute path '%s' in jar '%s'", path, jar))
                                 .collect(Collectors.toList())
                                 .stream();
                     } catch (IOException e) {
@@ -88,21 +110,78 @@ public final void run() {
                 })
                 .collect(Collectors.toSet());
 
-        JarFilesRelocator relocator = new JarFilesRelocator(pathsInJars, prefix.get() + ".");
-        shadowJar.relocate(relocator);
+        // The Relocator is responsible for fixing the bytecode at callsites *and* filenames of .class files,
+        // so we have to account for things _calling_ these weird multi-release classes.
+        Set<String> multiReleaseStuff = pathsInJars.stream()
+                .flatMap(input -> splitMultiReleasePath(input).stream().skip(1))
+                .collect(Collectors.toSet());
+
+        Set<String> relocatable = Stream.concat(pathsInJars.stream(), multiReleaseStuff.stream())
+                .filter(path -> !path.equals("META-INF/MANIFEST.MF")) // don't relocate this!
+                .collect(Collectors.toSet());
+
+        shadowJarTask.relocate(new JarFilesRelocator(relocatable, prefix.get() + "."));
+
+        if (!multiReleaseStuff.isEmpty()) {
+            try {
+                shadowJarTask.transform(ManifestAppenderTransformer.class, transformer -> {
+                    // JEP 238 requires this manifest entry
+                    transformer.append("Multi-Release", true);
+                });
+            } catch (ReflectiveOperationException e) {
+                throw new RuntimeException("Unable to construct ManifestAppenderTransformer", e);
+            }
+        }
+    }
+
+    /** Returns a pair of 'META-INF/versions/9/' and 'com/foo/whatever.class'. */
+    private static List<String> splitMultiReleasePath(String input) {
+        Matcher matcher = MULTIRELEASE_JAR_PREFIX.matcher(input);
+        if (matcher.find()) {
+            return ImmutableList.of(input.substring(0, matcher.end()), input.substring(matcher.end()));
+        } else {
+            return ImmutableList.of();
+        }
     }
 
+    @CacheableRelocator
     private static final class JarFilesRelocator extends SimpleRelocator {
-        private final Set<String> jarFilePaths;
+        private final Set<String> relocatable;
 
-        private JarFilesRelocator(Set<String> jarFilePaths, String shadedPrefix) {
+        private JarFilesRelocator(Set<String> relocatable, String shadedPrefix) {
             super("", shadedPrefix, ImmutableList.of(), ImmutableList.of());
-            this.jarFilePaths = jarFilePaths;
+            this.relocatable = relocatable;
         }
 
         @Override
         public boolean canRelocatePath(String path) {
-            return jarFilePaths.contains(path) || jarFilePaths.contains(path + CLASS_SUFFIX);
+            return relocatable.contains(path + CLASS_SUFFIX) || relocatable.contains(path);
+        }
+
+        @Override
+        public String relocatePath(RelocatePathContext context) {
+            List<String> maybePair = splitMultiReleasePath(context.getPath());
+            if (!maybePair.isEmpty()) {
+                return relocateMultiReleasePath(maybePair, context);
+            }
+
+            String output = super.relocatePath(context);
+            log.debug("relocatePath('{}') -> {}", context.getPath(), output);
+            return output;
+        }
+
+        private String relocateMultiReleasePath(List<String> pair, RelocatePathContext context) {
+            context.setPath(pair.get(1));
+            String out = pair.get(0) + super.relocatePath(context);
+            log.debug("relocateMultiReleasePath('{}') -> {}", context.getPath(), out);
+            return out;
+        }
+
+        @Override
+        public String relocateClass(RelocateClassContext context) {
+            String output = super.relocateClass(context);
+            log.debug("relocateClass('{}') -> {}", context.getClassName(), output);
+            return output;
         }
     }
 }

src/main/groovy/com/palantir/gradle/shadowjar/ShadowJarPlugin.java

@@ -238,7 +238,9 @@ private static void configureShadowJarTaskWithGoodDefaults(TaskProvider<ShadowJa
         shadowJarProvider.configure(shadowJar -> {
             // Enable archive with more than 2^16 files
             shadowJar.setZip64(true);
-            // This seems like a good default for every java project
+
+            // Multiple jars might have an entry in META-INF/services for the same interface, so we merge them.
+            // https://imperceptiblethoughts.com/shadow/configuration/merging/#merging-service-descriptor-files
             shadowJar.mergeServiceFiles();
         });
     }

src/test/groovy/com/palantir/gradle/shadowjar/ShadowJarPluginIntegrationSpec.groovy

@@ -150,6 +150,48 @@ class ShadowJarPluginIntegrationSpec extends IntegrationSpec {
         assert !jarEntryNames.contains(relocatedClass('org/slf4j/impl/Log4jLoggerFactory.class'))
     }
 
+    def 'should support multi-release jars'() {
+        // https://www.baeldung.com/java-multi-release-jar
+
+        when:
+        buildFile << """
+            repositories {
+                mavenCentral()
+            }
+            
+            dependencies {
+                shadeTransitively 'one.util:streamex:0.7.3'
+            }
+            
+            task extractForAssertions(type: Copy) {
+                dependsOn publishNebulaPublicationToTestRepoRepository
+                from zipTree("${MAVEN_ROOT}/com/palantir/bar-baz_quux/asd-fgh/2/asd-fgh-2.jar")
+                into "\$buildDir/extractForAssertions"
+            }
+        """.stripIndent()
+
+        then:
+        writeHelloWorld()
+        runTasksAndCheckSuccess('extractForAssertions')
+
+        def jarEntryNames = shadowJarFile().stream()
+                .map({it.name})
+                .collect(Collectors.toCollection({new LinkedHashSet()}))
+
+        assert jarEntryNames.contains(
+                'META-INF/versions/9/shadow/com/palantir/bar_baz_quux/asd_fgh/one/util/streamex/VerSpec.class')
+        assert jarEntryNames.contains(
+                'META-INF/versions/9/shadow/com/palantir/bar_baz_quux/asd_fgh/one/util/streamex/Java9Specific.class')
+        assert !jarEntryNames.contains(
+                'META-INF/versions/9/one/util/streamex/VerSpec.class')
+        assert !jarEntryNames.contains(
+                'META-INF/versions/9/one/util/streamex/Java9Specific.class')
+
+        assert shadowJarFile().isMultiRelease() ?:
+                "The jar manifest must include 'Multi-Release: true', but was '" +
+                        file("build/extractForAssertions/META-INF/MANIFEST.MF").text + "'"
+    }
+
     def 'should shade known logging implementations iff it is placed in shadeTransitively directly'() {
         when:
         def mavenRepo = generateMavenRepo(
@@ -320,8 +362,7 @@ class ShadowJarPluginIntegrationSpec extends IntegrationSpec {
 
     @CompileStatic
     private JarFile shadowJarFile() {
-        return new JarFile(
-                new File(projectDir, "${MAVEN_ROOT}/com/palantir/bar-baz_quux/asd-fgh/2/asd-fgh-2.jar"))
+        return new JarFile(file("${MAVEN_ROOT}/com/palantir/bar-baz_quux/asd-fgh/2/asd-fgh-2.jar"))
     }
 
     @CompileStatic