From f1c3c12dcb1b469ea83e57a528090c4885126578 Mon Sep 17 00:00:00 2001 From: iamdanfox Date: Tue, 1 Dec 2020 14:47:24 +0000 Subject: [PATCH] Support multi-release jars like `one.util:streamex` (#67) Support Multi-Release JARs (https://openjdk.java.net/jeps/238) like `one.util:streamex`. --- .../shadowjar/ShadowJarConfigurationTask.java | 99 +++++++++++++++++-- .../gradle/shadowjar/ShadowJarPlugin.java | 4 +- .../ShadowJarPluginIntegrationSpec.groovy | 45 ++++++++- 3 files changed, 135 insertions(+), 13 deletions(-) diff --git a/src/main/groovy/com/palantir/gradle/shadowjar/ShadowJarConfigurationTask.java b/src/main/groovy/com/palantir/gradle/shadowjar/ShadowJarConfigurationTask.java index 1c0ab4ce..712a55ef 100644 --- a/src/main/groovy/com/palantir/gradle/shadowjar/ShadowJarConfigurationTask.java +++ b/src/main/groovy/com/palantir/gradle/shadowjar/ShadowJarConfigurationTask.java @@ -16,15 +16,24 @@ package com.palantir.gradle.shadowjar; +import com.github.jengelman.gradle.plugins.shadow.relocation.CacheableRelocator; +import com.github.jengelman.gradle.plugins.shadow.relocation.RelocateClassContext; +import com.github.jengelman.gradle.plugins.shadow.relocation.RelocatePathContext; import com.github.jengelman.gradle.plugins.shadow.relocation.SimpleRelocator; import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar; +import com.github.jengelman.gradle.plugins.shadow.transformers.ManifestAppenderTransformer; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Set; import java.util.jar.JarFile; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.zip.ZipEntry; import org.gradle.api.DefaultTask; import org.gradle.api.artifacts.Configuration; import org.gradle.api.artifacts.ResolvedDependency; @@ -35,11 +44,20 @@ import org.gradle.api.tasks.Input; import org.gradle.api.tasks.Internal; import org.gradle.api.tasks.TaskAction; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; // Originally taken from https://github.com/johnrengelman/shadow/blob/d4e649d7dd014bfdd9575bfec92d7e74c3cf1aca/ // src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ConfigureShadowRelocation.groovy public abstract class ShadowJarConfigurationTask extends DefaultTask { + + private static final Logger log = LoggerFactory.getLogger(ShadowJarConfigurationTask.class); + private static final String CLASS_SUFFIX = ".class"; + + // Multi-Release JAR Files are defined in https://openjdk.java.net/jeps/238 + private static final Pattern MULTIRELEASE_JAR_PREFIX = Pattern.compile("^META-INF/versions/\\d+/"); + private final Property shadowJarProperty = getProject().getObjects().property(ShadowJar.class); @@ -69,17 +87,21 @@ public final SetProperty getAcceptedDependencies() { @TaskAction public final void run() { - ShadowJar shadowJar = shadowJarProperty.get(); + ShadowJar shadowJarTask = shadowJarProperty.get(); - shadowJar.getDependencyFilter().include(acceptedDependencies.get()::contains); + shadowJarTask.getDependencyFilter().include(acceptedDependencies.get()::contains); - FileCollection jars = shadowJar.getDependencyFilter().resolve(getConfigurations()); + FileCollection jars = shadowJarTask.getDependencyFilter().resolve(getConfigurations()); Set pathsInJars = jars.getFiles().stream() .flatMap(jar -> { try (JarFile jarFile = new JarFile(jar)) { return Collections.list(jarFile.entries()).stream() - .map(path -> path.getName()) + .filter(entry -> !entry.isDirectory()) + .map(ZipEntry::getName) + .peek(path -> log.debug("Jar '{}' contains entry '{}'", jar.getName(), path)) + .peek(path -> Preconditions.checkState( + !path.startsWith("/"), "Unexpected absolute path '%s' in jar '%s'", path, jar)) .collect(Collectors.toList()) .stream(); } catch (IOException e) { @@ -88,21 +110,78 @@ public final void run() { }) .collect(Collectors.toSet()); - JarFilesRelocator relocator = new JarFilesRelocator(pathsInJars, prefix.get() + "."); - shadowJar.relocate(relocator); + // The Relocator is responsible for fixing the bytecode at callsites *and* filenames of .class files, + // so we have to account for things _calling_ these weird multi-release classes. + Set multiReleaseStuff = pathsInJars.stream() + .flatMap(input -> splitMultiReleasePath(input).stream().skip(1)) + .collect(Collectors.toSet()); + + Set relocatable = Stream.concat(pathsInJars.stream(), multiReleaseStuff.stream()) + .filter(path -> !path.equals("META-INF/MANIFEST.MF")) // don't relocate this! + .collect(Collectors.toSet()); + + shadowJarTask.relocate(new JarFilesRelocator(relocatable, prefix.get() + ".")); + + if (!multiReleaseStuff.isEmpty()) { + try { + shadowJarTask.transform(ManifestAppenderTransformer.class, transformer -> { + // JEP 238 requires this manifest entry + transformer.append("Multi-Release", true); + }); + } catch (ReflectiveOperationException e) { + throw new RuntimeException("Unable to construct ManifestAppenderTransformer", e); + } + } + } + + /** Returns a pair of 'META-INF/versions/9/' and 'com/foo/whatever.class'. */ + private static List splitMultiReleasePath(String input) { + Matcher matcher = MULTIRELEASE_JAR_PREFIX.matcher(input); + if (matcher.find()) { + return ImmutableList.of(input.substring(0, matcher.end()), input.substring(matcher.end())); + } else { + return ImmutableList.of(); + } } + @CacheableRelocator private static final class JarFilesRelocator extends SimpleRelocator { - private final Set jarFilePaths; + private final Set relocatable; - private JarFilesRelocator(Set jarFilePaths, String shadedPrefix) { + private JarFilesRelocator(Set relocatable, String shadedPrefix) { super("", shadedPrefix, ImmutableList.of(), ImmutableList.of()); - this.jarFilePaths = jarFilePaths; + this.relocatable = relocatable; } @Override public boolean canRelocatePath(String path) { - return jarFilePaths.contains(path) || jarFilePaths.contains(path + CLASS_SUFFIX); + return relocatable.contains(path + CLASS_SUFFIX) || relocatable.contains(path); + } + + @Override + public String relocatePath(RelocatePathContext context) { + List maybePair = splitMultiReleasePath(context.getPath()); + if (!maybePair.isEmpty()) { + return relocateMultiReleasePath(maybePair, context); + } + + String output = super.relocatePath(context); + log.debug("relocatePath('{}') -> {}", context.getPath(), output); + return output; + } + + private String relocateMultiReleasePath(List pair, RelocatePathContext context) { + context.setPath(pair.get(1)); + String out = pair.get(0) + super.relocatePath(context); + log.debug("relocateMultiReleasePath('{}') -> {}", context.getPath(), out); + return out; + } + + @Override + public String relocateClass(RelocateClassContext context) { + String output = super.relocateClass(context); + log.debug("relocateClass('{}') -> {}", context.getClassName(), output); + return output; } } } diff --git a/src/main/groovy/com/palantir/gradle/shadowjar/ShadowJarPlugin.java b/src/main/groovy/com/palantir/gradle/shadowjar/ShadowJarPlugin.java index e375998c..390e9009 100644 --- a/src/main/groovy/com/palantir/gradle/shadowjar/ShadowJarPlugin.java +++ b/src/main/groovy/com/palantir/gradle/shadowjar/ShadowJarPlugin.java @@ -238,7 +238,9 @@ private static void configureShadowJarTaskWithGoodDefaults(TaskProvider { // Enable archive with more than 2^16 files shadowJar.setZip64(true); - // This seems like a good default for every java project + + // Multiple jars might have an entry in META-INF/services for the same interface, so we merge them. + // https://imperceptiblethoughts.com/shadow/configuration/merging/#merging-service-descriptor-files shadowJar.mergeServiceFiles(); }); } diff --git a/src/test/groovy/com/palantir/gradle/shadowjar/ShadowJarPluginIntegrationSpec.groovy b/src/test/groovy/com/palantir/gradle/shadowjar/ShadowJarPluginIntegrationSpec.groovy index 2ae527b4..2276132a 100644 --- a/src/test/groovy/com/palantir/gradle/shadowjar/ShadowJarPluginIntegrationSpec.groovy +++ b/src/test/groovy/com/palantir/gradle/shadowjar/ShadowJarPluginIntegrationSpec.groovy @@ -150,6 +150,48 @@ class ShadowJarPluginIntegrationSpec extends IntegrationSpec { assert !jarEntryNames.contains(relocatedClass('org/slf4j/impl/Log4jLoggerFactory.class')) } + def 'should support multi-release jars'() { + // https://www.baeldung.com/java-multi-release-jar + + when: + buildFile << """ + repositories { + mavenCentral() + } + + dependencies { + shadeTransitively 'one.util:streamex:0.7.3' + } + + task extractForAssertions(type: Copy) { + dependsOn publishNebulaPublicationToTestRepoRepository + from zipTree("${MAVEN_ROOT}/com/palantir/bar-baz_quux/asd-fgh/2/asd-fgh-2.jar") + into "\$buildDir/extractForAssertions" + } + """.stripIndent() + + then: + writeHelloWorld() + runTasksAndCheckSuccess('extractForAssertions') + + def jarEntryNames = shadowJarFile().stream() + .map({it.name}) + .collect(Collectors.toCollection({new LinkedHashSet()})) + + assert jarEntryNames.contains( + 'META-INF/versions/9/shadow/com/palantir/bar_baz_quux/asd_fgh/one/util/streamex/VerSpec.class') + assert jarEntryNames.contains( + 'META-INF/versions/9/shadow/com/palantir/bar_baz_quux/asd_fgh/one/util/streamex/Java9Specific.class') + assert !jarEntryNames.contains( + 'META-INF/versions/9/one/util/streamex/VerSpec.class') + assert !jarEntryNames.contains( + 'META-INF/versions/9/one/util/streamex/Java9Specific.class') + + assert shadowJarFile().isMultiRelease() ?: + "The jar manifest must include 'Multi-Release: true', but was '" + + file("build/extractForAssertions/META-INF/MANIFEST.MF").text + "'" + } + def 'should shade known logging implementations iff it is placed in shadeTransitively directly'() { when: def mavenRepo = generateMavenRepo( @@ -320,8 +362,7 @@ class ShadowJarPluginIntegrationSpec extends IntegrationSpec { @CompileStatic private JarFile shadowJarFile() { - return new JarFile( - new File(projectDir, "${MAVEN_ROOT}/com/palantir/bar-baz_quux/asd-fgh/2/asd-fgh-2.jar")) + return new JarFile(file("${MAVEN_ROOT}/com/palantir/bar-baz_quux/asd-fgh/2/asd-fgh-2.jar")) } @CompileStatic