Attempt to share/deduplicate jar contents

org.eclipse.jdt.core.javac/src/org/eclipse/jdt/core/dom/JavacBindingResolver.java

@@ -101,7 +101,7 @@
  */
 public class JavacBindingResolver extends BindingResolver {
 
-	private final JavacTask javac; // TODO evaluate memory cost of storing the instance
+	private JavacTask javac; // TODO evaluate memory cost of storing the instance
 	// it will probably be better to run the `Enter` and then only extract interesting
 	// date from it.
 	public final Context context;
@@ -443,7 +443,10 @@ private void resolve() {
 					ILog.get().error(e.getMessage(), e);
 				}
 			}
+			// some cleanups to encourage garbage collection
+			JavacCompilationUnitResolver.cleanup(context);
 		}
+		this.javac = null;
 		synchronized (this) {
 			if (this.symbolToDeclaration == null) {
 				Map<Symbol, ASTNode> wipSymbolToDeclaration = new HashMap<>();

org.eclipse.jdt.core.javac/src/org/eclipse/jdt/core/dom/JavacCompilationUnitResolver.java

@@ -25,12 +25,13 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.SortedSet;
+import java.util.TreeSet;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
 import javax.lang.model.element.TypeElement;
 import javax.tools.Diagnostic;
-import javax.tools.Diagnostic.Kind;
 import javax.tools.DiagnosticListener;
 import javax.tools.JavaFileManager;
 import javax.tools.JavaFileObject;
@@ -71,6 +72,7 @@
 import org.eclipse.jdt.internal.core.JavaProject;
 import org.eclipse.jdt.internal.core.dom.ICompilationUnitResolver;
 import org.eclipse.jdt.internal.core.util.BindingKeyParser;
+import org.eclipse.jdt.internal.javac.CachingJarsJavaFileManager;
 import org.eclipse.jdt.internal.javac.JavacProblemConverter;
 import org.eclipse.jdt.internal.javac.JavacUtils;
 import org.eclipse.jdt.internal.javac.UnusedProblemFactory;
@@ -104,12 +106,55 @@
  * @implNote Cannot move to another package because parent class is package visible only
  */
 public class JavacCompilationUnitResolver implements ICompilationUnitResolver {
-	public JavacCompilationUnitResolver() {
-		// 0-arg constructor
+
+	private final class ForwardDiagnosticsAsDOMProblems implements DiagnosticListener<JavaFileObject> {
+		public final Map<JavaFileObject, CompilationUnit> filesToUnits;
+		private final JavacProblemConverter problemConverter;
+
+		private ForwardDiagnosticsAsDOMProblems(Map<JavaFileObject, CompilationUnit> filesToUnits,
+				JavacProblemConverter problemConverter) {
+			this.filesToUnits = filesToUnits;
+			this.problemConverter = problemConverter;
+		}
+
+		@Override
+		public void report(Diagnostic<? extends JavaFileObject> diagnostic) {
+			findTargetDOM(filesToUnits, diagnostic).ifPresent(dom -> {
+				var newProblem = problemConverter.createJavacProblem(diagnostic);
+				if (newProblem != null) {
+					IProblem[] previous = dom.getProblems();
+					IProblem[] newProblems = Arrays.copyOf(previous, previous.length + 1);
+					newProblems[newProblems.length - 1] = newProblem;
+					dom.setProblems(newProblems);
+				}
+			});
+		}
+
+		private static Optional<CompilationUnit> findTargetDOM(Map<JavaFileObject, CompilationUnit> filesToUnits, Object obj) {
+			if (obj == null) {
+				return Optional.empty();
+			}
+			if (obj instanceof JavaFileObject o) {
+				return Optional.ofNullable(filesToUnits.get(o));
+			}
+			if (obj instanceof DiagnosticSource source) {
+				return findTargetDOM(filesToUnits, source.getFile());
+			}
+			if (obj instanceof Diagnostic diag) {
+				return findTargetDOM(filesToUnits, diag.getSource());
+			}
+			return Optional.empty();
+		}
 	}
+
 	private interface GenericRequestor {
 		public void acceptBinding(String bindingKey, IBinding binding);
 	}
+
+	public JavacCompilationUnitResolver() {
+		// 0-arg constructor
+	}
+
 	private List<org.eclipse.jdt.internal.compiler.env.ICompilationUnit> createSourceUnitList(String[] sourceFilePaths, String[] encodings) {
 		// make list of source unit
 		int length = sourceFilePaths.length;
@@ -489,23 +534,12 @@ private Map<org.eclipse.jdt.internal.compiler.env.ICompilationUnit, CompilationU
 		}
 		var compiler = ToolProvider.getSystemJavaCompiler();
 		Context context = new Context();
+		CachingJarsJavaFileManager.preRegister(context);
 		Map<org.eclipse.jdt.internal.compiler.env.ICompilationUnit, CompilationUnit> result = new HashMap<>(sourceUnits.length, 1.f);
 		Map<JavaFileObject, CompilationUnit> filesToUnits = new HashMap<>();
 		final UnusedProblemFactory unusedProblemFactory = new UnusedProblemFactory(new DefaultProblemFactory(), compilerOptions);
 		var problemConverter = new JavacProblemConverter(compilerOptions, context);
-		boolean[] hasParseError = new boolean[] { false };
-		DiagnosticListener<JavaFileObject> diagnosticListener = diagnostic -> {
-			findTargetDOM(filesToUnits, diagnostic).ifPresent(dom -> {
-				hasParseError[0] |= diagnostic.getKind() == Kind.ERROR;
-				var newProblem = problemConverter.createJavacProblem(diagnostic);
-				if (newProblem != null) {
-					IProblem[] previous = dom.getProblems();
-					IProblem[] newProblems = Arrays.copyOf(previous, previous.length + 1);
-					newProblems[newProblems.length - 1] = newProblem;
-					dom.setProblems(newProblems);
-				}
-			});
-		};
+		DiagnosticListener<JavaFileObject> diagnosticListener = new ForwardDiagnosticsAsDOMProblems(filesToUnits, problemConverter);
 		MultiTaskListener.instance(context).add(new TaskListener() {
 			@Override
 			public void finished(TaskEvent e) {
@@ -653,7 +687,8 @@ public Void visitClass(ClassTree node, Void p) {
 					try {
 						rawText = fileObjects.get(i).getCharContent(true).toString();
 					} catch( IOException ioe) {
-						// ignore
+						ILog.get().error(ioe.getMessage(), ioe);
+						return null;
 					}
 					CompilationUnit res = result.get(sourceUnits[i]);
 					AST ast = res.ast;
@@ -740,6 +775,9 @@ public void postVisit(ASTNode node) {
 					ILog.get().error(thrown.getMessage(), thrown);
 				}
 			}
+			if (!resolveBindings) {
+				destroy(context);
+			}
 			if (cachedThrown != null) {
 				throw new RuntimeException(cachedThrown);
 			}
@@ -750,6 +788,24 @@ public void postVisit(ASTNode node) {
 		return result;
 	}
 
+	/// cleans up context after analysis (nothing left to process)
+	/// but remain it usable by bindings by keeping filemanager available.
+	public static void cleanup(Context context) {
+		MultiTaskListener.instance(context).clear();
+		if (context.get(DiagnosticListener.class) instanceof ForwardDiagnosticsAsDOMProblems listener) {
+			listener.filesToUnits.clear(); // no need to keep handle on generated ASTs in the context
+		}
+	}
+	/// destroys the context, it's not usable at all after
+	public void destroy(Context context) {
+		cleanup(context);
+		try {
+			context.get(JavaFileManager.class).close();
+		} catch (IOException e) {
+			ILog.get().error(e.getMessage(), e);
+		}
+	}
+
 	private void addProblemsToDOM(CompilationUnit dom, Collection<CategorizedProblem> problems) {
 		if (problems == null) {
 			return;
@@ -764,22 +820,6 @@ private void addProblemsToDOM(CompilationUnit dom, Collection<CategorizedProblem
 		dom.setProblems(newProblems);
 	}
 
-	private Optional<CompilationUnit> findTargetDOM(Map<JavaFileObject, CompilationUnit> filesToUnits, Object obj) {
-		if (obj == null) {
-			return Optional.empty();
-		}
-		if (obj instanceof JavaFileObject o) {
-			return Optional.ofNullable(filesToUnits.get(o));
-		}
-		if (obj instanceof DiagnosticSource source) {
-			return findTargetDOM(filesToUnits, source.getFile());
-		}
-		if (obj instanceof Diagnostic diag) {
-			return findTargetDOM(filesToUnits, diag.getSource());
-		}
-		return Optional.empty();
-	}
-
 	private AST createAST(Map<String, String> options, int level, Context context, int flags) {
 		AST ast = AST.newAST(level, JavaCore.ENABLED.equals(options.get(JavaCore.COMPILER_PB_ENABLE_PREVIEW_FEATURES)));
 		ast.setFlag(flags);

org.eclipse.jdt.core.javac/src/org/eclipse/jdt/internal/javac/CachingJarsJavaFileManager.java

@@ -0,0 +1,99 @@
+/*******************************************************************************
+* Copyright (c) 2024 Red Hat, Inc. and others.
+* All rights reserved. This program and the accompanying materials
+* are made available under the terms of the Eclipse Public License 2.0
+* which accompanies this distribution, and is available at
+* https://www.eclipse.org/legal/epl-2.0/
+*
+* SPDX-License-Identifier: EPL-2.0
+*******************************************************************************/
+package org.eclipse.jdt.internal.javac;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.nio.file.FileSystem;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import javax.tools.JavaFileManager;
+
+import org.eclipse.core.runtime.ILog;
+
+import com.sun.tools.javac.api.ClientCodeWrapper;
+import com.sun.tools.javac.file.FSInfo;
+import com.sun.tools.javac.file.JavacFileManager;
+import com.sun.tools.javac.util.Context;
+import com.sun.tools.javac.util.Context.Factory;
+
+/// An implementation of [JavacFileManager] suitable for local parsing/resolution.
+/// It allows to reuse the filesystems for the referenced jars so they're not duplicated,
+/// 
+/// Note that the main goal is to override the [#close()] method so it does _not_ close
+/// the underlying ZipFileSystem which might still be in use. 
+public class CachingJarsJavaFileManager extends JavacFileManager {
+
+	private static final ZipFileSystemProviderWithCache zipCache = new ZipFileSystemProviderWithCache();
+
+	/**
+	 * Register a Context.Factory to create a JavacFileManager.
+	 */
+	public static void preRegister(Context context) {
+		context.put(FSInfo.class, new FSInfo() {
+			@Override
+			public synchronized java.nio.file.spi.FileSystemProvider getJarFSProvider() {
+				return CachingJarsJavaFileManager.zipCache;
+			}
+		});
+		context.put(ClientCodeWrapper.class, new ClientCodeWrapper(context) {
+				@Override
+				protected boolean isTrusted(Object o) {
+					return super.isTrusted(o) || o.getClass().getClassLoader() == CachingJarsJavaFileManager.class.getClassLoader();
+				}
+			});
+		context.put(JavaFileManager.class, (Factory<JavaFileManager>)c -> new CachingJarsJavaFileManager(c));
+	}
+
+	/**
+	 * Create a JavacFileManager using a given context, optionally registering
+	 * it as the JavaFileManager for that context.
+	 */
+	public CachingJarsJavaFileManager(Context context) {
+		super(context, true, null);
+		zipCache.register(this);
+	}
+
+	@Override
+	public void close() throws IOException {
+		// closes as much as possible, except the containers as they have a
+		// ref to the shared ZipFileSystem that we don't want to close
+		// To improve: close non-ArchiveContainer containers
+		flush();
+		locations.close();
+		resetOutputFilesWritten();
+		zipCache.closeFileManager(System.identityHashCode(this));
+	}
+
+	Collection<FileSystem> listFilesystemsInArchiveContainers() {
+		Set<FileSystem> res = new HashSet<>();
+		try {
+			Field containerField = JavacFileManager.class.getDeclaredField("containers");
+			containerField.setAccessible(true);
+			if (containerField.get(this) instanceof Map containersMap) {
+				for (Object o : containersMap.values()) {
+					if (o.getClass().getName().equals(JavacFileManager.class.getName() + "$ArchiveContainer")) {
+						Field filesystemField = o.getClass().getDeclaredField("fileSystem");
+						filesystemField.setAccessible(true);
+						if (filesystemField.get(o) instanceof FileSystem fs) {
+							res.add(fs);
+						}
+					}
+				}
+			}
+		} catch (Exception ex) {
+			ILog.get().error(ex.getMessage(), ex);
+		}
+		return res;
+	}
+}

org.eclipse.jdt.core.javac/src/org/eclipse/jdt/internal/javac/JavacUtils.java

@@ -28,6 +28,7 @@
 import java.util.stream.Stream;
 
 import javax.tools.JavaFileManager;
+import javax.tools.StandardJavaFileManager;
 import javax.tools.StandardLocation;
 
 import org.eclipse.core.resources.IProject;
@@ -205,7 +206,7 @@ private static void addDebugInfos(Map<String, String> compilerOptions, Options o
 
 	private static void configurePaths(JavaProject javaProject, Context context, JavacConfig compilerConfig,
 	        File output, boolean isTest) {
-		JavacFileManager fileManager = (JavacFileManager)context.get(JavaFileManager.class);
+		var fileManager = (StandardJavaFileManager)context.get(JavaFileManager.class);
 		try {
 			if (compilerConfig != null && !isEmpty(compilerConfig.annotationProcessorPaths())) {
 				fileManager.setLocation(StandardLocation.ANNOTATION_PROCESSOR_PATH,