From 4aee3bd9df97e2b96a444751ea26a3a9091223f5 Mon Sep 17 00:00:00 2001 From: "Ahmad K. Bawaneh" Date: Wed, 17 Jul 2024 21:44:55 +0300 Subject: [PATCH] Enable embedding sources inside the sourcemaps json as sourcesContent (#9936) This PR updates the rebased version of sourcemaps jar and other dependencies to support embedding the source files content into the source maps sourcesContent field. In doing so, it also removes the protobuf dependency from being needed at runtime on the server for deobfuscating stack traces - only legacy dev mode requires it now. Fixes #9752 Fixes #9778 Fixes #9790 --- dev/build.xml | 16 +-- .../linker/impl/StandardLinkerContext.java | 6 ++ .../gwt/core/linker/SymbolMapsLinker.java | 97 +++++++++++++++++-- .../src/com/google/gwt/dev/CompilePerms.java | 6 +- dev/core/src/com/google/gwt/dev/Compiler.java | 5 +- dev/core/src/com/google/gwt/dev/Link.java | 5 +- .../src/com/google/gwt/dev/Precompile.java | 2 +- .../com/google/gwt/dev/PrecompileOnePerm.java | 5 +- .../src/com/google/gwt/dev/cfg/ModuleDef.java | 19 ++++ .../gwt/dev/jjs/JavaToJavaScriptCompiler.java | 1 + eclipse/dev/.classpath | 2 +- eclipse/dev/codeserver/.classpath | 2 +- eclipse/user/.classpath | 2 +- requestfactory/build.xml | 3 + servlet/build.xml | 12 +-- .../gwt/core/CompilerParameters.gwt.xml | 6 ++ .../core/server/StackTraceDeobfuscator.java | 6 +- 17 files changed, 163 insertions(+), 32 deletions(-) diff --git a/dev/build.xml b/dev/build.xml index 1b5a2ac9ffc..0b181dd75a9 100755 --- a/dev/build.xml +++ b/dev/build.xml @@ -68,11 +68,12 @@ - + + - + @@ -129,7 +130,8 @@ src="${gwt.tools.lib}/eclipse/org.eclipse.jdt.core_3.32.0.v20221108-1853.jar"/> - + + @@ -140,7 +142,7 @@ - + @@ -227,9 +229,11 @@ + location="${gwt.tools.lib}/guava/guava-33.0/guava-33.0.0-jre-rebased.jar"/> + - + selectionProperties; + private final ModuleDef module; public StandardLinkerContext(TreeLogger logger, ModuleDef module, ResourceOracle publicResourceOracle, JsOutputOption outputOption) @@ -143,6 +144,7 @@ public StandardLinkerContext(TreeLogger logger, ModuleDef module, this.moduleLastModified = module.lastModified(); this.publicResourceOracle = publicResourceOracle; this.outputOption = outputOption; + this.module = module; // Sort the linkers into the order they should actually run. linkerClasses = new ArrayList>(); @@ -537,6 +539,10 @@ public void produceOutput(TreeLogger logger, ArtifactSet artifacts, } } + public ModuleDef getModule() { + return module; + } + /** * (Re)instantiate all linkers. */ diff --git a/dev/core/src/com/google/gwt/core/linker/SymbolMapsLinker.java b/dev/core/src/com/google/gwt/core/linker/SymbolMapsLinker.java index edffd14cc7f..b56a65d7f4a 100644 --- a/dev/core/src/com/google/gwt/core/linker/SymbolMapsLinker.java +++ b/dev/core/src/com/google/gwt/core/linker/SymbolMapsLinker.java @@ -15,6 +15,9 @@ */ package com.google.gwt.core.linker; +import static java.util.Objects.isNull; +import static java.util.Objects.nonNull; + import com.google.gwt.core.ext.LinkerContext; import com.google.gwt.core.ext.TreeLogger; import com.google.gwt.core.ext.UnableToCompleteException; @@ -31,17 +34,27 @@ import com.google.gwt.core.ext.linker.SoftPermutation; import com.google.gwt.core.ext.linker.SymbolData; import com.google.gwt.core.ext.linker.SyntheticArtifact; +import com.google.gwt.core.ext.linker.impl.StandardLinkerContext; +import com.google.gwt.dev.cfg.ResourceLoader; +import com.google.gwt.dev.cfg.ResourceLoaders; import com.google.gwt.dev.util.Util; import com.google.gwt.dev.util.collect.HashMap; import com.google.gwt.dev.util.log.speedtracer.CompilerEventType; import com.google.gwt.dev.util.log.speedtracer.SpeedTracerLogger; import com.google.gwt.dev.util.log.speedtracer.SpeedTracerLogger.Event; +import com.google.gwt.thirdparty.debugging.sourcemap.SourceMapConsumerV3; import com.google.gwt.thirdparty.debugging.sourcemap.SourceMapGeneratorV3; -import com.google.gwt.thirdparty.debugging.sourcemap.SourceMapGeneratorV3.ExtensionMergeAction; +import com.google.gwt.thirdparty.debugging.sourcemap.SourceMapParseException; +import com.google.gwt.thirdparty.guava.common.collect.Maps; +import com.google.gwt.util.tools.Utility; import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; import java.io.PrintWriter; import java.io.StringWriter; +import java.net.URISyntaxException; +import java.net.URL; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -162,7 +175,8 @@ public static class SourceMapArtifact extends SyntheticArtifact { private final String sourceRoot; public SourceMapArtifact(int permutationId, int fragment, byte[] js, String sourceRoot) { - super(SymbolMapsLinker.class, permutationId + '/' + sourceMapFilenameForFragment(fragment), js); + super(SymbolMapsLinker.class, permutationId + '/' + + sourceMapFilenameForFragment(fragment), js); this.permutationId = permutationId; this.fragment = fragment; this.js = js; @@ -277,6 +291,7 @@ public ArtifactSet link(TreeLogger logger, LinkerContext context, Event writeSourceMapsEvent = SpeedTracerLogger.start(CompilerEventType.WRITE_SOURCE_MAPS); + StandardLinkerContext stdContext = (StandardLinkerContext) context; for (SourceMapArtifact se : artifacts.find(SourceMapArtifact.class)) { // filename is permutation_id/sourceMap.json String sourceMapString = Util.readStreamAsString(se.getContents(logger)); @@ -312,14 +327,16 @@ public ArtifactSet link(TreeLogger logger, LinkerContext context, totalPrefixLines += op.getNumLines(); } } + // TODO(cromwellian): apply insert and remove edits - sourceMapGenerator.mergeMapSection(totalPrefixLines, 0, sourceMapString, - new ExtensionMergeAction() { - @Override - public Object merge(String extKey, Object oldVal, Object newVal) { - return newVal; - } - }); + if (stdContext.getModule().shouldEmbedSourceMapContents()) { + embedSourcesInSourceMaps(logger, stdContext, artifacts, sourceMapGenerator, + totalPrefixLines, sourceMapString, partialPath); + } else { + sourceMapGenerator.mergeMapSection(totalPrefixLines, 0, sourceMapString, + (extKey, oldVal, newVal) -> newVal); + } + StringWriter stringWriter = new StringWriter(); sourceMapGenerator.appendTo(stringWriter, "sourceMap"); emArt = emitSourceMapString(logger, stringWriter.toString(), partialPath); @@ -335,6 +352,68 @@ public Object merge(String extKey, Object oldVal, Object newVal) { return artifacts; } + private static void embedSourcesInSourceMaps(TreeLogger logger, StandardLinkerContext context, + ArtifactSet artifacts, + SourceMapGeneratorV3 sourceMapGenerator, + int totalPrefixLines, String sourceMapString, + String partialPath) + throws SourceMapParseException { + sourceMapGenerator.setStartingPosition(totalPrefixLines, 0); + SourceMapConsumerV3 section = new SourceMapConsumerV3(); + section.parse(sourceMapString); + section.visitMappings(sourceMapGenerator::addMapping); + + for (Entry entry : section.getExtensions().entrySet()) { + String extensionKey = entry.getKey(); + sourceMapGenerator.addExtension(extensionKey, entry.getValue()); + } + + ResourceLoader resourceLoader = ResourceLoaders.fromContextClassLoader(); + + Map generatedSources = Maps.newHashMap(); + artifacts.find(EmittedArtifact.class) + .forEach(emittedArtifact -> { + if (Visibility.Source == emittedArtifact.getVisibility()) { + generatedSources.put(emittedArtifact.getPartialPath(), emittedArtifact); + } + }); + + for (String sourceFileName : section.getOriginalSources()) { + String content; + InputStream cis = null; + try { + cis = loadSource(logger, sourceFileName, generatedSources, + resourceLoader); + if (isNull(cis)) { + cis = context.getModule().findSourceFile(sourceFileName).openContents(); + } + content = Util.readStreamAsString(cis); + sourceMapGenerator.addSourcesContent(sourceFileName, content); + } catch (UnableToCompleteException | URISyntaxException | IOException e) { + logger.log(TreeLogger.Type.WARN, "Can't write source map " + + partialPath, e); + } finally { + Utility.close(cis); + } + } + } + + private static InputStream loadSource(TreeLogger logger, String sourceFileName, + Map generatedSources, + ResourceLoader resourceLoader) + throws UnableToCompleteException, URISyntaxException, IOException { + if (generatedSources.containsKey(sourceFileName)) { + return generatedSources.get(sourceFileName).getContents(logger); + } else { + // ask the resourceOracle for the file contents and add it + URL resource = resourceLoader.getResource(sourceFileName); + if (nonNull(resource)) { + return resource.openStream(); + } + } + return null; + } + /** * Override to change the manner in which the symbol map is emitted. */ diff --git a/dev/core/src/com/google/gwt/dev/CompilePerms.java b/dev/core/src/com/google/gwt/dev/CompilePerms.java index 3b8a1b042f8..2a65b87b77b 100644 --- a/dev/core/src/com/google/gwt/dev/CompilePerms.java +++ b/dev/core/src/com/google/gwt/dev/CompilePerms.java @@ -380,8 +380,12 @@ private boolean precompileAndCompile(TreeLogger logger, String moduleName, if (precompilation == null) { return false; } + + boolean embedSourcesContent = compilerContext.getModule() + .shouldEmbedSourceMapContents(); + // TODO: move to precompile() after params are refactored - if (!options.shouldSaveSource()) { + if (!options.shouldSaveSource() && !embedSourcesContent) { precompilation.removeSourceArtifacts(logger); } diff --git a/dev/core/src/com/google/gwt/dev/Compiler.java b/dev/core/src/com/google/gwt/dev/Compiler.java index 6e122046bde..31a22f8c197 100644 --- a/dev/core/src/com/google/gwt/dev/Compiler.java +++ b/dev/core/src/com/google/gwt/dev/Compiler.java @@ -193,8 +193,11 @@ public static boolean compile( if (precompilation == null) { return false; } + + boolean embedSourcesContent = compilerContext.getModule() + .shouldEmbedSourceMapContents(); // TODO: move to precompile() after params are refactored - if (!options.shouldSaveSource()) { + if (!options.shouldSaveSource() && !embedSourcesContent) { precompilation.removeSourceArtifacts(branch); } diff --git a/dev/core/src/com/google/gwt/dev/Link.java b/dev/core/src/com/google/gwt/dev/Link.java index d7d00625010..a0e02c09db0 100644 --- a/dev/core/src/com/google/gwt/dev/Link.java +++ b/dev/core/src/com/google/gwt/dev/Link.java @@ -415,7 +415,10 @@ private static void doProduceOutput(TreeLogger logger, ArtifactSet artifacts, linkerContext.produceOutput(logger, artifacts, Visibility.Private, extraFileSet); - if (saveSources) { + boolean embedSourcesContent = linkerContext.getModule() + .shouldEmbedSourceMapContents(); + + if (saveSources && !embedSourcesContent) { // Assume that all source code is available in the compiler's classpath. // (This will have to be adjusted to work with Super Dev Mode.) ResourceLoader loader = ResourceLoaders.fromContextClassLoader(); diff --git a/dev/core/src/com/google/gwt/dev/Precompile.java b/dev/core/src/com/google/gwt/dev/Precompile.java index a81bdc80a0c..8386a61f231 100644 --- a/dev/core/src/com/google/gwt/dev/Precompile.java +++ b/dev/core/src/com/google/gwt/dev/Precompile.java @@ -443,7 +443,7 @@ public boolean run(TreeLogger logger) throws UnableToCompleteException { return false; } // TODO: move to precompile() after params are refactored - if (!options.shouldSaveSource()) { + if (!options.shouldSaveSource() && !module.shouldEmbedSourceMapContents()) { precompilation.removeSourceArtifacts(logger); } Util.writeObjectAsFile(logger, precompilationFile, precompilation); diff --git a/dev/core/src/com/google/gwt/dev/PrecompileOnePerm.java b/dev/core/src/com/google/gwt/dev/PrecompileOnePerm.java index dd679c6948d..9a123d6a5c0 100644 --- a/dev/core/src/com/google/gwt/dev/PrecompileOnePerm.java +++ b/dev/core/src/com/google/gwt/dev/PrecompileOnePerm.java @@ -233,8 +233,11 @@ private boolean precompilePermutation(TreeLogger logger, return false; } + boolean embedSourcesContent = compilerContext.getModule() + .shouldEmbedSourceMapContents(); + // TODO: precompile should do this after we get the parameter passing refactored. - if (!options.shouldSaveSource()) { + if (!options.shouldSaveSource() && embedSourcesContent) { precompilation.removeSourceArtifacts(logger); } diff --git a/dev/core/src/com/google/gwt/dev/cfg/ModuleDef.java b/dev/core/src/com/google/gwt/dev/cfg/ModuleDef.java index 9b91b3bb41a..ff68743b463 100644 --- a/dev/core/src/com/google/gwt/dev/cfg/ModuleDef.java +++ b/dev/core/src/com/google/gwt/dev/cfg/ModuleDef.java @@ -67,6 +67,9 @@ * XML for unit tests. */ public class ModuleDef implements DepsInfoProvider { + + public static final String EMBED_SOURCE_MAPS = "compiler.embedSourceMaps"; + private static final ResourceFilter NON_JAVA_RESOURCES = new ResourceFilter() { @Override public boolean allows(String path) { @@ -628,6 +631,22 @@ public synchronized void setNameOverride(String nameOverride) { this.nameOverride = nameOverride; } + /** + * Checks if embedding sources content inside sourceMaps json is enabled or not. + * @return the boolean value true/false of compiler.embedSourceMaps + * configuration property + */ + public boolean shouldEmbedSourceMapContents() { + return getProperties() + .getConfigurationProperties() + .stream() + .filter(configurationProperty -> EMBED_SOURCE_MAPS.equals( + configurationProperty.getName())) + .findFirst() + .map(configurationProperty -> Boolean.valueOf(configurationProperty.getValue())) + .orElse(false); + } + void addBindingPropertyDefinedValue(BindingProperty bindingProperty, String token) { bindingProperty.addDefinedValue(bindingProperty.getRootCondition(), token); } diff --git a/dev/core/src/com/google/gwt/dev/jjs/JavaToJavaScriptCompiler.java b/dev/core/src/com/google/gwt/dev/jjs/JavaToJavaScriptCompiler.java index a51cd7f9597..5895e377d82 100644 --- a/dev/core/src/com/google/gwt/dev/jjs/JavaToJavaScriptCompiler.java +++ b/dev/core/src/com/google/gwt/dev/jjs/JavaToJavaScriptCompiler.java @@ -1201,6 +1201,7 @@ private String buildEntryMethodHolder(StandardGeneratorContext context, } EntryMethodHolderGenerator entryMethodHolderGenerator = new EntryMethodHolderGenerator(); + context.setCurrentGenerator(EntryMethodHolderGenerator.class); String entryMethodHolderTypeName = entryMethodHolderGenerator.generate(logger, context, module.getCanonicalName()); context.finish(logger); diff --git a/eclipse/dev/.classpath b/eclipse/dev/.classpath index ac771cbca58..a84d801f71e 100644 --- a/eclipse/dev/.classpath +++ b/eclipse/dev/.classpath @@ -21,7 +21,7 @@ - + diff --git a/eclipse/dev/codeserver/.classpath b/eclipse/dev/codeserver/.classpath index 88a6968b93b..bc64bf5c128 100644 --- a/eclipse/dev/codeserver/.classpath +++ b/eclipse/dev/codeserver/.classpath @@ -3,7 +3,7 @@ - + diff --git a/eclipse/user/.classpath b/eclipse/user/.classpath index 75f936204d0..1fa63d0ffcc 100644 --- a/eclipse/user/.classpath +++ b/eclipse/user/.classpath @@ -44,7 +44,7 @@ - + diff --git a/requestfactory/build.xml b/requestfactory/build.xml index c835c86da4e..ba82e804e35 100755 --- a/requestfactory/build.xml +++ b/requestfactory/build.xml @@ -41,6 +41,9 @@ + + + diff --git a/servlet/build.xml b/servlet/build.xml index 47d554892f1..9f23dfcdf5f 100644 --- a/servlet/build.xml +++ b/servlet/build.xml @@ -24,11 +24,11 @@ - - + + + - @@ -70,11 +70,11 @@ - - + + + - diff --git a/user/src/com/google/gwt/core/CompilerParameters.gwt.xml b/user/src/com/google/gwt/core/CompilerParameters.gwt.xml index df4aca4b873..fdeb60705fc 100644 --- a/user/src/com/google/gwt/core/CompilerParameters.gwt.xml +++ b/user/src/com/google/gwt/core/CompilerParameters.gwt.xml @@ -96,6 +96,12 @@ + + + + diff --git a/user/src/com/google/gwt/core/server/StackTraceDeobfuscator.java b/user/src/com/google/gwt/core/server/StackTraceDeobfuscator.java index 776e4cfd964..f6f37ef59f4 100644 --- a/user/src/com/google/gwt/core/server/StackTraceDeobfuscator.java +++ b/user/src/com/google/gwt/core/server/StackTraceDeobfuscator.java @@ -15,9 +15,9 @@ */ package com.google.gwt.core.server; +import com.google.gwt.thirdparty.debugging.sourcemap.OriginalMapping; import com.google.gwt.thirdparty.debugging.sourcemap.SourceMapConsumerFactory; import com.google.gwt.thirdparty.debugging.sourcemap.SourceMapping; -import com.google.gwt.thirdparty.debugging.sourcemap.proto.Mapping; import java.io.BufferedReader; import java.io.File; @@ -299,13 +299,13 @@ public final StackTraceElement resymbolize(StackTraceElement ste, String strongN if (sourceMapCapable && fragmentId != -1 && column != -1) { SourceMapping sourceMapping = loadSourceMap(strongName, fragmentId); if (sourceMapping != null && ste.getLineNumber() > -1) { - Mapping.OriginalMapping mappingForLine = sourceMapping + OriginalMapping mappingForLine = sourceMapping .getMappingForLine(jsLineNumber, column); if (mappingForLine != null) { if (declaringClass == null || declaringClass.equals(ste.getClassName())) { declaringClass = mappingForLine.getOriginalFile(); - methodName = mappingForLine.getIdentifier(); + methodName = mappingForLine.getIdentifier().orElse(null); } fileName = mappingForLine.getOriginalFile(); lineNumber = mappingForLine.getLineNumber();