Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rough draft of bundling sourcemaps and original sources #253

Draft
wants to merge 4 commits into
base: main
Choose a base branch
from
Draft

Rough draft of bundling sourcemaps and original sources #253

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
a3a8436
Rough draft of bundling sourcemaps and original sources
niloc132 Feb 15, 2024
4e9754f
Don't copy sources before bundling, plus some cleanup
niloc132 Feb 16, 2024
99b53dc
Compute hashes of bundle and its sourcemap while writing
niloc132 Feb 17, 2024
9f82c26
Add a specific prefix for sourcemap sources
niloc132 Feb 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 8 additions & 10 deletions j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/BundleJarTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,10 @@
import com.google.gson.GsonBuilder;
import com.google.javascript.jscomp.deps.ClosureBundler;
import com.vertispan.j2cl.build.task.*;
import com.vertispan.j2cl.tools.Closure;
import org.apache.commons.io.FileUtils;

import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.PathMatcher;
Expand Down Expand Up @@ -53,7 +50,7 @@ public Task resolve(Project project, Config config) {
.map(inputs(OutputTypes.BUNDLED_JS)),
Stream.of(input(project, OutputTypes.BUNDLED_JS))
)
.map(i -> i.filter(BUNDLE_JS))
.map(i -> i.filter(BUNDLE_JS, withSuffix(".bundle.js.map")))
.collect(Collectors.toUnmodifiableList());

// Sort the projects, to try to include them in order. We can't be sure that all project
Expand Down Expand Up @@ -113,19 +110,20 @@ public void finish(TaskContext taskContext) throws IOException {
Files.copy(bundle.getAbsolutePath(), targetFile, StandardCopyOption.REPLACE_EXISTING);
}

File destSourcesDir = outputDir.toPath().resolve(Closure.SOURCES_DIRECTORY_NAME).toFile();
destSourcesDir.mkdirs();
for (Path dir : jsSources.stream().map(Input::getParentPaths).flatMap(Collection::stream).map(p -> p.resolve(Closure
.SOURCES_DIRECTORY_NAME)).collect(Collectors.toSet())) {
FileUtils.copyDirectory(dir.toFile(), destSourcesDir);
}
// File destSourcesDir = outputDir.toPath().resolve(Closure.SOURCES_DIRECTORY_NAME).toFile();
// destSourcesDir.mkdirs();
// for (Path dir : jsSources.stream().map(Input::getParentPaths).flatMap(Collection::stream).map(p -> p.resolve(Closure
// .SOURCES_DIRECTORY_NAME)).collect(Collectors.toSet())) {
// FileUtils.copyDirectory(dir.toFile(), destSourcesDir);
// }

try {
Gson gson = new GsonBuilder().setPrettyPrinting().create();
String scriptsArray = gson.toJson(sourceOrder.stream()
.flatMap(i -> i.getFilesAndHashes().stream())
.map(CachedPath::getSourcePath)
.map(Path::toString)
.filter(s -> s.endsWith(".js"))
.collect(Collectors.toUnmodifiableList())
);
// unconditionally set this to false, so that our dependency order works, since we're always in BUNDLE now
Expand Down
111 changes: 102 additions & 9 deletions j2cl-tasks/src/main/java/com/vertispan/j2cl/build/provided/ClosureBundleTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.reflect.TypeToken;
import com.google.debugging.sourcemap.SourceMapConsumerV3;
import com.google.debugging.sourcemap.SourceMapGeneratorV3;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.javascript.jscomp.Compiler;
Expand Down Expand Up @@ -32,11 +34,13 @@
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FilterWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.lang.reflect.Type;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
Expand Down Expand Up @@ -121,6 +125,7 @@ public Task resolve(Project project, Config config) {
List<DependencyInfoAndSource> dependencyInfos = new ArrayList<>();
Compiler jsCompiler = new Compiler(System.err);//TODO before merge, write this to the log

Path sourcesPath = context.outputPath().resolve(Closure.SOURCES_DIRECTORY_NAME);
if (incrementalEnabled && context.lastSuccessfulOutput().isPresent()) {
// collect any dep info from disk for existing files
final Map<String, DependencyInfoAndSource> depInfoMap;
Expand All @@ -145,7 +150,7 @@ public Task resolve(Project project, Config config) {
} else {
// ADD or MODIFY
CompilerInput input = new CompilerInput(SourceFile.builder()
.withPath(context.outputPath().resolve(Closure.SOURCES_DIRECTORY_NAME).resolve(change.getSourcePath()))
.withPath(sourcesPath.resolve(change.getSourcePath()))
.withOriginalPath(change.getSourcePath().toString())
.build());
input.setCompiler(jsCompiler);
Expand All @@ -166,7 +171,7 @@ public Task resolve(Project project, Config config) {
for (Input jsInput : js) {
for (CachedPath path : jsInput.getFilesAndHashes()) {
CompilerInput input = new CompilerInput(SourceFile.builder()
.withPath(context.outputPath().resolve(Closure.SOURCES_DIRECTORY_NAME).resolve(path.getSourcePath()))
.withPath(sourcesPath.resolve(path.getSourcePath()))
.withOriginalPath(path.getSourcePath().toString())
.build());
input.setCompiler(jsCompiler);
Expand All @@ -182,6 +187,8 @@ public Task resolve(Project project, Config config) {

// TODO optional/stretch-goal find first change in the list, so we can keep old prefix of bundle output

SourceMapGeneratorV3 sourceMapGenerator = new SourceMapGeneratorV3();

// rebundle all (optional: remaining) files using this already handled sort
ClosureBundler bundler = new ClosureBundler(Transpiler.NULL, new BaseTranspiler(
new BaseTranspiler.CompilerSupplier(
Expand All @@ -195,27 +202,52 @@ public Task resolve(Project project, Config config) {
ImmutableMap.of()
),
""
)).useEval(true);
)).useEval(false);

String sourcemapOutFileName = fileNameKey + ".bundle.js.map";

try (OutputStream outputStream = Files.newOutputStream(Paths.get(outputFile));
BufferedWriter bundleOut = new BufferedWriter(new OutputStreamWriter(outputStream, StandardCharsets.UTF_8))) {
try (OutputStream outputStream = Files.newOutputStream(outputFilePath);
BufferedWriter bundleOut = new BufferedWriter(new OutputStreamWriter(outputStream, StandardCharsets.UTF_8));
LineCountingWriter writer = new LineCountingWriter(bundleOut)) {
for (DependencyInfoAndSource info : sorter.getSortedList()) {
String code = info.getSource();
String name = info.getName();
String sourcemapContents = info.loadSourcemap(sourcesPath);

//TODO do we actually need this?
if (Compiler.isFillFileName(name) && code.isEmpty()) {
continue;
}

// append this file and a comment where it came from
bundleOut.append("//").append(name).append("\n");
bundler.withPath(name).withSourceUrl(Closure.SOURCES_DIRECTORY_NAME + "/" + name).appendTo(bundleOut, info, code);
bundleOut.append("\n");
writer.append("//").append(name).append("\n");

if (sourcemapContents != null) {
sourceMapGenerator.setStartingPosition(writer.getLine(), 0);
SourceMapConsumerV3 section = new SourceMapConsumerV3();
section.parse(sourcemapContents);
section.visitMappings((sourceName, symbolName, sourceStartPosition, startPosition, endPosition) -> sourceMapGenerator.addMapping(Paths.get(name).resolveSibling(sourceName).toString(), symbolName, sourceStartPosition, startPosition, endPosition));
for (String source : section.getOriginalSources()) {
String content = Files.readString(sourcesPath.resolve(name).resolveSibling(source));
sourceMapGenerator.addSourcesContent(Paths.get(name).resolveSibling(source).toString(), content);
}
}

// append this file and a comment where it came from
bundler.withPath(name).appendTo(writer, info, code);
writer.append("\n");
}

// write a reference to our new sourcemaps
// writer.append("// " + writer.getLine()).append("\n");
writer.append("//# sourceMappingURL=").append(sourcemapOutFileName).append('\n');
}

// TODO hash in the name
try (OutputStream outputStream = Files.newOutputStream(outputFilePath.resolveSibling(sourcemapOutFileName));
BufferedWriter smOut = new BufferedWriter(new OutputStreamWriter(outputStream, StandardCharsets.UTF_8))) {
sourceMapGenerator.appendTo(smOut, fileNameKey);
}

// append dependency info to deserialize on some incremental rebuild
try (OutputStream outputStream = Files.newOutputStream(context.outputPath().resolve("depInfo.json"));
BufferedWriter jsonOut = new BufferedWriter(new OutputStreamWriter(outputStream, StandardCharsets.UTF_8))) {
Expand All @@ -238,6 +270,56 @@ public Task resolve(Project project, Config config) {
};
}


public static class LineCountingWriter extends FilterWriter {
private int line;
protected LineCountingWriter(Writer out) {
super(out);
}

public int getLine() {
return line;
}

@Override
public void write(int c) throws IOException {
if (c == '\n') {
line++;
}
super.write(c);
}

@Override
public void write(char[] cbuf, int off, int len) throws IOException {
for (char c : cbuf) {
if (c == '\n') {
line++;
}
}
super.write(cbuf, off, len);
}

@Override
public void write(String str, int off, int len) throws IOException {
str.chars().skip(off).limit(len).forEach(c -> {
if (c == '\n') {
line++;
}
});
super.write(str, off, len);
}

@Override
public void write(char[] cbuf) throws IOException {
for (char c : cbuf) {
if (c == '\n') {
line++;
}
}
super.write(cbuf);
}
}

public interface SourceSupplier {
String get() throws IOException;
}
Expand Down Expand Up @@ -309,6 +391,17 @@ public boolean getHasExternsAnnotation() {
public boolean getHasNoCompileAnnotation() {
return delegate.getHasNoCompileAnnotation();
}

public String loadSourcemap(Path outPath) throws IOException {
String sourceMappingUrlMarker = "//# sourceMappingURL=";
int offset = getSource().lastIndexOf(sourceMappingUrlMarker);
if (offset == -1) {
return null;
}
int urlPos = offset + sourceMappingUrlMarker.length();
String sourcemapName = getSource().substring(urlPos).split("\\s")[0];
return Files.readString(outPath.resolve(getName()).resolveSibling(sourcemapName));
}
}

public static class DependencyInfoFormat implements DependencyInfo {
Expand Down