From 7979fab81893f8eb49b5639996864d7056f3be6f Mon Sep 17 00:00:00 2001 From: mmews Date: Wed, 3 Jan 2024 16:47:29 +0100 Subject: [PATCH] add parser for yaml files, check workspaces in pnpm-workspaces.yaml --- .../src/org/eclipse/n4js/N4JSGlobals.java | 5 + .../ProjectDescription.java | 22 ++- .../ProjectDescriptionBuilder.java | 12 +- .../n4js/utils/ProjectDescriptionLoader.java | 34 +++- .../src/org/eclipse/n4js/utils/YamlUtil.java | 102 +++++++++++ .../org/eclipse/n4js/utils/YamlUtilTest.java | 166 ++++++++++++++++++ 6 files changed, 334 insertions(+), 7 deletions(-) create mode 100644 plugins/org.eclipse.n4js/src/org/eclipse/n4js/utils/YamlUtil.java create mode 100644 tests/org.eclipse.n4js.utils.tests/src/org/eclipse/n4js/utils/YamlUtilTest.java diff --git a/plugins/org.eclipse.n4js/src/org/eclipse/n4js/N4JSGlobals.java b/plugins/org.eclipse.n4js/src/org/eclipse/n4js/N4JSGlobals.java index d6ae3d965e..e91e35cc33 100644 --- a/plugins/org.eclipse.n4js/src/org/eclipse/n4js/N4JSGlobals.java +++ b/plugins/org.eclipse.n4js/src/org/eclipse/n4js/N4JSGlobals.java @@ -274,6 +274,11 @@ public final class N4JSGlobals { */ public static final String TS_CONFIG = "tsconfig.json"; + /** + * The name of the pnpm-workspace.yaml file. + */ + public static final String PNPM_WORKSPACE = "pnpm-workspace.yaml"; + /** * All project names of n4js libraries. */ diff --git a/plugins/org.eclipse.n4js/src/org/eclipse/n4js/packagejson/projectDescription/ProjectDescription.java b/plugins/org.eclipse.n4js/src/org/eclipse/n4js/packagejson/projectDescription/ProjectDescription.java index ebc1af0e34..51f985fac1 100644 --- a/plugins/org.eclipse.n4js/src/org/eclipse/n4js/packagejson/projectDescription/ProjectDescription.java +++ b/plugins/org.eclipse.n4js/src/org/eclipse/n4js/packagejson/projectDescription/ProjectDescription.java @@ -66,6 +66,7 @@ public class ProjectDescription extends ImmutableDataClass { private final boolean esm; private final boolean moduleProperty; private final boolean n4jsNature; + private final boolean pnpmWorkspaceRoot; private final boolean yarnWorkspaceRoot; private final boolean isGeneratorEnabledSourceMaps; private final boolean isGeneratorEnabledDts; @@ -86,8 +87,8 @@ public ProjectDescription(FileURI location, FileURI relatedRootlocation, Iterable implementedProjects, String outputPath, Iterable sourceContainers, Iterable moduleFilters, Iterable testedProjects, String definesPackage, boolean nestedNodeModulesFolder, - boolean esm, boolean moduleProperty, boolean n4jsNature, boolean yarnWorkspaceRoot, - boolean isGeneratorEnabledSourceMaps, boolean isGeneratorEnabledDts, + boolean esm, boolean moduleProperty, boolean n4jsNature, boolean pnpmWorkspaceRoot, + boolean yarnWorkspaceRoot, boolean isGeneratorEnabledSourceMaps, boolean isGeneratorEnabledDts, Map generatorRewriteModuleSpecifiers, boolean isGeneratorEnabledRewriteCjsImports, Iterable workspaces, Iterable tsFiles, Iterable tsInclude, Iterable tsExclude) { @@ -121,6 +122,7 @@ public ProjectDescription(FileURI location, FileURI relatedRootlocation, this.esm = esm; this.moduleProperty = moduleProperty; this.n4jsNature = n4jsNature; + this.pnpmWorkspaceRoot = pnpmWorkspaceRoot; this.yarnWorkspaceRoot = yarnWorkspaceRoot; this.isGeneratorEnabledSourceMaps = isGeneratorEnabledSourceMaps; this.isGeneratorEnabledDts = isGeneratorEnabledDts; @@ -162,6 +164,7 @@ public ProjectDescription(ProjectDescription template) { this.esm = template.esm; this.moduleProperty = template.moduleProperty; this.n4jsNature = template.n4jsNature; + this.pnpmWorkspaceRoot = template.pnpmWorkspaceRoot; this.yarnWorkspaceRoot = template.yarnWorkspaceRoot; this.isGeneratorEnabledSourceMaps = template.isGeneratorEnabledSourceMaps; this.isGeneratorEnabledDts = template.isGeneratorEnabledDts; @@ -207,6 +210,7 @@ public ProjectDescriptionBuilder change() { builder.setESM(esm); builder.setModuleProperty(moduleProperty); builder.setN4JSNature(n4jsNature); + builder.setPnpmWorkspaceRoot(pnpmWorkspaceRoot); builder.setYarnWorkspaceRoot(yarnWorkspaceRoot); builder.setGeneratorEnabledSourceMaps(isGeneratorEnabledSourceMaps); builder.setGeneratorEnabledDts(isGeneratorEnabledDts); @@ -387,6 +391,16 @@ public boolean isYarnWorkspaceRoot() { return yarnWorkspaceRoot; } + /** + * True iff the project represented by this project description is the root of a pnpm workspace. This flag will be + * {@code true} iff the directory containing a package.json also contains the pnpm-workspace.yaml file with the + * top-level property "packages", no matter the value (i.e. will be {@code true} even if the value is the empty + * array). + */ + public boolean isPnpmWorkspaceRoot() { + return pnpmWorkspaceRoot; + } + /** Returns true iff source maps should be emitted. */ public boolean isGeneratorEnabledSourceMaps() { return isGeneratorEnabledSourceMaps; @@ -513,6 +527,7 @@ protected boolean computeEquals(Object obj) { && esm == other.esm && moduleProperty == other.moduleProperty && n4jsNature == other.n4jsNature + && pnpmWorkspaceRoot == other.pnpmWorkspaceRoot && yarnWorkspaceRoot == other.yarnWorkspaceRoot && isGeneratorEnabledSourceMaps == other.isGeneratorEnabledSourceMaps && isGeneratorEnabledDts == other.isGeneratorEnabledDts @@ -578,6 +593,9 @@ public void toStringAdditionalProperties(StringBuilder sb) { if (definesPackage != null) { sb.append(" definesPackage: " + definesPackage + "\n"); } + if (pnpmWorkspaceRoot) { + sb.append(" pnpmWorkspaceRoot: true\n"); + } if (yarnWorkspaceRoot) { sb.append(" yarnWorkspaceRoot: true\n"); } diff --git a/plugins/org.eclipse.n4js/src/org/eclipse/n4js/packagejson/projectDescription/ProjectDescriptionBuilder.java b/plugins/org.eclipse.n4js/src/org/eclipse/n4js/packagejson/projectDescription/ProjectDescriptionBuilder.java index cff7a0a6af..876188db42 100644 --- a/plugins/org.eclipse.n4js/src/org/eclipse/n4js/packagejson/projectDescription/ProjectDescriptionBuilder.java +++ b/plugins/org.eclipse.n4js/src/org/eclipse/n4js/packagejson/projectDescription/ProjectDescriptionBuilder.java @@ -57,6 +57,7 @@ public class ProjectDescriptionBuilder { private boolean esm; private boolean moduleProperty; private boolean n4jsNature; + private boolean pnpmWorkspaceRoot; private boolean yarnWorkspaceRoot; private Boolean isGeneratorEnabledSourceMaps; private Boolean isGeneratorEnabledDts; @@ -84,7 +85,7 @@ public ProjectDescription build() { exports, extendedRuntimeEnvironment, providedRuntimeLibraries, requiredRuntimeLibraries, dependencies, implementationId, implementedProjects, outputPath, sourceContainers, moduleFilters, testedProjects, definesPackage, - nestedNodeModulesFolder, esm, moduleProperty, n4jsNature, yarnWorkspaceRoot, + nestedNodeModulesFolder, esm, moduleProperty, n4jsNature, pnpmWorkspaceRoot, yarnWorkspaceRoot, isGeneratorEnabledSourceMaps, isGeneratorEnabledDts, generatorRewriteModuleSpecifiers, isGeneratorEnabledRewriteCjsImports, workspaces, tsFiles, tsInclude, tsExclude); } @@ -378,10 +379,19 @@ public ProjectDescriptionBuilder setN4JSNature(boolean n4jsNature) { return this; } + public boolean isPnpmWorkspaceRoot() { + return pnpmWorkspaceRoot; + } + public boolean isYarnWorkspaceRoot() { return yarnWorkspaceRoot; } + public ProjectDescriptionBuilder setPnpmWorkspaceRoot(boolean pnpmWorkspaceRoot) { + this.pnpmWorkspaceRoot = pnpmWorkspaceRoot; + return this; + } + public ProjectDescriptionBuilder setYarnWorkspaceRoot(boolean yarnWorkspaceRoot) { this.yarnWorkspaceRoot = yarnWorkspaceRoot; return this; diff --git a/plugins/org.eclipse.n4js/src/org/eclipse/n4js/utils/ProjectDescriptionLoader.java b/plugins/org.eclipse.n4js/src/org/eclipse/n4js/utils/ProjectDescriptionLoader.java index d518a4aa52..0e1ed3f551 100644 --- a/plugins/org.eclipse.n4js/src/org/eclipse/n4js/utils/ProjectDescriptionLoader.java +++ b/plugins/org.eclipse.n4js/src/org/eclipse/n4js/utils/ProjectDescriptionLoader.java @@ -23,11 +23,13 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.regex.Pattern; +import org.apache.log4j.Logger; import org.eclipse.emf.common.util.URI; import org.eclipse.emf.common.util.WrappedException; import org.eclipse.emf.ecore.EObject; @@ -54,10 +56,10 @@ import org.eclipse.xtext.resource.XtextResourceSet; import org.eclipse.xtext.util.LazyStringInputStream; import org.eclipse.xtext.util.Pair; -import org.eclipse.xtext.util.RuntimeIOException; import org.eclipse.xtext.util.Strings; import org.eclipse.xtext.util.Tuples; +import com.google.common.collect.Multimap; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; @@ -73,6 +75,7 @@ */ @Singleton public class ProjectDescriptionLoader { + private final static Logger LOGGER = Logger.getLogger(ProjectDescriptionLoader.class); @Inject private Provider resourceSetProvider; @@ -111,6 +114,7 @@ public ProjectDescription loadProjectDescriptionAtLocation(FileURI location, URI if (pdbFromPackageJSON != null) { setInformationFromFileSystem(location, pdbFromPackageJSON); setInformationFromTSConfig(location, pdbFromPackageJSON); + setInformationFromPnpmWorkspace(location, pdbFromPackageJSON); pdbFromPackageJSON.setLocation(location); pdbFromPackageJSON.setRelatedRootLocation(relatedRootLocation); @@ -228,7 +232,7 @@ private void setInformationFromFileSystem(FileURI location, ProjectDescriptionBu } /** - * Store some information from {@code tsconfig.json} files iff existent in the project folders root. + * Store some information from {@code tsconfig.json} file iff existent in the project folders root. */ private void setInformationFromTSConfig(FileURI location, ProjectDescriptionBuilder target) { ProjectType type = target.getProjectType(); @@ -248,7 +252,7 @@ private void setInformationFromTSConfig(FileURI location, ProjectDescriptionBuil } } JSONDocument tsconfig = loadJSONAtLocation(path); - JSONValue content = tsconfig.getContent(); + JSONValue content = tsconfig == null ? null : tsconfig.getContent(); if (!(content instanceof JSONObject)) { return; } @@ -273,6 +277,27 @@ private void setInformationFromTSConfig(FileURI location, ProjectDescriptionBuil } } + /** + * Store some information from {@code pnpm-workspaces.yaml} file iff existent in the project folders root. + */ + private void setInformationFromPnpmWorkspace(FileURI location, ProjectDescriptionBuilder target) { + Path path = location.appendSegment(N4JSGlobals.PNPM_WORKSPACE).toFileSystemPath(); + if (!Files.isReadable(path)) { + path = location.appendSegment(N4JSGlobals.PNPM_WORKSPACE + "." + N4JSGlobals.XT_FILE_EXTENSION) + .toFileSystemPath(); + if (!Files.isReadable(path)) { + return; + } + } + + Multimap pnpmWorkspacesYaml = YamlUtil.loadYamlAtLocation(path); + Collection packagesEntries = pnpmWorkspacesYaml.get("packages"); + if (!packagesEntries.isEmpty()) { + target.setPnpmWorkspaceRoot(true); + target.getWorkspaces().addAll(packagesEntries); + } + } + private JSONDocument loadPackageJSONAtLocation(FileURI location) { Path path = location.appendSegment(N4JSGlobals.PACKAGE_JSON).toFileSystemPath(); if (!Files.isReadable(path)) { @@ -299,7 +324,8 @@ private JSONDocument loadJSONAtLocation(Path path) { return packageJSON; } } catch (IOException e) { - throw new RuntimeIOException(e); + LOGGER.error("Could not load " + path.toString(), e); + return null; } } diff --git a/plugins/org.eclipse.n4js/src/org/eclipse/n4js/utils/YamlUtil.java b/plugins/org.eclipse.n4js/src/org/eclipse/n4js/utils/YamlUtil.java new file mode 100644 index 0000000000..64da04b64d --- /dev/null +++ b/plugins/org.eclipse.n4js/src/org/eclipse/n4js/utils/YamlUtil.java @@ -0,0 +1,102 @@ +/** + * Copyright (c) 2024 NumberFour AG. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * NumberFour AG - Initial API and implementation + */ +package org.eclipse.n4js.utils; + +import static com.google.common.base.Strings.isNullOrEmpty; +import static org.eclipse.n4js.utils.Strings.join; +import static org.eclipse.xtext.xbase.lib.IterableExtensions.filter; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.apache.log4j.Logger; + +import com.google.common.collect.LinkedHashMultimap; +import com.google.common.collect.Multimap; + +/** + * Utility for loading yaml files + */ +public class YamlUtil { + private final static Logger LOGGER = Logger.getLogger(YamlUtil.class); + + /** Loads a yaml file from the give location */ + static public Multimap loadYamlAtLocation(Path path) { + try { + List lines = Files.readAllLines(path, StandardCharsets.UTF_8); + return loadYamlFromLines(lines); + + } catch (IOException e) { + LOGGER.error("Could not load " + path.toString(), e); + } + return LinkedHashMultimap.create(); + } + + /** Loads yaml content from the given string */ + static public Multimap loadYamlFromString(String str) { + return loadYamlFromLines(Arrays.asList(str.split("\n"))); + } + + /** Loads yaml content from the given lines */ + static public Multimap loadYamlFromLines(List lines) { + Multimap result = LinkedHashMultimap.create(); + Map stack = new LinkedHashMap<>(); + for (String line : lines) { + if (line.startsWith("---") || line.startsWith("#")) { + continue; + } + int commentIdx = line.indexOf("#"); + if (commentIdx >= 0) { + line = line.substring(0, commentIdx); + } + int leadingSpaces = line.indexOf(line.trim()); + line = line.trim(); + + if (line.startsWith("-")) { + line = line.substring(1).trim(); + } + int propNameIdx = line.indexOf(":"); + if (propNameIdx >= 0) { + String propName = line.substring(0, propNameIdx).trim(); + + stack.put(leadingSpaces, propName); + stack.keySet().removeIf(k -> k > leadingSpaces); + + String value = line.substring(propNameIdx + 1).trim(); + if (!isNullOrEmpty(value)) { + value = trimQuotes(value); + String qpn = join(":", stack.values()); + result.put(qpn, value); + } + } else { + line = trimQuotes(line); + String qpn = join(":", e -> e.getValue(), filter(stack.entrySet(), e -> e.getKey() <= leadingSpaces)); + result.put(qpn, line); + } + } + return result; + } + + static private String trimQuotes(String str) { + if (str.startsWith("\"") && str.endsWith("\"")) { + return str.substring(1, str.length() - 1); // no trim + } else if (str.startsWith("'") && str.endsWith("'")) { + return str.substring(1, str.length() - 1); // no trim + } + return str; + } +} diff --git a/tests/org.eclipse.n4js.utils.tests/src/org/eclipse/n4js/utils/YamlUtilTest.java b/tests/org.eclipse.n4js.utils.tests/src/org/eclipse/n4js/utils/YamlUtilTest.java new file mode 100644 index 0000000000..74b51464fb --- /dev/null +++ b/tests/org.eclipse.n4js.utils.tests/src/org/eclipse/n4js/utils/YamlUtilTest.java @@ -0,0 +1,166 @@ +/** + * Copyright (c) 2024 NumberFour AG. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * NumberFour AG - Initial API and implementation + */ +package org.eclipse.n4js.utils; + +import org.junit.Assert; +import org.junit.Test; + +import com.google.common.collect.Multimap; + +/** + * + */ +public class YamlUtilTest { + + @Test + public void testPnpmWorkspaceYaml() { + Multimap map = YamlUtil.loadYamlFromString(""" + # NOTE: Experimental, local packages from the workspace are preferred over packages from the registry + prefer-workspace-packages: true + packages: + - 'packages/*' + """); + + String result = Strings.join("\n", e -> e.getKey() + " => " + e.getValue(), map.entries()); + + Assert.assertEquals(""" + prefer-workspace-packages => true + packages => packages/*""", result); + } + + @Test + public void testYaml1() { + Multimap map = YamlUtil.loadYamlFromString(""" + --- + # A sample yaml file + company: spacelift + domain: + - devops + - devsecops + tutorial: + - yaml: + name: "YAML Ain't Markup Language" + type: awesome + born: 2001 + - json: + name: JavaScript Object Notation + type: great + born: 2001 + - xml: + name: Extensible Markup Language + type: good + born: 1996 + author: omkarbirade + published: true + """); + + String result = Strings.join("\n", e -> e.getKey() + " => " + e.getValue(), map.entries()); + + Assert.assertEquals(""" + company => spacelift + domain => devops + domain => devsecops + tutorial:yaml:name => YAML Ain't Markup Language + tutorial:yaml:type => awesome + tutorial:yaml:born => 2001 + tutorial:json:name => JavaScript Object Notation + tutorial:json:type => great + tutorial:json:born => 2001 + tutorial:xml:name => Extensible Markup Language + tutorial:xml:type => good + tutorial:xml:born => 1996 + author => omkarbirade + published => true""", result); + } + + @Test + public void testYaml1WithComments() { + Multimap map = YamlUtil.loadYamlFromString(""" + --- + # key: value [mapping] + company: spacelift + # key: value is an array [sequence] + domain: + - devops + - devsecops + tutorial: + - yaml: + name: "YAML Ain't Markup Language" #string [literal] + type: awesome #string [literal] + born: 2001 #number [literal] + - json: + name: JavaScript Object Notation #string [literal] + type: great #string [literal] + born: 2001 #number [literal] + - xml: + name: Extensible Markup Language #string [literal] + type: good #string [literal] + born: 1996 #number [literal] + author: omkarbirade + published: true + """); + + String result = Strings.join("\n", e -> e.getKey() + " => " + e.getValue(), map.entries()); + + Assert.assertEquals(""" + company => spacelift + domain => devops + domain => devsecops + tutorial:yaml:name => YAML Ain't Markup Language + tutorial:yaml:type => awesome + tutorial:yaml:born => 2001 + tutorial:json:name => JavaScript Object Notation + tutorial:json:type => great + tutorial:json:born => 2001 + tutorial:xml:name => Extensible Markup Language + tutorial:xml:type => good + tutorial:xml:born => 1996 + author => omkarbirade + published => true""", result); + } + + /** Actually this is not correct since the multi line string should be a single entry */ + @Test + public void testYamlSimpleMultiline() { + Multimap map = YamlUtil.loadYamlFromString(""" + message: this is + a real multiline + message + """); + + String result = Strings.join("\n", e -> e.getKey() + " => " + e.getValue(), map.entries()); + + Assert.assertEquals(""" + message => this is + message => a real multiline + message => message""", result); + } + + @Test + public void testYamlBacktrack() { + Multimap map = YamlUtil.loadYamlFromString(""" + tutorial: + - yaml: + - one: empty + - two: empty + - json: empty + - xml: empty + """); + + String result = Strings.join("\n", e -> e.getKey() + " => " + e.getValue(), map.entries()); + + Assert.assertEquals(""" + tutorial:yaml:one => empty + tutorial:yaml:two => empty + tutorial:json => empty + tutorial:xml => empty""", result); + } +}