From b3c6e1f474d41b3a9c0ae55c926d3d46bb0e442c Mon Sep 17 00:00:00 2001 From: Robert burner Schadek Date: Fri, 18 Dec 2020 15:54:16 +0100 Subject: [PATCH 1/5] Test data generation for pubgrub impl To test the pubgrub implementation test data is needed. It would be nice to use the real world data available from code.dlang.org. But reading in the multiple MB json dumb for unit testing is likely more work than actually running the tests with that data. Or at least hopefully that is the case. The functionality contained in inlinetestdatagen will read in the dump from code.dlang.org once and then generate a D source file that contains all packages spelled out as struct constructor calls. A poor mans binary format so to speak. --- testdata/dub.json | 4 + .../source/dud/testdata/inlinetestdatagen.d | 407 ++++++++++++++++++ 2 files changed, 411 insertions(+) create mode 100644 testdata/source/dud/testdata/inlinetestdatagen.d diff --git a/testdata/dub.json b/testdata/dub.json index 00277c7..a462f30 100644 --- a/testdata/dub.json +++ b/testdata/dub.json @@ -9,6 +9,10 @@ "descriptiongetter": { "path": "../descriptiongetter", "version": ">=0.0.0" + }, + "semver": { + "path": "../semver", + "version": ">=0.0.0" } }, "configurations": [ diff --git a/testdata/source/dud/testdata/inlinetestdatagen.d b/testdata/source/dud/testdata/inlinetestdatagen.d new file mode 100644 index 0000000..43b684d --- /dev/null +++ b/testdata/source/dud/testdata/inlinetestdatagen.d @@ -0,0 +1,407 @@ +module dud.testdata.inlinetestdatagen; + +import std.array : array, empty, front; +import std.algorithm.searching : startsWith; +import std.algorithm.iteration : filter; +import std.exception : enforce; +import std.format : format; +import std.json; +import std.stdio; + +import dud.semver.semver; +import dud.semver.parse; +import dud.semver.versionrange; + +template ToJSONType(T) { + import std.traits : isIntegral, isFloatingPoint; + static if(is(T == string)) { + enum ToJSONType = JSONType.string; + } else static if(isIntegral!T) { + enum ToJSONType = JSONType.integer; + } else static if(isFloatingPoint!T) { + enum ToJSONType = JSONType.float_; + } else { + static assert(false, T.stringof ~ " not handled"); + } +} + +JSONValue getNested(JSONValue j, string[] path) @safe { + string[] copy = path; + JSONValue ptr = j; + while(!copy.empty) { + enforce(ptr.type == JSONType.object, format( + "Path '%--(%s.%)' does not consists of objects '%s'" + , path, j.toPrettyString())); + if(copy.front !in ptr) { + return JSONValue(null); + } + + ptr = ptr[copy.front]; + copy = copy[1 .. $]; + } + return ptr; +} + +T getNested(T)(JSONValue j, string member) @safe { + enforce(j.type == JSONType.object, format( + "Trying to get member of name '%s' but JSONValue is of type '%s'" + , member, j.type)); + enforce(member in j, () @trusted { + return format( + "No member of name '%s' found in keys '[%(%s,%)]' JSONValue of '%s' " + , member + , j.objectNoRef().keys + , j.toPrettyString); + }()); + + enum JSONType expJT = ToJSONType!T; + enforce(j[member].type == expJT, format( + "Member '%s' expected to have type '%s' but got '%s'\n%s" + , member, expJT, j[member].type, j.toPrettyString())); + + return j[member].get!T(); +} + +T getNested(T)(JSONValue j, string[] path) @safe { + JSONValue acc = j.getNested(path); + + enum JSONType expJT = ToJSONType!T; + enforce(acc.type == expJT, format( + "Member '%--(%s.%)' expected to have type '%s' but got '%s'\n%s" + , path, expJT, acc.type, j.toPrettyString())); + + return acc.get!T(); +} + +struct DepImpl { + string name; + VersionRange ver; +} + +alias PackageDep = DepImpl; +alias ToolDep = DepImpl; + +struct PackageVersion { + string name; + SemVer ver; + string branchName; + + PackageDep[] pkgDeps; + ToolDep[] toolDeps; + + PackageVersion[] subPackages; + PackageVersion[] configurations; +} + +PackageVersion toPackageVersion(JSONValue j, string prefix = "") @trusted { + PackageVersion ret; + JSONValue ver = j.getNested(["version"]); + if(ver.type == JSONType.string) { + string verStr = ver.get!string(); + if(verStr.startsWith("~")) { + ret.branchName = verStr; + } else if(!verStr.empty) { + ret.ver = verStr.parseSemVer(); + } + } + + JSONValue nameJ = j.getNested([prefix, "name"].filter!(i => !i.empty).array); + if(nameJ.type == JSONType.string) { + ret.name = nameJ.get!string(); + } + + auto deps = j.getNested([prefix, "dependencies"].filter!(i => !i.empty).array); + if(deps.type == JSONType.object) { + foreach(string key, JSONValue value; deps.objectNoRef()) { + PackageDep pd; + pd.name = key; + if(value.type == JSONType.string) { + pd.ver = value.get!string().parseVersionRange(); + ret.pkgDeps ~= pd; + } else { + writefln("%s %s", key, value.toString()); + } + } + } else if(deps.type != JSONType.null_) { + assert(false, deps.toPrettyString()); + } + + auto toolChain = j.getNested([prefix, "toolchainRequirements"] + .filter!(i => !i.empty) + .array); + + if(toolChain.type == JSONType.object) { + foreach(string key, JSONValue value; toolChain.objectNoRef()) { + ToolDep td; + td.name = key; + if(value.type == JSONType.string) { + td.ver = value.get!string().parseVersionRange(); + ret.toolDeps ~= td; + } else { + writefln("%s %s", key, value.toString()); + } + } + } else if(toolChain.type != JSONType.null_) { + assert(false, deps.toPrettyString()); + } + + auto subPackages = j.getNested([prefix, "subPackages"] + .filter!(i => !i.empty) + .array); + + if(subPackages.type == JSONType.array) { + foreach(JSONValue value; subPackages.arrayNoRef()) { + ret.subPackages ~= toPackageVersion(value); + } + } + + auto configurations = j.getNested([prefix, "configurations"] + .filter!(i => !i.empty) + .array); + if(configurations.type == JSONType.array) { + foreach(JSONValue value; configurations.arrayNoRef()) { + ret.configurations ~= toPackageVersion(value); + } + } + + return ret; +} + +unittest { + JSONValue jv = parseJSON( +` +{ + "packageDescription": { + "authors": [ + "Thomas Stuart Bockman" + ], + "buildOptions": [ + "ignoreUnknownPragmas" + ], + "copyright": "Copyright © 2015, Thomas Stuart Bockman", + "description": "Checked integer math types and operations.", + "license": "BSL-1.0", + "name": "checkedint", + "subPackages": [ + { + "authors": [ + "Thomas Stuart Bockman" + ], + "buildOptions": [ + "ignoreUnknownPragmas" + ], + "copyright": "Copyright © 2015, Thomas Stuart Bockman", + "dependencies": { + "checkedint": ">=0.0.0" + }, + "description": "Exhaustive tests for the checkedint package. Verifies correctness and measures performance.", + "license": "BSL-1.0", + "mainSourceFile": "source\/package.d", + "name": "tests", + "path": ".\/tests\/", + "targetType": "executable" + } + ], + "targetType": "library", + "toolchainRequirements": { + "frontend": ">=2.71.0" + } + }, + "version": "2.2.1" +}`); + PackageVersion pv = toPackageVersion(jv, "packageDescription"); + writeln(pv); +} + +unittest { + JSONValue jv = parseJSON( +` + { + "packageDescription": { + "authors": [ + "Andrej Petrović" + ], + "configurations": [ + { + "name": "library", + "targetType": "library" + }, + { + "dependencies": { + "imageformats": "7.0.2", + "silly": "1.0.2" + }, + "name": "unittest", + "targetType": "library" + } + ], + "description": "A D language implementation of https:\/\/github.com\/KdotJPG\/OpenSimplex2", + "license": "public domain", + "name": "open-simplex-2" + }, + "version": "1.0.1" +}`); + PackageVersion pv = toPackageVersion(jv, "packageDescription"); + writeln(pv); +} + +struct Package { + string name; + PackageVersion[] versions; +} + +Package toPackage(JSONValue jv) { + Package ret; + ret.name = jv.getNested!string("name"); + + auto versions = jv.getNested(["versions"]); + if(versions.type == JSONType.array) { + foreach(JSONValue value; versions.arrayNoRef()) { + ret.versions ~= value.toPackageVersion("packageDescription"); + } + } + return ret; +} + +unittest { + JSONValue jv = parseJSON( +`{ + "name": "vibenews", + "repository": { + "kind": "github", + "project": "vibenews" + }, + "versions": [ + { + "packageDescription": { + "authors": [ + "Sönke Ludwig" + ], + "copyright": "Copyright (c) 2012-2018 Sönke Ludwig", + "dependencies": { + "antispam": "~>0.1.2", + "userman": "~>0.4.0", + "vibe-d": ">=0.8.0 <0.10.0-0" + }, + "description": "Combined web forum and NNTP server implementation for stand-alone newsgroups", + "homepage": "https:\/\/github.com\/rejectedsoftware\/vibenews", + "license": "AGPL-3.0", + "name": "vibenews" + }, + "version": "~master" + }, + { + "packageDescription": { + "authors": [ + "Sönke Ludwig" + ], + "copyright": "Copyright (c) 2012-2014 Sönke Ludwig", + "dependencies": { + "antispam": "~>0.0.5", + "userman": "~>0.2.3", + "vibe-d": "~>0.7.22" + }, + "description": "Combined web forum and NNTP server implementation for stand-alone newsgroups", + "homepage": "https:\/\/github.com\/rejectedsoftware\/vibenews", + "license": "AGPL-3.0", + "name": "vibenews", + "versions": [ + "VibeDefaultMain" + ] + }, + "version": "0.6.7" + } + ] + }`); + Package pv = toPackage(jv); + writeln(pv); +} + +Package[string] toPackages(JSONValue jv) { + Package[string] ret; + foreach(JSONValue it; jv.arrayNoRef()) { + Package p = toPackage(it); + enforce(p.name !in ret, p.name); + ret[p.name] = p; + } + return ret; +} + +unittest { + JSONValue jv = parseJSON( +`[ { + "name": "vibenews", + "repository": { + "kind": "github", + "project": "vibenews" + }, + "versions": [ + { + "packageDescription": { + "authors": [ + "Sönke Ludwig" + ], + "copyright": "Copyright (c) 2012-2014 Sönke Ludwig", + "dependencies": { + "antispam": "~>0.0.5", + "userman": "~>0.2.3", + "vibe-d": "~>0.7.22" + }, + "description": "Combined web forum and NNTP server implementation for stand-alone newsgroups", + "homepage": "https:\/\/github.com\/rejectedsoftware\/vibenews", + "license": "AGPL-3.0", + "name": "vibenews", + "versions": [ + "VibeDefaultMain" + ] + }, + "version": "0.6.7" + } + ] +} +, { + "name": "vibelog", + "repository": { + "kind": "github", + "project": "vibelog" + }, + "versions": [ + { + "packageDescription": { + "authors": [ + "Sönke Ludwig" + ], + "configurations": [ + { + "name": "standalone", + "targetType": "executable", + "versions": [ + "VibeDefaultMain" + ] + }, + { + "excludedSourceFiles": [ + "source\/app.d" + ], + "name": "library", + "targetType": "library" + } + ], + "dependencies": { + "diskuto": "~>1.5", + "stringex": "~>0.1.0", + "vibe-d": ">=0.7.31 <0.10.0" + }, + "description": "A light-weight embeddable blog implementation", + "homepage": "https:\/\/github.com\/rejectedsoftware\/vibelog", + "license": "AGPL-3.0", + "name": "vibelog" + }, + "version": "~master" + } + ] + } +]`); + Package[string] pkgs = toPackages(jv); + writeln(pkgs); +} From e6720a016753f24eb9274b827836f21d2beddbc6 Mon Sep 17 00:00:00 2001 From: Robert burner Schadek Date: Fri, 18 Dec 2020 16:51:24 +0100 Subject: [PATCH 2/5] ignoring some things for know the ignored things are printed --- .../source/dud/testdata/inlinetestdatagen.d | 17 ++++++++++++----- testdata/source/dud/testdata/main.d | 13 +++++++++++-- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/testdata/source/dud/testdata/inlinetestdatagen.d b/testdata/source/dud/testdata/inlinetestdatagen.d index 43b684d..56b9eb9 100644 --- a/testdata/source/dud/testdata/inlinetestdatagen.d +++ b/testdata/source/dud/testdata/inlinetestdatagen.d @@ -116,10 +116,14 @@ PackageVersion toPackageVersion(JSONValue j, string prefix = "") @trusted { PackageDep pd; pd.name = key; if(value.type == JSONType.string) { - pd.ver = value.get!string().parseVersionRange(); + string s = value.get!string(); + if(s.startsWith("v")) { + continue; + } + pd.ver = s.parseVersionRange(); ret.pkgDeps ~= pd; } else { - writefln("%s %s", key, value.toString()); + writefln("dep %s %s", key, value.toString()); } } } else if(deps.type != JSONType.null_) { @@ -138,7 +142,7 @@ PackageVersion toPackageVersion(JSONValue j, string prefix = "") @trusted { td.ver = value.get!string().parseVersionRange(); ret.toolDeps ~= td; } else { - writefln("%s %s", key, value.toString()); + writefln("tool %s %s", key, value.toString()); } } } else if(toolChain.type != JSONType.null_) { @@ -257,7 +261,11 @@ Package toPackage(JSONValue jv) { auto versions = jv.getNested(["versions"]); if(versions.type == JSONType.array) { foreach(JSONValue value; versions.arrayNoRef()) { - ret.versions ~= value.toPackageVersion("packageDescription"); + try { + ret.versions ~= value.toPackageVersion("packageDescription"); + } catch(Exception e) { + writeln(e.toString()); + } } } return ret; @@ -403,5 +411,4 @@ unittest { } ]`); Package[string] pkgs = toPackages(jv); - writeln(pkgs); } diff --git a/testdata/source/dud/testdata/main.d b/testdata/source/dud/testdata/main.d index 4fd9ac3..df77fa6 100644 --- a/testdata/source/dud/testdata/main.d +++ b/testdata/source/dud/testdata/main.d @@ -7,11 +7,13 @@ import std.getopt; import std.file : readText; import dud.descriptiongetter.code; +import dud.testdata.inlinetestdatagen; private struct Options { bool getCodeDump; string outFilename; string inFilename; + string dOutFilename; } version(App): @@ -23,8 +25,10 @@ void main(string[] args) { "o|outFilename", "The filename of the output of the trimed dump", &options.outFilename, "i|inFilename", "In filename to a code.dlang.org dump.json file", - &options.inFilename - ); + &options.inFilename, + "d|dOutFilename", "The filename of the file to write a D source file " + ~ "repesentation of the inFilename too", + &options.dOutFilename); if(helpWanted.helpWanted) { defaultGetoptPrinter("CLI to handle the code.dlang.org api dump", @@ -50,4 +54,9 @@ void main(string[] args) { } else { writeln(shorter.toPrettyString()); } + + if(!options.dOutFilename.empty) { + //auto f = File(options.dOutFilename, "w"); + writeln(toPackages(shorter)); + } } From d40e1c9a009a28a440d8c653925ca92f999cd903 Mon Sep 17 00:00:00 2001 From: Robert burner Schadek Date: Fri, 12 Feb 2021 16:45:28 +0100 Subject: [PATCH 3/5] the program now can turn the json dump from code.dlang.org into a D file that builds up the same data. When using reggae, at some build, linking in the object file should be a lot faster than parsing the json every time. --- semver/source/dud/semver/semver.d | 9 +- .../source/dud/testdata/inlinetestdatagen.d | 183 +++++++++++++++++- testdata/source/dud/testdata/main.d | 14 +- 3 files changed, 195 insertions(+), 11 deletions(-) diff --git a/semver/source/dud/semver/semver.d b/semver/source/dud/semver/semver.d index bb1f267..e46d3f6 100644 --- a/semver/source/dud/semver/semver.d +++ b/semver/source/dud/semver/semver.d @@ -3,6 +3,8 @@ module dud.semver.semver; @safe: struct SemVer { + import std.array : empty; + import std.format : format; @safe: uint major; @@ -50,8 +52,6 @@ struct SemVer { } string toString() const @safe pure { - import std.array : appender, empty; - import std.format : format; string ret = format("%s.%s.%s", this.major, this.minor, this.patch); if(!this.preRelease.empty) { ret ~= format("-%-(%s.%)", this.preRelease); @@ -61,4 +61,9 @@ struct SemVer { } return ret; } + + string toStringD() const @safe pure { + return format("SemVer(%s, %s, %s, %s, %s)", this.major, this.minor + , this.patch, this.preRelease, this.buildIdentifier); + } } diff --git a/testdata/source/dud/testdata/inlinetestdatagen.d b/testdata/source/dud/testdata/inlinetestdatagen.d index 56b9eb9..6104d5a 100644 --- a/testdata/source/dud/testdata/inlinetestdatagen.d +++ b/testdata/source/dud/testdata/inlinetestdatagen.d @@ -1,10 +1,12 @@ module dud.testdata.inlinetestdatagen; +@safe: + import std.array : array, empty, front; import std.algorithm.searching : startsWith; import std.algorithm.iteration : filter; import std.exception : enforce; -import std.format : format; +import std.format : format, formattedWrite; import std.json; import std.stdio; @@ -93,6 +95,176 @@ struct PackageVersion { PackageVersion[] configurations; } +void formIndent(Out, T...)(auto ref Out o, size_t indent, string form, T t) { + foreach(i; 0 .. indent) { + formattedWrite(o, "\t"); + } + formattedWrite(o, form, t); +} + +enum BranchOrSemVerMixin = q{ +struct BranchOrSemVer { + pure @safe: + import std.typecons : Nullable; + Nullable!SemVer sv; + string s; + + bool opEquals(const(BranchOrSemVer) other) const nothrow pure { + return !this.sv.isNull() && !other.sv.isNull() + ? this.sv.get() == other.sv.get() + : this.sv.isNull() && !other.sv.isNull() + ? false + : !this.sv.isNull() && other.sv.isNull() + ? false + : this.s == other.s; + } + + size_t toHash() const nothrow @nogc pure { + return this.sv.isNull() + ? this.sv.toHash() + : hashOf(this.s); + } + + string toString() const pure { + return this.sv.isNull() + ? this.s + : this.sv.toString(); + } +} + +BranchOrSemVer toBranchOrSemVer(T)(T s) pure @safe { + BranchOrSemVer ret; + static if(is(T == SemVer)) { + ret.sv = s; + } else { + ret.s = s; + } + return ret; +} +}; + +unittest { + pragma(msg, BranchOrSemVerMixin); + mixin(BranchOrSemVerMixin); + + bool[BranchOrSemVer] aa; + + auto a = toBranchOrSemVer("hello"); + auto b = toBranchOrSemVer("world"); + auto c = toBranchOrSemVer(SemVer(1, 0, 0)); + auto d = toBranchOrSemVer(SemVer(1, 1, 0)); + + auto arr = [a, b, c, d]; + + foreach(idx, it; arr) { + assert(it !in aa); + aa[it] = true; + assert(it in aa, it.toString()); + foreach(jt; arr[idx + 1 .. $]) { + assert(jt !in aa, jt.toString()); + } + } +} + +string replaceInvalidName(string s) { + import std.array : replace; + s = s.replace("-", "_"); + return s; +} + +void toDCode(Out)(auto ref Out o, const string modName,Package[string] pvs) { + formattedWrite(o, "module %s;\n\n", modName); + formattedWrite(o, +`import dud.pkgdescription; +import dud.semver.semver; + +`); + + formattedWrite(o, BranchOrSemVerMixin); + formattedWrite(o, "\n"); + foreach(key, ref value; pvs) { + formattedWrite(o + , "void build%s(ref PackageDescription[string][BranchOrSemVer] result) {\n" + , replaceInvalidName(key)); + foreach(pv; value.versions) { + toDCode(o, pv); + } + formattedWrite(o, "}\n\n"); + } + + formattedWrite(o, +`PackageDescription[string][BranchOrSemVer] buildAll() { + PackageDescription[string][BranchOrSemVer] ret; + +`); + foreach(key, ref value; pvs) { + formIndent(o, 1, "build%s(ret);\n", replaceInvalidName(key)); + } + formattedWrite(o, "\treturn ret;\n}\n"); +} + +void toDCode(Out)(auto ref Out o, PackageVersion vr , const string nested = "") { + const indent = nested == "" ? 0 : 1; + + const name = nested == "" ? "pkg" : "pkg" ~ nested; + + formIndent(o, 1 + indent, "{\n"); + formIndent(o, 2 + indent, "auto %s = PackageDescription.init;\n", name); + formIndent(o, 2 + indent, "%s.name = \"%s\";\n", name, vr.name); + formIndent(o, 2 + indent, "%s.dependencies = [%s", name + , vr.pkgDeps.empty ? "" : "\n"); + foreach(idx; 0 .. vr.pkgDeps.length) { + formIndent(o, 3 + indent, "%s makeDependency(\"%s\", " + , idx == 0 ? '[' : ',', vr.pkgDeps[idx].name); + toDCode(o, vr.pkgDeps[idx].ver); + formattedWrite(o, ")\n"); + } + formIndent(o, vr.pkgDeps.empty ? 0 : 3 + indent, "];\n", vr.name); + foreach(t; vr.toolDeps) { + formIndent(o, 2 + indent + , "%s.toolchainRequirements[Toolchain.%s] = makeToolDep(" + , name, t.name); + toDCode(o, t.ver); + formattedWrite(o, ");\n"); + } + + foreach(sP; vr.subPackages) { + toDCode(o, sP, "subPackages"); + } + + foreach(conf; vr.configurations) { + toDCode(o, conf, "configuration"); + } + + switch(nested) { + case "subPackages": + formIndent(o, 2 + indent, "pkg.subPackages ~= %s;\n", name); + break; + case "configuration": + formIndent(o, 2 + indent, "pkg.configuration[\"%s\"] ~= %s;\n" + , vr.name, name); + break; + default: + formIndent(o, 2 + indent + , "result[\"%s\"][toBranchOrSemVer(%s)] = %s;\n" + , vr.name + , vr.branchName.empty + ? vr.ver.toStringD() + : format("\"%s\"", vr.branchName) + , name); + } + + formIndent(o, 1 + indent, "}\n"); +} + + +void toDCode(Out)(auto ref Out o, VersionRange vr) { + formattedWrite(o, "VersionRange("); + formattedWrite(o, "%s, Inclusive.%s, ", vr.low.toStringD(), vr.inclusiveLow); + formattedWrite(o, "%s, Inclusive.%s", vr.high.toStringD(), vr.inclusiveHigh); + formattedWrite(o, ")"); +} + PackageVersion toPackageVersion(JSONValue j, string prefix = "") @trusted { PackageVersion ret; JSONValue ver = j.getNested(["version"]); @@ -214,7 +386,6 @@ unittest { "version": "2.2.1" }`); PackageVersion pv = toPackageVersion(jv, "packageDescription"); - writeln(pv); } unittest { @@ -246,7 +417,6 @@ unittest { "version": "1.0.1" }`); PackageVersion pv = toPackageVersion(jv, "packageDescription"); - writeln(pv); } struct Package { @@ -264,7 +434,7 @@ Package toPackage(JSONValue jv) { try { ret.versions ~= value.toPackageVersion("packageDescription"); } catch(Exception e) { - writeln(e.toString()); + () @trusted { writeln(e.toString()); }(); } } } @@ -322,7 +492,6 @@ unittest { ] }`); Package pv = toPackage(jv); - writeln(pv); } Package[string] toPackages(JSONValue jv) { @@ -411,4 +580,8 @@ unittest { } ]`); Package[string] pkgs = toPackages(jv); + () @trusted { writeln(pkgs); }(); + foreach(key, value; pkgs) { + () @trusted { toDCode(stdout.lockingTextWriter(), value.versions.front); }(); + } } diff --git a/testdata/source/dud/testdata/main.d b/testdata/source/dud/testdata/main.d index df77fa6..bffeb56 100644 --- a/testdata/source/dud/testdata/main.d +++ b/testdata/source/dud/testdata/main.d @@ -14,6 +14,7 @@ private struct Options { string outFilename; string inFilename; string dOutFilename; + string dOutModuleName; } version(App): @@ -28,7 +29,11 @@ void main(string[] args) { &options.inFilename, "d|dOutFilename", "The filename of the file to write a D source file " ~ "repesentation of the inFilename too", - &options.dOutFilename); + &options.dOutFilename, + "m|dOutModuleName", "The module name of the file " + ~ "repesentation of the inFilename", + &options.dOutModuleName + ); if(helpWanted.helpWanted) { defaultGetoptPrinter("CLI to handle the code.dlang.org api dump", @@ -52,11 +57,12 @@ void main(string[] args) { auto f = File(options.outFilename, "w"); f.writeln(shorter.toPrettyString()); } else { - writeln(shorter.toPrettyString()); + //writeln(shorter.toPrettyString()); } if(!options.dOutFilename.empty) { - //auto f = File(options.dOutFilename, "w"); - writeln(toPackages(shorter)); + auto f = File(options.dOutFilename, "w"); + toDCode(f.lockingTextWriter(), options.dOutModuleName + , toPackages(shorter)); } } From 889c2adc056e27d08addd149a55291ebba9d6561 Mon Sep 17 00:00:00 2001 From: Robert burner Schadek Date: Fri, 5 Mar 2021 17:06:25 +0100 Subject: [PATCH 4/5] inlinetestdatagen actuall seems to generate a compileable file now --- compiledtestdata/README.md | 7 +++ compiledtestdata/dub.json | 10 ++++ .../source/dud/compiledtestdata/.gitkeep | 0 .../source/dud/pkgdescription/path.d | 4 ++ .../source/dud/testdata/inlinetestdatagen.d | 55 ++++++++++++++----- 5 files changed, 63 insertions(+), 13 deletions(-) create mode 100644 compiledtestdata/README.md create mode 100644 compiledtestdata/dub.json create mode 100644 compiledtestdata/source/dud/compiledtestdata/.gitkeep diff --git a/compiledtestdata/README.md b/compiledtestdata/README.md new file mode 100644 index 0000000..becb5a3 --- /dev/null +++ b/compiledtestdata/README.md @@ -0,0 +1,7 @@ +Run this + +```sh +dub -- -i newdump.json -d ../compiledtestdata/source/dud/compiledtestdata/data.d -m "dud.compiledtestdata.data" +``` + +in the testdata folder if you have a dump called newdump.json in it diff --git a/compiledtestdata/dub.json b/compiledtestdata/dub.json new file mode 100644 index 0000000..c9a8c82 --- /dev/null +++ b/compiledtestdata/dub.json @@ -0,0 +1,10 @@ +{ + "dependencies": { + "pkgdescription" : { + "path": "../pkgdescription", + "version": ">=0.0.0" + } + }, + "name": "compiledtestdata", + "targetType": "library" +} diff --git a/compiledtestdata/source/dud/compiledtestdata/.gitkeep b/compiledtestdata/source/dud/compiledtestdata/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/pkgdescription/source/dud/pkgdescription/path.d b/pkgdescription/source/dud/pkgdescription/path.d index c7152df..3389bcf 100644 --- a/pkgdescription/source/dud/pkgdescription/path.d +++ b/pkgdescription/source/dud/pkgdescription/path.d @@ -12,6 +12,10 @@ struct Path { PathPlatform[] platforms; } +Path pathFromString(string s) { + return Path(PathPlatform(UnprocessedPath(s, [Platform.all]))); +} + struct PathPlatform { UnprocessedPath path; Platform[] platforms; diff --git a/testdata/source/dud/testdata/inlinetestdatagen.d b/testdata/source/dud/testdata/inlinetestdatagen.d index 6104d5a..4b7a725 100644 --- a/testdata/source/dud/testdata/inlinetestdatagen.d +++ b/testdata/source/dud/testdata/inlinetestdatagen.d @@ -169,23 +169,38 @@ unittest { string replaceInvalidName(string s) { import std.array : replace; s = s.replace("-", "_"); + s = s.replace("+", "_plus_"); + s = s.replace(":", "_colon_"); + s = s.replace(" ", "_space_"); + s = s.replace(".", "_dot_"); return s; } void toDCode(Out)(auto ref Out o, const string modName,Package[string] pvs) { formattedWrite(o, "module %s;\n\n", modName); formattedWrite(o, -`import dud.pkgdescription; +`import std.typecons : nullable; +import dud.pkgdescription; import dud.semver.semver; +import dud.semver.semver; +import dud.semver.versionrange; +Dependency makeDependency(string name, VersionRange vr) { + Dependency ret; + ret.name = name; + ret.version_ = nullable(vr); + return ret; +} `); formattedWrite(o, BranchOrSemVerMixin); formattedWrite(o, "\n"); + size_t cnt; foreach(key, ref value; pvs) { formattedWrite(o - , "void build%s(ref PackageDescription[string][BranchOrSemVer] result) {\n" - , replaceInvalidName(key)); + , "void build%s%s(ref PackageDescription[BranchOrSemVer][string] result) {\n" + , replaceInvalidName(key), cnt); + ++cnt; foreach(pv; value.versions) { toDCode(o, pv); } @@ -193,12 +208,13 @@ import dud.semver.semver; } formattedWrite(o, -`PackageDescription[string][BranchOrSemVer] buildAll() { - PackageDescription[string][BranchOrSemVer] ret; +`PackageDescription[BranchOrSemVer][string] buildAll() { + PackageDescription[BranchOrSemVer][string] ret; `); + cnt = 0; foreach(key, ref value; pvs) { - formIndent(o, 1, "build%s(ret);\n", replaceInvalidName(key)); + formIndent(o, 1, "build%s%s(ret);\n", replaceInvalidName(key), cnt++); } formattedWrite(o, "\treturn ret;\n}\n"); } @@ -206,13 +222,15 @@ import dud.semver.semver; void toDCode(Out)(auto ref Out o, PackageVersion vr , const string nested = "") { const indent = nested == "" ? 0 : 1; - const name = nested == "" ? "pkg" : "pkg" ~ nested; + const name = nested == "" + ? "pkg" + : "pkg" ~ vr.name.replaceInvalidName() ~ nested; formIndent(o, 1 + indent, "{\n"); formIndent(o, 2 + indent, "auto %s = PackageDescription.init;\n", name); formIndent(o, 2 + indent, "%s.name = \"%s\";\n", name, vr.name); - formIndent(o, 2 + indent, "%s.dependencies = [%s", name - , vr.pkgDeps.empty ? "" : "\n"); + formIndent(o, 2 + indent, "%s.dependencies = %s", name + , vr.pkgDeps.empty ? "[" : "\n"); foreach(idx; 0 .. vr.pkgDeps.length) { formIndent(o, 3 + indent, "%s makeDependency(\"%s\", " , idx == 0 ? '[' : ',', vr.pkgDeps[idx].name); @@ -222,27 +240,38 @@ void toDCode(Out)(auto ref Out o, PackageVersion vr , const string nested = "") formIndent(o, vr.pkgDeps.empty ? 0 : 3 + indent, "];\n", vr.name); foreach(t; vr.toolDeps) { formIndent(o, 2 + indent - , "%s.toolchainRequirements[Toolchain.%s] = makeToolDep(" + , "%s.toolchainRequirements[Toolchain.%s] = ToolchainRequirement(" + ~ "false, " , name, t.name); toDCode(o, t.ver); formattedWrite(o, ");\n"); } + if(!vr.subPackages.empty) { + formIndent(o, 2 + indent, "// subpackages\n"); + } foreach(sP; vr.subPackages) { toDCode(o, sP, "subPackages"); } + if(!vr.configurations.empty) { + formIndent(o, 2 + indent, "// configurations\n"); + } foreach(conf; vr.configurations) { toDCode(o, conf, "configuration"); } switch(nested) { case "subPackages": - formIndent(o, 2 + indent, "pkg.subPackages ~= %s;\n", name); + formIndent(o, 2 + indent, "auto p = pathFromString(\"%s\");\n" + , name); + formIndent(o, 2 + indent + , "pkg.subPackages ~= SubPackage(p, nullable(%s));\n" + , name); break; case "configuration": - formIndent(o, 2 + indent, "pkg.configuration[\"%s\"] ~= %s;\n" - , vr.name, name); + formIndent(o, 2 + indent, "%s.configurations[\"%s\"] = %s;\n" + , name, vr.name, name); break; default: formIndent(o, 2 + indent From ba22676dd2a735ff751a45e96287f183eb50a920 Mon Sep 17 00:00:00 2001 From: Robert burner Schadek Date: Mon, 8 Mar 2021 09:42:21 +0100 Subject: [PATCH 5/5] trying to compile this faster --- testdata/source/dud/testdata/inlinetestdatagen.d | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/testdata/source/dud/testdata/inlinetestdatagen.d b/testdata/source/dud/testdata/inlinetestdatagen.d index 4b7a725..4d8ea07 100644 --- a/testdata/source/dud/testdata/inlinetestdatagen.d +++ b/testdata/source/dud/testdata/inlinetestdatagen.d @@ -210,12 +210,19 @@ Dependency makeDependency(string name, VersionRange vr) { formattedWrite(o, `PackageDescription[BranchOrSemVer][string] buildAll() { PackageDescription[BranchOrSemVer][string] ret; - -`); + foreach(it; [ `); cnt = 0; foreach(key, ref value; pvs) { - formIndent(o, 1, "build%s%s(ret);\n", replaceInvalidName(key), cnt++); + if(cnt) { + formIndent(o, 2, ", "); + } + formattedWrite(o, "&build%s%s%s", replaceInvalidName(key) + , cnt, cnt < pvs.length - 1 ? "\n" : ""); + ++cnt; } + formattedWrite(o, "])\n\t{\n"); + formIndent(o, 2, "it(ret);\n"); + formIndent(o, 1, "}\n"); formattedWrite(o, "\treturn ret;\n}\n"); }