From 4172b5b29081bb8ff0b113bbfa55f7983589c6e0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 8 May 2019 13:28:02 +0200 Subject: [PATCH 01/73] Add flake.nix --- flake.lock | 9 +++++++++ flake.nix | 21 +++++++++++++++++++++ release.nix | 3 ++- 3 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000..9848cb996 --- /dev/null +++ b/flake.lock @@ -0,0 +1,9 @@ +{ + "nonFlakeRequires": {}, + "requires": { + "nixpkgs": { + "uri": "github:edolstra/nixpkgs/a4d896e89932e873c4117908d558db6210fa3b56" + } + }, + "version": 1 +} \ No newline at end of file diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000..1e9bc3125 --- /dev/null +++ b/flake.nix @@ -0,0 +1,21 @@ +{ + name = "hydra"; + + description = "A Nix-based continuous build system"; + + epoch = 2019; + + requires = [ "nixpkgs" ]; + + provides = deps: rec { + + hydraJobs = import ./release.nix { + hydraSrc = deps.self; + nixpkgs = deps.nixpkgs; + }; + + packages.hydra = hydraJobs.build.x86_64-linux; + + defaultPackage = packages.hydra; + }; +} diff --git a/release.nix b/release.nix index dd28cef0e..9d060d6d6 100644 --- a/release.nix +++ b/release.nix @@ -30,7 +30,8 @@ let environment.systemPackages = [ pkgs.perlPackages.LWP pkgs.perlPackages.JSON ]; }; - version = builtins.readFile ./version + "." + toString hydraSrc.revCount + "." + hydraSrc.rev; + # FIXME: use commit date. + version = builtins.readFile ./version + "." + toString hydraSrc.revCount or 0 + "." + hydraSrc.shortRev or "0000000"; in From 2ecc06e5575c01ec929e580d570cb71a1baaf778 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 8 May 2019 18:12:24 +0200 Subject: [PATCH 02/73] Replace shell.nix with a flake devShell attribute --- flake.nix | 7 +++++++ release.nix | 4 +--- shell.nix | 1 - 3 files changed, 8 insertions(+), 4 deletions(-) delete mode 100644 shell.nix diff --git a/flake.nix b/flake.nix index 1e9bc3125..15e30151d 100644 --- a/flake.nix +++ b/flake.nix @@ -17,5 +17,12 @@ packages.hydra = hydraJobs.build.x86_64-linux; defaultPackage = packages.hydra; + + devShell = (import ./release.nix { + hydraSrc = deps.self; + nixpkgs = deps.nixpkgs; + shell = true; + }).build.x86_64-linux; + }; } diff --git a/release.nix b/release.nix index 9d060d6d6..4747b9ba7 100644 --- a/release.nix +++ b/release.nix @@ -138,9 +138,7 @@ rec { preConfigure = "autoreconf -vfi"; - NIX_LDFLAGS = [ - "-lpthread" - ]; + NIX_LDFLAGS = [ "-lpthread" ]; enableParallelBuilding = true; diff --git a/shell.nix b/shell.nix deleted file mode 100644 index 454c00bff..000000000 --- a/shell.nix +++ /dev/null @@ -1 +0,0 @@ -(import ./release.nix { shell = true; }).build.x86_64-linux From 950e8bef6cd90befcf14e96826053d1d154e39fe Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 8 May 2019 21:22:52 +0200 Subject: [PATCH 03/73] Disable deprecation warnings --- src/hydra-evaluator/Makefile.am | 2 +- src/hydra-queue-runner/Makefile.am | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hydra-evaluator/Makefile.am b/src/hydra-evaluator/Makefile.am index 161f8567f..73638cfee 100644 --- a/src/hydra-evaluator/Makefile.am +++ b/src/hydra-evaluator/Makefile.am @@ -2,4 +2,4 @@ bin_PROGRAMS = hydra-evaluator hydra_evaluator_SOURCES = hydra-evaluator.cc hydra_evaluator_LDADD = $(NIX_LIBS) -lpqxx -hydra_evaluator_CXXFLAGS = $(NIX_CFLAGS) -Wall -I ../libhydra +hydra_evaluator_CXXFLAGS = $(NIX_CFLAGS) -Wall -I ../libhydra -Wno-deprecated-declarations diff --git a/src/hydra-queue-runner/Makefile.am b/src/hydra-queue-runner/Makefile.am index b360faed7..1726d0df9 100644 --- a/src/hydra-queue-runner/Makefile.am +++ b/src/hydra-queue-runner/Makefile.am @@ -4,4 +4,4 @@ hydra_queue_runner_SOURCES = hydra-queue-runner.cc queue-monitor.cc dispatcher.c builder.cc build-result.cc build-remote.cc \ build-result.hh counter.hh token-server.hh state.hh db.hh hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx -hydra_queue_runner_CXXFLAGS = $(NIX_CFLAGS) -Wall -I ../libhydra +hydra_queue_runner_CXXFLAGS = $(NIX_CFLAGS) -Wall -I ../libhydra -Wno-deprecated-declarations From 80fe42e77e15bc12fa4f0ccee39c0c668f1024a1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 9 May 2019 15:10:16 +0200 Subject: [PATCH 04/73] Update flake.lock --- flake.lock | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flake.lock b/flake.lock index 9848cb996..965d038ce 100644 --- a/flake.lock +++ b/flake.lock @@ -2,8 +2,9 @@ "nonFlakeRequires": {}, "requires": { "nixpkgs": { + "contentHash": "sha256-vy2UmXQM66aS/Kn2tCtjt9RwxfBvV+nQVb5tJQFwi8E=", "uri": "github:edolstra/nixpkgs/a4d896e89932e873c4117908d558db6210fa3b56" } }, "version": 1 -} \ No newline at end of file +} From 8f3114960cd4178669c54e7785973ac6198315ab Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 10 May 2019 21:57:19 +0200 Subject: [PATCH 05/73] Remove the hydra-eval-jobs restart hack It's not very effective. --- src/hydra-eval-jobs/hydra-eval-jobs.cc | 136 +++---------------------- 1 file changed, 12 insertions(+), 124 deletions(-) diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index ce6967b5e..40112c72b 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -55,41 +55,9 @@ static string queryMetaStrings(EvalState & state, DrvInfo & drv, const string & } -static std::string lastAttrPath; -static bool comma = false; -static size_t maxHeapSize; - - -struct BailOut { }; - - -bool lte(const std::string & s1, const std::string & s2) -{ - size_t p1 = 0, p2 = 0; - - while (true) { - if (p1 == s1.size()) return p2 == s2.size(); - if (p2 == s2.size()) return true; - - auto d1 = s1.find('.', p1); - auto d2 = s2.find('.', p2); - - auto c = s1.compare(p1, d1 - p1, s2, p2, d2 - p2); - - if (c < 0) return true; - if (c > 0) return false; - - p1 = d1 == std::string::npos ? s1.size() : d1 + 1; - p2 = d2 == std::string::npos ? s2.size() : d2 + 1; - } -} - - static void findJobsWrapped(EvalState & state, JSONObject & top, Bindings & autoArgs, Value & vIn, const string & attrPath) { - if (lastAttrPath != "" && lte(attrPath, lastAttrPath)) return; - debug(format("at path `%1%'") % attrPath); checkInterrupt(); @@ -109,8 +77,6 @@ static void findJobsWrapped(EvalState & state, JSONObject & top, if (drv->querySystem() == "unknown") throw EvalError("derivation must have a ‘system’ attribute"); - if (comma) { std::cout << ","; comma = false; } - { auto res = top.object(attrPath); res.attr("nixName", drv->queryName()); @@ -156,15 +122,6 @@ static void findJobsWrapped(EvalState & state, JSONObject & top, res2.attr(j.first, j.second); } - - GC_prof_stats_s gc; - GC_get_prof_stats(&gc, sizeof(gc)); - - if (gc.heapsize_full > maxHeapSize) { - printInfo("restarting hydra-eval-jobs after job '%s' because heap size is at %d bytes", attrPath, gc.heapsize_full); - lastAttrPath = attrPath; - throw BailOut(); - } } else { @@ -200,7 +157,6 @@ static void findJobs(EvalState & state, JSONObject & top, try { findJobsWrapped(state, top, autoArgs, v, attrPath); } catch (EvalError & e) { - if (comma) { std::cout << ","; comma = false; } auto res = top.object(attrPath); res.attr("error", filterANSIEscapes(e.msg(), true)); } @@ -209,14 +165,6 @@ static void findJobs(EvalState & state, JSONObject & top, int main(int argc, char * * argv) { - assert(lte("abc", "def")); - assert(lte("abc", "def.foo")); - assert(!lte("def", "abc")); - assert(lte("nixpkgs.hello", "nixpkgs")); - assert(lte("nixpkgs.hello", "nixpkgs.hellooo")); - assert(lte("gitAndTools.git-annex.x86_64-darwin", "gitAndTools.git-annex.x86_64-linux")); - assert(lte("gitAndTools.git-annex.x86_64-linux", "gitAndTools.git-annex-remote-b2.aarch64-linux")); - /* Prevent undeclared dependencies in the evaluation via $NIX_PATH. */ unsetenv("NIX_PATH"); @@ -229,23 +177,9 @@ int main(int argc, char * * argv) if (initialHeapSize != "") setenv("GC_INITIAL_HEAP_SIZE", initialHeapSize.c_str(), 1); - maxHeapSize = config->getIntOption("evaluator_max_heap_size", 1UL << 30); - initNix(); initGC(); - /* Read the current heap size, which is the initial heap size. */ - GC_prof_stats_s gc; - GC_get_prof_stats(&gc, sizeof(gc)); - auto initialHeapSizeInt = gc.heapsize_full; - - /* Then make sure the maximum heap size will be bigger than the initial heap size. */ - if (initialHeapSizeInt > maxHeapSize) { - printInfo("warning: evaluator_initial_heap_size (%d) bigger than evaluator_max_heap_size (%d).", initialHeapSizeInt, maxHeapSize); - maxHeapSize = initialHeapSizeInt * 1.1; - printInfo(" evaluator_max_heap_size now set to %d.", maxHeapSize); - } - Path releaseExpr; struct MyArgs : LegacyArgs, MixEvalArgs @@ -270,71 +204,25 @@ int main(int argc, char * * argv) JSONObject json(std::cout, true); std::cout.flush(); - do { - - Pipe pipe; - pipe.create(); - - ProcessOptions options; - options.allowVfork = false; - - GC_atfork_prepare(); - - auto pid = startProcess([&]() { - pipe.readSide = -1; - - GC_atfork_child(); - GC_start_mark_threads(); - - if (lastAttrPath != "") debug("resuming from '%s'", lastAttrPath); + /* FIXME: The build hook in conjunction with import-from-derivation is causing "unexpected EOF" during eval */ + settings.builders = ""; - /* FIXME: The build hook in conjunction with import-from-derivation is causing "unexpected EOF" during eval */ - settings.builders = ""; + /* Prevent access to paths outside of the Nix search path and + to the environment. */ + evalSettings.restrictEval = true; - /* Prevent access to paths outside of the Nix search path and - to the environment. */ - evalSettings.restrictEval = true; + if (releaseExpr == "") throw UsageError("no expression specified"); - if (releaseExpr == "") throw UsageError("no expression specified"); + if (gcRootsDir == "") printMsg(lvlError, "warning: `--gc-roots-dir' not specified"); - if (gcRootsDir == "") printMsg(lvlError, "warning: `--gc-roots-dir' not specified"); + EvalState state(myArgs.searchPath, openStore()); - EvalState state(myArgs.searchPath, openStore()); - - Bindings & autoArgs = *myArgs.getAutoArgs(state); - - Value v; - state.evalFile(lookupFileArg(state, releaseExpr), v); - - comma = lastAttrPath != ""; - - try { - findJobs(state, json, autoArgs, v, ""); - lastAttrPath = ""; - } catch (BailOut &) { } - - writeFull(pipe.writeSide.get(), lastAttrPath); - - exit(0); - }, options); - - GC_atfork_parent(); - - pipe.writeSide = -1; - - int status; - while (true) { - checkInterrupt(); - if (waitpid(pid, &status, 0) == pid) break; - if (errno != EINTR) continue; - } + Bindings & autoArgs = *myArgs.getAutoArgs(state); - if (status != 0) - throw Exit(WIFEXITED(status) ? WEXITSTATUS(status) : 99); + Value v; + state.evalFile(lookupFileArg(state, releaseExpr), v); - maxHeapSize += 64 * 1024 * 1024; + findJobs(state, json, autoArgs, v, ""); - lastAttrPath = drainFD(pipe.readSide.get()); - } while (lastAttrPath != ""); }); } From 2c60019910038cfda905db7dc1d6c8ca01df758d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 10 May 2019 22:05:49 +0200 Subject: [PATCH 06/73] hydra-eval-jobs: Modernize argument parser --- src/hydra-eval-jobs/hydra-eval-jobs.cc | 47 ++++++++++++++++---------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index 40112c72b..6b4636343 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -180,25 +180,36 @@ int main(int argc, char * * argv) initNix(); initGC(); - Path releaseExpr; - - struct MyArgs : LegacyArgs, MixEvalArgs + struct MyArgs : MixEvalArgs, MixCommonArgs { - using LegacyArgs::LegacyArgs; - }; + Path releaseExpr; - MyArgs myArgs(baseNameOf(argv[0]), [&](Strings::iterator & arg, const Strings::iterator & end) { - if (*arg == "--gc-roots-dir") - gcRootsDir = getArg(*arg, arg, end); - else if (*arg == "--dry-run") - settings.readOnlyMode = true; - else if (*arg != "" && arg->at(0) == '-') - return false; - else - releaseExpr = *arg; - return true; - }); + MyArgs() : MixCommonArgs("hydra-eval-jobs") + { + mkFlag() + .longName("help") + .description("show usage information") + .handler([&]() { + printHelp(programName, std::cout); + throw Exit(); + }); + + mkFlag() + .longName("gc-roots-dir") + .description("garbage collector roots directory") + .labels({"path"}) + .dest(&gcRootsDir); + + mkFlag() + .longName("dry-run") + .description("don't create store derivations") + .set(&settings.readOnlyMode, true); + + expectArg("expr", &releaseExpr); + } + }; + MyArgs myArgs; myArgs.parseCmdline(argvToStrings(argc, argv)); JSONObject json(std::cout, true); @@ -211,7 +222,7 @@ int main(int argc, char * * argv) to the environment. */ evalSettings.restrictEval = true; - if (releaseExpr == "") throw UsageError("no expression specified"); + if (myArgs.releaseExpr == "") throw UsageError("no expression specified"); if (gcRootsDir == "") printMsg(lvlError, "warning: `--gc-roots-dir' not specified"); @@ -220,7 +231,7 @@ int main(int argc, char * * argv) Bindings & autoArgs = *myArgs.getAutoArgs(state); Value v; - state.evalFile(lookupFileArg(state, releaseExpr), v); + state.evalFile(lookupFileArg(state, myArgs.releaseExpr), v); findJobs(state, json, autoArgs, v, ""); From ddcf05ac2226ae5632f4b6f6c481b30eb6164375 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 10 May 2019 22:18:15 +0200 Subject: [PATCH 07/73] release.nix: Do not rely on currentSystem --- release.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release.nix b/release.nix index 4747b9ba7..20e983763 100644 --- a/release.nix +++ b/release.nix @@ -8,7 +8,7 @@ with import (nixpkgs + "/lib"); let - pkgs = import nixpkgs {}; + pkgs = import nixpkgs { system = "x86_64-linux"; }; genAttrs' = genAttrs [ "x86_64-linux" /* "i686-linux" */ ]; From 6ee6ec3bda5c20c4920135bac090ea9af7600719 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 10 May 2019 22:18:37 +0200 Subject: [PATCH 08/73] hydra-eval-jobs: Support flakes E.g. 'hydra-eval-jobs ~/Dev/patchelf' is enough to evaluate patchelf - no need to set up a $NIX_PATH, pass arguments, etc. --- src/hydra-eval-jobs/hydra-eval-jobs.cc | 32 ++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index 6b4636343..fa5fc176b 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -13,6 +13,8 @@ #include "get-drvs.hh" #include "globals.hh" #include "common-eval-args.hh" +#include "flakeref.hh" +#include "flake.hh" #include "hydra-config.hh" @@ -183,6 +185,7 @@ int main(int argc, char * * argv) struct MyArgs : MixEvalArgs, MixCommonArgs { Path releaseExpr; + bool flake = false; MyArgs() : MixCommonArgs("hydra-eval-jobs") { @@ -205,6 +208,11 @@ int main(int argc, char * * argv) .description("don't create store derivations") .set(&settings.readOnlyMode, true); + mkFlag() + .longName("flake") + .description("build a flake") + .set(&flake, true); + expectArg("expr", &releaseExpr); } }; @@ -222,6 +230,10 @@ int main(int argc, char * * argv) to the environment. */ evalSettings.restrictEval = true; + /* When building a flake, use pure evaluation (no access to + 'getEnv', 'currentSystem' etc. */ + evalSettings.pureEval = myArgs.flake; + if (myArgs.releaseExpr == "") throw UsageError("no expression specified"); if (gcRootsDir == "") printMsg(lvlError, "warning: `--gc-roots-dir' not specified"); @@ -231,9 +243,25 @@ int main(int argc, char * * argv) Bindings & autoArgs = *myArgs.getAutoArgs(state); Value v; - state.evalFile(lookupFileArg(state, myArgs.releaseExpr), v); - findJobs(state, json, autoArgs, v, ""); + if (myArgs.flake) { + FlakeRef flakeRef(myArgs.releaseExpr); + auto vFlake = state.allocValue(); + makeFlakeValue(state, flakeRef, AllowRegistryAtTop, *vFlake); + + auto vProvides = (*vFlake->attrs->get(state.symbols.create("provides")))->value; + state.forceValue(*vProvides); + auto aHydraJobs = vProvides->attrs->get(state.symbols.create("hydraJobs")); + if (!aHydraJobs) + throw Error("flake '%s' does not provide any Hydra jobs", flakeRef); + + v = *(*aHydraJobs)->value; + + } else { + state.evalFile(lookupFileArg(state, myArgs.releaseExpr), v); + } + + findJobs(state, json, autoArgs, v, ""); }); } From f9f595cd21d4fbb11148c2e46f48930bfc2e18b3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 10 May 2019 23:39:55 +0200 Subject: [PATCH 09/73] Add flake configuration to the web interface --- src/lib/Hydra/Controller/Jobset.pm | 51 ++++++++++++++++++------------ src/lib/Hydra/Schema/Builds.pm | 11 +++++-- src/lib/Hydra/Schema/Jobsets.pm | 34 ++++++++++++++++---- src/root/edit-jobset.tt | 35 +++++++++++++++++++- src/root/jobset.tt | 12 +++++++ src/sql/hydra.sql | 8 +++-- src/sql/upgrade-57.sql | 6 ++++ 7 files changed, 125 insertions(+), 32 deletions(-) create mode 100644 src/sql/upgrade-57.sql diff --git a/src/lib/Hydra/Controller/Jobset.pm b/src/lib/Hydra/Controller/Jobset.pm index 75284c100..0be734187 100644 --- a/src/lib/Hydra/Controller/Jobset.pm +++ b/src/lib/Hydra/Controller/Jobset.pm @@ -223,12 +223,19 @@ sub updateJobset { error($c, "Cannot rename jobset to ‘$jobsetName’ since that identifier is already taken.") if $jobsetName ne $oldName && defined $c->stash->{project}->jobsets->find({ name => $jobsetName }); - # When the expression is in a .scm file, assume it's a Guile + Guix - # build expression. - my $exprType = - $c->stash->{params}->{"nixexprpath"} =~ /.scm$/ ? "guile" : "nix"; - - my ($nixExprPath, $nixExprInput) = nixExprPathFromParams $c; + my $type = int($c->stash->{params}->{"type"}) // 0; + + my ($nixExprPath, $nixExprInput); + my $flake; + + if ($type == 0) { + ($nixExprPath, $nixExprInput) = nixExprPathFromParams $c; + } elsif ($type == 1) { + $flake = trim($c->stash->{params}->{"flakeref"}); + error($c, "Invalid flake URI ‘$flake’.") if $flake !~ /^[a-zA-Z]/; + } else { + error($c, "Invalid jobset type."); + } my $enabled = int($c->stash->{params}->{enabled}); die if $enabled < 0 || $enabled > 2; @@ -251,6 +258,8 @@ sub updateJobset { , checkinterval => $checkinterval , triggertime => ($enabled && $checkinterval > 0) ? $jobset->triggertime // time() : undef , schedulingshares => $shares + , type => $type + , flake => $flake }); $jobset->project->jobsetrenames->search({ from_ => $jobsetName })->delete; @@ -260,23 +269,25 @@ sub updateJobset { # Set the inputs of this jobset. $jobset->jobsetinputs->delete; - foreach my $name (keys %{$c->stash->{params}->{inputs}}) { - my $inputData = $c->stash->{params}->{inputs}->{$name}; - my $type = $inputData->{type}; - my $value = $inputData->{value}; - my $emailresponsible = defined $inputData->{emailresponsible} ? 1 : 0; + if ($type == 0) { + foreach my $name (keys %{$c->stash->{params}->{inputs}}) { + my $inputData = $c->stash->{params}->{inputs}->{$name}; + my $type = $inputData->{type}; + my $value = $inputData->{value}; + my $emailresponsible = defined $inputData->{emailresponsible} ? 1 : 0; - error($c, "Invalid input name ‘$name’.") unless $name =~ /^[[:alpha:]][\w-]*$/; - error($c, "Invalid input type ‘$type’.") unless defined $c->stash->{inputTypes}->{$type}; + error($c, "Invalid input name ‘$name’.") unless $name =~ /^[[:alpha:]][\w-]*$/; + error($c, "Invalid input type ‘$type’.") unless defined $c->stash->{inputTypes}->{$type}; - my $input = $jobset->jobsetinputs->create( - { name => $name, - type => $type, - emailresponsible => $emailresponsible - }); + my $input = $jobset->jobsetinputs->create( + { name => $name, + type => $type, + emailresponsible => $emailresponsible + }); - $value = checkInputValue($c, $name, $type, $value); - $input->jobsetinputalts->create({altnr => 0, value => $value}); + $value = checkInputValue($c, $name, $type, $value); + $input->jobsetinputalts->create({altnr => 0, value => $value}); + } } } diff --git a/src/lib/Hydra/Schema/Builds.pm b/src/lib/Hydra/Schema/Builds.pm index 1c80f7103..a4de7f554 100644 --- a/src/lib/Hydra/Schema/Builds.pm +++ b/src/lib/Hydra/Schema/Builds.pm @@ -191,6 +191,11 @@ __PACKAGE__->table("Builds"); default_value: 0 is_nullable: 0 +=head2 notificationpendingsince + + data_type: 'integer' + is_nullable: 1 + =cut __PACKAGE__->add_columns( @@ -252,6 +257,8 @@ __PACKAGE__->add_columns( { data_type => "text", is_nullable => 1 }, "keep", { data_type => "integer", default_value => 0, is_nullable => 0 }, + "notificationpendingsince", + { data_type => "integer", is_nullable => 1 }, ); =head1 PRIMARY KEY @@ -537,8 +544,8 @@ __PACKAGE__->many_to_many( ); -# Created by DBIx::Class::Schema::Loader v0.07043 @ 2016-02-12 17:20:42 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:/8aVtXu/+o0jmKHnSzwt+g +# Created by DBIx::Class::Schema::Loader v0.07049 @ 2019-05-10 22:30:12 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:YK8Fc+37UAcL0u6ziOc5xQ __PACKAGE__->has_many( "dependents", diff --git a/src/lib/Hydra/Schema/Jobsets.pm b/src/lib/Hydra/Schema/Jobsets.pm index 17b4ab935..3ca67e18d 100644 --- a/src/lib/Hydra/Schema/Jobsets.pm +++ b/src/lib/Hydra/Schema/Jobsets.pm @@ -56,12 +56,12 @@ __PACKAGE__->table("Jobsets"); data_type: 'text' is_foreign_key: 1 - is_nullable: 0 + is_nullable: 1 =head2 nixexprpath data_type: 'text' - is_nullable: 0 + is_nullable: 1 =head2 errormsg @@ -139,6 +139,17 @@ __PACKAGE__->table("Jobsets"); data_type: 'integer' is_nullable: 1 +=head2 type + + data_type: 'integer' + default_value: 0 + is_nullable: 0 + +=head2 flake + + data_type: 'text' + is_nullable: 1 + =cut __PACKAGE__->add_columns( @@ -149,9 +160,9 @@ __PACKAGE__->add_columns( "description", { data_type => "text", is_nullable => 1 }, "nixexprinput", - { data_type => "text", is_foreign_key => 1, is_nullable => 0 }, + { data_type => "text", is_foreign_key => 1, is_nullable => 1 }, "nixexprpath", - { data_type => "text", is_nullable => 0 }, + { data_type => "text", is_nullable => 1 }, "errormsg", { data_type => "text", is_nullable => 1 }, "errortime", @@ -180,6 +191,10 @@ __PACKAGE__->add_columns( { data_type => "boolean", is_nullable => 1 }, "starttime", { data_type => "integer", is_nullable => 1 }, + "type", + { data_type => "integer", default_value => 0, is_nullable => 0 }, + "flake", + { data_type => "text", is_nullable => 1 }, ); =head1 PRIMARY KEY @@ -282,7 +297,12 @@ __PACKAGE__->belongs_to( "jobsetinput", "Hydra::Schema::JobsetInputs", { jobset => "name", name => "nixexprinput", project => "project" }, - { is_deferrable => 0, on_delete => "NO ACTION", on_update => "NO ACTION" }, + { + is_deferrable => 0, + join_type => "LEFT", + on_delete => "NO ACTION", + on_update => "NO ACTION", + }, ); =head2 jobsetinputs @@ -352,8 +372,8 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.07045 @ 2017-03-09 13:03:05 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:ivYvsUyhEeaeI4EmRQ0/QQ +# Created by DBIx::Class::Schema::Loader v0.07049 @ 2019-05-11 00:03:52 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:UVG1D59bXaQ1TUEF237tXQ my %hint = ( columns => [ diff --git a/src/root/edit-jobset.tt b/src/root/edit-jobset.tt index 6c380a3a3..95b1afaf6 100644 --- a/src/root/edit-jobset.tt +++ b/src/root/edit-jobset.tt @@ -42,7 +42,7 @@ [% END %] [% BLOCK renderJobsetInputs %] - +
@@ -96,6 +96,24 @@
+ +
+
+ + + +
+
+
+ +
+ +
+ jobset.flake) %]/> +
+
+ +
jobset.nixexprpath) %]/> @@ -167,6 +185,21 @@ $(document).ready(function() { var id = 0; + function update() { + if ($("#type").val() == 0) { + $(".show-on-legacy").show(); + $(".show-on-flake").hide(); + } else { + $(".show-on-legacy").hide(); + $(".show-on-flake").show(); + } + } + + $("#type-flake").click(function() { update(); }); + $("#type-legacy").click(function() { update(); }); + + update(); + $(".add-input").click(function() { var newid = "input-" + id++; var x = $("#input-template").clone(true).attr("id", "").insertBefore($(this).parents("tr")).show(); diff --git a/src/root/jobset.tt b/src/root/jobset.tt index 9cf1202a4..4b891ff63 100644 --- a/src/root/jobset.tt +++ b/src/root/jobset.tt @@ -135,6 +135,15 @@
+ [% IF jobset.type == 1 %] + + + + + [% END %] + [% IF jobset.type == 0 %] + [% END %] @@ -166,7 +176,9 @@
Input nameTypeValueNotify committers
Description: [% HTML.escape(jobset.description) %]
Flake URI: + [% HTML.escape(jobset.flake) %] +
Nix expression: @@ -142,6 +151,7 @@ [% HTML.escape(jobset.nixexprinput) %]
Check interval: [% jobset.checkinterval || "disabled" %]
+ [% IF jobset.type == 0 %] [% INCLUDE renderJobsetInputs %] + [% END %] [% INCLUDE makeLazyTab tabName="tabs-jobs" uri=c.uri_for('/jobset' project.name jobset.name "jobs-tab") %] diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index 4c8710b86..4b37f6e1c 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -54,8 +54,8 @@ create table Jobsets ( name text not null, project text not null, description text, - nixExprInput text not null, -- name of the jobsetInput containing the Nix or Guix expression - nixExprPath text not null, -- relative path of the Nix or Guix expression + nixExprInput text, -- name of the jobsetInput containing the Nix or Guix expression + nixExprPath text, -- relative path of the Nix or Guix expression errorMsg text, -- used to signal the last evaluation error etc. for this jobset errorTime integer, -- timestamp associated with errorMsg lastCheckedTime integer, -- last time the evaluator looked at this jobset @@ -70,7 +70,11 @@ create table Jobsets ( fetchErrorMsg text, forceEval boolean, startTime integer, -- if jobset is currently running + type integer not null default 0, -- 0 == legacy, 1 == flake + flake text, check (schedulingShares > 0), + check ((type = 0) = (nixExprInput is not null and nixExprPath is not null)), + check ((type = 1) = (flake is not null)), primary key (project, name), foreign key (project) references Projects(name) on delete cascade on update cascade #ifdef SQLITE diff --git a/src/sql/upgrade-57.sql b/src/sql/upgrade-57.sql new file mode 100644 index 000000000..15553bebd --- /dev/null +++ b/src/sql/upgrade-57.sql @@ -0,0 +1,6 @@ +alter table Jobsets alter column nixExprInput drop not null; +alter table Jobsets alter column nixExprPath drop not null; +alter table Jobsets add column type integer default 0; +alter table Jobsets add column flake text; +alter table Jobsets add check ((type = 0) = (nixExprInput is not null and nixExprPath is not null)); +alter table Jobsets add check ((type = 1) = (flake is not null)); From 842ef9af24f0a595983afb29b69cdc524f17bf27 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 11 May 2019 00:10:46 +0200 Subject: [PATCH 10/73] hydra-eval-jobset: Support flakes --- src/script/hydra-eval-jobset | 37 ++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset index 9d375c1b7..1e5c81e7f 100755 --- a/src/script/hydra-eval-jobset +++ b/src/script/hydra-eval-jobset @@ -352,18 +352,27 @@ sub inputsToArgs { sub evalJobs { - my ($inputInfo, $exprType, $nixExprInputName, $nixExprPath) = @_; + my ($inputInfo, $exprType, $nixExprInputName, $nixExprPath, $flakeRef) = @_; - my $nixExprInput = $inputInfo->{$nixExprInputName}->[0] - or die "cannot find the input containing the job expression\n"; + my @cmd; - my $evaluator = ($exprType eq "guile") ? "hydra-eval-guile-jobs" : "hydra-eval-jobs"; + if (defined $flakeRef) { + @cmd = ("hydra-eval-jobs", + "--flake", $flakeRef, + "--gc-roots-dir", getGCRootsDir, + "--max-jobs", 1); + } else { + my $nixExprInput = $inputInfo->{$nixExprInputName}->[0] + or die "cannot find the input containing the job expression\n"; + + my $evaluator = ($exprType eq "guile") ? "hydra-eval-guile-jobs" : "hydra-eval-jobs"; - my @cmd = ($evaluator, - "<" . $nixExprInputName . "/" . $nixExprPath . ">", - "--gc-roots-dir", getGCRootsDir, - "-j", 1, - inputsToArgs($inputInfo, $exprType)); + @cmd = ($evaluator, + "<" . $nixExprInputName . "/" . $nixExprPath . ">", + "--gc-roots-dir", getGCRootsDir, + "--max-jobs", 1, + inputsToArgs($inputInfo, $exprType)); + } if (defined $ENV{'HYDRA_DEBUG'}) { sub escape { @@ -376,13 +385,13 @@ sub evalJobs { } (my $res, my $jobsJSON, my $stderr) = captureStdoutStderr(21600, @cmd); - die "$evaluator returned " . ($res & 127 ? "signal $res" : "exit code " . ($res >> 8)) + die "hydra-eval-jobs returned " . ($res & 127 ? "signal $res" : "exit code " . ($res >> 8)) . ":\n" . ($stderr ? decode("utf-8", $stderr) : "(no output)\n") if $res; print STDERR "$stderr"; - return (decode_json($jobsJSON), $nixExprInput); + return decode_json($jobsJSON); } @@ -400,7 +409,7 @@ sub getPrevJobsetEval { # Check whether to add the build described by $buildInfo. sub checkBuild { - my ($db, $jobset, $inputInfo, $nixExprInput, $buildInfo, $buildMap, $prevEval, $jobOutPathMap, $plugins) = @_; + my ($db, $jobset, $inputInfo, $buildInfo, $buildMap, $prevEval, $jobOutPathMap, $plugins) = @_; my @outputNames = sort keys %{$buildInfo->{outputs}}; die unless scalar @outputNames; @@ -628,7 +637,7 @@ sub checkJobsetWrapped { # Evaluate the job expression. my $evalStart = clock_gettime(CLOCK_MONOTONIC); - my ($jobs, $nixExprInput) = evalJobs($inputInfo, $exprType, $jobset->nixexprinput, $jobset->nixexprpath); + my $jobs = evalJobs($inputInfo, $exprType, $jobset->nixexprinput, $jobset->nixexprpath, $jobset->flake); my $evalStop = clock_gettime(CLOCK_MONOTONIC); if ($jobsetsJobset) { @@ -673,7 +682,7 @@ sub checkJobsetWrapped { foreach my $job (permute(values %{$jobs})) { next if defined $job->{error}; #print STDERR "considering job " . $project->name, ":", $jobset->name, ":", $job->{jobName} . "\n"; - checkBuild($db, $jobset, $inputInfo, $nixExprInput, $job, \%buildMap, $prevEval, $jobOutPathMap, $plugins); + checkBuild($db, $jobset, $inputInfo, $job, \%buildMap, $prevEval, $jobOutPathMap, $plugins); } # Have any builds been added or removed since last time? From 0387787e7edd33efcbb46d1969d4e3868d2b9734 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 11 May 2019 00:14:18 +0200 Subject: [PATCH 11/73] TODO --- src/sql/hydra.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index 4b37f6e1c..8998fd572 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -185,7 +185,8 @@ create table Builds ( -- Copy of the nixExprInput/nixExprPath fields of the jobset that -- instantiated this build. Needed if we want to reproduce this - -- build. + -- build. FIXME: this should be stored in JobsetEvals, storing it + -- here is denormal. nixExprInput text, nixExprPath text, From 30e8fe951b94c0ac0819b14a38d120fc340d2f1c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 11 May 2019 00:40:40 +0200 Subject: [PATCH 12/73] Improve error message --- src/hydra-evaluator/hydra-evaluator.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hydra-evaluator/hydra-evaluator.cc b/src/hydra-evaluator/hydra-evaluator.cc index 86ac5c399..e3004c2ef 100644 --- a/src/hydra-evaluator/hydra-evaluator.cc +++ b/src/hydra-evaluator/hydra-evaluator.cc @@ -83,7 +83,7 @@ struct Evaluator } if (evalOne && seen.empty()) { - printError("the specified jobset does not exist"); + printError("the specified jobset does not exist or is disabled"); std::_Exit(1); } From ed00f0b25e5cee10a434f170d18f62a6692681a1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 11 May 2019 00:41:13 +0200 Subject: [PATCH 13/73] Cache flake-based jobset evaluations --- src/lib/Hydra/Schema/JobsetEvals.pm | 11 +++++++++-- src/script/hydra-eval-jobset | 15 +++++++++++++-- src/sql/hydra.sql | 2 ++ src/sql/upgrade-57.sql | 1 + 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/lib/Hydra/Schema/JobsetEvals.pm b/src/lib/Hydra/Schema/JobsetEvals.pm index 0bd21da2f..ef0b15232 100644 --- a/src/lib/Hydra/Schema/JobsetEvals.pm +++ b/src/lib/Hydra/Schema/JobsetEvals.pm @@ -88,6 +88,11 @@ __PACKAGE__->table("JobsetEvals"); data_type: 'integer' is_nullable: 1 +=head2 flake + + data_type: 'text' + is_nullable: 1 + =cut __PACKAGE__->add_columns( @@ -111,6 +116,8 @@ __PACKAGE__->add_columns( { data_type => "integer", is_nullable => 1 }, "nrsucceeded", { data_type => "integer", is_nullable => 1 }, + "flake", + { data_type => "text", is_nullable => 1 }, ); =head1 PRIMARY KEY @@ -188,8 +195,8 @@ __PACKAGE__->belongs_to( ); -# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-06-13 01:54:50 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:SlEiF8oN6FBK262uSiMKiw +# Created by DBIx::Class::Schema::Loader v0.07049 @ 2019-05-11 00:16:10 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:XwlJFCJiS0LHsLg2fFqfUg __PACKAGE__->has_many( "buildIds", diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset index 1e5c81e7f..71d24a833 100755 --- a/src/script/hydra-eval-jobset +++ b/src/script/hydra-eval-jobset @@ -607,6 +607,16 @@ sub checkJobsetWrapped { }; my $fetchError = $@; + my $flakeRef = $jobset->flake; + if (defined $flakeRef) { + (my $res, my $json, my $stderr) = captureStdoutStderr( + 600, "nix", "flake", "info", "--tarball-ttl", 0, "--json", "--", $flakeRef); + die "'nix flake info' returned " . ($res & 127 ? "signal $res" : "exit code " . ($res >> 8)) + . ":\n" . ($stderr ? decode("utf-8", $stderr) : "(no output)\n") + if $res; + $flakeRef = decode_json($json)->{'uri'}; + } + Net::Statsd::increment("hydra.evaluator.checkouts"); my $checkoutStop = clock_gettime(CLOCK_MONOTONIC); Net::Statsd::timing("hydra.evaluator.checkout_time", int(($checkoutStop - $checkoutStart) * 1000)); @@ -626,7 +636,7 @@ sub checkJobsetWrapped { my @args = ($jobset->nixexprinput, $jobset->nixexprpath, inputsToArgs($inputInfo, $exprType)); my $argsHash = sha256_hex("@args"); my $prevEval = getPrevJobsetEval($db, $jobset, 0); - if (defined $prevEval && $prevEval->hash eq $argsHash && !$dryRun && !$jobset->forceeval) { + if (defined $prevEval && $prevEval->hash eq $argsHash && !$dryRun && !$jobset->forceeval && $prevEval->flake eq $flakeRef) { print STDERR " jobset is unchanged, skipping\n"; Net::Statsd::increment("hydra.evaluator.unchanged_checkouts"); txn_do($db, sub { @@ -637,7 +647,7 @@ sub checkJobsetWrapped { # Evaluate the job expression. my $evalStart = clock_gettime(CLOCK_MONOTONIC); - my $jobs = evalJobs($inputInfo, $exprType, $jobset->nixexprinput, $jobset->nixexprpath, $jobset->flake); + my $jobs = evalJobs($inputInfo, $exprType, $jobset->nixexprinput, $jobset->nixexprpath, $flakeRef); my $evalStop = clock_gettime(CLOCK_MONOTONIC); if ($jobsetsJobset) { @@ -697,6 +707,7 @@ sub checkJobsetWrapped { , evaltime => abs(int($evalStop - $evalStart)) , hasnewbuilds => $jobsetChanged ? 1 : 0 , nrbuilds => $jobsetChanged ? scalar(keys %buildMap) : undef + , flake => $flakeRef }); if ($jobsetChanged) { diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index 8998fd572..f81bbad43 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -527,6 +527,8 @@ create table JobsetEvals ( nrBuilds integer, nrSucceeded integer, -- set lazily when all builds are finished + flake text, -- immutable flake reference + foreign key (project) references Projects(name) on delete cascade on update cascade, foreign key (project, jobset) references Jobsets(project, name) on delete cascade on update cascade ); diff --git a/src/sql/upgrade-57.sql b/src/sql/upgrade-57.sql index 15553bebd..b59ee9497 100644 --- a/src/sql/upgrade-57.sql +++ b/src/sql/upgrade-57.sql @@ -4,3 +4,4 @@ alter table Jobsets add column type integer default 0; alter table Jobsets add column flake text; alter table Jobsets add check ((type = 0) = (nixExprInput is not null and nixExprPath is not null)); alter table Jobsets add check ((type = 1) = (flake is not null)); +alter table JobsetEvals add column flake text; From 4ec51032eedc8fabd108266d5eee08fb3e62f962 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 11 May 2019 00:52:06 +0200 Subject: [PATCH 14/73] Jobset eval page: Show immutable flake URI --- src/root/jobset-eval.tt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/root/jobset-eval.tt b/src/root/jobset-eval.tt index 979d9862d..c091ec590 100644 --- a/src/root/jobset-eval.tt +++ b/src/root/jobset-eval.tt @@ -18,7 +18,8 @@ -

This evaluation was performed on [% INCLUDE renderDateTime +

This evaluation was performed [% IF eval.flake %]from the flake +[%HTML.escape(eval.flake)%][%END%] on [% INCLUDE renderDateTime timestamp=eval.timestamp %]. Fetching the dependencies took [% eval.checkouttime %]s and evaluation took [% eval.evaltime %]s.

From f68cb7b57e29370b2de641091ee8be72ff5b6ed2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 11 May 2019 00:52:25 +0200 Subject: [PATCH 15/73] "Reproduce" action: Support flakes No more need for a reproduction script! It just says something like If you have Nix installed, you can reproduce this build on your own machine by running the following command: # nix build github:edolstra/dwarffs/09c823e977946668b63ad6c88ed358b48220f124:hydraJobs.build.x86_64-linux --- src/root/build.tt | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/root/build.tt b/src/root/build.tt index 750ae4742..b563ebb12 100644 --- a/src/root/build.tt +++ b/src/root/build.tt @@ -120,7 +120,7 @@ END;
    - [% IF build.nixexprinput %] + [% IF build.nixexprinput || eval.flake %]
  • Reproduce locally
    • [% END %] [% IF c.user_exists %] @@ -532,18 +532,33 @@ END;
    -

    You can reproduce this build on your own machine by downloading - url) %]>a script that checks out - all inputs of the build and then invokes Nix to perform the build. - This script requires that you have Nix on your system.

    + [% IF eval.flake %] + +

    If you have Nix + installed, you can reproduce this build on your own machine by + running the following command:

    + +
    +# nix build [% HTML.escape(eval.flake) %]:hydraJobs.[% HTML.escape(job.name) %]
+
    + + [% ELSE %] + +

    If you have Nix + installed, you can reproduce this build on your own machine by + downloading url) %]>a script + that checks out all inputs of the build and then invokes Nix to + perform the build.

    To download and execute the script from the command line, run the following command:

    -$ curl  url) %]>[% HTML.escape(url) %] | bash
+# curl  url) %]>[% HTML.escape(url) %] | bash
    + [% END %] +
    From 89fa82930af32ae8d7543fe73500084f6b3c2066 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 11 May 2019 00:55:26 +0200 Subject: [PATCH 16/73] Create projects in enabled state --- src/root/edit-project.tt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/root/edit-project.tt b/src/root/edit-project.tt index cedc36ee2..b2d7264a9 100644 --- a/src/root/edit-project.tt +++ b/src/root/edit-project.tt @@ -8,7 +8,7 @@
    From 438ddf52898cddf100cf59ca91f32581368c9b2e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 15 May 2019 09:53:23 +0200 Subject: [PATCH 17/73] Fix error in GitlabStatus plugin May 15 09:20:10 chef hydra-queue-runner[27523]: Hydra::Plugin::GitlabStatus=HASH(0x519a7b8)->buildFinished: Can't call method "value" on an undefined value at /nix/store/858hinflxcl2jd12wv1r3a8j11ybsf6w-hydra-0.1.2629.89fa829/libexec/hydra/lib/Hydra/Plugin/GitlabStatus.pm line 57. --- src/lib/Hydra/Plugin/GitlabStatus.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/Hydra/Plugin/GitlabStatus.pm b/src/lib/Hydra/Plugin/GitlabStatus.pm index 4aef2f090..82f2fb20a 100644 --- a/src/lib/Hydra/Plugin/GitlabStatus.pm +++ b/src/lib/Hydra/Plugin/GitlabStatus.pm @@ -54,7 +54,7 @@ sub common { }); while (my $eval = $evals->next) { my $gitlabstatusInput = $eval->jobsetevalinputs->find({ name => "gitlab_status_repo" }); - next unless defined $gitlabstatusInput->value; + next unless defined $gitlabstatusInput && defined $gitlabstatusInput->value; my $i = $eval->jobsetevalinputs->find({ name => $gitlabstatusInput->value, altnr => 0 }); next unless defined $i; my $projectId = $eval->jobsetevalinputs->find({ name => "gitlab_project_id" })->value; From 8717d69a36bb944f3d97a7876a4e17ccc96f3902 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 May 2019 14:44:10 +0200 Subject: [PATCH 18/73] Build using the nix flake --- flake.lock | 10 ++++++++++ flake.nix | 5 ++++- release.nix | 8 +++----- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 965d038ce..3b733fc8b 100644 --- a/flake.lock +++ b/flake.lock @@ -1,6 +1,16 @@ { "nonFlakeRequires": {}, "requires": { + "nix": { + "contentHash": "sha256-rUM0KiuCRHv8WKBOfv4XKOu7WP8nsYdPdnTPZFBzjyc=", + "requires": { + "nixpkgs": { + "contentHash": "sha256-vy2UmXQM66aS/Kn2tCtjt9RwxfBvV+nQVb5tJQFwi8E=", + "uri": "github:edolstra/nixpkgs/a4d896e89932e873c4117908d558db6210fa3b56" + } + }, + "uri": "github:NixOS/nix/3cecf3f39c5c580ed6d0c5f0bcbb09dcc53dfae4" + }, "nixpkgs": { "contentHash": "sha256-vy2UmXQM66aS/Kn2tCtjt9RwxfBvV+nQVb5tJQFwi8E=", "uri": "github:edolstra/nixpkgs/a4d896e89932e873c4117908d558db6210fa3b56" diff --git a/flake.nix b/flake.nix index 15e30151d..09d0bc92e 100644 --- a/flake.nix +++ b/flake.nix @@ -5,13 +5,16 @@ epoch = 2019; - requires = [ "nixpkgs" ]; + requires = [ "nixpkgs" "nix" ]; provides = deps: rec { hydraJobs = import ./release.nix { hydraSrc = deps.self; nixpkgs = deps.nixpkgs; + nix = deps.nix.provides.hydraJobs.build.x86_64-linux // { + perl-bindings = deps.nix.provides.hydraJobs.perlBindings.x86_64-linux; + }; }; packages.hydra = hydraJobs.build.x86_64-linux; diff --git a/release.nix b/release.nix index 20e983763..c01518b2d 100644 --- a/release.nix +++ b/release.nix @@ -2,14 +2,14 @@ , nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-19.03-small"; } , officialRelease ? false , shell ? false +, pkgs ? import nixpkgs { system = "x86_64-linux"; } +, nix ? pkgs.nixUnstable or pkgs.nix }: -with import (nixpkgs + "/lib"); +with pkgs.lib; let - pkgs = import nixpkgs { system = "x86_64-linux"; }; - genAttrs' = genAttrs [ "x86_64-linux" /* "i686-linux" */ ]; hydraServer = hydraPkg: @@ -44,8 +44,6 @@ rec { let - nix = pkgs.nixUnstable or pkgs.nix; - perlDeps = buildEnv { name = "hydra-perl-deps"; paths = with perlPackages; From b7be2bf9adc8a21990bb3bddfcbcc1957cfa8286 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 May 2019 14:51:55 +0200 Subject: [PATCH 19/73] Fix build against the latest flakes branch --- src/hydra-eval-jobs/hydra-eval-jobs.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index fa5fc176b..ce219db54 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -247,7 +247,7 @@ int main(int argc, char * * argv) if (myArgs.flake) { FlakeRef flakeRef(myArgs.releaseExpr); auto vFlake = state.allocValue(); - makeFlakeValue(state, flakeRef, AllowRegistryAtTop, *vFlake); + makeFlakeValue(state, flakeRef, AllPure, *vFlake); auto vProvides = (*vFlake->attrs->get(state.symbols.create("provides")))->value; state.forceValue(*vProvides); From bc11ab22a3654c09fc956114efe471cc40f98fc2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 3 Jun 2019 12:09:18 +0200 Subject: [PATCH 20/73] Update flake.nix --- flake.lock | 18 +++++++++--------- flake.nix | 16 ++++++++-------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/flake.lock b/flake.lock index 3b733fc8b..fa4dec65f 100644 --- a/flake.lock +++ b/flake.lock @@ -1,20 +1,20 @@ { - "nonFlakeRequires": {}, - "requires": { + "inputs": { "nix": { - "contentHash": "sha256-rUM0KiuCRHv8WKBOfv4XKOu7WP8nsYdPdnTPZFBzjyc=", - "requires": { + "inputs": { "nixpkgs": { - "contentHash": "sha256-vy2UmXQM66aS/Kn2tCtjt9RwxfBvV+nQVb5tJQFwi8E=", - "uri": "github:edolstra/nixpkgs/a4d896e89932e873c4117908d558db6210fa3b56" + "narHash": "sha256-rMiWaLXkhizEEMEeMDutUl0Y/c+VEjfjvMkvBwvuQJU=", + "uri": "github:edolstra/nixpkgs/eeeffd24cd7e407cfaa99e98cfbb8f93bf4cc033" } }, - "uri": "github:NixOS/nix/3cecf3f39c5c580ed6d0c5f0bcbb09dcc53dfae4" + "narHash": "sha256-Gu5o0nUsQHthWNXGQ3iZviecgNGcEKY+4R0okapfZ54=", + "uri": "github:NixOS/nix/507c150034256117cd56e31944feedb8f2f3ff3e" }, "nixpkgs": { - "contentHash": "sha256-vy2UmXQM66aS/Kn2tCtjt9RwxfBvV+nQVb5tJQFwi8E=", - "uri": "github:edolstra/nixpkgs/a4d896e89932e873c4117908d558db6210fa3b56" + "narHash": "sha256-rMiWaLXkhizEEMEeMDutUl0Y/c+VEjfjvMkvBwvuQJU=", + "uri": "github:edolstra/nixpkgs/eeeffd24cd7e407cfaa99e98cfbb8f93bf4cc033" } }, + "nonFlakeInputs": {}, "version": 1 } diff --git a/flake.nix b/flake.nix index 09d0bc92e..c25b92b22 100644 --- a/flake.nix +++ b/flake.nix @@ -5,15 +5,15 @@ epoch = 2019; - requires = [ "nixpkgs" "nix" ]; + inputs = [ "nixpkgs" "nix" ]; - provides = deps: rec { + outputs = inputs: rec { hydraJobs = import ./release.nix { - hydraSrc = deps.self; - nixpkgs = deps.nixpkgs; - nix = deps.nix.provides.hydraJobs.build.x86_64-linux // { - perl-bindings = deps.nix.provides.hydraJobs.perlBindings.x86_64-linux; + hydraSrc = inputs.self; + nixpkgs = inputs.nixpkgs; + nix = inputs.nix.outputs.hydraJobs.build.x86_64-linux // { + perl-bindings = inputs.nix.outputs.hydraJobs.perlBindings.x86_64-linux; }; }; @@ -22,8 +22,8 @@ defaultPackage = packages.hydra; devShell = (import ./release.nix { - hydraSrc = deps.self; - nixpkgs = deps.nixpkgs; + hydraSrc = inputs.self; + nixpkgs = inputs.nixpkgs; shell = true; }).build.x86_64-linux; From 00421837176e39711c8cb42b853166e8cc01857a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 3 Jun 2019 12:23:49 +0200 Subject: [PATCH 21/73] Fix build --- src/hydra-eval-jobs/hydra-eval-jobs.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index ce219db54..f865e922c 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -245,9 +245,10 @@ int main(int argc, char * * argv) Value v; if (myArgs.flake) { + using namespace flake; FlakeRef flakeRef(myArgs.releaseExpr); auto vFlake = state.allocValue(); - makeFlakeValue(state, flakeRef, AllPure, *vFlake); + callFlake(state, resolveFlake(state, flakeRef, AllPure), *vFlake); auto vProvides = (*vFlake->attrs->get(state.symbols.create("provides")))->value; state.forceValue(*vProvides); From 517acb4ca85587479e9e246853901f057fe08aa1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 3 Jun 2019 12:29:42 +0200 Subject: [PATCH 22/73] Add some flake checks --- flake.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flake.nix b/flake.nix index c25b92b22..48cf104b7 100644 --- a/flake.nix +++ b/flake.nix @@ -17,6 +17,9 @@ }; }; + checks.build = hydraJobs.build.x86_64-linux; + checks.install = hydraJobs.tests.install.x86_64-linux; + packages.hydra = hydraJobs.build.x86_64-linux; defaultPackage = packages.hydra; From 9a8880fd289ece3462009f164f7204813273e76f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 3 Jun 2019 13:53:22 +0200 Subject: [PATCH 23/73] Doh --- src/hydra-eval-jobs/hydra-eval-jobs.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index f865e922c..e104fb5f8 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -250,10 +250,10 @@ int main(int argc, char * * argv) auto vFlake = state.allocValue(); callFlake(state, resolveFlake(state, flakeRef, AllPure), *vFlake); - auto vProvides = (*vFlake->attrs->get(state.symbols.create("provides")))->value; - state.forceValue(*vProvides); + auto vOutputs = (*vFlake->attrs->get(state.symbols.create("outputs")))->value; + state.forceValue(*vOutputs); - auto aHydraJobs = vProvides->attrs->get(state.symbols.create("hydraJobs")); + auto aHydraJobs = vOutputs->attrs->get(state.symbols.create("hydraJobs")); if (!aHydraJobs) throw Error("flake '%s' does not provide any Hydra jobs", flakeRef); From 7936a293643ae52b1af1747ee806316e28133b86 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 3 Jun 2019 13:55:32 +0200 Subject: [PATCH 24/73] hydra-eval-jobs: Use 'checks' output if there is no 'hydraJobs' output --- src/hydra-eval-jobs/hydra-eval-jobs.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index e104fb5f8..a309abc75 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -255,7 +255,9 @@ int main(int argc, char * * argv) auto aHydraJobs = vOutputs->attrs->get(state.symbols.create("hydraJobs")); if (!aHydraJobs) - throw Error("flake '%s' does not provide any Hydra jobs", flakeRef); + aHydraJobs = vOutputs->attrs->get(state.symbols.create("checks")); + if (!aHydraJobs) + throw Error("flake '%s' does not provide any Hydra jobs or checks", flakeRef); v = *(*aHydraJobs)->value; From d834ac9c1446bf84c516bf33b65047e7e37e98e5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 18 Jun 2019 18:00:40 +0200 Subject: [PATCH 25/73] Update flake.{nix,lock} --- flake.lock | 22 +++++++++++++++------- flake.nix | 2 +- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/flake.lock b/flake.lock index fa4dec65f..acad413e7 100644 --- a/flake.lock +++ b/flake.lock @@ -1,20 +1,28 @@ { "inputs": { "nix": { + "id": "nix", "inputs": { "nixpkgs": { - "narHash": "sha256-rMiWaLXkhizEEMEeMDutUl0Y/c+VEjfjvMkvBwvuQJU=", - "uri": "github:edolstra/nixpkgs/eeeffd24cd7e407cfaa99e98cfbb8f93bf4cc033" + "id": "nixpkgs", + "inputs": {}, + "narHash": "sha256-5/HXWs25BLukwG9VaxmdmUf/9o79e32aW/tmhcWEbKk=", + "nonFlakeInputs": {}, + "uri": "github:edolstra/nixpkgs/62ac6f7f504c8d3998558d9b269d22d26f13f1f0" } }, - "narHash": "sha256-Gu5o0nUsQHthWNXGQ3iZviecgNGcEKY+4R0okapfZ54=", - "uri": "github:NixOS/nix/507c150034256117cd56e31944feedb8f2f3ff3e" + "narHash": "sha256-MjLaMAh/KtAksnMwOH2GJn+YRJ4szwx24ybFXHTmShU=", + "nonFlakeInputs": {}, + "uri": "github:NixOS/nix/5d8ec94d7fe4eded9f836bc49449d6eaff8fbc8f" }, "nixpkgs": { - "narHash": "sha256-rMiWaLXkhizEEMEeMDutUl0Y/c+VEjfjvMkvBwvuQJU=", - "uri": "github:edolstra/nixpkgs/eeeffd24cd7e407cfaa99e98cfbb8f93bf4cc033" + "id": "nixpkgs", + "inputs": {}, + "narHash": "sha256-5/HXWs25BLukwG9VaxmdmUf/9o79e32aW/tmhcWEbKk=", + "nonFlakeInputs": {}, + "uri": "github:edolstra/nixpkgs/62ac6f7f504c8d3998558d9b269d22d26f13f1f0" } }, "nonFlakeInputs": {}, - "version": 1 + "version": 2 } diff --git a/flake.nix b/flake.nix index 48cf104b7..06b3657fe 100644 --- a/flake.nix +++ b/flake.nix @@ -3,7 +3,7 @@ description = "A Nix-based continuous build system"; - epoch = 2019; + epoch = 201906; inputs = [ "nixpkgs" "nix" ]; From 1c9e22e73c402a311a456fc559eb3ea1286aaabe Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 9 Aug 2019 13:10:50 +0200 Subject: [PATCH 26/73] Use nixFlakes in devShell --- flake.lock | 6 +++--- flake.nix | 41 ++++++++++++++++++++++++----------------- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/flake.lock b/flake.lock index acad413e7..dfbe5b29b 100644 --- a/flake.lock +++ b/flake.lock @@ -1,6 +1,6 @@ { "inputs": { - "nix": { + "nix/ab16b3d076e9cd3ecfdcde128f43dd486b072557": { "id": "nix", "inputs": { "nixpkgs": { @@ -11,9 +11,9 @@ "uri": "github:edolstra/nixpkgs/62ac6f7f504c8d3998558d9b269d22d26f13f1f0" } }, - "narHash": "sha256-MjLaMAh/KtAksnMwOH2GJn+YRJ4szwx24ybFXHTmShU=", + "narHash": "sha256-I9YtUO9Ia5CwiTFo4QzSYdZHO2G+BQ7sM4MSq29NDqc=", "nonFlakeInputs": {}, - "uri": "github:NixOS/nix/5d8ec94d7fe4eded9f836bc49449d6eaff8fbc8f" + "uri": "github:NixOS/nix/ab16b3d076e9cd3ecfdcde128f43dd486b072557" }, "nixpkgs": { "id": "nixpkgs", diff --git a/flake.nix b/flake.nix index 06b3657fe..780b43cb9 100644 --- a/flake.nix +++ b/flake.nix @@ -5,30 +5,37 @@ epoch = 201906; - inputs = [ "nixpkgs" "nix" ]; + inputs = + [ "nixpkgs" + "nix/ab16b3d076e9cd3ecfdcde128f43dd486b072557" + ]; - outputs = inputs: rec { - - hydraJobs = import ./release.nix { - hydraSrc = inputs.self; - nixpkgs = inputs.nixpkgs; + outputs = inputs: + let nix = inputs.nix.outputs.hydraJobs.build.x86_64-linux // { perl-bindings = inputs.nix.outputs.hydraJobs.perlBindings.x86_64-linux; }; - }; + in rec { - checks.build = hydraJobs.build.x86_64-linux; - checks.install = hydraJobs.tests.install.x86_64-linux; + hydraJobs = import ./release.nix { + hydraSrc = inputs.self; + nixpkgs = inputs.nixpkgs; + inherit nix; + }; - packages.hydra = hydraJobs.build.x86_64-linux; + checks.build = hydraJobs.build.x86_64-linux; + checks.install = hydraJobs.tests.install.x86_64-linux; - defaultPackage = packages.hydra; + packages.hydra = hydraJobs.build.x86_64-linux; - devShell = (import ./release.nix { - hydraSrc = inputs.self; - nixpkgs = inputs.nixpkgs; - shell = true; - }).build.x86_64-linux; + defaultPackage = packages.hydra; - }; + devShell = (import ./release.nix { + hydraSrc = inputs.self; + nixpkgs = inputs.nixpkgs; + shell = true; + inherit nix; + }).build.x86_64-linux; + + }; } From ada640d340b15658eab38dc1c1d79924dd10e2d2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 30 Aug 2019 17:23:05 +0200 Subject: [PATCH 27/73] Update flake.nix --- flake.lock | 21 +++++++-------------- flake.nix | 23 +++++++++-------------- 2 files changed, 16 insertions(+), 28 deletions(-) diff --git a/flake.lock b/flake.lock index dfbe5b29b..936075770 100644 --- a/flake.lock +++ b/flake.lock @@ -1,28 +1,21 @@ { "inputs": { - "nix/ab16b3d076e9cd3ecfdcde128f43dd486b072557": { - "id": "nix", + "nix": { "inputs": { "nixpkgs": { - "id": "nixpkgs", "inputs": {}, - "narHash": "sha256-5/HXWs25BLukwG9VaxmdmUf/9o79e32aW/tmhcWEbKk=", - "nonFlakeInputs": {}, - "uri": "github:edolstra/nixpkgs/62ac6f7f504c8d3998558d9b269d22d26f13f1f0" + "narHash": "sha256-TrLhI3xPkTTznE9gcMHhkHirGNN7N02zM4DxJ/U3WRs=", + "uri": "github:edolstra/nixpkgs/24bf27fc215e8300877dfa1c426b9966bbfbd150" } }, - "narHash": "sha256-I9YtUO9Ia5CwiTFo4QzSYdZHO2G+BQ7sM4MSq29NDqc=", - "nonFlakeInputs": {}, - "uri": "github:NixOS/nix/ab16b3d076e9cd3ecfdcde128f43dd486b072557" + "narHash": "sha256-oj3c5cpfoiwoKEaevSfOLGdcyS3LynAZkM9tuUNWcdE=", + "uri": "github:NixOS/nix/80c36d4562af71a90c67b3adb886a1003834890e" }, "nixpkgs": { - "id": "nixpkgs", "inputs": {}, - "narHash": "sha256-5/HXWs25BLukwG9VaxmdmUf/9o79e32aW/tmhcWEbKk=", - "nonFlakeInputs": {}, - "uri": "github:edolstra/nixpkgs/62ac6f7f504c8d3998558d9b269d22d26f13f1f0" + "narHash": "sha256-TrLhI3xPkTTznE9gcMHhkHirGNN7N02zM4DxJ/U3WRs=", + "uri": "github:edolstra/nixpkgs/24bf27fc215e8300877dfa1c426b9966bbfbd150" } }, - "nonFlakeInputs": {}, "version": 2 } diff --git a/flake.nix b/flake.nix index 780b43cb9..3ba5b3c59 100644 --- a/flake.nix +++ b/flake.nix @@ -1,25 +1,20 @@ { - name = "hydra"; - description = "A Nix-based continuous build system"; - epoch = 201906; + epoch = 201909; - inputs = - [ "nixpkgs" - "nix/ab16b3d076e9cd3ecfdcde128f43dd486b072557" - ]; + inputs.nix.uri = "nix/80c36d4562af71a90c67b3adb886a1003834890e"; - outputs = inputs: + outputs = { self, nixpkgs, nix }@inputs: let - nix = inputs.nix.outputs.hydraJobs.build.x86_64-linux // { - perl-bindings = inputs.nix.outputs.hydraJobs.perlBindings.x86_64-linux; + nix = inputs.nix.hydraJobs.build.x86_64-linux // { + perl-bindings = inputs.nix.hydraJobs.perlBindings.x86_64-linux; }; in rec { hydraJobs = import ./release.nix { - hydraSrc = inputs.self; - nixpkgs = inputs.nixpkgs; + hydraSrc = self; + nixpkgs = nixpkgs; inherit nix; }; @@ -31,8 +26,8 @@ defaultPackage = packages.hydra; devShell = (import ./release.nix { - hydraSrc = inputs.self; - nixpkgs = inputs.nixpkgs; + hydraSrc = self; + nixpkgs = nixpkgs; shell = true; inherit nix; }).build.x86_64-linux; From 66d71689ec43814004f7f741c87347068a2f5074 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 17 Sep 2019 17:25:59 +0200 Subject: [PATCH 28/73] flake.nix improvements The hydra flake now exports a Nixpkgs overlay and a NixOS module. --- flake.nix | 300 +++++++++++++++++++++++++++++++++++++++--- hydra-module.nix | 4 +- release.nix | 331 ----------------------------------------------- 3 files changed, 285 insertions(+), 350 deletions(-) delete mode 100644 release.nix diff --git a/flake.nix b/flake.nix index 3ba5b3c59..affc0a185 100644 --- a/flake.nix +++ b/flake.nix @@ -5,32 +5,300 @@ inputs.nix.uri = "nix/80c36d4562af71a90c67b3adb886a1003834890e"; - outputs = { self, nixpkgs, nix }@inputs: + outputs = { self, nixpkgs, nix }: let - nix = inputs.nix.hydraJobs.build.x86_64-linux // { - perl-bindings = inputs.nix.hydraJobs.perlBindings.x86_64-linux; + + version = "${builtins.readFile ./version}.${builtins.substring 0 8 self.lastModified}.${self.shortRev}"; + + # FIXME: use nix overlay? + nix' = nix.hydraJobs.build.x86_64-linux // { + perl-bindings = nix.hydraJobs.perlBindings.x86_64-linux; + }; + + pkgs = import nixpkgs { + system = "x86_64-linux"; + overlays = [ self.overlay ]; }; + + # NixOS configuration used for VM tests. + hydraServer = + { config, pkgs, ... }: + { imports = [ self.nixosModules.hydra ]; + + virtualisation.memorySize = 1024; + virtualisation.writableStore = true; + + services.hydra-dev.enable = true; + services.hydra-dev.hydraURL = "http://hydra.example.org"; + services.hydra-dev.notificationSender = "admin@hydra.example.org"; + + services.postgresql.enable = true; + services.postgresql.package = pkgs.postgresql95; + + environment.systemPackages = [ pkgs.perlPackages.LWP pkgs.perlPackages.JSON ]; + + # The following is to work around the following error from hydra-server: + # [error] Caught exception in engine "Cannot determine local time zone" + time.timeZone = "UTC"; + + nix = { + # The following is to work around: https://github.com/NixOS/hydra/pull/432 + buildMachines = [ + { hostName = "localhost"; + system = "x86_64-linux"; + } + ]; + # Without this nix tries to fetch packages from the default + # cache.nixos.org which is not reachable from this sandboxed NixOS test. + binaryCaches = []; + }; + }; + in rec { - hydraJobs = import ./release.nix { - hydraSrc = self; - nixpkgs = nixpkgs; - inherit nix; + # A Nixpkgs overlay that provides a 'hydra' package. + overlay = final: prev: { + + hydra = with final; let + + perlDeps = buildEnv { + name = "hydra-perl-deps"; + paths = with perlPackages; + [ ModulePluggable + CatalystActionREST + CatalystAuthenticationStoreDBIxClass + CatalystDevel + CatalystDispatchTypeRegex + CatalystPluginAccessLog + CatalystPluginAuthorizationRoles + CatalystPluginCaptcha + CatalystPluginSessionStateCookie + CatalystPluginSessionStoreFastMmap + CatalystPluginStackTrace + CatalystPluginUnicodeEncoding + CatalystTraitForRequestProxyBase + CatalystViewDownload + CatalystViewJSON + CatalystViewTT + CatalystXScriptServerStarman + CatalystXRoleApplicator + CryptRandPasswd + DBDPg + DBDSQLite + DataDump + DateTime + DigestSHA1 + EmailMIME + EmailSender + FileSlurp + IOCompress + IPCRun + JSON + JSONAny + JSONXS + LWP + LWPProtocolHttps + NetAmazonS3 + NetStatsd + PadWalker + Readonly + SQLSplitStatement + SetScalar + Starman + SysHostnameLong + TestMore + TextDiff + TextTable + XMLSimple + nix' + nix'.perl-bindings + git + boehmgc + ]; + }; + + in stdenv.mkDerivation { + + name = "hydra-${version}"; + + src = self; + + buildInputs = + [ makeWrapper autoconf automake libtool unzip nukeReferences pkgconfig sqlite libpqxx + gitAndTools.topGit mercurial darcs subversion bazaar openssl bzip2 libxslt + guile # optional, for Guile + Guix support + perlDeps perl nix' + postgresql95 # for running the tests + boost + nlohmann_json + ]; + + hydraPath = lib.makeBinPath ( + [ sqlite subversion openssh nix' coreutils findutils pixz + gzip bzip2 lzma gnutar unzip git gitAndTools.topGit mercurial darcs gnused bazaar + ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] ); + + configureFlags = [ "--with-docbook-xsl=${docbook_xsl}/xml/xsl/docbook" ]; + + shellHook = '' + PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$PATH + PERL5LIB=$(pwd)/src/lib:$PERL5LIB + ''; + + preConfigure = "autoreconf -vfi"; + + NIX_LDFLAGS = [ "-lpthread" ]; + + enableParallelBuilding = true; + + preCheck = '' + patchShebangs . + export LOGNAME=''${LOGNAME:-foo} + ''; + + postInstall = '' + mkdir -p $out/nix-support + + for i in $out/bin/*; do + read -n 4 chars < $i + if [[ $chars =~ ELF ]]; then continue; fi + wrapProgram $i \ + --prefix PERL5LIB ':' $out/libexec/hydra/lib:$PERL5LIB \ + --prefix PATH ':' $out/bin:$hydraPath \ + --set HYDRA_RELEASE ${version} \ + --set HYDRA_HOME $out/libexec/hydra \ + --set NIX_RELEASE ${nix'.name or "unknown"} + done + ''; + + dontStrip = true; + + meta.description = "Build of Hydra on ${system}"; + passthru.perlDeps = perlDeps; + }; + }; + + hydraJobs = { + + build.x86_64-linux = packages.hydra; + + manual = + pkgs.runCommand "hydra-manual-${version}" {} + '' + mkdir -p $out/share + cp -prvd ${pkgs.hydra}/share/doc $out/share/ + + mkdir $out/nix-support + echo "doc manual $out/share/doc/hydra" >> $out/nix-support/hydra-build-products + ''; + + tests.install.x86_64-linux = + with import (nixpkgs + "/nixos/lib/testing.nix") { system = "x86_64-linux"; }; + simpleTest { + machine = hydraServer; + testScript = + '' + $machine->waitForJob("hydra-init"); + $machine->waitForJob("hydra-server"); + $machine->waitForJob("hydra-evaluator"); + $machine->waitForJob("hydra-queue-runner"); + $machine->waitForOpenPort("3000"); + $machine->succeed("curl --fail http://localhost:3000/"); + ''; + }; + + tests.api.x86_64-linux = + with import (nixpkgs + "/nixos/lib/testing.nix") { system = "x86_64-linux"; }; + simpleTest { + machine = hydraServer; + testScript = + let dbi = "dbi:Pg:dbname=hydra;user=root;"; in + '' + $machine->waitForJob("hydra-init"); + + # Create an admin account and some other state. + $machine->succeed + ( "su - hydra -c \"hydra-create-user root --email-address 'alice\@example.org' --password foobar --role admin\"" + , "mkdir /run/jobset /tmp/nix" + , "chmod 755 /run/jobset /tmp/nix" + , "cp ${./tests/api-test.nix} /run/jobset/default.nix" + , "chmod 644 /run/jobset/default.nix" + , "chown -R hydra /run/jobset /tmp/nix" + ); + + $machine->succeed("systemctl stop hydra-evaluator hydra-queue-runner"); + $machine->waitForJob("hydra-server"); + $machine->waitForOpenPort("3000"); + + # Run the API tests. + $machine->mustSucceed("su - hydra -c 'perl -I ${pkgs.hydra.perlDeps}/lib/perl5/site_perl ${./tests/api-test.pl}' >&2"); + ''; + }; + + tests.notifications.x86_64-linux = + with import (nixpkgs + "/nixos/lib/testing.nix") { system = "x86_64-linux"; }; + simpleTest { + machine = { pkgs, ... }: { + imports = [ hydraServer ]; + services.hydra-dev.extraConfig = '' + + url = http://127.0.0.1:8086 + db = hydra + + ''; + services.influxdb.enable = true; + }; + testScript = '' + $machine->waitForJob("hydra-init"); + + # Create an admin account and some other state. + $machine->succeed + ( "su - hydra -c \"hydra-create-user root --email-address 'alice\@example.org' --password foobar --role admin\"" + , "mkdir /run/jobset" + , "chmod 755 /run/jobset" + , "cp ${./tests/api-test.nix} /run/jobset/default.nix" + , "chmod 644 /run/jobset/default.nix" + , "chown -R hydra /run/jobset" + ); + + # Wait until InfluxDB can receive web requests + $machine->waitForJob("influxdb"); + $machine->waitForOpenPort("8086"); + + # Create an InfluxDB database where hydra will write to + $machine->succeed( + "curl -XPOST 'http://127.0.0.1:8086/query' \\ + --data-urlencode 'q=CREATE DATABASE hydra'"); + + # Wait until hydra-server can receive HTTP requests + $machine->waitForJob("hydra-server"); + $machine->waitForOpenPort("3000"); + + # Setup the project and jobset + $machine->mustSucceed( + "su - hydra -c 'perl -I ${pkgs.hydra.perlDeps}/lib/perl5/site_perl ${./tests/setup-notifications-jobset.pl}' >&2"); + + # Wait until hydra has build the job and + # the InfluxDBNotification plugin uploaded its notification to InfluxDB + $machine->waitUntilSucceeds( + "curl -s -H 'Accept: application/csv' \\ + -G 'http://127.0.0.1:8086/query?db=hydra' \\ + --data-urlencode 'q=SELECT * FROM hydra_build_status' | grep success"); + ''; + }; + }; checks.build = hydraJobs.build.x86_64-linux; checks.install = hydraJobs.tests.install.x86_64-linux; - packages.hydra = hydraJobs.build.x86_64-linux; - - defaultPackage = packages.hydra; + packages.hydra = pkgs.hydra; + defaultPackage = pkgs.hydra; - devShell = (import ./release.nix { - hydraSrc = self; - nixpkgs = nixpkgs; - shell = true; - inherit nix; - }).build.x86_64-linux; + nixosModules.hydra = { + imports = [ ./hydra-module.nix ]; + nixpkgs.overlays = [ self.overlay ]; + }; }; } diff --git a/hydra-module.nix b/hydra-module.nix index d601ac37d..4fa2d63f6 100644 --- a/hydra-module.nix +++ b/hydra-module.nix @@ -64,7 +64,7 @@ in package = mkOption { type = types.path; - #default = pkgs.hydra; + default = pkgs.hydra; description = "The Hydra package."; }; @@ -218,8 +218,6 @@ in nix.trustedUsers = [ "hydra-queue-runner" ]; - services.hydra-dev.package = mkDefault ((import ./release.nix {}).build.x86_64-linux); - services.hydra-dev.extraConfig = '' using_frontend_proxy = 1 diff --git a/release.nix b/release.nix deleted file mode 100644 index 4ae73091b..000000000 --- a/release.nix +++ /dev/null @@ -1,331 +0,0 @@ -{ hydraSrc ? builtins.fetchGit ./. -, nixpkgs ? builtins.fetchGit { url = https://github.com/NixOS/nixpkgs-channels.git; ref = "nixos-19.03-small"; } -, officialRelease ? false -, shell ? false -, pkgs ? import nixpkgs { system = "x86_64-linux"; } -, nix ? pkgs.nixUnstable or pkgs.nix -}: - -with pkgs.lib; - -let - - genAttrs' = genAttrs [ "x86_64-linux" /* "i686-linux" */ ]; - - hydraServer = hydraPkg: - { config, pkgs, ... }: - { imports = [ ./hydra-module.nix ]; - - virtualisation.memorySize = 1024; - virtualisation.writableStore = true; - - services.hydra-dev.enable = true; - services.hydra-dev.package = hydraPkg; - services.hydra-dev.hydraURL = "http://hydra.example.org"; - services.hydra-dev.notificationSender = "admin@hydra.example.org"; - - services.postgresql.enable = true; - services.postgresql.package = pkgs.postgresql95; - - environment.systemPackages = [ pkgs.perlPackages.LWP pkgs.perlPackages.JSON ]; - - # The following is to work around the following error from hydra-server: - # [error] Caught exception in engine "Cannot determine local time zone" - time.timeZone = "UTC"; - - nix = { - # The following is to work around: https://github.com/NixOS/hydra/pull/432 - buildMachines = [ - { hostName = "localhost"; - system = "x86_64-linux"; - } - ]; - # Without this nix tries to fetch packages from the default - # cache.nixos.org which is not reachable from this sandboxed NixOS test. - binaryCaches = []; - }; - }; - - # FIXME: use commit date. - version = builtins.readFile ./version + "." + toString hydraSrc.revCount or 0 + "." + hydraSrc.shortRev or "0000000"; - -in - -rec { - - build = genAttrs' (system: - let pkgs = import nixpkgs { inherit system; }; in - - with pkgs; - - let - - perlDeps = buildEnv { - name = "hydra-perl-deps"; - paths = with perlPackages; - [ ModulePluggable - CatalystActionREST - CatalystAuthenticationStoreDBIxClass - CatalystDevel - CatalystDispatchTypeRegex - CatalystPluginAccessLog - CatalystPluginAuthorizationRoles - CatalystPluginCaptcha - CatalystPluginSessionStateCookie - CatalystPluginSessionStoreFastMmap - CatalystPluginStackTrace - CatalystPluginUnicodeEncoding - CatalystTraitForRequestProxyBase - CatalystViewDownload - CatalystViewJSON - CatalystViewTT - CatalystXScriptServerStarman - CatalystXRoleApplicator - CryptRandPasswd - DBDPg - DBDSQLite - DataDump - DateTime - DigestSHA1 - EmailMIME - EmailSender - FileSlurp - IOCompress - IPCRun - JSON - JSONAny - JSONXS - LWP - LWPProtocolHttps - NetAmazonS3 - NetStatsd - PadWalker - Readonly - SQLSplitStatement - SetScalar - Starman - SysHostnameLong - TestMore - TextDiff - TextTable - XMLSimple - nix - nix.perl-bindings - git - boehmgc - ]; - }; - - in - - releaseTools.nixBuild { - name = "hydra-${version}"; - - src = if shell then null else hydraSrc; - - buildInputs = - [ makeWrapper autoconf automake libtool unzip nukeReferences pkgconfig sqlite libpqxx - gitAndTools.topGit mercurial darcs subversion bazaar openssl bzip2 libxslt - guile # optional, for Guile + Guix support - perlDeps perl nix - postgresql95 # for running the tests - boost - nlohmann_json - ]; - - hydraPath = lib.makeBinPath ( - [ sqlite subversion openssh nix coreutils findutils pixz - gzip bzip2 lzma gnutar unzip git gitAndTools.topGit mercurial darcs gnused bazaar - ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] ); - - postUnpack = optionalString (!shell) '' - # Clean up when building from a working tree. - (cd $sourceRoot && (git ls-files -o --directory | xargs -r rm -rfv)) || true - ''; - - configureFlags = [ "--with-docbook-xsl=${docbook_xsl}/xml/xsl/docbook" ]; - - shellHook = '' - PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$PATH - ${lib.optionalString shell "PERL5LIB=$(pwd)/src/lib:$PERL5LIB"} - ''; - - preConfigure = "autoreconf -vfi"; - - NIX_LDFLAGS = [ "-lpthread" ]; - - enableParallelBuilding = true; - - preCheck = '' - patchShebangs . - export LOGNAME=''${LOGNAME:-foo} - ''; - - postInstall = '' - mkdir -p $out/nix-support - - for i in $out/bin/*; do - read -n 4 chars < $i - if [[ $chars =~ ELF ]]; then continue; fi - wrapProgram $i \ - --prefix PERL5LIB ':' $out/libexec/hydra/lib:$PERL5LIB \ - --prefix PATH ':' $out/bin:$hydraPath \ - --set HYDRA_RELEASE ${version} \ - --set HYDRA_HOME $out/libexec/hydra \ - --set NIX_RELEASE ${nix.name or "unknown"} - done - ''; # */ - - dontStrip = true; - - meta.description = "Build of Hydra on ${system}"; - passthru.perlDeps = perlDeps; - }); - - manual = pkgs.runCommand "hydra-manual-${version}" - { build = build.x86_64-linux; - } - '' - mkdir -p $out/share - cp -prvd $build/share/doc $out/share/ - - mkdir $out/nix-support - echo "doc manual $out/share/doc/hydra" >> $out/nix-support/hydra-build-products - ''; - - tests.install = genAttrs' (system: - with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; }; - simpleTest { - machine = hydraServer build.${system}; - testScript = - '' - $machine->waitForJob("hydra-init"); - $machine->waitForJob("hydra-server"); - $machine->waitForJob("hydra-evaluator"); - $machine->waitForJob("hydra-queue-runner"); - $machine->waitForOpenPort("3000"); - $machine->succeed("curl --fail http://localhost:3000/"); - ''; - }); - - tests.api = genAttrs' (system: - with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; }; - simpleTest { - machine = hydraServer build.${system}; - testScript = - let dbi = "dbi:Pg:dbname=hydra;user=root;"; in - '' - $machine->waitForJob("hydra-init"); - - # Create an admin account and some other state. - $machine->succeed - ( "su - hydra -c \"hydra-create-user root --email-address 'alice\@example.org' --password foobar --role admin\"" - , "mkdir /run/jobset /tmp/nix" - , "chmod 755 /run/jobset /tmp/nix" - , "cp ${./tests/api-test.nix} /run/jobset/default.nix" - , "chmod 644 /run/jobset/default.nix" - , "chown -R hydra /run/jobset /tmp/nix" - ); - - $machine->succeed("systemctl stop hydra-evaluator hydra-queue-runner"); - $machine->waitForJob("hydra-server"); - $machine->waitForOpenPort("3000"); - - # Run the API tests. - $machine->mustSucceed("su - hydra -c 'perl -I ${build.${system}.perlDeps}/lib/perl5/site_perl ${./tests/api-test.pl}' >&2"); - ''; - }); - - tests.notifications = genAttrs' (system: - with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; }; - simpleTest { - machine = { pkgs, ... }: { - imports = [ (hydraServer build.${system}) ]; - services.hydra-dev.extraConfig = '' - - url = http://127.0.0.1:8086 - db = hydra - - ''; - services.influxdb.enable = true; - }; - testScript = '' - $machine->waitForJob("hydra-init"); - - # Create an admin account and some other state. - $machine->succeed - ( "su - hydra -c \"hydra-create-user root --email-address 'alice\@example.org' --password foobar --role admin\"" - , "mkdir /run/jobset" - , "chmod 755 /run/jobset" - , "cp ${./tests/api-test.nix} /run/jobset/default.nix" - , "chmod 644 /run/jobset/default.nix" - , "chown -R hydra /run/jobset" - ); - - # Wait until InfluxDB can receive web requests - $machine->waitForJob("influxdb"); - $machine->waitForOpenPort("8086"); - - # Create an InfluxDB database where hydra will write to - $machine->succeed( - "curl -XPOST 'http://127.0.0.1:8086/query' \\ - --data-urlencode 'q=CREATE DATABASE hydra'"); - - # Wait until hydra-server can receive HTTP requests - $machine->waitForJob("hydra-server"); - $machine->waitForOpenPort("3000"); - - # Setup the project and jobset - $machine->mustSucceed( - "su - hydra -c 'perl -I ${build.${system}.perlDeps}/lib/perl5/site_perl ${./tests/setup-notifications-jobset.pl}' >&2"); - - # Wait until hydra has build the job and - # the InfluxDBNotification plugin uploaded its notification to InfluxDB - $machine->waitUntilSucceeds( - "curl -s -H 'Accept: application/csv' \\ - -G 'http://127.0.0.1:8086/query?db=hydra' \\ - --data-urlencode 'q=SELECT * FROM hydra_build_status' | grep success"); - ''; - }); - - /* - tests.s3backup = genAttrs' (system: - with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; }; - let hydra = build.${system} - simpleTest { - machine = - { config, pkgs, ... }: - { services.postgresql.enable = true; - services.postgresql.package = pkgs.postgresql95; - environment.systemPackages = [ hydra pkgs.rubyLibs.fakes3 ]; - virtualisation.memorySize = 2047; - boot.kernelPackages = pkgs.linuxPackages_3_10; - virtualisation.writableStore = true; - networking.extraHosts = '' - 127.0.0.1 hydra.s3.amazonaws.com - ''; - }; - - testScript = - '' - $machine->waitForJob("postgresql"); - - # Initialise the database and the state. - $machine->succeed - ( "createdb -O root hydra" - , "psql hydra -f ${hydra}/libexec/hydra/sql/hydra-postgresql.sql" - , "mkdir /var/lib/hydra" - , "mkdir /tmp/jobs" - , "cp ${./tests/s3-backup-test.pl} /tmp/s3-backup-test.pl" - , "cp ${./tests/api-test.nix} /tmp/jobs/default.nix" - ); - - # start fakes3 - $machine->succeed("fakes3 --root /tmp/s3 --port 80 &>/dev/null &"); - $machine->waitForOpenPort("80"); - - $machine->succeed("cd /tmp && LOGNAME=root AWS_ACCESS_KEY_ID=foo AWS_SECRET_ACCESS_KEY=bar HYDRA_DBI='dbi:Pg:dbname=hydra;user=root;' HYDRA_CONFIG=${./tests/s3-backup-test.config} perl -I ${hydra}/libexec/hydra/lib -I ${hydra.perlDeps}/lib/perl5/site_perl ./s3-backup-test.pl >&2"); - ''; - }); - */ -} From 1ab3b26b677ea3924991ba5e82b8f2266a8a69ab Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 19 Sep 2019 00:10:12 +0200 Subject: [PATCH 29/73] Update lock file --- flake.lock | 17 ++++++++++------- flake.nix | 2 -- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/flake.lock b/flake.lock index 936075770..8b63b8ec3 100644 --- a/flake.lock +++ b/flake.lock @@ -4,18 +4,21 @@ "inputs": { "nixpkgs": { "inputs": {}, - "narHash": "sha256-TrLhI3xPkTTznE9gcMHhkHirGNN7N02zM4DxJ/U3WRs=", - "uri": "github:edolstra/nixpkgs/24bf27fc215e8300877dfa1c426b9966bbfbd150" + "narHash": "sha256-HGlE2VNbdEjCP76hWAS72kHBlMWhpvqWo58Obg1Vy6s=", + "originalUri": "nixpkgs", + "uri": "github:edolstra/nixpkgs/13e1bce51f4aebdf3db58ce8c4a93e904a272bff" } }, - "narHash": "sha256-oj3c5cpfoiwoKEaevSfOLGdcyS3LynAZkM9tuUNWcdE=", - "uri": "github:NixOS/nix/80c36d4562af71a90c67b3adb886a1003834890e" + "narHash": "sha256-bzSMXhxVX5awBSTO4v6Fe0dd1igEMVFskJ5RKSSJ+/4=", + "originalUri": "nix", + "uri": "github:NixOS/nix/aeb7148afd56b228604b79373a45793d36d660a3" }, "nixpkgs": { "inputs": {}, - "narHash": "sha256-TrLhI3xPkTTznE9gcMHhkHirGNN7N02zM4DxJ/U3WRs=", - "uri": "github:edolstra/nixpkgs/24bf27fc215e8300877dfa1c426b9966bbfbd150" + "narHash": "sha256-HGlE2VNbdEjCP76hWAS72kHBlMWhpvqWo58Obg1Vy6s=", + "originalUri": "nixpkgs", + "uri": "github:edolstra/nixpkgs/13e1bce51f4aebdf3db58ce8c4a93e904a272bff" } }, - "version": 2 + "version": 3 } diff --git a/flake.nix b/flake.nix index affc0a185..7b05c2d74 100644 --- a/flake.nix +++ b/flake.nix @@ -3,8 +3,6 @@ epoch = 201909; - inputs.nix.uri = "nix/80c36d4562af71a90c67b3adb886a1003834890e"; - outputs = { self, nixpkgs, nix }: let From 40e7b6e8846c5acd1d3f462541b5bafa9acb8bfd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 24 Sep 2019 19:58:10 +0200 Subject: [PATCH 30/73] Add a NixOS container to demo Hydra It contains Hydra, PostgreSQL and an frontend proxy. So you can get a running Hydra instance by doing $ nixos-container create hydra --flake hydra $ nixos-container start hydra The web interface is available on port 80. --- flake.lock | 6 ++--- flake.nix | 77 +++++++++++++++++++++++++++++++++++++++++------------- 2 files changed, 62 insertions(+), 21 deletions(-) diff --git a/flake.lock b/flake.lock index 8b63b8ec3..598e1acad 100644 --- a/flake.lock +++ b/flake.lock @@ -15,9 +15,9 @@ }, "nixpkgs": { "inputs": {}, - "narHash": "sha256-HGlE2VNbdEjCP76hWAS72kHBlMWhpvqWo58Obg1Vy6s=", - "originalUri": "nixpkgs", - "uri": "github:edolstra/nixpkgs/13e1bce51f4aebdf3db58ce8c4a93e904a272bff" + "narHash": "sha256-qWEM8iYRYUdaVCHwTq13w69/11hAFWZplp5mqeU2qBM=", + "originalUri": "nixpkgs/release-19.09", + "uri": "github:edolstra/nixpkgs/7d9219a0adea88942b8d609f89cf4a1f86df1a9c" } }, "version": 3 diff --git a/flake.nix b/flake.nix index a027b5cbb..01efa0170 100644 --- a/flake.nix +++ b/flake.nix @@ -3,6 +3,8 @@ epoch = 201909; + inputs.nixpkgs.uri = "nixpkgs/release-19.09"; + outputs = { self, nixpkgs, nix }: let @@ -21,31 +23,14 @@ # NixOS configuration used for VM tests. hydraServer = { config, pkgs, ... }: - { imports = [ self.nixosModules.hydra ]; + { imports = [ self.nixosModules.hydraTest ]; virtualisation.memorySize = 1024; virtualisation.writableStore = true; - services.hydra-dev.enable = true; - services.hydra-dev.hydraURL = "http://hydra.example.org"; - services.hydra-dev.notificationSender = "admin@hydra.example.org"; - - services.postgresql.enable = true; - services.postgresql.package = pkgs.postgresql95; - environment.systemPackages = [ pkgs.perlPackages.LWP pkgs.perlPackages.JSON ]; - # The following is to work around the following error from hydra-server: - # [error] Caught exception in engine "Cannot determine local time zone" - time.timeZone = "UTC"; - nix = { - # The following is to work around: https://github.com/NixOS/hydra/pull/432 - buildMachines = [ - { hostName = "localhost"; - system = "x86_64-linux"; - } - ]; # Without this nix tries to fetch packages from the default # cache.nixos.org which is not reachable from this sandboxed NixOS test. binaryCaches = []; @@ -297,6 +282,62 @@ nixosModules.hydra = { imports = [ ./hydra-module.nix ]; nixpkgs.overlays = [ self.overlay ]; + nix.package = nix'; + }; + + nixosModules.hydraTest = { + imports = [ self.nixosModules.hydra ]; + + services.hydra-dev.enable = true; + services.hydra-dev.hydraURL = "http://hydra.example.org"; + services.hydra-dev.notificationSender = "admin@hydra.example.org"; + + systemd.services.hydra-send-stats.enable = false; + + services.postgresql.enable = true; + services.postgresql.package = pkgs.postgresql95; + + # The following is to work around the following error from hydra-server: + # [error] Caught exception in engine "Cannot determine local time zone" + time.timeZone = "UTC"; + }; + + nixosConfigurations.container = nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + modules = + [ { imports = [ self.nixosModules.hydraTest ]; + system.configurationRevision = self.rev; + + boot.isContainer = true; + networking.useDHCP = false; + networking.firewall.allowedTCPPorts = [ 80 ]; + networking.hostName = "hydra"; + + nix.extraOptions = '' + allowed-uris = https://github.com/ + ''; + + services.hydra-dev.useSubstitutes = true; + + services.httpd = { + enable = true; + adminAddr = "hydra-admin@example.org"; + extraConfig = '' + + Order deny,allow + Allow from all + + + ProxyRequests Off + ProxyPreserveHost On + ProxyPass /apache-errors ! + ErrorDocument 503 /apache-errors/503.html + ProxyPass / http://127.0.0.1:3000/ retry=5 disablereuse=on + ProxyPassReverse / http://127.0.0.1:3000/ + ''; + }; + } + ]; }; }; From e110d5a38f8322ddd6c196e26e85195141efcc7b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 24 Sep 2019 21:48:28 +0200 Subject: [PATCH 31/73] Build the container in Hydra --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index 01efa0170..bb5bc3da0 100644 --- a/flake.nix +++ b/flake.nix @@ -271,6 +271,7 @@ ''; }; + container = nixosConfigurations.container.config.system.build.toplevel; }; checks.build = hydraJobs.build.x86_64-linux; From 8d2896b36b490ae9b058f80edbdce7b8e0152cf7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 26 Sep 2019 18:35:00 +0200 Subject: [PATCH 32/73] epoch -> edition --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index bb5bc3da0..fa5fec18b 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,7 @@ { description = "A Nix-based continuous build system"; - epoch = 201909; + edition = 201909; inputs.nixpkgs.uri = "nixpkgs/release-19.09"; From d0b972bdd01512573bd90be02e3980fe2ae5032b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 4 Oct 2019 17:32:29 +0200 Subject: [PATCH 33/73] Use the overlay from the nix flake --- flake.lock | 10 +++++----- flake.nix | 20 +++++++------------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/flake.lock b/flake.lock index 598e1acad..c7560d35a 100644 --- a/flake.lock +++ b/flake.lock @@ -4,14 +4,14 @@ "inputs": { "nixpkgs": { "inputs": {}, - "narHash": "sha256-HGlE2VNbdEjCP76hWAS72kHBlMWhpvqWo58Obg1Vy6s=", - "originalUri": "nixpkgs", - "uri": "github:edolstra/nixpkgs/13e1bce51f4aebdf3db58ce8c4a93e904a272bff" + "narHash": "sha256-ltGlDPfwicH/u4orj1n4JXgRsA+jvKQsGnekObi0TV4=", + "originalUri": "nixpkgs/release-19.03", + "uri": "github:edolstra/nixpkgs/9a593b575e4044f9aff939b512e7cb1cf1e76a65" } }, - "narHash": "sha256-bzSMXhxVX5awBSTO4v6Fe0dd1igEMVFskJ5RKSSJ+/4=", + "narHash": "sha256-m89j1BztPNjhdaDBj3Uolc11qRagbo+kpJwSsc/vbkA=", "originalUri": "nix", - "uri": "github:NixOS/nix/aeb7148afd56b228604b79373a45793d36d660a3" + "uri": "github:NixOS/nix/a323b7826c1104a6404c5fdfd4a824a0d5598618" }, "nixpkgs": { "inputs": {}, diff --git a/flake.nix b/flake.nix index fa5fec18b..b47612b0f 100644 --- a/flake.nix +++ b/flake.nix @@ -10,14 +10,9 @@ version = "${builtins.readFile ./version}.${builtins.substring 0 8 self.lastModified}.${self.shortRev}"; - # FIXME: use nix overlay? - nix' = nix.hydraJobs.build.x86_64-linux // { - perl-bindings = nix.hydraJobs.perlBindings.x86_64-linux; - }; - pkgs = import nixpkgs { system = "x86_64-linux"; - overlays = [ self.overlay ]; + overlays = [ self.overlay nix.overlay ]; }; # NixOS configuration used for VM tests. @@ -94,8 +89,8 @@ TextDiff TextTable XMLSimple - nix' - nix'.perl-bindings + final.nix + final.nix.perl-bindings git boehmgc ]; @@ -111,14 +106,14 @@ [ makeWrapper autoconf automake libtool unzip nukeReferences pkgconfig sqlite libpqxx gitAndTools.topGit mercurial darcs subversion bazaar openssl bzip2 libxslt guile # optional, for Guile + Guix support - perlDeps perl nix' + perlDeps perl final.nix postgresql95 # for running the tests boost nlohmann_json ]; hydraPath = lib.makeBinPath ( - [ sqlite subversion openssh nix' coreutils findutils pixz + [ sqlite subversion openssh final.nix coreutils findutils pixz gzip bzip2 lzma gnutar unzip git gitAndTools.topGit mercurial darcs gnused bazaar ] ++ lib.optionals stdenv.isLinux [ rpm dpkg cdrkit ] ); @@ -151,7 +146,7 @@ --prefix PATH ':' $out/bin:$hydraPath \ --set HYDRA_RELEASE ${version} \ --set HYDRA_HOME $out/libexec/hydra \ - --set NIX_RELEASE ${nix'.name or "unknown"} + --set NIX_RELEASE ${final.nix.name or "unknown"} done ''; @@ -282,8 +277,7 @@ nixosModules.hydra = { imports = [ ./hydra-module.nix ]; - nixpkgs.overlays = [ self.overlay ]; - nix.package = nix'; + nixpkgs.overlays = [ self.overlay nix.overlay ]; }; nixosModules.hydraTest = { From 80738fadb132db054aef6e60c1af0b17a3ab92c2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 4 Oct 2019 21:31:27 +0200 Subject: [PATCH 34/73] Add hydraProxy module --- flake.nix | 50 ++++++++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/flake.nix b/flake.nix index b47612b0f..460722dd1 100644 --- a/flake.nix +++ b/flake.nix @@ -3,7 +3,7 @@ edition = 201909; - inputs.nixpkgs.uri = "nixpkgs/release-19.09"; + inputs.nixpkgs.uri = "nixpkgs/release-19.03"; outputs = { self, nixpkgs, nix }: let @@ -295,12 +295,36 @@ # The following is to work around the following error from hydra-server: # [error] Caught exception in engine "Cannot determine local time zone" time.timeZone = "UTC"; + + nix.extraOptions = '' + allowed-uris = https://github.com/ + ''; + }; + + nixosModules.hydraProxy = { + services.httpd = { + enable = true; + adminAddr = "hydra-admin@example.org"; + extraConfig = '' + + Order deny,allow + Allow from all + + + ProxyRequests Off + ProxyPreserveHost On + ProxyPass /apache-errors ! + ErrorDocument 503 /apache-errors/503.html + ProxyPass / http://127.0.0.1:3000/ retry=5 disablereuse=on + ProxyPassReverse / http://127.0.0.1:3000/ + ''; + }; }; nixosConfigurations.container = nixpkgs.lib.nixosSystem { system = "x86_64-linux"; modules = - [ { imports = [ self.nixosModules.hydraTest ]; + [ { imports = [ self.nixosModules.hydraTest self.nixosModules.hydraProxy ]; system.configurationRevision = self.rev; boot.isContainer = true; @@ -308,29 +332,7 @@ networking.firewall.allowedTCPPorts = [ 80 ]; networking.hostName = "hydra"; - nix.extraOptions = '' - allowed-uris = https://github.com/ - ''; - services.hydra-dev.useSubstitutes = true; - - services.httpd = { - enable = true; - adminAddr = "hydra-admin@example.org"; - extraConfig = '' - - Order deny,allow - Allow from all - - - ProxyRequests Off - ProxyPreserveHost On - ProxyPass /apache-errors ! - ErrorDocument 503 /apache-errors/503.html - ProxyPass / http://127.0.0.1:3000/ retry=5 disablereuse=on - ProxyPassReverse / http://127.0.0.1:3000/ - ''; - }; } ]; }; From 4b280689256bb62aff9982ae1a0f9d26bcbb6395 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 4 Oct 2019 21:43:59 +0200 Subject: [PATCH 35/73] Update flake.lock --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index c7560d35a..69ecc70c3 100644 --- a/flake.lock +++ b/flake.lock @@ -15,9 +15,9 @@ }, "nixpkgs": { "inputs": {}, - "narHash": "sha256-qWEM8iYRYUdaVCHwTq13w69/11hAFWZplp5mqeU2qBM=", - "originalUri": "nixpkgs/release-19.09", - "uri": "github:edolstra/nixpkgs/7d9219a0adea88942b8d609f89cf4a1f86df1a9c" + "narHash": "sha256-ltGlDPfwicH/u4orj1n4JXgRsA+jvKQsGnekObi0TV4=", + "originalUri": "nixpkgs/release-19.03", + "uri": "github:edolstra/nixpkgs/9a593b575e4044f9aff939b512e7cb1cf1e76a65" } }, "version": 3 From 551e936facb31ac09d4a032e2191d8bf678bbe57 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 15 Oct 2019 20:21:42 +0200 Subject: [PATCH 36/73] Update flake.nix Currently Hydra is only supported on x86_64-linux. --- flake.nix | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/flake.nix b/flake.nix index 460722dd1..e580a1078 100644 --- a/flake.nix +++ b/flake.nix @@ -159,7 +159,7 @@ hydraJobs = { - build.x86_64-linux = packages.hydra; + build.x86_64-linux = packages.x86_64-linux.hydra; manual = pkgs.runCommand "hydra-manual-${version}" {} @@ -269,11 +269,11 @@ container = nixosConfigurations.container.config.system.build.toplevel; }; - checks.build = hydraJobs.build.x86_64-linux; - checks.install = hydraJobs.tests.install.x86_64-linux; + checks.x86_64-linux.build = hydraJobs.build.x86_64-linux; + checks.x86_64-linux.install = hydraJobs.tests.install.x86_64-linux; - packages.hydra = pkgs.hydra; - defaultPackage = pkgs.hydra; + packages.x86_64-linux.hydra = pkgs.hydra; + defaultPackage.x86_64-linux = pkgs.hydra; nixosModules.hydra = { imports = [ ./hydra-module.nix ]; From 7ba06c13ac22efb79aad19da7b82f3660fcc2d5a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Oct 2019 20:19:40 +0200 Subject: [PATCH 37/73] Bump flake dependencies --- flake.lock | 18 +++++++++--------- flake.nix | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/flake.lock b/flake.lock index 69ecc70c3..0287b8af5 100644 --- a/flake.lock +++ b/flake.lock @@ -4,20 +4,20 @@ "inputs": { "nixpkgs": { "inputs": {}, - "narHash": "sha256-ltGlDPfwicH/u4orj1n4JXgRsA+jvKQsGnekObi0TV4=", - "originalUri": "nixpkgs/release-19.03", - "uri": "github:edolstra/nixpkgs/9a593b575e4044f9aff939b512e7cb1cf1e76a65" + "narHash": "sha256-D1YSQzR6Xo2cNZ4sf+WCak2fqE/CIbwpZbPKd2B2Syc=", + "originalUrl": "nixpkgs/release-19.09", + "url": "github:edolstra/nixpkgs/e3d44c4fc62c99b850c792429cb61b946b66a704" } }, - "narHash": "sha256-m89j1BztPNjhdaDBj3Uolc11qRagbo+kpJwSsc/vbkA=", - "originalUri": "nix", - "uri": "github:NixOS/nix/a323b7826c1104a6404c5fdfd4a824a0d5598618" + "narHash": "sha256-Y1cdnCNoJmjqyC/a+Nt2N+5L3Ttg7K7zOD7gmtg1QzA=", + "originalUrl": "nix", + "url": "github:NixOS/nix/9cac895406724e0304dff140379783c4d786e855" }, "nixpkgs": { "inputs": {}, - "narHash": "sha256-ltGlDPfwicH/u4orj1n4JXgRsA+jvKQsGnekObi0TV4=", - "originalUri": "nixpkgs/release-19.03", - "uri": "github:edolstra/nixpkgs/9a593b575e4044f9aff939b512e7cb1cf1e76a65" + "narHash": "sha256-D1YSQzR6Xo2cNZ4sf+WCak2fqE/CIbwpZbPKd2B2Syc=", + "originalUrl": "nixpkgs/release-19.09", + "url": "github:edolstra/nixpkgs/e3d44c4fc62c99b850c792429cb61b946b66a704" } }, "version": 3 diff --git a/flake.nix b/flake.nix index e580a1078..149f64185 100644 --- a/flake.nix +++ b/flake.nix @@ -3,7 +3,7 @@ edition = 201909; - inputs.nixpkgs.uri = "nixpkgs/release-19.03"; + inputs.nixpkgs.uri = "nixpkgs/release-19.09"; outputs = { self, nixpkgs, nix }: let From 43d4bc910890e251cebb9216891924eb08e19236 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Oct 2019 16:29:12 +0200 Subject: [PATCH 38/73] Fix reproduction instructions --- src/root/build.tt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/root/build.tt b/src/root/build.tt index b563ebb12..723f66dda 100644 --- a/src/root/build.tt +++ b/src/root/build.tt @@ -539,7 +539,7 @@ END; running the following command:

    -# nix build [% HTML.escape(eval.flake) %]:hydraJobs.[% HTML.escape(job.name) %]
+# nix build [% HTML.escape(eval.flake) %]#hydraJobs.[% HTML.escape(job.name) %]
    [% ELSE %] From a816730d8b3b95a6cbb96fc64c483957d6e8573e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Oct 2019 20:31:27 +0200 Subject: [PATCH 39/73] uri -> url --- src/script/hydra-eval-jobset | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset index 1c3c0e255..a8f542206 100755 --- a/src/script/hydra-eval-jobset +++ b/src/script/hydra-eval-jobset @@ -614,7 +614,7 @@ sub checkJobsetWrapped { die "'nix flake info' returned " . ($res & 127 ? "signal $res" : "exit code " . ($res >> 8)) . ":\n" . ($stderr ? decode("utf-8", $stderr) : "(no output)\n") if $res; - $flakeRef = decode_json($json)->{'uri'}; + $flakeRef = decode_json($json)->{'url'}; } Net::Statsd::increment("hydra.evaluator.checkouts"); From 26ea7e493de7cc6cc7043067ae47531aa1fd661a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 23 Oct 2019 21:32:37 +0200 Subject: [PATCH 40/73] Update flake.lock --- flake.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index 0287b8af5..806f6d17c 100644 --- a/flake.lock +++ b/flake.lock @@ -4,9 +4,9 @@ "inputs": { "nixpkgs": { "inputs": {}, - "narHash": "sha256-D1YSQzR6Xo2cNZ4sf+WCak2fqE/CIbwpZbPKd2B2Syc=", + "narHash": "sha256-ALTr2z3J1mE/2OAuEIALIq5pxgADfKvq7rF7gqxT4c0=", "originalUrl": "nixpkgs/release-19.09", - "url": "github:edolstra/nixpkgs/e3d44c4fc62c99b850c792429cb61b946b66a704" + "url": "github:edolstra/nixpkgs/bcceb882ccdfa818c1e18f4c1e88f2db68964305" } }, "narHash": "sha256-Y1cdnCNoJmjqyC/a+Nt2N+5L3Ttg7K7zOD7gmtg1QzA=", @@ -15,9 +15,9 @@ }, "nixpkgs": { "inputs": {}, - "narHash": "sha256-D1YSQzR6Xo2cNZ4sf+WCak2fqE/CIbwpZbPKd2B2Syc=", + "narHash": "sha256-ALTr2z3J1mE/2OAuEIALIq5pxgADfKvq7rF7gqxT4c0=", "originalUrl": "nixpkgs/release-19.09", - "url": "github:edolstra/nixpkgs/e3d44c4fc62c99b850c792429cb61b946b66a704" + "url": "github:edolstra/nixpkgs/bcceb882ccdfa818c1e18f4c1e88f2db68964305" } }, "version": 3 From 6f99d958bc58f378ef4890856c31ef65d8d74d5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janne=20He=C3=9F?= Date: Sun, 27 Oct 2019 14:57:53 +0100 Subject: [PATCH 41/73] Fix declarative flake builds --- src/lib/Hydra/Helper/AddBuilds.pm | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/Hydra/Helper/AddBuilds.pm b/src/lib/Hydra/Helper/AddBuilds.pm index 40375c6ba..3b8869d66 100644 --- a/src/lib/Hydra/Helper/AddBuilds.pm +++ b/src/lib/Hydra/Helper/AddBuilds.pm @@ -30,6 +30,8 @@ sub updateDeclarativeJobset { my @allowed_keys = qw( enabled hidden + type + flake description nixexprinput nixexprpath From 1784c362ed00fd8d6fe3f69e513d3776081ab729 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 7 Nov 2019 18:46:32 +0100 Subject: [PATCH 42/73] Simplify flake.nix --- flake.nix | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 1e757f20d..aa2c6762e 100644 --- a/flake.nix +++ b/flake.nix @@ -325,8 +325,9 @@ nixosConfigurations.container = nixpkgs.lib.nixosSystem { system = "x86_64-linux"; modules = - [ { imports = [ self.nixosModules.hydraTest self.nixosModules.hydraProxy ]; - system.configurationRevision = self.rev; + [ self.nixosModules.hydraTest + self.nixosModules.hydraProxy + { system.configurationRevision = self.rev; boot.isContainer = true; networking.useDHCP = false; From 4c8f600eaa0b184b39f2525cc76b6d92c065343c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 7 Nov 2019 23:16:05 +0100 Subject: [PATCH 43/73] Update to latest Nix and fix compilation --- flake.lock | 12 ++++++------ flake.nix | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/flake.lock b/flake.lock index 806f6d17c..8d8b6741d 100644 --- a/flake.lock +++ b/flake.lock @@ -4,20 +4,20 @@ "inputs": { "nixpkgs": { "inputs": {}, - "narHash": "sha256-ALTr2z3J1mE/2OAuEIALIq5pxgADfKvq7rF7gqxT4c0=", + "narHash": "sha256-ZzR2l1dovxeZ555KXxz7SAXrC72BfaR4BeqvJzRdmwQ=", "originalUrl": "nixpkgs/release-19.09", - "url": "github:edolstra/nixpkgs/bcceb882ccdfa818c1e18f4c1e88f2db68964305" + "url": "github:edolstra/nixpkgs/d37927a77e70a2b3408ceaa2e763b6df1f4d941a" } }, - "narHash": "sha256-Y1cdnCNoJmjqyC/a+Nt2N+5L3Ttg7K7zOD7gmtg1QzA=", + "narHash": "sha256-jdPbZC16ky1nhh1P0E6RGnEjtDofgfbWXFn1v6GyYXA=", "originalUrl": "nix", - "url": "github:NixOS/nix/9cac895406724e0304dff140379783c4d786e855" + "url": "github:NixOS/nix/0ba8a4e942fbbc6ed71fa3aa76c6f923de3923ee" }, "nixpkgs": { "inputs": {}, - "narHash": "sha256-ALTr2z3J1mE/2OAuEIALIq5pxgADfKvq7rF7gqxT4c0=", + "narHash": "sha256-ZzR2l1dovxeZ555KXxz7SAXrC72BfaR4BeqvJzRdmwQ=", "originalUrl": "nixpkgs/release-19.09", - "url": "github:edolstra/nixpkgs/bcceb882ccdfa818c1e18f4c1e88f2db68964305" + "url": "github:edolstra/nixpkgs/d37927a77e70a2b3408ceaa2e763b6df1f4d941a" } }, "version": 3 diff --git a/flake.nix b/flake.nix index aa2c6762e..acf1035c8 100644 --- a/flake.nix +++ b/flake.nix @@ -110,7 +110,7 @@ perlDeps perl final.nix postgresql95 # for running the tests boost - nlohmann_json + (nlohmann_json.override { multipleHeaders = true; }) ]; hydraPath = lib.makeBinPath ( From 9e0e625bfa417c69abe4a321f05751d89c3c61f4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 Nov 2019 14:41:13 +0100 Subject: [PATCH 44/73] Update flake.lock --- flake.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flake.lock b/flake.lock index 8d8b6741d..d04f4e248 100644 --- a/flake.lock +++ b/flake.lock @@ -9,9 +9,9 @@ "url": "github:edolstra/nixpkgs/d37927a77e70a2b3408ceaa2e763b6df1f4d941a" } }, - "narHash": "sha256-jdPbZC16ky1nhh1P0E6RGnEjtDofgfbWXFn1v6GyYXA=", + "narHash": "sha256-VWsHDiuEV/3apjBta2/QcL7Woqb0eE8gGuES4hpyCno=", "originalUrl": "nix", - "url": "github:NixOS/nix/0ba8a4e942fbbc6ed71fa3aa76c6f923de3923ee" + "url": "github:NixOS/nix/0bc0d35b6ba252a1f3ec5d92acecdd041328d811" }, "nixpkgs": { "inputs": {}, From 47797576838974c8209536b67bb45e953a50900f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 Nov 2019 15:45:21 +0100 Subject: [PATCH 45/73] Update flake.lock --- flake.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flake.lock b/flake.lock index d04f4e248..93811e5e7 100644 --- a/flake.lock +++ b/flake.lock @@ -9,9 +9,9 @@ "url": "github:edolstra/nixpkgs/d37927a77e70a2b3408ceaa2e763b6df1f4d941a" } }, - "narHash": "sha256-VWsHDiuEV/3apjBta2/QcL7Woqb0eE8gGuES4hpyCno=", + "narHash": "sha256-zfFjX4SmTKGponLg7X1Lc4XHOUNEDz3czSjGaZTHXOk=", "originalUrl": "nix", - "url": "github:NixOS/nix/0bc0d35b6ba252a1f3ec5d92acecdd041328d811" + "url": "github:NixOS/nix/2c1e05ae9389742dac637a6f051f718397eff2db" }, "nixpkgs": { "inputs": {}, From e7f2139e251cb73195eea6fb84e2a6167b4db968 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 30 Dec 2019 22:49:26 +0100 Subject: [PATCH 46/73] Build against nix-master --- flake.lock | 12 +-- src/hydra-eval-jobs/Makefile.am | 2 +- src/hydra-eval-jobs/hydra-eval-jobs.cc | 5 +- src/hydra-queue-runner/Makefile.am | 2 +- src/hydra-queue-runner/build-remote.cc | 76 +++++++++-------- src/hydra-queue-runner/build-result.cc | 28 +++---- src/hydra-queue-runner/builder.cc | 44 +++++----- src/hydra-queue-runner/dispatcher.cc | 4 +- src/hydra-queue-runner/hydra-queue-runner.cc | 29 ++++--- src/hydra-queue-runner/queue-monitor.cc | 87 +++++++++++--------- src/hydra-queue-runner/state.hh | 20 ++--- src/hydra-queue-runner/token-server.hh | 2 +- src/libhydra/db.hh | 2 +- src/libhydra/hydra-config.hh | 4 +- 14 files changed, 172 insertions(+), 145 deletions(-) diff --git a/flake.lock b/flake.lock index 93811e5e7..e96b9e43b 100644 --- a/flake.lock +++ b/flake.lock @@ -4,20 +4,20 @@ "inputs": { "nixpkgs": { "inputs": {}, - "narHash": "sha256-ZzR2l1dovxeZ555KXxz7SAXrC72BfaR4BeqvJzRdmwQ=", + "narHash": "sha256-HKuPcp/pBpKNBOnDlrSzObee5eB0LdzhI0RpRjTVxik=", "originalUrl": "nixpkgs/release-19.09", - "url": "github:edolstra/nixpkgs/d37927a77e70a2b3408ceaa2e763b6df1f4d941a" + "url": "github:edolstra/nixpkgs/03f3def66a104a221aac8b751eeb7075374848fd" } }, - "narHash": "sha256-zfFjX4SmTKGponLg7X1Lc4XHOUNEDz3czSjGaZTHXOk=", + "narHash": "sha256-wiOmdFFMhuBGEcAURvj7c1LAcVbO82QjcFr+5WjuNzM=", "originalUrl": "nix", - "url": "github:NixOS/nix/2c1e05ae9389742dac637a6f051f718397eff2db" + "url": "github:NixOS/nix/c7866733d7ce2836fbb43de90dd64d17b0d20753" }, "nixpkgs": { "inputs": {}, - "narHash": "sha256-ZzR2l1dovxeZ555KXxz7SAXrC72BfaR4BeqvJzRdmwQ=", + "narHash": "sha256-HKuPcp/pBpKNBOnDlrSzObee5eB0LdzhI0RpRjTVxik=", "originalUrl": "nixpkgs/release-19.09", - "url": "github:edolstra/nixpkgs/d37927a77e70a2b3408ceaa2e763b6df1f4d941a" + "url": "github:edolstra/nixpkgs/03f3def66a104a221aac8b751eeb7075374848fd" } }, "version": 3 diff --git a/src/hydra-eval-jobs/Makefile.am b/src/hydra-eval-jobs/Makefile.am index 7a4e9c913..b41a4eb80 100644 --- a/src/hydra-eval-jobs/Makefile.am +++ b/src/hydra-eval-jobs/Makefile.am @@ -1,5 +1,5 @@ bin_PROGRAMS = hydra-eval-jobs hydra_eval_jobs_SOURCES = hydra-eval-jobs.cc -hydra_eval_jobs_LDADD = $(NIX_LIBS) +hydra_eval_jobs_LDADD = $(NIX_LIBS) -lnixrust hydra_eval_jobs_CXXFLAGS = $(NIX_CFLAGS) -I ../libhydra diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index a309abc75..334864d54 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -115,8 +115,9 @@ static void findJobsWrapped(EvalState & state, JSONObject & top, done. */ auto localStore = state.store.dynamic_pointer_cast(); if (gcRootsDir != "" && localStore) { - Path root = gcRootsDir + "/" + baseNameOf(drvPath); - if (!pathExists(root)) localStore->addPermRoot(drvPath, root, false); + Path root = gcRootsDir + "/" + std::string(baseNameOf(drvPath)); + if (!pathExists(root)) + localStore->addPermRoot(localStore->parseStorePath(drvPath), root, false); } auto res2 = res.object("outputs"); diff --git a/src/hydra-queue-runner/Makefile.am b/src/hydra-queue-runner/Makefile.am index 1726d0df9..af95a3f99 100644 --- a/src/hydra-queue-runner/Makefile.am +++ b/src/hydra-queue-runner/Makefile.am @@ -3,5 +3,5 @@ bin_PROGRAMS = hydra-queue-runner hydra_queue_runner_SOURCES = hydra-queue-runner.cc queue-monitor.cc dispatcher.cc \ builder.cc build-result.cc build-remote.cc \ build-result.hh counter.hh token-server.hh state.hh db.hh -hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx +hydra_queue_runner_LDADD = $(NIX_LIBS) -lpqxx -lnixrust hydra_queue_runner_CXXFLAGS = $(NIX_CFLAGS) -Wall -I ../libhydra -Wno-deprecated-declarations diff --git a/src/hydra-queue-runner/build-remote.cc b/src/hydra-queue-runner/build-remote.cc index 81692849c..6070bd4e2 100644 --- a/src/hydra-queue-runner/build-remote.cc +++ b/src/hydra-queue-runner/build-remote.cc @@ -82,10 +82,10 @@ static void openConnection(Machine::ptr machine, Path tmpDir, int stderrFD, Chil static void copyClosureTo(std::timed_mutex & sendMutex, ref destStore, - FdSource & from, FdSink & to, const PathSet & paths, + FdSource & from, FdSink & to, const StorePathSet & paths, bool useSubstitutes = false) { - PathSet closure; + StorePathSet closure; for (auto & path : paths) destStore->computeFSClosure(path, closure); @@ -94,20 +94,21 @@ static void copyClosureTo(std::timed_mutex & sendMutex, ref destStore, garbage-collect paths that are already there. Optionally, ask the remote host to substitute missing paths. */ // FIXME: substitute output pollutes our build log - to << cmdQueryValidPaths << 1 << useSubstitutes << closure; + to << cmdQueryValidPaths << 1 << useSubstitutes; + writeStorePaths(*destStore, to, closure); to.flush(); /* Get back the set of paths that are already valid on the remote host. */ - auto present = readStorePaths(*destStore, from); + auto present = readStorePaths(*destStore, from); if (present.size() == closure.size()) return; - Paths sorted = destStore->topoSortPaths(closure); + auto sorted = destStore->topoSortPaths(closure); - Paths missing; + StorePathSet missing; for (auto i = sorted.rbegin(); i != sorted.rend(); ++i) - if (present.find(*i) == present.end()) missing.push_back(*i); + if (!present.count(*i)) missing.insert(i->clone()); printMsg(lvlDebug, format("sending %1% missing paths") % missing.size()); @@ -131,7 +132,7 @@ void State::buildRemote(ref destStore, { assert(BuildResult::TimedOut == 8); - string base = baseNameOf(step->drvPath); + string base(step->drvPath.to_string()); result.logFile = logDir + "/" + string(base, 0, 2) + "/" + string(base, 2); AutoDelete autoDelete(result.logFile, false); @@ -217,22 +218,22 @@ void State::buildRemote(ref destStore, outputs of the input derivations. */ updateStep(ssSendingInputs); - PathSet inputs; - BasicDerivation basicDrv(step->drv); + StorePathSet inputs; + BasicDerivation basicDrv(*step->drv); if (sendDerivation) - inputs.insert(step->drvPath); + inputs.insert(step->drvPath.clone()); else - for (auto & p : step->drv.inputSrcs) - inputs.insert(p); + for (auto & p : step->drv->inputSrcs) + inputs.insert(p.clone()); - for (auto & input : step->drv.inputDrvs) { - Derivation drv2 = readDerivation(input.first); + for (auto & input : step->drv->inputDrvs) { + Derivation drv2 = readDerivation(*localStore, localStore->printStorePath(input.first)); for (auto & name : input.second) { auto i = drv2.outputs.find(name); if (i == drv2.outputs.end()) continue; - inputs.insert(i->second.path); - basicDrv.inputSrcs.insert(i->second.path); + inputs.insert(i->second.path.clone()); + basicDrv.inputSrcs.insert(i->second.path.clone()); } } @@ -241,14 +242,15 @@ void State::buildRemote(ref destStore, this will copy the inputs to the binary cache from the local store. */ if (localStore != std::shared_ptr(destStore)) - copyClosure(ref(localStore), destStore, step->drv.inputSrcs, NoRepair, NoCheckSigs); + copyClosure(ref(localStore), destStore, step->drv->inputSrcs, NoRepair, NoCheckSigs); /* Copy the input closure. */ if (!machine->isLocalhost()) { auto mc1 = std::make_shared>(nrStepsWaiting); mc1.reset(); MaintainCount mc2(nrStepsCopyingTo); - printMsg(lvlDebug, format("sending closure of ‘%1%’ to ‘%2%’") % step->drvPath % machine->sshName); + printMsg(lvlDebug, "sending closure of ‘%s’ to ‘%s’", + localStore->printStorePath(step->drvPath), machine->sshName); auto now1 = std::chrono::steady_clock::now(); @@ -272,14 +274,19 @@ void State::buildRemote(ref destStore, logFD = -1; /* Do the build. */ - printMsg(lvlDebug, format("building ‘%1%’ on ‘%2%’") % step->drvPath % machine->sshName); + printMsg(lvlDebug, "building ‘%s’ on ‘%s’", + localStore->printStorePath(step->drvPath), + machine->sshName); updateStep(ssBuilding); - if (sendDerivation) - to << cmdBuildPaths << PathSet({step->drvPath}); - else - to << cmdBuildDerivation << step->drvPath << basicDrv; + if (sendDerivation) { + to << cmdBuildPaths; + writeStorePaths(*localStore, to, singleton(step->drvPath)); + } else { + to << cmdBuildDerivation << localStore->printStorePath(step->drvPath); + writeDerivation(to, *localStore, basicDrv); + } to << maxSilentTime << buildTimeout; if (GET_PROTOCOL_MINOR(remoteVersion) >= 2) to << maxLogSize; @@ -380,7 +387,8 @@ void State::buildRemote(ref destStore, /* If the path was substituted or already valid, then we didn't get a build log. */ if (result.isCached) { - printMsg(lvlInfo, format("outputs of ‘%1%’ substituted or already valid on ‘%2%’") % step->drvPath % machine->sshName); + printMsg(lvlInfo, "outputs of ‘%s’ substituted or already valid on ‘%s’", + localStore->printStorePath(step->drvPath), machine->sshName); unlink(result.logFile.c_str()); result.logFile = ""; } @@ -395,13 +403,12 @@ void State::buildRemote(ref destStore, auto now1 = std::chrono::steady_clock::now(); - PathSet outputs; - for (auto & output : step->drv.outputs) - outputs.insert(output.second.path); + auto outputs = step->drv->outputPaths(); /* Query the size of the output paths. */ size_t totalNarSize = 0; - to << cmdQueryPathInfos << outputs; + to << cmdQueryPathInfos; + writeStorePaths(*localStore, to, outputs); to.flush(); while (true) { if (readString(from) == "") break; @@ -416,8 +423,8 @@ void State::buildRemote(ref destStore, return; } - printMsg(lvlDebug, format("copying outputs of ‘%s’ from ‘%s’ (%d bytes)") - % step->drvPath % machine->sshName % totalNarSize); + printMsg(lvlDebug, "copying outputs of ‘%s’ from ‘%s’ (%d bytes)", + localStore->printStorePath(step->drvPath), machine->sshName, totalNarSize); /* Block until we have the required amount of memory available, which is twice the NAR size (namely the @@ -431,10 +438,11 @@ void State::buildRemote(ref destStore, auto resMs = std::chrono::duration_cast(resStop - resStart).count(); if (resMs >= 1000) - printMsg(lvlError, format("warning: had to wait %d ms for %d memory tokens for %s") - % resMs % totalNarSize % step->drvPath); + printMsg(lvlError, "warning: had to wait %d ms for %d memory tokens for %s", + resMs, totalNarSize, localStore->printStorePath(step->drvPath)); - to << cmdExportPaths << 0 << outputs; + to << cmdExportPaths << 0; + writeStorePaths(*localStore, to, outputs); to.flush(); destStore->importPaths(from, result.accessor, NoCheckSigs); diff --git a/src/hydra-queue-runner/build-result.cc b/src/hydra-queue-runner/build-result.cc index 25e8c41a1..cd8f0a392 100644 --- a/src/hydra-queue-runner/build-result.cc +++ b/src/hydra-queue-runner/build-result.cc @@ -14,16 +14,14 @@ BuildOutput getBuildOutput(nix::ref store, BuildOutput res; /* Compute the closure size. */ - PathSet outputs; - for (auto & output : drv.outputs) - outputs.insert(output.second.path); - PathSet closure; + auto outputs = drv.outputPaths(); + StorePathSet closure; for (auto & output : outputs) - store->computeFSClosure(output, closure); + store->computeFSClosure(singleton(output), closure); for (auto & path : closure) { auto info = store->queryPathInfo(path); res.closureSize += info->narSize; - if (outputs.find(path) != outputs.end()) res.size += info->narSize; + if (outputs.count(path)) res.size += info->narSize; } /* Get build products. */ @@ -39,11 +37,13 @@ BuildOutput getBuildOutput(nix::ref store, , std::regex::extended); for (auto & output : outputs) { - Path failedFile = output + "/nix-support/failed"; + auto outputS = store->printStorePath(output); + + Path failedFile = outputS + "/nix-support/failed"; if (accessor->stat(failedFile).type == FSAccessor::Type::tRegular) res.failed = true; - Path productsFile = output + "/nix-support/hydra-build-products"; + Path productsFile = outputS + "/nix-support/hydra-build-products"; if (accessor->stat(productsFile).type != FSAccessor::Type::tRegular) continue; @@ -72,7 +72,7 @@ BuildOutput getBuildOutput(nix::ref store, auto st = accessor->stat(product.path); if (st.type == FSAccessor::Type::tMissing) continue; - product.name = product.path == output ? "" : baseNameOf(product.path); + product.name = product.path == store->printStorePath(output) ? "" : baseNameOf(product.path); if (st.type == FSAccessor::Type::tRegular) { product.isRegular = true; @@ -91,14 +91,14 @@ BuildOutput getBuildOutput(nix::ref store, if (!explicitProducts) { for (auto & output : drv.outputs) { BuildProduct product; - product.path = output.second.path; + product.path = store->printStorePath(output.second.path); product.type = "nix-build"; product.subtype = output.first == "out" ? "" : output.first; - product.name = storePathToName(product.path); + product.name = output.second.path.name(); auto st = accessor->stat(product.path); if (st.type == FSAccessor::Type::tMissing) - throw Error(format("getting status of ‘%1%’") % product.path); + throw Error("getting status of ‘%s’", product.path); if (st.type == FSAccessor::Type::tDirectory) res.products.push_back(product); } @@ -106,7 +106,7 @@ BuildOutput getBuildOutput(nix::ref store, /* Get the release name from $output/nix-support/hydra-release-name. */ for (auto & output : outputs) { - Path p = output + "/nix-support/hydra-release-name"; + auto p = store->printStorePath(output) + "/nix-support/hydra-release-name"; if (accessor->stat(p).type != FSAccessor::Type::tRegular) continue; try { res.releaseName = trim(accessor->readFile(p)); @@ -116,7 +116,7 @@ BuildOutput getBuildOutput(nix::ref store, /* Get metrics. */ for (auto & output : outputs) { - Path metricsFile = output + "/nix-support/hydra-metrics"; + auto metricsFile = store->printStorePath(output) + "/nix-support/hydra-metrics"; if (accessor->stat(metricsFile).type != FSAccessor::Type::tRegular) continue; for (auto & line : tokenizeString(accessor->readFile(metricsFile), "\n")) { auto fields = tokenizeString>(line); diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc index edd4b1f76..a0fc01c33 100644 --- a/src/hydra-queue-runner/builder.cc +++ b/src/hydra-queue-runner/builder.cc @@ -18,7 +18,7 @@ void setThreadName(const std::string & name) void State::builder(MachineReservation::ptr reservation) { - setThreadName("bld~" + baseNameOf(reservation->step->drvPath)); + setThreadName("bld~" + std::string(reservation->step->drvPath.to_string())); StepResult res = sRetry; @@ -39,8 +39,10 @@ void State::builder(MachineReservation::ptr reservation) auto destStore = getDestStore(); res = doBuildStep(destStore, reservation, activeStep); } catch (std::exception & e) { - printMsg(lvlError, format("uncaught exception building ‘%1%’ on ‘%2%’: %3%") - % reservation->step->drvPath % reservation->machine->sshName % e.what()); + printMsg(lvlError, "uncaught exception building ‘%s’ on ‘%s’: %s", + localStore->printStorePath(reservation->step->drvPath), + reservation->machine->sshName, + e.what()); } } @@ -60,7 +62,7 @@ void State::builder(MachineReservation::ptr reservation) nrRetries++; if (step_->tries > maxNrRetries) maxNrRetries = step_->tries; // yeah yeah, not atomic int delta = retryInterval * std::pow(retryBackoff, step_->tries - 1) + (rand() % 10); - printMsg(lvlInfo, format("will retry ‘%1%’ after %2%s") % step->drvPath % delta); + printMsg(lvlInfo, "will retry ‘%s’ after %ss", localStore->printStorePath(step->drvPath), delta); step_->after = std::chrono::system_clock::now() + std::chrono::seconds(delta); } @@ -95,7 +97,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, cancelled (namely if there are no more Builds referring to it). */ BuildID buildId; - Path buildDrvPath; + std::optional buildDrvPath; unsigned int maxSilentTime, buildTimeout; unsigned int repeats = step->isDeterministic ? 1 : 0; @@ -116,7 +118,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, possibility, we retry this step (putting it back in the runnable queue). If there are really no strong pointers to the step, it will be deleted. */ - printMsg(lvlInfo, format("maybe cancelling build step ‘%1%’") % step->drvPath); + printMsg(lvlInfo, "maybe cancelling build step ‘%s’", localStore->printStorePath(step->drvPath)); return sMaybeCancelled; } @@ -138,15 +140,15 @@ State::StepResult State::doBuildStep(nix::ref destStore, if (!build) build = *dependents.begin(); buildId = build->id; - buildDrvPath = build->drvPath; + buildDrvPath = build->drvPath.clone(); maxSilentTime = build->maxSilentTime; buildTimeout = build->buildTimeout; printInfo("performing step ‘%s’ %d times on ‘%s’ (needed by build %d and %d others)", - step->drvPath, repeats + 1, machine->sshName, buildId, (dependents.size() - 1)); + localStore->printStorePath(step->drvPath), repeats + 1, machine->sshName, buildId, (dependents.size() - 1)); } - bool quit = buildId == buildOne && step->drvPath == buildDrvPath; + bool quit = buildId == buildOne && step->drvPath == *buildDrvPath; RemoteResult result; BuildOutput res; @@ -166,7 +168,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, try { auto store = destStore.dynamic_pointer_cast(); if (uploadLogsToBinaryCache && store && pathExists(result.logFile)) { - store->upsertFile("log/" + baseNameOf(step->drvPath), readFile(result.logFile), "text/plain; charset=utf-8"); + store->upsertFile("log/" + std::string(step->drvPath.to_string()), readFile(result.logFile), "text/plain; charset=utf-8"); unlink(result.logFile.c_str()); } } catch (...) { @@ -218,7 +220,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, if (result.stepStatus == bsSuccess) { updateStep(ssPostProcessing); - res = getBuildOutput(destStore, ref(result.accessor), step->drv); + res = getBuildOutput(destStore, ref(result.accessor), *step->drv); } result.accessor = 0; @@ -255,8 +257,8 @@ State::StepResult State::doBuildStep(nix::ref destStore, /* The step had a hopefully temporary failure (e.g. network issue). Retry a number of times. */ if (result.canRetry) { - printMsg(lvlError, format("possibly transient failure building ‘%1%’ on ‘%2%’: %3%") - % step->drvPath % machine->sshName % result.errorMsg); + printMsg(lvlError, "possibly transient failure building ‘%s’ on ‘%s’: %s", + localStore->printStorePath(step->drvPath), machine->sshName, result.errorMsg); assert(stepNr); bool retry; { @@ -275,7 +277,7 @@ State::StepResult State::doBuildStep(nix::ref destStore, assert(stepNr); - for (auto & path : step->drv.outputPaths()) + for (auto & path : step->drv->outputPaths()) addRoot(path); /* Register success in the database for all Build objects that @@ -308,7 +310,8 @@ State::StepResult State::doBuildStep(nix::ref destStore, no new referrers can have been added in the meantime or be added afterwards. */ if (direct.empty()) { - printMsg(lvlDebug, format("finishing build step ‘%1%’") % step->drvPath); + printMsg(lvlDebug, "finishing build step ‘%s’", + localStore->printStorePath(step->drvPath)); steps_->erase(step->drvPath); } } @@ -393,7 +396,8 @@ State::StepResult State::doBuildStep(nix::ref destStore, be certain no new referrers can be added. */ if (indirect.empty()) { for (auto & s : steps) { - printMsg(lvlDebug, format("finishing build step ‘%1%’") % s->drvPath); + printMsg(lvlDebug, "finishing build step ‘%s’", + localStore->printStorePath(s->drvPath)); steps_->erase(s->drvPath); } } @@ -437,8 +441,8 @@ State::StepResult State::doBuildStep(nix::ref destStore, /* Remember failed paths in the database so that they won't be built again. */ if (result.stepStatus != bsCachedFailure && result.canCache) - for (auto & path : step->drv.outputPaths()) - txn.parameterized("insert into FailedPaths values ($1)")(path).exec(); + for (auto & path : step->drv->outputPaths()) + txn.parameterized("insert into FailedPaths values ($1)")(localStore->printStorePath(path)).exec(); txn.commit(); } @@ -478,8 +482,8 @@ State::StepResult State::doBuildStep(nix::ref destStore, } -void State::addRoot(const Path & storePath) +void State::addRoot(const StorePath & storePath) { - auto root = rootsDir + "/" + baseNameOf(storePath); + auto root = rootsDir + "/" + std::string(storePath.to_string()); if (!pathExists(root)) writeFile(root, ""); } diff --git a/src/hydra-queue-runner/dispatcher.cc b/src/hydra-queue-runner/dispatcher.cc index 068d5c57a..86596ff53 100644 --- a/src/hydra-queue-runner/dispatcher.cc +++ b/src/hydra-queue-runner/dispatcher.cc @@ -10,7 +10,7 @@ using namespace nix; void State::makeRunnable(Step::ptr step) { - printMsg(lvlChatty, format("step ‘%1%’ is now runnable") % step->drvPath); + printMsg(lvlChatty, "step ‘%s’ is now runnable", localStore->printStorePath(step->drvPath)); { auto step_(step->state.lock()); @@ -248,7 +248,7 @@ system_time State::doDispatch() /* Can this machine do this step? */ if (!mi.machine->supportsStep(step)) { debug("machine '%s' does not support step '%s' (system type '%s')", - mi.machine->sshName, step->drvPath, step->drv.platform); + mi.machine->sshName, localStore->printStorePath(step->drvPath), step->drv->platform); continue; } diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index fa7515d30..5a30c08e0 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -37,9 +37,9 @@ static uint64_t getMemSize() std::string getEnvOrDie(const std::string & key) { - char * value = getenv(key.c_str()); + auto value = getEnv(key); if (!value) throw Error("environment variable '%s' is not set", key); - return value; + return *value; } @@ -159,7 +159,7 @@ void State::monitorMachinesFile() { string defaultMachinesFile = "/etc/nix/machines"; auto machinesFiles = tokenizeString>( - getEnv("NIX_REMOTE_SYSTEMS", pathExists(defaultMachinesFile) ? defaultMachinesFile : ""), ":"); + getEnv("NIX_REMOTE_SYSTEMS").value_or(pathExists(defaultMachinesFile) ? defaultMachinesFile : ""), ":"); if (machinesFiles.empty()) { parseMachines("localhost " + @@ -251,10 +251,10 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID (buildId) (stepNr) (0) // == build - (step->drvPath) + (localStore->printStorePath(step->drvPath)) (status == bsBusy ? 1 : 0) (startTime, startTime != 0) - (step->drv.platform) + (step->drv->platform) ((int) status, status != bsBusy) (propagatedFrom, propagatedFrom != 0) (errorMsg, errorMsg != "") @@ -263,10 +263,10 @@ unsigned int State::createBuildStep(pqxx::work & txn, time_t startTime, BuildID if (r.affected_rows() == 0) goto restart; - for (auto & output : step->drv.outputs) + for (auto & output : step->drv->outputs) txn.parameterized ("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)") - (buildId)(stepNr)(output.first)(output.second.path).exec(); + (buildId)(stepNr)(output.first)(localStore->printStorePath(output.second.path)).exec(); if (status == bsBusy) txn.exec(fmt("notify step_started, '%d\t%d'", buildId, stepNr)); @@ -309,7 +309,7 @@ void State::finishBuildStep(pqxx::work & txn, const RemoteResult & result, int State::createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime, - Build::ptr build, const Path & drvPath, const string & outputName, const Path & storePath) + Build::ptr build, const StorePath & drvPath, const string & outputName, const StorePath & storePath) { restart: auto stepNr = allocBuildStep(txn, build->id); @@ -319,7 +319,7 @@ int State::createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t sto (build->id) (stepNr) (1) // == substitution - (drvPath) + (localStore->printStorePath(drvPath)) (0) (0) (startTime) @@ -329,7 +329,10 @@ int State::createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t sto txn.parameterized ("insert into BuildStepOutputs (build, stepnr, name, path) values ($1, $2, $3, $4)") - (build->id)(stepNr)(outputName)(storePath).exec(); + (build->id) + (stepNr) + (outputName) + (localStore->printStorePath(storePath)).exec(); return stepNr; } @@ -449,8 +452,8 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build, bool State::checkCachedFailure(Step::ptr step, Connection & conn) { pqxx::work txn(conn); - for (auto & path : step->drv.outputPaths()) - if (!txn.parameterized("select 1 from FailedPaths where path = $1")(path).exec().empty()) + for (auto & path : step->drv->outputPaths()) + if (!txn.parameterized("select 1 from FailedPaths where path = $1")(localStore->printStorePath(path)).exec().empty()) return true; return false; } @@ -762,7 +765,7 @@ void State::run(BuildID buildOne) Store::Params localParams; localParams["max-connections"] = "16"; localParams["max-connection-age"] = "600"; - localStore = openStore(getEnv("NIX_REMOTE"), localParams); + localStore = openStore(getEnv("NIX_REMOTE").value_or(""), localParams); auto storeUri = config->getStrOption("store_uri"); _destStore = storeUri == "" ? localStore : openStore(storeUri); diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index e657a4b88..62de134a7 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -83,7 +83,7 @@ bool State::getQueuedBuilds(Connection & conn, them yet (since we don't want a long-running transaction). */ std::vector newIDs; std::map newBuildsByID; - std::multimap newBuildsByPath; + std::multimap newBuildsByPath; unsigned int newLastBuildId = lastBuildId; @@ -104,7 +104,7 @@ bool State::getQueuedBuilds(Connection & conn, auto build = std::make_shared(); build->id = id; - build->drvPath = row["drvPath"].as(); + build->drvPath = localStore->parseStorePath(row["drvPath"].as()); build->projectName = row["project"].as(); build->jobsetName = row["jobset"].as(); build->jobName = row["job"].as(); @@ -117,14 +117,14 @@ bool State::getQueuedBuilds(Connection & conn, newIDs.push_back(id); newBuildsByID[id] = build; - newBuildsByPath.emplace(std::make_pair(build->drvPath, id)); + newBuildsByPath.emplace(std::make_pair(build->drvPath.clone(), id)); } } std::set newRunnable; unsigned int nrAdded; std::function createBuild; - std::set finishedDrvs; + std::set finishedDrvs; createBuild = [&](Build::ptr build) { printMsg(lvlTalkative, format("loading build %1% (%2%)") % build->id % build->fullJobName()); @@ -160,7 +160,8 @@ bool State::getQueuedBuilds(Connection & conn, /* Some step previously failed, so mark the build as failed right away. */ - printMsg(lvlError, format("marking build %d as cached failure due to ‘%s’") % build->id % ex.step->drvPath); + printMsg(lvlError, "marking build %d as cached failure due to ‘%s’", + build->id, localStore->printStorePath(ex.step->drvPath)); if (!build->finishedInDB) { auto mc = startDbUpdate(); pqxx::work txn(conn); @@ -171,14 +172,14 @@ bool State::getQueuedBuilds(Connection & conn, auto res = txn.parameterized ("select max(build) from BuildSteps where drvPath = $1 and startTime != 0 and stopTime != 0 and status = 1") - (ex.step->drvPath).exec(); + (localStore->printStorePath(ex.step->drvPath)).exec(); if (!res[0][0].is_null()) propagatedFrom = res[0][0].as(); if (!propagatedFrom) { - for (auto & output : ex.step->drv.outputs) { + for (auto & output : ex.step->drv->outputs) { auto res = txn.parameterized ("select max(s.build) from BuildSteps s join BuildStepOutputs o on s.build = o.build where path = $1 and startTime != 0 and stopTime != 0 and status = 1") - (output.second.path).exec(); + (localStore->printStorePath(output.second.path)).exec(); if (!res[0][0].is_null()) { propagatedFrom = res[0][0].as(); break; @@ -217,7 +218,7 @@ bool State::getQueuedBuilds(Connection & conn, /* If we didn't get a step, it means the step's outputs are all valid. So we mark this as a finished, cached build. */ if (!step) { - Derivation drv = readDerivation(build->drvPath); + Derivation drv = readDerivation(*localStore, localStore->printStorePath(build->drvPath)); BuildOutput res = getBuildOutputCached(conn, destStore, drv); for (auto & path : drv.outputPaths()) @@ -227,7 +228,7 @@ bool State::getQueuedBuilds(Connection & conn, auto mc = startDbUpdate(); pqxx::work txn(conn); time_t now = time(0); - printMsg(lvlInfo, format("marking build %1% as succeeded (cached)") % build->id); + printMsg(lvlInfo, "marking build %1% as succeeded (cached)", build->id); markSucceededBuild(txn, build, res, true, now, now); notifyBuildFinished(txn, build->id, {}); txn.commit(); @@ -250,8 +251,8 @@ bool State::getQueuedBuilds(Connection & conn, build->propagatePriorities(); - printMsg(lvlChatty, format("added build %1% (top-level step %2%, %3% new steps)") - % build->id % step->drvPath % newSteps.size()); + printMsg(lvlChatty, "added build %1% (top-level step %2%, %3% new steps)", + build->id, localStore->printStorePath(step->drvPath), newSteps.size()); }; /* Now instantiate build steps for each new build. The builder @@ -271,7 +272,7 @@ bool State::getQueuedBuilds(Connection & conn, try { createBuild(build); } catch (Error & e) { - e.addPrefix(format("while loading build %1%: ") % build->id); + e.addPrefix(fmt("while loading build %1%: ", build->id)); throw; } @@ -358,10 +359,12 @@ void State::processQueueChange(Connection & conn) activeStepState->cancelled = true; if (activeStepState->pid != -1) { printInfo("killing builder process %d of build step ‘%s’", - activeStepState->pid, activeStep->step->drvPath); + activeStepState->pid, + localStore->printStorePath(activeStep->step->drvPath)); if (kill(activeStepState->pid, SIGINT) == -1) printError("error killing build step ‘%s’: %s", - activeStep->step->drvPath, strerror(errno)); + localStore->printStorePath(activeStep->step->drvPath), + strerror(errno)); } } } @@ -370,8 +373,8 @@ void State::processQueueChange(Connection & conn) Step::ptr State::createStep(ref destStore, - Connection & conn, Build::ptr build, const Path & drvPath, - Build::ptr referringBuild, Step::ptr referringStep, std::set & finishedDrvs, + Connection & conn, Build::ptr build, const StorePath & drvPath, + Build::ptr referringBuild, Step::ptr referringStep, std::set & finishedDrvs, std::set & newSteps, std::set & newRunnable) { if (finishedDrvs.find(drvPath) != finishedDrvs.end()) return 0; @@ -400,7 +403,7 @@ Step::ptr State::createStep(ref destStore, /* If it doesn't exist, create it. */ if (!step) { step = std::make_shared(); - step->drvPath = drvPath; + step->drvPath = drvPath.clone(); isNew = true; } @@ -414,28 +417,28 @@ Step::ptr State::createStep(ref destStore, if (referringStep) step_->rdeps.push_back(referringStep); - (*steps_)[drvPath] = step; + steps_->insert_or_assign(drvPath.clone(), step); } if (!isNew) return step; - printMsg(lvlDebug, format("considering derivation ‘%1%’") % drvPath); + printMsg(lvlDebug, "considering derivation ‘%1%’", localStore->printStorePath(drvPath)); /* Initialize the step. Note that the step may be visible in ‘steps’ before this point, but that doesn't matter because it's not runnable yet, and other threads won't make it runnable while step->created == false. */ - step->drv = readDerivation(drvPath); - step->parsedDrv = std::make_unique(drvPath, step->drv); + step->drv = std::make_unique(readDerivation(*localStore, localStore->printStorePath(drvPath))); + step->parsedDrv = std::make_unique(drvPath.clone(), *step->drv); step->preferLocalBuild = step->parsedDrv->willBuildLocally(); - step->isDeterministic = get(step->drv.env, "isDetermistic", "0") == "1"; + step->isDeterministic = get(step->drv->env, "isDetermistic").value_or("0") == "1"; - step->systemType = step->drv.platform; + step->systemType = step->drv->platform; { - auto i = step->drv.env.find("requiredSystemFeatures"); + auto i = step->drv->env.find("requiredSystemFeatures"); StringSet features; - if (i != step->drv.env.end()) + if (i != step->drv->env.end()) features = step->requiredSystemFeatures = tokenizeString>(i->second); if (step->preferLocalBuild) features.insert("local"); @@ -451,12 +454,13 @@ Step::ptr State::createStep(ref destStore, /* Are all outputs valid? */ bool valid = true; - PathSet outputs = step->drv.outputPaths(); + auto outputs = step->drv->outputPaths(); DerivationOutputs missing; - for (auto & i : step->drv.outputs) + for (auto & i : step->drv->outputs) if (!destStore->isValidPath(i.second.path)) { valid = false; - missing[i.first] = i.second; + missing.insert_or_assign(i.first, + DerivationOutput(i.second.path.clone(), std::string(i.second.hashAlgo), std::string(i.second.hash))); } /* Try to copy the missing paths from the local store or from @@ -469,7 +473,7 @@ Step::ptr State::createStep(ref destStore, avail++; else if (useSubstitutes) { SubstitutablePathInfos infos; - localStore->querySubstitutablePathInfos({i.second.path}, infos); + localStore->querySubstitutablePathInfos(singleton(i.second.path), infos); if (infos.size() == 1) avail++; } @@ -482,14 +486,18 @@ Step::ptr State::createStep(ref destStore, time_t startTime = time(0); if (localStore->isValidPath(i.second.path)) - printInfo("copying output ‘%1%’ of ‘%2%’ from local store", i.second.path, drvPath); + printInfo("copying output ‘%1%’ of ‘%2%’ from local store", + localStore->printStorePath(i.second.path), + localStore->printStorePath(drvPath)); else { - printInfo("substituting output ‘%1%’ of ‘%2%’", i.second.path, drvPath); + printInfo("substituting output ‘%1%’ of ‘%2%’", + localStore->printStorePath(i.second.path), + localStore->printStorePath(drvPath)); localStore->ensurePath(i.second.path); // FIXME: should copy directly from substituter to destStore. } - copyClosure(ref(localStore), destStore, {i.second.path}); + copyClosure(ref(localStore), destStore, singleton(i.second.path)); time_t stopTime = time(0); @@ -501,7 +509,10 @@ Step::ptr State::createStep(ref destStore, } } catch (Error & e) { - printError("while copying/substituting output ‘%s’ of ‘%s’: %s", i.second.path, drvPath, e.what()); + printError("while copying/substituting output ‘%s’ of ‘%s’: %s", + localStore->printStorePath(i.second.path), + localStore->printStorePath(drvPath), + e.what()); valid = false; break; } @@ -511,15 +522,15 @@ Step::ptr State::createStep(ref destStore, // FIXME: check whether all outputs are in the binary cache. if (valid) { - finishedDrvs.insert(drvPath); + finishedDrvs.insert(drvPath.clone()); return 0; } /* No, we need to build. */ - printMsg(lvlDebug, format("creating build step ‘%1%’") % drvPath); + printMsg(lvlDebug, "creating build step ‘%1%’", localStore->printStorePath(drvPath)); /* Create steps for the dependencies. */ - for (auto & i : step->drv.inputDrvs) { + for (auto & i : step->drv->inputDrvs) { auto dep = createStep(destStore, conn, build, i.first, 0, step, finishedDrvs, newSteps, newRunnable); if (dep) { auto step_(step->state.lock()); @@ -607,7 +618,7 @@ BuildOutput State::getBuildOutputCached(Connection & conn, nix::ref ("select id, buildStatus, releaseName, closureSize, size from Builds b " "join BuildOutputs o on b.id = o.build " "where finished = 1 and (buildStatus = 0 or buildStatus = 6) and path = $1") - (output.second.path).exec(); + (localStore->printStorePath(output.second.path)).exec(); if (r.empty()) continue; BuildID id = r[0][0].as(); diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index a95cdb61d..de74b768c 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -123,8 +123,8 @@ struct Build typedef std::weak_ptr wptr; BuildID id; - nix::Path drvPath; - std::map outputs; + nix::StorePath drvPath; + std::map outputs; std::string projectName, jobsetName, jobName; time_t timestamp; unsigned int maxSilentTime, buildTimeout; @@ -150,8 +150,8 @@ struct Step typedef std::shared_ptr ptr; typedef std::weak_ptr wptr; - nix::Path drvPath; - nix::Derivation drv; + nix::StorePath drvPath; + std::unique_ptr drv; std::unique_ptr parsedDrv; std::set requiredSystemFeatures; bool preferLocalBuild; @@ -252,7 +252,7 @@ struct Machine { /* Check that this machine is of the type required by the step. */ - if (!systemTypes.count(step->drv.platform == "builtin" ? nix::settings.thisSystem : step->drv.platform)) + if (!systemTypes.count(step->drv->platform == "builtin" ? nix::settings.thisSystem : step->drv->platform)) return false; /* Check that the step requires all mandatory features of this @@ -313,7 +313,7 @@ private: queued builds). Note that these are weak pointers. Steps are kept alive by being reachable from Builds or by being in progress. */ - typedef std::map Steps; + typedef std::map Steps; nix::Sync steps; /* Build steps that have no unbuilt dependencies. */ @@ -454,7 +454,7 @@ private: const std::string & machine); int createSubstitutionStep(pqxx::work & txn, time_t startTime, time_t stopTime, - Build::ptr build, const nix::Path & drvPath, const std::string & outputName, const nix::Path & storePath); + Build::ptr build, const nix::StorePath & drvPath, const std::string & outputName, const nix::StorePath & storePath); void updateBuild(pqxx::work & txn, Build::ptr build, BuildStatus status); @@ -473,8 +473,8 @@ private: const nix::Derivation & drv); Step::ptr createStep(nix::ref store, - Connection & conn, Build::ptr build, const nix::Path & drvPath, - Build::ptr referringBuild, Step::ptr referringStep, std::set & finishedDrvs, + Connection & conn, Build::ptr build, const nix::StorePath & drvPath, + Build::ptr referringBuild, Step::ptr referringStep, std::set & finishedDrvs, std::set & newSteps, std::set & newRunnable); Jobset::ptr createJobset(pqxx::work & txn, @@ -523,7 +523,7 @@ private: void dumpStatus(Connection & conn, bool log); - void addRoot(const nix::Path & storePath); + void addRoot(const nix::StorePath & storePath); public: diff --git a/src/hydra-queue-runner/token-server.hh b/src/hydra-queue-runner/token-server.hh index e00004d00..d8004f731 100644 --- a/src/hydra-queue-runner/token-server.hh +++ b/src/hydra-queue-runner/token-server.hh @@ -7,7 +7,7 @@ namespace nix { -MakeError(NoTokens, Error) +MakeError(NoTokens, Error); /* This class hands out tokens. There are only ‘maxTokens’ tokens available. Calling get(N) will return a Token object, representing diff --git a/src/libhydra/db.hh b/src/libhydra/db.hh index 5bb5aac50..b7bed8151 100644 --- a/src/libhydra/db.hh +++ b/src/libhydra/db.hh @@ -12,7 +12,7 @@ struct Connection : pqxx::connection std::string getFlags() { using namespace nix; - auto s = getEnv("HYDRA_DBI", "dbi:Pg:dbname=hydra;"); + auto s = getEnv("HYDRA_DBI").value_or("dbi:Pg:dbname=hydra;"); std::string prefix = "dbi:Pg:"; if (std::string(s, 0, prefix.size()) != prefix) throw Error("$HYDRA_DBI does not denote a PostgreSQL database"); diff --git a/src/libhydra/hydra-config.hh b/src/libhydra/hydra-config.hh index a40506662..98d73d47d 100644 --- a/src/libhydra/hydra-config.hh +++ b/src/libhydra/hydra-config.hh @@ -14,9 +14,9 @@ struct Config /* Read hydra.conf. */ auto hydraConfigFile = getEnv("HYDRA_CONFIG"); - if (pathExists(hydraConfigFile)) { + if (hydraConfigFile && pathExists(*hydraConfigFile)) { - for (auto line : tokenizeString(readFile(hydraConfigFile), "\n")) { + for (auto line : tokenizeString(readFile(*hydraConfigFile), "\n")) { line = trim(string(line, 0, line.find('#'))); auto eq = line.find('='); From f0e8e56f8963cac54e8dd07573acc3de79769f58 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 3 Feb 2020 17:53:51 +0100 Subject: [PATCH 47/73] Update flake.lock --- flake.lock | 73 ++++++++++++++++++++++++++++++++++++++++-------------- flake.nix | 2 +- 2 files changed, 55 insertions(+), 20 deletions(-) diff --git a/flake.lock b/flake.lock index e96b9e43b..5d8912233 100644 --- a/flake.lock +++ b/flake.lock @@ -1,24 +1,59 @@ { - "inputs": { - "nix": { - "inputs": { - "nixpkgs": { - "inputs": {}, - "narHash": "sha256-HKuPcp/pBpKNBOnDlrSzObee5eB0LdzhI0RpRjTVxik=", - "originalUrl": "nixpkgs/release-19.09", - "url": "github:edolstra/nixpkgs/03f3def66a104a221aac8b751eeb7075374848fd" - } - }, - "narHash": "sha256-wiOmdFFMhuBGEcAURvj7c1LAcVbO82QjcFr+5WjuNzM=", - "originalUrl": "nix", - "url": "github:NixOS/nix/c7866733d7ce2836fbb43de90dd64d17b0d20753" - }, + "inputs": { + "nix": { + "info": { + "lastModified": 1580749449, + "narHash": "sha256-wEPYfs8ULx0dK6R4hrsXC/r8omxJLc86OhQg39eCo0c=", + "revCount": 7729 + }, + "inputs": { "nixpkgs": { - "inputs": {}, - "narHash": "sha256-HKuPcp/pBpKNBOnDlrSzObee5eB0LdzhI0RpRjTVxik=", - "originalUrl": "nixpkgs/release-19.09", - "url": "github:edolstra/nixpkgs/03f3def66a104a221aac8b751eeb7075374848fd" + "info": { + "narHash": "sha256-V4jz8Hbt+mZkXhH+3KmUQcRGETOFd8mVPhgQlS4Lu5E=" + }, + "inputs": {}, + "locked": { + "owner": "edolstra", + "repo": "nixpkgs", + "rev": "dd45a16733f4469a0dded6ad0bf9a662ea39bdea", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "ref": "release-19.09", + "type": "indirect" + } } + }, + "locked": { + "ref": "HEAD", + "rev": "0a4e911cf478d0d38082a8840a5acc8f57d00086", + "type": "git", + "url": "file:///home/eelco/Dev/nix-flake" + }, + "original": { + "type": "git", + "url": "file:///home/eelco/Dev/nix-flake" + } }, - "version": 3 + "nixpkgs": { + "info": { + "lastModified": 1580555706, + "narHash": "sha256-ly5Gxl9yZgW2w1F3aDU5bvg34Ct+juUaOg8vue+Mkbo=" + }, + "inputs": {}, + "locked": { + "owner": "edolstra", + "repo": "nixpkgs", + "rev": "43e283790fbe330b723ac44b584824cf6d85c1ee", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "ref": "release-19.09", + "type": "indirect" + } + } + }, + "version": 4 } diff --git a/flake.nix b/flake.nix index acf1035c8..85d8e5e4a 100644 --- a/flake.nix +++ b/flake.nix @@ -8,7 +8,7 @@ outputs = { self, nixpkgs, nix }: let - version = "${builtins.readFile ./version}.${builtins.substring 0 8 self.lastModified}.${self.shortRev}"; + version = "${builtins.readFile ./version}.${builtins.substring 0 8 self.lastModified}.${self.shortRev or "DIRTY"}"; pkgs = import nixpkgs { system = "x86_64-linux"; From d1bc950de281c0687c33ab8eeb1e455168437434 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 3 Feb 2020 18:38:05 +0100 Subject: [PATCH 48/73] Doh --- flake.lock | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/flake.lock b/flake.lock index 5d8912233..eee69d4cd 100644 --- a/flake.lock +++ b/flake.lock @@ -3,8 +3,7 @@ "nix": { "info": { "lastModified": 1580749449, - "narHash": "sha256-wEPYfs8ULx0dK6R4hrsXC/r8omxJLc86OhQg39eCo0c=", - "revCount": 7729 + "narHash": "sha256-wEPYfs8ULx0dK6R4hrsXC/r8omxJLc86OhQg39eCo0c=" }, "inputs": { "nixpkgs": { @@ -26,14 +25,14 @@ } }, "locked": { - "ref": "HEAD", + "owner": "NixOS", + "repo": "nix", "rev": "0a4e911cf478d0d38082a8840a5acc8f57d00086", - "type": "git", - "url": "file:///home/eelco/Dev/nix-flake" + "type": "github" }, "original": { - "type": "git", - "url": "file:///home/eelco/Dev/nix-flake" + "id": "nix", + "type": "indirect" } }, "nixpkgs": { From ba8814a245fe1e82408ec8591eeff3275f593f84 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 3 Feb 2020 18:43:45 +0100 Subject: [PATCH 49/73] Fix build --- src/hydra-eval-jobs/hydra-eval-jobs.cc | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index 334864d54..695a020af 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -13,8 +13,8 @@ #include "get-drvs.hh" #include "globals.hh" #include "common-eval-args.hh" -#include "flakeref.hh" -#include "flake.hh" +#include "flake/flakeref.hh" +#include "flake/flake.hh" #include "hydra-config.hh" @@ -247,9 +247,19 @@ int main(int argc, char * * argv) if (myArgs.flake) { using namespace flake; - FlakeRef flakeRef(myArgs.releaseExpr); + + auto flakeRef = parseFlakeRef(myArgs.releaseExpr); + auto vFlake = state.allocValue(); - callFlake(state, resolveFlake(state, flakeRef, AllPure), *vFlake); + + auto lockedFlake = lockFlake(state, flakeRef, + LockFlags { + .updateLockFile = false, + .useRegistries = false, + .allowMutable = false, + }); + + callFlake(state, lockedFlake, *vFlake); auto vOutputs = (*vFlake->attrs->get(state.symbols.create("outputs")))->value; state.forceValue(*vOutputs); From 93ed7f81cbc579013d5dd810ec5ea500974b93ea Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 14 Feb 2020 23:26:56 +0100 Subject: [PATCH 50/73] flake.lock: Update Flake input changes: * Updated 'nix': 'github:NixOS/nix/d2032edb2f86e955a8a7724a27c0c3225f386500' -> 'github:NixOS/nix/eb19ff3b82240326fc0e999e09f81b6c8ed98640' --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index d53637436..79bf46f3b 100644 --- a/flake.lock +++ b/flake.lock @@ -2,8 +2,8 @@ "inputs": { "nix": { "info": { - "lastModified": 1581081721, - "narHash": "sha256-Axhas6uySMqevQrjdLaIfFQZgyCueBngzUV7CEYhbWI=" + "lastModified": 1581719021, + "narHash": "sha256-x3c+ShqMxlEeuRuo44MEFhuTde1SsVKWKpbS7umesnU=" }, "inputs": { "nixpkgs": { @@ -28,7 +28,7 @@ "locked": { "owner": "NixOS", "repo": "nix", - "rev": "d2032edb2f86e955a8a7724a27c0c3225f386500", + "rev": "eb19ff3b82240326fc0e999e09f81b6c8ed98640", "type": "github" }, "original": { From 639c660abfd5de62ecfcd8d3cbc2eb6924c7ec75 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 15 Feb 2020 14:54:21 +0100 Subject: [PATCH 51/73] Fix build --- src/hydra-queue-runner/queue-monitor.cc | 7 +++---- src/hydra-queue-runner/state.hh | 6 ++++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index 62de134a7..3c02e1aa5 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -102,9 +102,9 @@ bool State::getQueuedBuilds(Connection & conn, if (id > newLastBuildId) newLastBuildId = id; if (builds_->count(id)) continue; - auto build = std::make_shared(); + auto build = std::make_shared( + localStore->parseStorePath(row["drvPath"].as())); build->id = id; - build->drvPath = localStore->parseStorePath(row["drvPath"].as()); build->projectName = row["project"].as(); build->jobsetName = row["jobset"].as(); build->jobName = row["job"].as(); @@ -402,8 +402,7 @@ Step::ptr State::createStep(ref destStore, /* If it doesn't exist, create it. */ if (!step) { - step = std::make_shared(); - step->drvPath = drvPath.clone(); + step = std::make_shared(drvPath.clone()); isNew = true; } diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index de74b768c..180907e9f 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -136,6 +136,9 @@ struct Build std::atomic_bool finishedInDB{false}; + Build(nix::StorePath && drvPath) : drvPath(std::move(drvPath)) + { } + std::string fullJobName() { return projectName + ":" + jobsetName + ":" + jobName; @@ -201,6 +204,9 @@ struct Step nix::Sync state; + Step(nix::StorePath && drvPath) : drvPath(std::move(drvPath)) + { } + ~Step() { //printMsg(lvlError, format("destroying step %1%") % drvPath); From 23c9ca3e94669087d463642baea0cf35a0b8d72f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 15 Feb 2020 15:12:14 +0100 Subject: [PATCH 52/73] Remove hydra-eval-guile-jobs This hasn't been used in a long time (Guix uses its own CI system), and it probably doesn't work anymore. --- .gitignore | 1 - configure.ac | 13 +- flake.nix | 2 +- src/script/Makefile.am | 6 +- src/script/hydra-eval-guile-jobs.in | 249 ---------------------------- src/script/hydra-eval-jobset | 75 +++------ 6 files changed, 28 insertions(+), 318 deletions(-) delete mode 100644 src/script/hydra-eval-guile-jobs.in diff --git a/.gitignore b/.gitignore index e53ae6a83..5663a9d67 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,6 @@ Makefile.in /aclocal.m4 /missing /install-sh -/src/script/hydra-eval-guile-jobs /src/sql/hydra-postgresql.sql /src/sql/hydra-sqlite.sql /src/sql/tmp.sqlite diff --git a/configure.ac b/configure.ac index f992e49ed..f99d4c9da 100644 --- a/configure.ac +++ b/configure.ac @@ -53,15 +53,6 @@ fi PKG_CHECK_MODULES([NIX], [nix-main nix-expr nix-store]) -PKG_CHECK_MODULES([GUILE], [guile-2.0], [HAVE_GUILE=yes], [HAVE_GUILE=no]) - -if test "x$HAVE_GUILE" = xyes; then - AC_PATH_PROG([GUILE], [guile]) -else - GUILE="guile" -fi -AC_SUBST([GUILE]) - testPath="$(dirname $(type -p expr))" AC_SUBST(testPath) @@ -80,13 +71,11 @@ AC_CONFIG_FILES([ src/lib/Makefile src/root/Makefile src/script/Makefile - src/script/hydra-eval-guile-jobs tests/Makefile tests/jobs/config.nix ]) -AC_CONFIG_COMMANDS([executable-scripts], - [chmod +x src/script/hydra-eval-guile-jobs]) +AC_CONFIG_COMMANDS([executable-scripts], []) AC_CONFIG_HEADER([hydra-config.h]) diff --git a/flake.nix b/flake.nix index 85d8e5e4a..3037a1e32 100644 --- a/flake.nix +++ b/flake.nix @@ -106,8 +106,8 @@ buildInputs = [ makeWrapper autoconf automake libtool unzip nukeReferences pkgconfig sqlite libpqxx gitAndTools.topGit mercurial darcs subversion bazaar openssl bzip2 libxslt - guile # optional, for Guile + Guix support perlDeps perl final.nix + boehmgc postgresql95 # for running the tests boost (nlohmann_json.override { multipleHeaders = true; }) diff --git a/src/script/Makefile.am b/src/script/Makefile.am index 9deb6f29a..466d3153a 100644 --- a/src/script/Makefile.am +++ b/src/script/Makefile.am @@ -1,6 +1,5 @@ EXTRA_DIST = \ - $(distributable_scripts) \ - hydra-eval-guile-jobs.in + $(distributable_scripts) distributable_scripts = \ hydra-backfill-ids \ @@ -17,5 +16,4 @@ distributable_scripts = \ nix-prefetch-hg bin_SCRIPTS = \ - $(distributable_scripts) \ - hydra-eval-guile-jobs + $(distributable_scripts) diff --git a/src/script/hydra-eval-guile-jobs.in b/src/script/hydra-eval-guile-jobs.in deleted file mode 100644 index 8c5df1252..000000000 --- a/src/script/hydra-eval-guile-jobs.in +++ /dev/null @@ -1,249 +0,0 @@ -#!/bin/sh -# Aside from this initial boilerplate, this is actually -*- scheme -*- code. -main="(module-ref (resolve-interface '(hydra-eval-guile-jobs)) 'eval-guile-jobs)" - -# Keep the host's GUILE_LOAD_PATH unchanged to allow the installed Guix to -# be used. This moves Guix modules possibly out of control, but solves -# bootstrapping issues. -# -# Use `--fresh-auto-compile' to ignore any available .go, and force -# recompilation. This is because checkouts in the store has mtime set to -# the epoch, and thus .go files look newer, even though they may not -# correspond. - -exec ${GUILE:-@GUILE@} --no-auto-compile --fresh-auto-compile \ - -l "$0" -c "(apply $main (cdr (command-line)))" "$@" -!# -;;; Copyright © 2012, 2013, 2014 Ludovic Courtès -;;; -;;; This file is part of Hydra. -;;; -;;; Hydra is free software: you can redistribute it and/or modify -;;; it under the terms of the GNU General Public License as published by -;;; the Free Software Foundation, either version 3 of the License, or -;;; (at your option) any later version. -;;; -;;; Hydra is distributed in the hope that it will be useful, -;;; but WITHOUT ANY WARRANTY; without even the implied warranty of -;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -;;; GNU General Public License for more details. -;;; -;;; You should have received a copy of the GNU General Public License -;;; along with Hydra. If not, see . - -(define-module (hydra-eval-guile-jobs) - #:use-module (sxml simple) - #:use-module (ice-9 match) - #:use-module (ice-9 regex) - #:use-module (srfi srfi-1) - #:use-module (srfi srfi-11) - #:export (job-evaluations->xml - eval-guile-jobs)) - -(define (guix-variable module name) - "Dynamically link variable NAME under Guix module MODULE and return it. -Note: this is used instead of `@', because when using `@' in an uncompiled -file, Guile tries to load the module directly as it reads the source, which -fails in our case, leading to the creation of empty (guix ...) modules." - ;; TODO: fail with an XML error description - (let ((m (resolve-interface `(guix ,module)))) - (module-ref m name))) - -(define (%derivation-system drv) - ;; XXX: Awful hack to workaround the fact that `derivation-system', which - ;; is a macro, cannot be referred to dynamically. - (struct-ref drv 3)) - -(define strip-store-path - (let* ((store (or (getenv "NIX_STORE_DIR") "/nix/store")) - (store-path-rx - (make-regexp (string-append "^.*" (regexp-quote store) - "/[^-]+-(.+)$")))) - (lambda (path) - (or (and=> (regexp-exec store-path-rx path) - (lambda (match) - (let ((path (match:substring match 1))) - path))) - path)))) - -(define (derivation-path->name drv) - "Return the base name of DRV, sans hash and `.drv' extension." - (let ((d (strip-store-path drv))) - (if (string-suffix? ".drv" d) - (string-drop-right d 4) - d))) - -(define (register-gc-root drv roots-dir) - "Register a permanent garbage collector root under ROOTS-DIR for DRV." - (let ((root (string-append roots-dir "/" (basename drv)))) - (unless (file-exists? root) - (symlink drv root)))) - -(define* (job-evaluations->sxml jobs - #:key gc-roots-dir) - "Return the hydra-eval-jobs SXML form for the result of JOBS, a list of -symbol/thunk pairs." - `(*TOP* - (*PI* xml "version='1.0' encoding='utf-8'") - "\n" - (jobs "\n" - ,@(map (match-lambda - (((? symbol? name) . (? thunk? thunk)) - (let* ((result (save-module-excursion - (lambda () - (set-current-module %user-module) - (with-output-to-port (%make-void-port "w") - thunk)))) - (drv (assoc-ref result 'derivation))) - (define (opt-attr xml-name name) - (match (assoc name result) - ((_ . value) - `((,xml-name ,value))) - (_ - '()))) - - (when gc-roots-dir - ;; Register DRV as a GC root so that it's not collected by - ;; the time 'hydra-queue-runner' attempts to build it. - (register-gc-root drv gc-roots-dir)) - - ;; XXX: Add tags? - `(job (@ (jobName ,name) - (drvPath ,drv) - ,@(opt-attr 'homepage 'home-page) - (license - ,(let loop ((license (assoc-ref result 'license))) - (match license - ((? struct?) - (struct-ref license 0)) - ((l ...) - (string-join (map loop l))) - (_ "")))) - ,@(opt-attr 'description 'description) - (maintainers - ,(string-join (or (assoc-ref result 'maintainers) - '()) - ", ")) - (maxSilent - ,(number->string (or (assoc-ref result - 'max-silent-time) - 3600))) - (timeout - ,(number->string (or (assoc-ref result 'timeout) - 72000))) - (nixName ,(derivation-path->name drv)) - (schedulingPriority - ,(number->string (or (assoc-ref result - 'scheduling-priority) - 10))) - (system - ,(call-with-input-file drv - (compose %derivation-system - (guix-variable 'derivations - 'read-derivation))))) - ;; Resolve Guix modules lazily. - ,(map (match-lambda - ((name . path) - `(output (@ (name ,name) (path ,path))))) - ((guix-variable 'derivations - 'derivation-path->output-paths) - drv)) - - "\n")))) - jobs)))) - -(define* (job-evaluations->xml jobs port - #:key gc-roots-dir) - (set-port-encoding! port "UTF-8") - (sxml->xml (job-evaluations->sxml jobs #:gc-roots-dir gc-roots-dir) - port)) - - -;;; -;;; Command-line entry point. -;;; - -(define (parse-arguments args) - "Traverse ARGS, a list of command-line arguments compatible with -`hydra-eval-jobs', and return the name of the file that defines the jobs, an -expression that returns the entry point in that file (a unary procedure), the -list of name/value pairs passed to that entry point, as well as a GC root -directory or #f." - (define (module-directory dir) - (let ((d (string-append dir "/share/guile/site/2.0"))) - (if (file-exists? d) - d - dir))) - - (let loop ((args args) - (result '()) - (file #f) - (entry 'hydra-jobs) - (roots-dir #f)) - (match args - (() - (if (not file) - (error "hydra-eval-guile-jobs: no expression file given") - (values file entry (reverse result) roots-dir))) - (("-I" name=dir rest ...) - (let* ((dir (match (string-tokenize name=dir - (char-set-complement (char-set - #\=))) - ((_ dir) dir) - ((dir) dir))) - (dir* (module-directory dir))) - (format (current-error-port) "adding `~a' to the load path~%" dir*) - (set! %load-path (cons dir* %load-path)) - (set! %load-compiled-path (cons dir* %load-compiled-path))) - (loop rest result file entry roots-dir)) - (("--argstr" name value rest ...) - (loop rest (alist-cons (string->symbol name) value result) - file entry roots-dir)) - (("--arg" name expr rest ...) - (let ((value (eval (call-with-input-string expr read) - (current-module)))) - (loop rest (alist-cons (string->symbol name) value result) - file entry roots-dir))) - (("--gc-roots-dir" dir rest ...) - (loop rest result file entry dir)) - (("-j" _ rest ...) ; XXX: what's this? - (loop rest result file entry roots-dir)) - (("--entry" expr rest ...) ; entry point, like `guile -e' - (let ((expr (call-with-input-string expr read))) - (loop rest result file expr roots-dir))) - ((file rest ...) ; source file that defines the jobs - (loop rest result file entry roots-dir)) - (_ - (error "hydra-eval-guile-jobs: invalid arguments" args))))) - -(define %user-module - ;; Hydra user module. - ;; TODO: Make it a sandbox. - (let ((m (make-module))) - (beautify-user-module! m) - m)) - -(define (eval-guile-jobs . args) - (setlocale LC_ALL "") - - (let-values (((file entry args gc-roots-dir) - (parse-arguments args))) - - (save-module-excursion - (lambda () - (set-current-module %user-module) - - ;; The standard output must contain only XML. - (with-output-to-port (%make-void-port "w") - (lambda () - (primitive-load file))))) - - (let* ((entry (eval entry %user-module)) - (store ((guix-variable 'store 'open-connection))) - (jobs (entry store args))) - (unless (string? gc-roots-dir) - (format (current-error-port) - "warning: --gc-roots-dir not specified~%")) - - (job-evaluations->xml jobs (current-output-port) - #:gc-roots-dir gc-roots-dir)))) diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset index 9074955fd..4ee716bad 100755 --- a/src/script/hydra-eval-jobset +++ b/src/script/hydra-eval-jobset @@ -264,53 +264,31 @@ sub fetchInput { sub booleanToString { - my ($exprType, $value) = @_; - my $result; - if ($exprType eq "guile") { - if ($value eq "true") { - $result = "#t"; - } else { - $result = "#f"; - } - $result = $value; - } else { - $result = $value; - } - return $result; + my ($value) = @_; + return $value; } sub buildInputToString { - my ($exprType, $input) = @_; - my $result; - if ($exprType eq "guile") { - $result = "'((file-name . \"" . ${input}->{storePath} . "\")" . - (defined $input->{revision} ? "(revision . \"" . $input->{revision} . "\")" : "") . - (defined $input->{revCount} ? "(revision-count . " . $input->{revCount} . ")" : "") . - (defined $input->{gitTag} ? "(git-tag . \"" . $input->{gitTag} . "\")" : "") . - (defined $input->{shortRev} ? "(short-revision . \"" . $input->{shortRev} . "\")" : "") . - (defined $input->{version} ? "(version . \"" . $input->{version} . "\")" : "") . - ")"; - } else { - $result = "{ outPath = builtins.storePath " . $input->{storePath} . "" . - "; inputType = \"" . $input->{type} . "\"" . - (defined $input->{uri} ? "; uri = \"" . $input->{uri} . "\"" : "") . - (defined $input->{revNumber} ? "; rev = " . $input->{revNumber} . "" : "") . - (defined $input->{revision} ? "; rev = \"" . $input->{revision} . "\"" : "") . - (defined $input->{revCount} ? "; revCount = " . $input->{revCount} . "" : "") . - (defined $input->{gitTag} ? "; gitTag = \"" . $input->{gitTag} . "\"" : "") . - (defined $input->{shortRev} ? "; shortRev = \"" . $input->{shortRev} . "\"" : "") . - (defined $input->{version} ? "; version = \"" . $input->{version} . "\"" : "") . - (defined $input->{outputName} ? "; outputName = \"" . $input->{outputName} . "\"" : "") . - (defined $input->{drvPath} ? "; drvPath = builtins.storePath " . $input->{drvPath} . "" : "") . - ";}"; - } - return $result; + my ($input) = @_; + return + "{ outPath = builtins.storePath " . $input->{storePath} . "" . + "; inputType = \"" . $input->{type} . "\"" . + (defined $input->{uri} ? "; uri = \"" . $input->{uri} . "\"" : "") . + (defined $input->{revNumber} ? "; rev = " . $input->{revNumber} . "" : "") . + (defined $input->{revision} ? "; rev = \"" . $input->{revision} . "\"" : "") . + (defined $input->{revCount} ? "; revCount = " . $input->{revCount} . "" : "") . + (defined $input->{gitTag} ? "; gitTag = \"" . $input->{gitTag} . "\"" : "") . + (defined $input->{shortRev} ? "; shortRev = \"" . $input->{shortRev} . "\"" : "") . + (defined $input->{version} ? "; version = \"" . $input->{version} . "\"" : "") . + (defined $input->{outputName} ? "; outputName = \"" . $input->{outputName} . "\"" : "") . + (defined $input->{drvPath} ? "; drvPath = builtins.storePath " . $input->{drvPath} . "" : "") . + ";}"; } sub inputsToArgs { - my ($inputInfo, $exprType) = @_; + my ($inputInfo) = @_; my @res = (); foreach my $input (sort keys %{$inputInfo}) { @@ -327,14 +305,12 @@ sub inputsToArgs { push @res, "--argstr", $input, $alt->{value}; } elsif ($alt->{type} eq "boolean") { - push @res, "--arg", $input, booleanToString($exprType, $alt->{value}); + push @res, "--arg", $input, booleanToString($alt->{value}); } elsif ($alt->{type} eq "nix") { - die "input type ‘nix’ only supported for Nix-based jobsets\n" unless $exprType eq "nix"; push @res, "--arg", $input, $alt->{value}; } elsif ($alt->{type} eq "eval") { - die "input type ‘eval’ only supported for Nix-based jobsets\n" unless $exprType eq "nix"; my $s = "{ "; # FIXME: escape $_. But dots should not be escaped. $s .= "$_ = builtins.storePath ${\$alt->{jobs}->{$_}}; " @@ -343,7 +319,7 @@ sub inputsToArgs { push @res, "--arg", $input, $s; } else { - push @res, "--arg", $input, buildInputToString($exprType, $alt); + push @res, "--arg", $input, buildInputToString($alt); } } @@ -352,7 +328,7 @@ sub inputsToArgs { sub evalJobs { - my ($inputInfo, $exprType, $nixExprInputName, $nixExprPath, $flakeRef) = @_; + my ($inputInfo, $nixExprInputName, $nixExprPath, $flakeRef) = @_; my @cmd; @@ -365,13 +341,11 @@ sub evalJobs { my $nixExprInput = $inputInfo->{$nixExprInputName}->[0] or die "cannot find the input containing the job expression\n"; - my $evaluator = ($exprType eq "guile") ? "hydra-eval-guile-jobs" : "hydra-eval-jobs"; - - @cmd = ($evaluator, + @cmd = ("hydra-eval-jobs", "<" . $nixExprInputName . "/" . $nixExprPath . ">", "--gc-roots-dir", getGCRootsDir, "--max-jobs", 1, - inputsToArgs($inputInfo, $exprType)); + inputsToArgs($inputInfo)); } if (defined $ENV{'HYDRA_DEBUG'}) { @@ -604,7 +578,6 @@ sub checkJobsetWrapped { $jobset->discard_changes; $inputInfo->{"declInput"} = [ $declInput ]; } - my $exprType = $jobset->nixexprpath =~ /.scm$/ ? "guile" : "nix"; # Fetch all values for all inputs. my $checkoutStart = clock_gettime(CLOCK_MONOTONIC); @@ -640,7 +613,7 @@ sub checkJobsetWrapped { # Hash the arguments to hydra-eval-jobs and check the # JobsetInputHashes to see if the previous evaluation had the same # inputs. If so, bail out. - my @args = ($jobset->nixexprinput, $jobset->nixexprpath, inputsToArgs($inputInfo, $exprType)); + my @args = ($jobset->nixexprinput, $jobset->nixexprpath, inputsToArgs($inputInfo)); my $argsHash = sha256_hex("@args"); my $prevEval = getPrevJobsetEval($db, $jobset, 0); if (defined $prevEval && $prevEval->hash eq $argsHash && !$dryRun && !$jobset->forceeval && $prevEval->flake eq $flakeRef) { @@ -655,7 +628,7 @@ sub checkJobsetWrapped { # Evaluate the job expression. my $evalStart = clock_gettime(CLOCK_MONOTONIC); - my $jobs = evalJobs($inputInfo, $exprType, $jobset->nixexprinput, $jobset->nixexprpath, $flakeRef); + my $jobs = evalJobs($inputInfo, $jobset->nixexprinput, $jobset->nixexprpath, $flakeRef); my $evalStop = clock_gettime(CLOCK_MONOTONIC); if ($jobsetsJobset) { From 345512a6d0743eaac9d451833ecc1a71d12809c9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 15 Feb 2020 00:03:58 +0100 Subject: [PATCH 53/73] hydra-eval-jobs -> nix eval-hydra-jobs --- .gitignore | 1 - configure.ac | 1 - flake.nix | 2 +- src/Makefile.am | 2 +- src/hydra-eval-jobs/Makefile.am | 5 - src/hydra-eval-jobs/hydra-eval-jobs.cc | 281 ------------------------- src/script/hydra-eval-jobset | 27 ++- tests/Makefile.am | 2 +- 8 files changed, 19 insertions(+), 302 deletions(-) delete mode 100644 src/hydra-eval-jobs/Makefile.am delete mode 100644 src/hydra-eval-jobs/hydra-eval-jobs.cc diff --git a/.gitignore b/.gitignore index 5663a9d67..bcf4bcc1d 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,6 @@ Makefile.in /src/sql/hydra-postgresql.sql /src/sql/hydra-sqlite.sql /src/sql/tmp.sqlite -/src/hydra-eval-jobs/hydra-eval-jobs /src/root/static/bootstrap /src/root/static/js/flot /doc/manual/images diff --git a/configure.ac b/configure.ac index f99d4c9da..256bfb727 100644 --- a/configure.ac +++ b/configure.ac @@ -64,7 +64,6 @@ AC_CONFIG_FILES([ doc/manual/Makefile src/Makefile src/hydra-evaluator/Makefile - src/hydra-eval-jobs/Makefile src/hydra-queue-runner/Makefile src/sql/Makefile src/ttf/Makefile diff --git a/flake.nix b/flake.nix index 3037a1e32..1fed5e4fd 100644 --- a/flake.nix +++ b/flake.nix @@ -121,7 +121,7 @@ configureFlags = [ "--with-docbook-xsl=${docbook_xsl}/xml/xsl/docbook" ]; shellHook = '' - PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$PATH + PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-queue-runner:$PATH PERL5LIB=$(pwd)/src/lib:$PERL5LIB ''; diff --git a/src/Makefile.am b/src/Makefile.am index a28780b68..ccd52eb93 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,3 +1,3 @@ -SUBDIRS = hydra-evaluator hydra-eval-jobs hydra-queue-runner sql script lib root ttf +SUBDIRS = hydra-evaluator hydra-queue-runner sql script lib root ttf BOOTCLEAN_SUBDIRS = $(SUBDIRS) DIST_SUBDIRS = $(SUBDIRS) diff --git a/src/hydra-eval-jobs/Makefile.am b/src/hydra-eval-jobs/Makefile.am deleted file mode 100644 index b41a4eb80..000000000 --- a/src/hydra-eval-jobs/Makefile.am +++ /dev/null @@ -1,5 +0,0 @@ -bin_PROGRAMS = hydra-eval-jobs - -hydra_eval_jobs_SOURCES = hydra-eval-jobs.cc -hydra_eval_jobs_LDADD = $(NIX_LIBS) -lnixrust -hydra_eval_jobs_CXXFLAGS = $(NIX_CFLAGS) -I ../libhydra diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc deleted file mode 100644 index 695a020af..000000000 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ /dev/null @@ -1,281 +0,0 @@ -#include -#include - -#define GC_LINUX_THREADS 1 -#include - -#include "shared.hh" -#include "store-api.hh" -#include "eval.hh" -#include "eval-inline.hh" -#include "util.hh" -#include "json.hh" -#include "get-drvs.hh" -#include "globals.hh" -#include "common-eval-args.hh" -#include "flake/flakeref.hh" -#include "flake/flake.hh" - -#include "hydra-config.hh" - -#include -#include - -using namespace nix; - - -static Path gcRootsDir; - - -static void findJobs(EvalState & state, JSONObject & top, - Bindings & autoArgs, Value & v, const string & attrPath); - - -static string queryMetaStrings(EvalState & state, DrvInfo & drv, const string & name, const string & subAttribute) -{ - Strings res; - std::function rec; - - rec = [&](Value & v) { - state.forceValue(v); - if (v.type == tString) - res.push_back(v.string.s); - else if (v.isList()) - for (unsigned int n = 0; n < v.listSize(); ++n) - rec(*v.listElems()[n]); - else if (v.type == tAttrs) { - auto a = v.attrs->find(state.symbols.create(subAttribute)); - if (a != v.attrs->end()) - res.push_back(state.forceString(*a->value)); - } - }; - - Value * v = drv.queryMeta(name); - if (v) rec(*v); - - return concatStringsSep(", ", res); -} - - -static void findJobsWrapped(EvalState & state, JSONObject & top, - Bindings & autoArgs, Value & vIn, const string & attrPath) -{ - debug(format("at path `%1%'") % attrPath); - - checkInterrupt(); - - Value v; - state.autoCallFunction(autoArgs, vIn, v); - - if (v.type == tAttrs) { - - auto drv = getDerivation(state, v, false); - - if (drv) { - Path drvPath; - - DrvInfo::Outputs outputs = drv->queryOutputs(); - - if (drv->querySystem() == "unknown") - throw EvalError("derivation must have a ‘system’ attribute"); - - { - auto res = top.object(attrPath); - res.attr("nixName", drv->queryName()); - res.attr("system", drv->querySystem()); - res.attr("drvPath", drvPath = drv->queryDrvPath()); - res.attr("description", drv->queryMetaString("description")); - res.attr("license", queryMetaStrings(state, *drv, "license", "shortName")); - res.attr("homepage", drv->queryMetaString("homepage")); - res.attr("maintainers", queryMetaStrings(state, *drv, "maintainers", "email")); - res.attr("schedulingPriority", drv->queryMetaInt("schedulingPriority", 100)); - res.attr("timeout", drv->queryMetaInt("timeout", 36000)); - res.attr("maxSilent", drv->queryMetaInt("maxSilent", 7200)); - res.attr("isChannel", drv->queryMetaBool("isHydraChannel", false)); - - /* If this is an aggregate, then get its constituents. */ - Bindings::iterator a = v.attrs->find(state.symbols.create("_hydraAggregate")); - if (a != v.attrs->end() && state.forceBool(*a->value, *a->pos)) { - Bindings::iterator a = v.attrs->find(state.symbols.create("constituents")); - if (a == v.attrs->end()) - throw EvalError("derivation must have a ‘constituents’ attribute"); - PathSet context; - state.coerceToString(*a->pos, *a->value, context, true, false); - PathSet drvs; - for (auto & i : context) - if (i.at(0) == '!') { - size_t index = i.find("!", 1); - drvs.insert(string(i, index + 1)); - } - res.attr("constituents", concatStringsSep(" ", drvs)); - } - - /* Register the derivation as a GC root. !!! This - registers roots for jobs that we may have already - done. */ - auto localStore = state.store.dynamic_pointer_cast(); - if (gcRootsDir != "" && localStore) { - Path root = gcRootsDir + "/" + std::string(baseNameOf(drvPath)); - if (!pathExists(root)) - localStore->addPermRoot(localStore->parseStorePath(drvPath), root, false); - } - - auto res2 = res.object("outputs"); - for (auto & j : outputs) - res2.attr(j.first, j.second); - - } - } - - else { - if (!state.isDerivation(v)) { - for (auto & i : v.attrs->lexicographicOrder()) { - std::string name(i->name); - - /* Skip jobs with dots in the name. */ - if (name.find('.') != std::string::npos) { - printError("skipping job with illegal name '%s'", name); - continue; - } - - findJobs(state, top, autoArgs, *i->value, - (attrPath.empty() ? "" : attrPath + ".") + name); - } - } - } - } - - else if (v.type == tNull) { - // allow null values, meaning 'do nothing' - } - - else - throw TypeError(format("unsupported value: %1%") % v); -} - - -static void findJobs(EvalState & state, JSONObject & top, - Bindings & autoArgs, Value & v, const string & attrPath) -{ - try { - findJobsWrapped(state, top, autoArgs, v, attrPath); - } catch (EvalError & e) { - auto res = top.object(attrPath); - res.attr("error", filterANSIEscapes(e.msg(), true)); - } -} - - -int main(int argc, char * * argv) -{ - /* Prevent undeclared dependencies in the evaluation via - $NIX_PATH. */ - unsetenv("NIX_PATH"); - - return handleExceptions(argv[0], [&]() { - - auto config = std::make_unique<::Config>(); - - auto initialHeapSize = config->getStrOption("evaluator_initial_heap_size", ""); - if (initialHeapSize != "") - setenv("GC_INITIAL_HEAP_SIZE", initialHeapSize.c_str(), 1); - - initNix(); - initGC(); - - struct MyArgs : MixEvalArgs, MixCommonArgs - { - Path releaseExpr; - bool flake = false; - - MyArgs() : MixCommonArgs("hydra-eval-jobs") - { - mkFlag() - .longName("help") - .description("show usage information") - .handler([&]() { - printHelp(programName, std::cout); - throw Exit(); - }); - - mkFlag() - .longName("gc-roots-dir") - .description("garbage collector roots directory") - .labels({"path"}) - .dest(&gcRootsDir); - - mkFlag() - .longName("dry-run") - .description("don't create store derivations") - .set(&settings.readOnlyMode, true); - - mkFlag() - .longName("flake") - .description("build a flake") - .set(&flake, true); - - expectArg("expr", &releaseExpr); - } - }; - - MyArgs myArgs; - myArgs.parseCmdline(argvToStrings(argc, argv)); - - JSONObject json(std::cout, true); - std::cout.flush(); - - /* FIXME: The build hook in conjunction with import-from-derivation is causing "unexpected EOF" during eval */ - settings.builders = ""; - - /* Prevent access to paths outside of the Nix search path and - to the environment. */ - evalSettings.restrictEval = true; - - /* When building a flake, use pure evaluation (no access to - 'getEnv', 'currentSystem' etc. */ - evalSettings.pureEval = myArgs.flake; - - if (myArgs.releaseExpr == "") throw UsageError("no expression specified"); - - if (gcRootsDir == "") printMsg(lvlError, "warning: `--gc-roots-dir' not specified"); - - EvalState state(myArgs.searchPath, openStore()); - - Bindings & autoArgs = *myArgs.getAutoArgs(state); - - Value v; - - if (myArgs.flake) { - using namespace flake; - - auto flakeRef = parseFlakeRef(myArgs.releaseExpr); - - auto vFlake = state.allocValue(); - - auto lockedFlake = lockFlake(state, flakeRef, - LockFlags { - .updateLockFile = false, - .useRegistries = false, - .allowMutable = false, - }); - - callFlake(state, lockedFlake, *vFlake); - - auto vOutputs = (*vFlake->attrs->get(state.symbols.create("outputs")))->value; - state.forceValue(*vOutputs); - - auto aHydraJobs = vOutputs->attrs->get(state.symbols.create("hydraJobs")); - if (!aHydraJobs) - aHydraJobs = vOutputs->attrs->get(state.symbols.create("checks")); - if (!aHydraJobs) - throw Error("flake '%s' does not provide any Hydra jobs or checks", flakeRef); - - v = *(*aHydraJobs)->value; - - } else { - state.evalFile(lookupFileArg(state, myArgs.releaseExpr), v); - } - - findJobs(state, json, autoArgs, v, ""); - }); -} diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset index 4ee716bad..235e3a96b 100755 --- a/src/script/hydra-eval-jobset +++ b/src/script/hydra-eval-jobset @@ -330,22 +330,27 @@ sub inputsToArgs { sub evalJobs { my ($inputInfo, $nixExprInputName, $nixExprPath, $flakeRef) = @_; - my @cmd; + my @cmd = ( + "nix", "eval-hydra-jobs", + "--json", + "--gc-roots-dir", getGCRootsDir, + "--max-jobs", 1, + "--workers", $config->{evaluator_workers} // 1, + "--max-memory-size", $config->{evaluator_max_memory_size} // 4096, + "--builders", "", + ); if (defined $flakeRef) { - @cmd = ("hydra-eval-jobs", - "--flake", $flakeRef, - "--gc-roots-dir", getGCRootsDir, - "--max-jobs", 1); + push @cmd, + ($flakeRef, + "--no-update-lock-file", + "--no-registries"); } else { my $nixExprInput = $inputInfo->{$nixExprInputName}->[0] or die "cannot find the input containing the job expression\n"; - - @cmd = ("hydra-eval-jobs", - "<" . $nixExprInputName . "/" . $nixExprPath . ">", - "--gc-roots-dir", getGCRootsDir, - "--max-jobs", 1, - inputsToArgs($inputInfo)); + push @cmd, + ("--file", "<" . $nixExprInputName . "/" . $nixExprPath . ">", + inputsToArgs($inputInfo)); } if (defined $ENV{'HYDRA_DEBUG'}) { diff --git a/tests/Makefile.am b/tests/Makefile.am index e56e528ef..3fe032c7d 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -14,7 +14,7 @@ TESTS_ENVIRONMENT = \ NIX_BUILD_HOOK= \ PGHOST=/tmp \ PERL5LIB="$(srcdir):$(abs_top_srcdir)/src/lib:$$PERL5LIB" \ - PATH=$(abs_top_srcdir)/src/hydra-evaluator:$(abs_top_srcdir)/src/script:$(abs_top_srcdir)/src/hydra-eval-jobs:$(abs_top_srcdir)/src/hydra-queue-runner:$$PATH \ + PATH=$(abs_top_srcdir)/src/hydra-evaluator:$(abs_top_srcdir)/src/script:$(abs_top_srcdir)/src/hydra-queue-runner:$$PATH \ perl -w EXTRA_DIST = \ From 68bdf834ee7fe4208f287aa5d14f86e50086de1a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 15 Feb 2020 16:40:16 +0100 Subject: [PATCH 54/73] Fix hydra-evaluator --unlock --- src/hydra-evaluator/hydra-evaluator.cc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/hydra-evaluator/hydra-evaluator.cc b/src/hydra-evaluator/hydra-evaluator.cc index 9bee8ce78..fd9eb9d79 100644 --- a/src/hydra-evaluator/hydra-evaluator.cc +++ b/src/hydra-evaluator/hydra-evaluator.cc @@ -361,14 +361,15 @@ int main(int argc, char * * argv) return true; }); - if (!args.empty()) { - if (args.size() != 2) throw UsageError("Syntax: hydra-evaluator [ ]"); - evaluator.evalOne = JobsetName(args[0], args[1]); - } if (unlock) evaluator.unlock(); - else + else { + if (!args.empty()) { + if (args.size() != 2) throw UsageError("Syntax: hydra-evaluator [ ]"); + evaluator.evalOne = JobsetName(args[0], args[1]); + } evaluator.run(); + } }); } From b0a5365602712ee3acac9f0fb89b6cd3af932a34 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 15 Feb 2020 21:55:33 +0100 Subject: [PATCH 55/73] flake.lock: Update Flake input changes: * Updated 'nix': 'github:NixOS/nix/eb19ff3b82240326fc0e999e09f81b6c8ed98640' -> 'github:NixOS/nix/144bb3ef7ddca6502ea3e1878f7fc0a3f9013aba' --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 79bf46f3b..4cb91d532 100644 --- a/flake.lock +++ b/flake.lock @@ -2,8 +2,8 @@ "inputs": { "nix": { "info": { - "lastModified": 1581719021, - "narHash": "sha256-x3c+ShqMxlEeuRuo44MEFhuTde1SsVKWKpbS7umesnU=" + "lastModified": 1581799708, + "narHash": "sha256-lkFbDyBvk6DaFCA7rwAefSWhx6tOS2FnRDaNBmRA4cE=" }, "inputs": { "nixpkgs": { @@ -28,7 +28,7 @@ "locked": { "owner": "NixOS", "repo": "nix", - "rev": "eb19ff3b82240326fc0e999e09f81b6c8ed98640", + "rev": "144bb3ef7ddca6502ea3e1878f7fc0a3f9013aba", "type": "github" }, "original": { From 4b9cdc493300d5cb67e03a719f275ca5295a0e60 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 15 Feb 2020 21:56:35 +0100 Subject: [PATCH 56/73] Remove boehmgc dependency This is now propagated by Nix. --- flake.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/flake.nix b/flake.nix index 1fed5e4fd..c87503e74 100644 --- a/flake.nix +++ b/flake.nix @@ -93,7 +93,6 @@ final.nix final.nix.perl-bindings git - boehmgc ]; }; @@ -107,7 +106,6 @@ [ makeWrapper autoconf automake libtool unzip nukeReferences pkgconfig sqlite libpqxx gitAndTools.topGit mercurial darcs subversion bazaar openssl bzip2 libxslt perlDeps perl final.nix - boehmgc postgresql95 # for running the tests boost (nlohmann_json.override { multipleHeaders = true; }) From 4824b0fd9b44ccc1dae9d5b5874808b34aac4915 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Feb 2020 16:23:39 +0100 Subject: [PATCH 57/73] flake.lock: Update Flake input changes: * Updated 'nix': 'github:NixOS/nix/144bb3ef7ddca6502ea3e1878f7fc0a3f9013aba' -> 'github:NixOS/nix/6529490cc10018d5191e50c482ac1180b96b1a3c' --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 4cb91d532..d13c8b3a5 100644 --- a/flake.lock +++ b/flake.lock @@ -2,8 +2,8 @@ "inputs": { "nix": { "info": { - "lastModified": 1581799708, - "narHash": "sha256-lkFbDyBvk6DaFCA7rwAefSWhx6tOS2FnRDaNBmRA4cE=" + "lastModified": 1581951239, + "narHash": "sha256-DImAA/qFbgGNCtARoSJvOmeRhRIkuEN25N7KP6ZT6mk=" }, "inputs": { "nixpkgs": { @@ -28,7 +28,7 @@ "locked": { "owner": "NixOS", "repo": "nix", - "rev": "144bb3ef7ddca6502ea3e1878f7fc0a3f9013aba", + "rev": "6529490cc10018d5191e50c482ac1180b96b1a3c", "type": "github" }, "original": { From cf961ac8933b76d9f0a3ac3eb49cc08879f5bcc9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Feb 2020 16:33:25 +0100 Subject: [PATCH 58/73] Update aggregate handling --- src/script/hydra-eval-jobset | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset index 235e3a96b..d037dedd8 100755 --- a/src/script/hydra-eval-jobset +++ b/src/script/hydra-eval-jobset @@ -722,7 +722,7 @@ sub checkJobsetWrapped { foreach my $job (values %{$jobs}) { next unless $job->{constituents}; my $x = $drvPathToId{$job->{drvPath}} or die; - foreach my $drvPath (split / /, $job->{constituents}) { + foreach my $drvPath (@{$job->{constituents}}) { my $constituent = $drvPathToId{$drvPath}; if (defined $constituent) { $db->resultset('AggregateConstituents')->update_or_create({aggregate => $x->{id}, constituent => $constituent->{id}}); From 6f1d68bda483b2ed3b1c1f0a696e8fe300108853 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 19 Feb 2020 20:36:52 +0100 Subject: [PATCH 59/73] Revert "hydra-eval-jobs -> nix eval-hydra-jobs" This reverts commit 345512a6d0743eaac9d451833ecc1a71d12809c9. --- .gitignore | 1 + configure.ac | 1 + flake.nix | 2 +- src/Makefile.am | 2 +- src/hydra-eval-jobs/Makefile.am | 5 + src/hydra-eval-jobs/hydra-eval-jobs.cc | 281 +++++++++++++++++++++++++ src/script/hydra-eval-jobset | 27 +-- tests/Makefile.am | 2 +- 8 files changed, 302 insertions(+), 19 deletions(-) create mode 100644 src/hydra-eval-jobs/Makefile.am create mode 100644 src/hydra-eval-jobs/hydra-eval-jobs.cc diff --git a/.gitignore b/.gitignore index bcf4bcc1d..5663a9d67 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ Makefile.in /src/sql/hydra-postgresql.sql /src/sql/hydra-sqlite.sql /src/sql/tmp.sqlite +/src/hydra-eval-jobs/hydra-eval-jobs /src/root/static/bootstrap /src/root/static/js/flot /doc/manual/images diff --git a/configure.ac b/configure.ac index 256bfb727..f99d4c9da 100644 --- a/configure.ac +++ b/configure.ac @@ -64,6 +64,7 @@ AC_CONFIG_FILES([ doc/manual/Makefile src/Makefile src/hydra-evaluator/Makefile + src/hydra-eval-jobs/Makefile src/hydra-queue-runner/Makefile src/sql/Makefile src/ttf/Makefile diff --git a/flake.nix b/flake.nix index c87503e74..1af431765 100644 --- a/flake.nix +++ b/flake.nix @@ -119,7 +119,7 @@ configureFlags = [ "--with-docbook-xsl=${docbook_xsl}/xml/xsl/docbook" ]; shellHook = '' - PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-queue-runner:$PATH + PATH=$(pwd)/src/hydra-evaluator:$(pwd)/src/script:$(pwd)/src/hydra-eval-jobs:$(pwd)/src/hydra-queue-runner:$PATH PERL5LIB=$(pwd)/src/lib:$PERL5LIB ''; diff --git a/src/Makefile.am b/src/Makefile.am index ccd52eb93..a28780b68 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,3 +1,3 @@ -SUBDIRS = hydra-evaluator hydra-queue-runner sql script lib root ttf +SUBDIRS = hydra-evaluator hydra-eval-jobs hydra-queue-runner sql script lib root ttf BOOTCLEAN_SUBDIRS = $(SUBDIRS) DIST_SUBDIRS = $(SUBDIRS) diff --git a/src/hydra-eval-jobs/Makefile.am b/src/hydra-eval-jobs/Makefile.am new file mode 100644 index 000000000..b41a4eb80 --- /dev/null +++ b/src/hydra-eval-jobs/Makefile.am @@ -0,0 +1,5 @@ +bin_PROGRAMS = hydra-eval-jobs + +hydra_eval_jobs_SOURCES = hydra-eval-jobs.cc +hydra_eval_jobs_LDADD = $(NIX_LIBS) -lnixrust +hydra_eval_jobs_CXXFLAGS = $(NIX_CFLAGS) -I ../libhydra diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc new file mode 100644 index 000000000..695a020af --- /dev/null +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -0,0 +1,281 @@ +#include +#include + +#define GC_LINUX_THREADS 1 +#include + +#include "shared.hh" +#include "store-api.hh" +#include "eval.hh" +#include "eval-inline.hh" +#include "util.hh" +#include "json.hh" +#include "get-drvs.hh" +#include "globals.hh" +#include "common-eval-args.hh" +#include "flake/flakeref.hh" +#include "flake/flake.hh" + +#include "hydra-config.hh" + +#include +#include + +using namespace nix; + + +static Path gcRootsDir; + + +static void findJobs(EvalState & state, JSONObject & top, + Bindings & autoArgs, Value & v, const string & attrPath); + + +static string queryMetaStrings(EvalState & state, DrvInfo & drv, const string & name, const string & subAttribute) +{ + Strings res; + std::function rec; + + rec = [&](Value & v) { + state.forceValue(v); + if (v.type == tString) + res.push_back(v.string.s); + else if (v.isList()) + for (unsigned int n = 0; n < v.listSize(); ++n) + rec(*v.listElems()[n]); + else if (v.type == tAttrs) { + auto a = v.attrs->find(state.symbols.create(subAttribute)); + if (a != v.attrs->end()) + res.push_back(state.forceString(*a->value)); + } + }; + + Value * v = drv.queryMeta(name); + if (v) rec(*v); + + return concatStringsSep(", ", res); +} + + +static void findJobsWrapped(EvalState & state, JSONObject & top, + Bindings & autoArgs, Value & vIn, const string & attrPath) +{ + debug(format("at path `%1%'") % attrPath); + + checkInterrupt(); + + Value v; + state.autoCallFunction(autoArgs, vIn, v); + + if (v.type == tAttrs) { + + auto drv = getDerivation(state, v, false); + + if (drv) { + Path drvPath; + + DrvInfo::Outputs outputs = drv->queryOutputs(); + + if (drv->querySystem() == "unknown") + throw EvalError("derivation must have a ‘system’ attribute"); + + { + auto res = top.object(attrPath); + res.attr("nixName", drv->queryName()); + res.attr("system", drv->querySystem()); + res.attr("drvPath", drvPath = drv->queryDrvPath()); + res.attr("description", drv->queryMetaString("description")); + res.attr("license", queryMetaStrings(state, *drv, "license", "shortName")); + res.attr("homepage", drv->queryMetaString("homepage")); + res.attr("maintainers", queryMetaStrings(state, *drv, "maintainers", "email")); + res.attr("schedulingPriority", drv->queryMetaInt("schedulingPriority", 100)); + res.attr("timeout", drv->queryMetaInt("timeout", 36000)); + res.attr("maxSilent", drv->queryMetaInt("maxSilent", 7200)); + res.attr("isChannel", drv->queryMetaBool("isHydraChannel", false)); + + /* If this is an aggregate, then get its constituents. */ + Bindings::iterator a = v.attrs->find(state.symbols.create("_hydraAggregate")); + if (a != v.attrs->end() && state.forceBool(*a->value, *a->pos)) { + Bindings::iterator a = v.attrs->find(state.symbols.create("constituents")); + if (a == v.attrs->end()) + throw EvalError("derivation must have a ‘constituents’ attribute"); + PathSet context; + state.coerceToString(*a->pos, *a->value, context, true, false); + PathSet drvs; + for (auto & i : context) + if (i.at(0) == '!') { + size_t index = i.find("!", 1); + drvs.insert(string(i, index + 1)); + } + res.attr("constituents", concatStringsSep(" ", drvs)); + } + + /* Register the derivation as a GC root. !!! This + registers roots for jobs that we may have already + done. */ + auto localStore = state.store.dynamic_pointer_cast(); + if (gcRootsDir != "" && localStore) { + Path root = gcRootsDir + "/" + std::string(baseNameOf(drvPath)); + if (!pathExists(root)) + localStore->addPermRoot(localStore->parseStorePath(drvPath), root, false); + } + + auto res2 = res.object("outputs"); + for (auto & j : outputs) + res2.attr(j.first, j.second); + + } + } + + else { + if (!state.isDerivation(v)) { + for (auto & i : v.attrs->lexicographicOrder()) { + std::string name(i->name); + + /* Skip jobs with dots in the name. */ + if (name.find('.') != std::string::npos) { + printError("skipping job with illegal name '%s'", name); + continue; + } + + findJobs(state, top, autoArgs, *i->value, + (attrPath.empty() ? "" : attrPath + ".") + name); + } + } + } + } + + else if (v.type == tNull) { + // allow null values, meaning 'do nothing' + } + + else + throw TypeError(format("unsupported value: %1%") % v); +} + + +static void findJobs(EvalState & state, JSONObject & top, + Bindings & autoArgs, Value & v, const string & attrPath) +{ + try { + findJobsWrapped(state, top, autoArgs, v, attrPath); + } catch (EvalError & e) { + auto res = top.object(attrPath); + res.attr("error", filterANSIEscapes(e.msg(), true)); + } +} + + +int main(int argc, char * * argv) +{ + /* Prevent undeclared dependencies in the evaluation via + $NIX_PATH. */ + unsetenv("NIX_PATH"); + + return handleExceptions(argv[0], [&]() { + + auto config = std::make_unique<::Config>(); + + auto initialHeapSize = config->getStrOption("evaluator_initial_heap_size", ""); + if (initialHeapSize != "") + setenv("GC_INITIAL_HEAP_SIZE", initialHeapSize.c_str(), 1); + + initNix(); + initGC(); + + struct MyArgs : MixEvalArgs, MixCommonArgs + { + Path releaseExpr; + bool flake = false; + + MyArgs() : MixCommonArgs("hydra-eval-jobs") + { + mkFlag() + .longName("help") + .description("show usage information") + .handler([&]() { + printHelp(programName, std::cout); + throw Exit(); + }); + + mkFlag() + .longName("gc-roots-dir") + .description("garbage collector roots directory") + .labels({"path"}) + .dest(&gcRootsDir); + + mkFlag() + .longName("dry-run") + .description("don't create store derivations") + .set(&settings.readOnlyMode, true); + + mkFlag() + .longName("flake") + .description("build a flake") + .set(&flake, true); + + expectArg("expr", &releaseExpr); + } + }; + + MyArgs myArgs; + myArgs.parseCmdline(argvToStrings(argc, argv)); + + JSONObject json(std::cout, true); + std::cout.flush(); + + /* FIXME: The build hook in conjunction with import-from-derivation is causing "unexpected EOF" during eval */ + settings.builders = ""; + + /* Prevent access to paths outside of the Nix search path and + to the environment. */ + evalSettings.restrictEval = true; + + /* When building a flake, use pure evaluation (no access to + 'getEnv', 'currentSystem' etc. */ + evalSettings.pureEval = myArgs.flake; + + if (myArgs.releaseExpr == "") throw UsageError("no expression specified"); + + if (gcRootsDir == "") printMsg(lvlError, "warning: `--gc-roots-dir' not specified"); + + EvalState state(myArgs.searchPath, openStore()); + + Bindings & autoArgs = *myArgs.getAutoArgs(state); + + Value v; + + if (myArgs.flake) { + using namespace flake; + + auto flakeRef = parseFlakeRef(myArgs.releaseExpr); + + auto vFlake = state.allocValue(); + + auto lockedFlake = lockFlake(state, flakeRef, + LockFlags { + .updateLockFile = false, + .useRegistries = false, + .allowMutable = false, + }); + + callFlake(state, lockedFlake, *vFlake); + + auto vOutputs = (*vFlake->attrs->get(state.symbols.create("outputs")))->value; + state.forceValue(*vOutputs); + + auto aHydraJobs = vOutputs->attrs->get(state.symbols.create("hydraJobs")); + if (!aHydraJobs) + aHydraJobs = vOutputs->attrs->get(state.symbols.create("checks")); + if (!aHydraJobs) + throw Error("flake '%s' does not provide any Hydra jobs or checks", flakeRef); + + v = *(*aHydraJobs)->value; + + } else { + state.evalFile(lookupFileArg(state, myArgs.releaseExpr), v); + } + + findJobs(state, json, autoArgs, v, ""); + }); +} diff --git a/src/script/hydra-eval-jobset b/src/script/hydra-eval-jobset index d037dedd8..7ed1ffa96 100755 --- a/src/script/hydra-eval-jobset +++ b/src/script/hydra-eval-jobset @@ -330,27 +330,22 @@ sub inputsToArgs { sub evalJobs { my ($inputInfo, $nixExprInputName, $nixExprPath, $flakeRef) = @_; - my @cmd = ( - "nix", "eval-hydra-jobs", - "--json", - "--gc-roots-dir", getGCRootsDir, - "--max-jobs", 1, - "--workers", $config->{evaluator_workers} // 1, - "--max-memory-size", $config->{evaluator_max_memory_size} // 4096, - "--builders", "", - ); + my @cmd; if (defined $flakeRef) { - push @cmd, - ($flakeRef, - "--no-update-lock-file", - "--no-registries"); + @cmd = ("hydra-eval-jobs", + "--flake", $flakeRef, + "--gc-roots-dir", getGCRootsDir, + "--max-jobs", 1); } else { my $nixExprInput = $inputInfo->{$nixExprInputName}->[0] or die "cannot find the input containing the job expression\n"; - push @cmd, - ("--file", "<" . $nixExprInputName . "/" . $nixExprPath . ">", - inputsToArgs($inputInfo)); + + @cmd = ("hydra-eval-jobs", + "<" . $nixExprInputName . "/" . $nixExprPath . ">", + "--gc-roots-dir", getGCRootsDir, + "--max-jobs", 1, + inputsToArgs($inputInfo)); } if (defined $ENV{'HYDRA_DEBUG'}) { diff --git a/tests/Makefile.am b/tests/Makefile.am index 3fe032c7d..e56e528ef 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -14,7 +14,7 @@ TESTS_ENVIRONMENT = \ NIX_BUILD_HOOK= \ PGHOST=/tmp \ PERL5LIB="$(srcdir):$(abs_top_srcdir)/src/lib:$$PERL5LIB" \ - PATH=$(abs_top_srcdir)/src/hydra-evaluator:$(abs_top_srcdir)/src/script:$(abs_top_srcdir)/src/hydra-queue-runner:$$PATH \ + PATH=$(abs_top_srcdir)/src/hydra-evaluator:$(abs_top_srcdir)/src/script:$(abs_top_srcdir)/src/hydra-eval-jobs:$(abs_top_srcdir)/src/hydra-queue-runner:$$PATH \ perl -w EXTRA_DIST = \ From c53cfb523480e5d8f46a1b67b69c59709f3c7796 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 19 Feb 2020 20:37:14 +0100 Subject: [PATCH 60/73] flake.lock: Update Flake input changes: * Updated 'nix': 'github:NixOS/nix/6529490cc10018d5191e50c482ac1180b96b1a3c' -> 'github:NixOS/nix/4ad5826a182f8eeb4cbc61e27d2128b08050b0d4' --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index d13c8b3a5..16cc32b2d 100644 --- a/flake.lock +++ b/flake.lock @@ -2,8 +2,8 @@ "inputs": { "nix": { "info": { - "lastModified": 1581951239, - "narHash": "sha256-DImAA/qFbgGNCtARoSJvOmeRhRIkuEN25N7KP6ZT6mk=" + "lastModified": 1582125169, + "narHash": "sha256-rgoyhvUVM78gLkvviTA/9f6esC7ZJtnDLt1hzAaq2Ds=" }, "inputs": { "nixpkgs": { @@ -28,7 +28,7 @@ "locked": { "owner": "NixOS", "repo": "nix", - "rev": "6529490cc10018d5191e50c482ac1180b96b1a3c", + "rev": "4ad5826a182f8eeb4cbc61e27d2128b08050b0d4", "type": "github" }, "original": { From be8eb9d00d6a3fcac520bc7dfd5740c0b3ade746 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 19 Feb 2020 21:10:22 +0100 Subject: [PATCH 61/73] hydra-eval-jobs: Parallelize --- src/hydra-eval-jobs/hydra-eval-jobs.cc | 524 +++++++++++++++++-------- 1 file changed, 357 insertions(+), 167 deletions(-) diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index 695a020af..cbb6872ce 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -1,37 +1,71 @@ #include #include -#define GC_LINUX_THREADS 1 -#include - #include "shared.hh" #include "store-api.hh" #include "eval.hh" #include "eval-inline.hh" #include "util.hh" -#include "json.hh" #include "get-drvs.hh" #include "globals.hh" #include "common-eval-args.hh" #include "flake/flakeref.hh" #include "flake/flake.hh" +#include "attr-path.hh" +#include "derivations.hh" #include "hydra-config.hh" #include #include +#include -using namespace nix; +#include +using namespace nix; static Path gcRootsDir; +static size_t maxMemorySize; +struct MyArgs : MixEvalArgs, MixCommonArgs +{ + Path releaseExpr; + bool flake = false; + bool dryRun = false; + + MyArgs() : MixCommonArgs("hydra-eval-jobs") + { + mkFlag() + .longName("help") + .description("show usage information") + .handler([&]() { + printHelp(programName, std::cout); + throw Exit(); + }); + + mkFlag() + .longName("gc-roots-dir") + .description("garbage collector roots directory") + .labels({"path"}) + .dest(&gcRootsDir); + + mkFlag() + .longName("dry-run") + .description("don't create store derivations") + .set(&dryRun, true); + + mkFlag() + .longName("flake") + .description("build a flake") + .set(&flake, true); + + expectArg("expr", &releaseExpr); + } +}; -static void findJobs(EvalState & state, JSONObject & top, - Bindings & autoArgs, Value & v, const string & attrPath); - +static MyArgs myArgs; -static string queryMetaStrings(EvalState & state, DrvInfo & drv, const string & name, const string & subAttribute) +static std::string queryMetaStrings(EvalState & state, DrvInfo & drv, const string & name, const string & subAttribute) { Strings res; std::function rec; @@ -56,115 +90,162 @@ static string queryMetaStrings(EvalState & state, DrvInfo & drv, const string & return concatStringsSep(", ", res); } - -static void findJobsWrapped(EvalState & state, JSONObject & top, - Bindings & autoArgs, Value & vIn, const string & attrPath) +static void worker( + EvalState & state, + Bindings & autoArgs, + AutoCloseFD & to, + AutoCloseFD & from) { - debug(format("at path `%1%'") % attrPath); - - checkInterrupt(); - - Value v; - state.autoCallFunction(autoArgs, vIn, v); - - if (v.type == tAttrs) { - - auto drv = getDerivation(state, v, false); - - if (drv) { - Path drvPath; - - DrvInfo::Outputs outputs = drv->queryOutputs(); - - if (drv->querySystem() == "unknown") - throw EvalError("derivation must have a ‘system’ attribute"); - - { - auto res = top.object(attrPath); - res.attr("nixName", drv->queryName()); - res.attr("system", drv->querySystem()); - res.attr("drvPath", drvPath = drv->queryDrvPath()); - res.attr("description", drv->queryMetaString("description")); - res.attr("license", queryMetaStrings(state, *drv, "license", "shortName")); - res.attr("homepage", drv->queryMetaString("homepage")); - res.attr("maintainers", queryMetaStrings(state, *drv, "maintainers", "email")); - res.attr("schedulingPriority", drv->queryMetaInt("schedulingPriority", 100)); - res.attr("timeout", drv->queryMetaInt("timeout", 36000)); - res.attr("maxSilent", drv->queryMetaInt("maxSilent", 7200)); - res.attr("isChannel", drv->queryMetaBool("isHydraChannel", false)); - - /* If this is an aggregate, then get its constituents. */ - Bindings::iterator a = v.attrs->find(state.symbols.create("_hydraAggregate")); - if (a != v.attrs->end() && state.forceBool(*a->value, *a->pos)) { - Bindings::iterator a = v.attrs->find(state.symbols.create("constituents")); - if (a == v.attrs->end()) - throw EvalError("derivation must have a ‘constituents’ attribute"); - PathSet context; - state.coerceToString(*a->pos, *a->value, context, true, false); - PathSet drvs; - for (auto & i : context) - if (i.at(0) == '!') { - size_t index = i.find("!", 1); - drvs.insert(string(i, index + 1)); + Value vTop; + + if (myArgs.flake) { + using namespace flake; + + auto flakeRef = parseFlakeRef(myArgs.releaseExpr); + + auto vFlake = state.allocValue(); + + auto lockedFlake = lockFlake(state, flakeRef, + LockFlags { + .updateLockFile = false, + .useRegistries = false, + .allowMutable = false, + }); + + callFlake(state, lockedFlake, *vFlake); + + auto vOutputs = vFlake->attrs->get(state.symbols.create("outputs"))->value; + state.forceValue(*vOutputs); + + auto aHydraJobs = vOutputs->attrs->get(state.symbols.create("hydraJobs")); + if (!aHydraJobs) + aHydraJobs = vOutputs->attrs->get(state.symbols.create("checks")); + if (!aHydraJobs) + throw Error("flake '%s' does not provide any Hydra jobs or checks", flakeRef); + + vTop = *aHydraJobs->value; + + } else { + state.evalFile(lookupFileArg(state, myArgs.releaseExpr), vTop); + } + + auto vRoot = state.allocValue(); + state.autoCallFunction(autoArgs, vTop, *vRoot); + + while (true) { + /* Wait for the master to send us a job name. */ + writeLine(to.get(), "next"); + + auto s = readLine(from.get()); + if (s == "exit") break; + if (!hasPrefix(s, "do ")) abort(); + std::string attrPath(s, 3); + + debug("worker process %d at '%s'", getpid(), attrPath); + + /* Evaluate it and send info back to the master. */ + nlohmann::json reply; + + try { + auto v = findAlongAttrPath(state, attrPath, autoArgs, *vRoot).first; + + state.forceValue(*v); + + if (auto drv = getDerivation(state, *v, false)) { + + DrvInfo::Outputs outputs = drv->queryOutputs(); + + if (drv->querySystem() == "unknown") + throw EvalError("derivation must have a 'system' attribute"); + + auto drvPath = drv->queryDrvPath(); + + nlohmann::json job; + + job["nixName"] = drv->queryName(); + job["system"] =drv->querySystem(); + job["drvPath"] = drvPath; + job["description"] = drv->queryMetaString("description"); + job["license"] = queryMetaStrings(state, *drv, "license", "shortName"); + job["homepage"] = drv->queryMetaString("homepage"); + job["maintainers"] = queryMetaStrings(state, *drv, "maintainers", "email"); + job["schedulingPriority"] = drv->queryMetaInt("schedulingPriority", 100); + job["timeout"] = drv->queryMetaInt("timeout", 36000); + job["maxSilent"] = drv->queryMetaInt("maxSilent", 7200); + job["isChannel"] = drv->queryMetaBool("isHydraChannel", false); + + /* If this is an aggregate, then get its constituents. */ + auto a = v->attrs->get(state.symbols.create("_hydraAggregate")); + if (a && state.forceBool(*a->value, *a->pos)) { + auto a = v->attrs->get(state.symbols.create("constituents")); + if (!a) + throw EvalError("derivation must have a ‘constituents’ attribute"); + + + PathSet context; + state.coerceToString(*a->pos, *a->value, context, true, false); + for (auto & i : context) + if (i.at(0) == '!') { + size_t index = i.find("!", 1); + job["constituents"].push_back(string(i, index + 1)); + } + + state.forceList(*a->value, *a->pos); + for (unsigned int n = 0; n < a->value->listSize(); ++n) { + auto v = a->value->listElems()[n]; + state.forceValue(*v); + if (v->type == tString) + job["namedConstituents"].push_back(state.forceStringNoCtx(*v)); } - res.attr("constituents", concatStringsSep(" ", drvs)); - } + } - /* Register the derivation as a GC root. !!! This - registers roots for jobs that we may have already - done. */ - auto localStore = state.store.dynamic_pointer_cast(); - if (gcRootsDir != "" && localStore) { - Path root = gcRootsDir + "/" + std::string(baseNameOf(drvPath)); - if (!pathExists(root)) - localStore->addPermRoot(localStore->parseStorePath(drvPath), root, false); - } + /* Register the derivation as a GC root. !!! This + registers roots for jobs that we may have already + done. */ + auto localStore = state.store.dynamic_pointer_cast(); + if (gcRootsDir != "" && localStore) { + Path root = gcRootsDir + "/" + std::string(baseNameOf(drvPath)); + if (!pathExists(root)) + localStore->addPermRoot(localStore->parseStorePath(drvPath), root, false); + } - auto res2 = res.object("outputs"); - for (auto & j : outputs) - res2.attr(j.first, j.second); + nlohmann::json out; + for (auto & j : outputs) + out[j.first] = j.second; + job["outputs"] = std::move(out); + reply["job"] = std::move(job); } - } - else { - if (!state.isDerivation(v)) { - for (auto & i : v.attrs->lexicographicOrder()) { + else if (v->type == tAttrs) { + auto attrs = nlohmann::json::array(); + StringSet ss; + for (auto & i : v->attrs->lexicographicOrder()) { std::string name(i->name); - - /* Skip jobs with dots in the name. */ - if (name.find('.') != std::string::npos) { + if (name.find('.') != std::string::npos || name.find(' ') != std::string::npos) { printError("skipping job with illegal name '%s'", name); continue; } - - findJobs(state, top, autoArgs, *i->value, - (attrPath.empty() ? "" : attrPath + ".") + name); + attrs.push_back(name); } + reply["attrs"] = std::move(attrs); } - } - } - else if (v.type == tNull) { - // allow null values, meaning 'do nothing' - } - - else - throw TypeError(format("unsupported value: %1%") % v); -} + } catch (EvalError & e) { + reply["error"] = filterANSIEscapes(e.msg(), true); + } + writeLine(to.get(), reply.dump()); -static void findJobs(EvalState & state, JSONObject & top, - Bindings & autoArgs, Value & v, const string & attrPath) -{ - try { - findJobsWrapped(state, top, autoArgs, v, attrPath); - } catch (EvalError & e) { - auto res = top.object(attrPath); - res.attr("error", filterANSIEscapes(e.msg(), true)); + /* If our RSS exceeds the maximum, exit. The master will + start a new process. */ + struct rusage r; + getrusage(RUSAGE_SELF, &r); + if ((size_t) r.ru_maxrss > maxMemorySize * 1024) break; } -} + writeLine(to.get(), "restart"); +} int main(int argc, char * * argv) { @@ -176,54 +257,14 @@ int main(int argc, char * * argv) auto config = std::make_unique<::Config>(); - auto initialHeapSize = config->getStrOption("evaluator_initial_heap_size", ""); - if (initialHeapSize != "") - setenv("GC_INITIAL_HEAP_SIZE", initialHeapSize.c_str(), 1); + auto nrWorkers = config->getIntOption("evaluator_workers", 1); + maxMemorySize = config->getIntOption("evaluator_max_memory_size", 4096); initNix(); initGC(); - struct MyArgs : MixEvalArgs, MixCommonArgs - { - Path releaseExpr; - bool flake = false; - - MyArgs() : MixCommonArgs("hydra-eval-jobs") - { - mkFlag() - .longName("help") - .description("show usage information") - .handler([&]() { - printHelp(programName, std::cout); - throw Exit(); - }); - - mkFlag() - .longName("gc-roots-dir") - .description("garbage collector roots directory") - .labels({"path"}) - .dest(&gcRootsDir); - - mkFlag() - .longName("dry-run") - .description("don't create store derivations") - .set(&settings.readOnlyMode, true); - - mkFlag() - .longName("flake") - .description("build a flake") - .set(&flake, true); - - expectArg("expr", &releaseExpr); - } - }; - - MyArgs myArgs; myArgs.parseCmdline(argvToStrings(argc, argv)); - JSONObject json(std::cout, true); - std::cout.flush(); - /* FIXME: The build hook in conjunction with import-from-derivation is causing "unexpected EOF" during eval */ settings.builders = ""; @@ -235,47 +276,196 @@ int main(int argc, char * * argv) 'getEnv', 'currentSystem' etc. */ evalSettings.pureEval = myArgs.flake; + if (myArgs.dryRun) settings.readOnlyMode = true; + if (myArgs.releaseExpr == "") throw UsageError("no expression specified"); if (gcRootsDir == "") printMsg(lvlError, "warning: `--gc-roots-dir' not specified"); - EvalState state(myArgs.searchPath, openStore()); + struct State + { + std::set todo{""}; + std::set active; + nlohmann::json jobs; + std::exception_ptr exc; + }; - Bindings & autoArgs = *myArgs.getAutoArgs(state); + std::condition_variable wakeup; - Value v; + Sync state_; - if (myArgs.flake) { - using namespace flake; + /* Start a handler thread per worker process. */ + auto handler = [&]() + { + try { + pid_t pid = -1; + AutoCloseFD from, to; + + while (true) { + + /* Start a new worker process if necessary. */ + if (pid == -1) { + Pipe toPipe, fromPipe; + toPipe.create(); + fromPipe.create(); + pid = startProcess( + [&, + to{std::make_shared(std::move(fromPipe.writeSide))}, + from{std::make_shared(std::move(toPipe.readSide))} + ]() + { + try { + EvalState state(myArgs.searchPath, openStore()); + Bindings & autoArgs = *myArgs.getAutoArgs(state); + worker(state, autoArgs, *to, *from); + } catch (std::exception & e) { + nlohmann::json err; + err["error"] = e.what(); + writeLine(to->get(), err.dump()); + } + }, + ProcessOptions { .allowVfork = false }); + from = std::move(fromPipe.readSide); + to = std::move(toPipe.writeSide); + debug("created worker process %d", pid); + } - auto flakeRef = parseFlakeRef(myArgs.releaseExpr); + /* Check whether the existing worker process is still there. */ + auto s = readLine(from.get()); + if (s == "restart") { + pid = -1; + continue; + } else if (s != "next") { + auto json = nlohmann::json::parse(s); + throw Error("worker error: %s", (std::string) json["error"]); + } - auto vFlake = state.allocValue(); + /* Wait for a job name to become available. */ + std::string attrPath; + + while (true) { + checkInterrupt(); + auto state(state_.lock()); + if ((state->todo.empty() && state->active.empty()) || state->exc) { + writeLine(to.get(), "exit"); + return; + } + if (!state->todo.empty()) { + attrPath = *state->todo.begin(); + state->todo.erase(state->todo.begin()); + state->active.insert(attrPath); + break; + } else + state.wait(wakeup); + } - auto lockedFlake = lockFlake(state, flakeRef, - LockFlags { - .updateLockFile = false, - .useRegistries = false, - .allowMutable = false, - }); + /* Tell the worker to evaluate it. */ + writeLine(to.get(), "do " + attrPath); - callFlake(state, lockedFlake, *vFlake); + /* Wait for the response. */ + auto response = nlohmann::json::parse(readLine(from.get())); - auto vOutputs = (*vFlake->attrs->get(state.symbols.create("outputs")))->value; - state.forceValue(*vOutputs); + /* Handle the response. */ + StringSet newAttrs; - auto aHydraJobs = vOutputs->attrs->get(state.symbols.create("hydraJobs")); - if (!aHydraJobs) - aHydraJobs = vOutputs->attrs->get(state.symbols.create("checks")); - if (!aHydraJobs) - throw Error("flake '%s' does not provide any Hydra jobs or checks", flakeRef); + if (response.find("job") != response.end()) { + auto state(state_.lock()); + state->jobs[attrPath] = response["job"]; + } - v = *(*aHydraJobs)->value; + if (response.find("attrs") != response.end()) { + for (auto & i : response["attrs"]) { + auto s = (attrPath.empty() ? "" : attrPath + ".") + (std::string) i; + newAttrs.insert(s); + } + } + + if (response.find("error") != response.end()) { + auto state(state_.lock()); + state->jobs[attrPath]["error"] = response["error"]; + } + + /* Add newly discovered job names to the queue. */ + { + auto state(state_.lock()); + state->active.erase(attrPath); + for (auto & s : newAttrs) + state->todo.insert(s); + wakeup.notify_all(); + } + } + } catch (...) { + auto state(state_.lock()); + state->exc = std::current_exception(); + wakeup.notify_all(); + } + }; + + std::vector threads; + for (size_t i = 0; i < nrWorkers; i++) + threads.emplace_back(std::thread(handler)); + + for (auto & thread : threads) + thread.join(); + + auto state(state_.lock()); + + if (state->exc) + std::rethrow_exception(state->exc); + + /* For aggregate jobs that have named consistuents + (i.e. constituents that are a job name rather than a + derivation), look up the referenced job and add it to the + dependencies of the aggregate derivation. */ + auto store = openStore(); + + for (auto i = state->jobs.begin(); i != state->jobs.end(); ++i) { + auto jobName = i.key(); + auto & job = i.value(); + + auto named = job.find("namedConstituents"); + if (named == job.end()) continue; + + if (myArgs.dryRun) { + for (std::string jobName2 : *named) { + auto job2 = state->jobs.find(jobName2); + if (job2 == state->jobs.end()) + throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2); + std::string drvPath2 = (*job2)["drvPath"]; + job["constituents"].push_back(drvPath2); + } + } else { + std::string drvPath = job["drvPath"]; + auto drv = readDerivation(*store, drvPath); + + for (std::string jobName2 : *named) { + auto job2 = state->jobs.find(jobName2); + if (job2 == state->jobs.end()) + throw Error("aggregate job '%s' references non-existent job '%s'", jobName, jobName2); + std::string drvPath2 = (*job2)["drvPath"]; + auto drv2 = readDerivation(*store, drvPath2); + job["constituents"].push_back(drvPath2); + drv.inputDrvs[store->parseStorePath(drvPath2)] = {drv2.outputs.begin()->first}; + } + + std::string drvName(store->parseStorePath(drvPath).name()); + assert(hasSuffix(drvName, drvExtension)); + drvName.resize(drvName.size() - drvExtension.size()); + auto h = hashDerivationModulo(*store, drv, true); + auto outPath = store->makeOutputPath("out", h, drvName); + drv.env["out"] = store->printStorePath(outPath); + drv.outputs.insert_or_assign("out", DerivationOutput(outPath.clone(), "", "")); + auto newDrvPath = store->printStorePath(writeDerivation(store, drv, drvName)); + + debug("rewrote aggregate derivation %s -> %s", drvPath, newDrvPath); + + job["drvPath"] = newDrvPath; + job["outputs"]["out"] = store->printStorePath(outPath); + } - } else { - state.evalFile(lookupFileArg(state, myArgs.releaseExpr), v); + job.erase("namedConstituents"); } - findJobs(state, json, autoArgs, v, ""); + std::cout << state->jobs.dump(2) << "\n"; }); } From 101a9b3797cb94d8177c997b3db5af97b0f48e6b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Mar 2020 15:39:21 +0100 Subject: [PATCH 62/73] Run 'make check' This got lost when I converted from releaseTools.nixBuild to stdenv.mkDerivation. Doh! --- flake.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flake.nix b/flake.nix index 1af431765..8acea7c33 100644 --- a/flake.nix +++ b/flake.nix @@ -129,6 +129,8 @@ enableParallelBuilding = true; + doCheck = true; + preCheck = '' patchShebangs . export LOGNAME=''${LOGNAME:-foo} From 5b731004da89a9b94f4ea7945ea6538837a64e41 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 26 Mar 2020 14:59:52 +0100 Subject: [PATCH 63/73] Remove tabs --- src/root/common.tt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/root/common.tt b/src/root/common.tt index 83b1c736f..bc777ede2 100644 --- a/src/root/common.tt +++ b/src/root/common.tt @@ -584,10 +584,10 @@ BLOCK renderJobsetOverview %] [% IF showProject; INCLUDE renderFullJobsetName project=j.get_column('project') jobset=j.name inRow=1; ELSE; INCLUDE renderJobsetName project=j.get_column('project') jobset=j.name inRow=1; END %] [% HTML.escape(j.description) %] [% IF j.lastcheckedtime; - INCLUDE renderDateTime timestamp = j.lastcheckedtime; - IF j.errormsg || j.fetcherrormsg; %] Error[% END; - ELSE; "-"; - END %] + INCLUDE renderDateTime timestamp = j.lastcheckedtime; + IF j.errormsg || j.fetcherrormsg; %] Error[% END; + ELSE; "-"; + END %] [% IF j.get_column('nrtotal') > 0 %] [% successrate = ( j.get_column('nrsucceeded') / j.get_column('nrtotal') )*100 %] [% IF j.get_column('nrscheduled') > 0 %] From f5cdbfe21d930db43d3812c7d8e87746d6378ef9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 26 Mar 2020 15:00:04 +0100 Subject: [PATCH 64/73] Abort unsupported build steps If we don't see machine that supports a build step for 'max_unsupported_time' seconds, the step is aborted. The default is 0, which is appropriate for Hydra installations that don't provision missing machines dynamically. --- src/hydra-queue-runner/builder.cc | 178 ++++++++++--------- src/hydra-queue-runner/dispatcher.cc | 85 +++++++++ src/hydra-queue-runner/hydra-queue-runner.cc | 1 + src/hydra-queue-runner/state.hh | 20 ++- 4 files changed, 200 insertions(+), 84 deletions(-) diff --git a/src/hydra-queue-runner/builder.cc b/src/hydra-queue-runner/builder.cc index a0fc01c33..0a005321c 100644 --- a/src/hydra-queue-runner/builder.cc +++ b/src/hydra-queue-runner/builder.cc @@ -376,109 +376,121 @@ State::StepResult State::doBuildStep(nix::ref destStore, } } - } else { + } else + failStep(*conn, step, buildId, result, machine, stepFinished, quit); - /* Register failure in the database for all Build objects that - directly or indirectly depend on this step. */ + // FIXME: keep stats about aborted steps? + nrStepsDone++; + totalStepTime += stepStopTime - stepStartTime; + totalStepBuildTime += result.stopTime - result.startTime; + machine->state->nrStepsDone++; + machine->state->totalStepTime += stepStopTime - stepStartTime; + machine->state->totalStepBuildTime += result.stopTime - result.startTime; - std::vector dependentIDs; + if (quit) exit(0); // testing hack; FIXME: this won't run plugins - while (true) { - /* Get the builds and steps that depend on this step. */ - std::set indirect; - { - auto steps_(steps.lock()); - std::set steps; - getDependents(step, indirect, steps); - - /* If there are no builds left, delete all referring - steps from ‘steps’. As for the success case, we can - be certain no new referrers can be added. */ - if (indirect.empty()) { - for (auto & s : steps) { - printMsg(lvlDebug, "finishing build step ‘%s’", - localStore->printStorePath(s->drvPath)); - steps_->erase(s->drvPath); - } - } - } + return sDone; +} - if (indirect.empty() && stepFinished) break; - /* Update the database. */ - { - auto mc = startDbUpdate(); +void State::failStep( + Connection & conn, + Step::ptr step, + BuildID buildId, + const RemoteResult & result, + Machine::ptr machine, + bool & stepFinished, + bool & quit) +{ + /* Register failure in the database for all Build objects that + directly or indirectly depend on this step. */ - pqxx::work txn(*conn); + std::vector dependentIDs; - /* Create failed build steps for every build that - depends on this, except when this step is cached - and is the top-level of that build (since then it's - redundant with the build's isCachedBuild field). */ - for (auto & build2 : indirect) { - if ((result.stepStatus == bsCachedFailure && build2->drvPath == step->drvPath) || - (result.stepStatus != bsCachedFailure && buildId == build2->id) || - build2->finishedInDB) - continue; - createBuildStep(txn, 0, build2->id, step, machine->sshName, - result.stepStatus, result.errorMsg, buildId == build2->id ? 0 : buildId); + while (true) { + /* Get the builds and steps that depend on this step. */ + std::set indirect; + { + auto steps_(steps.lock()); + std::set steps; + getDependents(step, indirect, steps); + + /* If there are no builds left, delete all referring + steps from ‘steps’. As for the success case, we can + be certain no new referrers can be added. */ + if (indirect.empty()) { + for (auto & s : steps) { + printMsg(lvlDebug, "finishing build step ‘%s’", + localStore->printStorePath(s->drvPath)); + steps_->erase(s->drvPath); } + } + } - /* Mark all builds that depend on this derivation as failed. */ - for (auto & build2 : indirect) { - if (build2->finishedInDB) continue; - printMsg(lvlError, format("marking build %1% as failed") % build2->id); - txn.parameterized - ("update Builds set finished = 1, buildStatus = $2, startTime = $3, stopTime = $4, isCachedBuild = $5, notificationPendingSince = $4 where id = $1 and finished = 0") - (build2->id) - ((int) (build2->drvPath != step->drvPath && result.buildStatus() == bsFailed ? bsDepFailed : result.buildStatus())) - (result.startTime) - (result.stopTime) - (result.stepStatus == bsCachedFailure ? 1 : 0).exec(); - nrBuildsDone++; - } + if (indirect.empty() && stepFinished) break; - /* Remember failed paths in the database so that they - won't be built again. */ - if (result.stepStatus != bsCachedFailure && result.canCache) - for (auto & path : step->drv->outputPaths()) - txn.parameterized("insert into FailedPaths values ($1)")(localStore->printStorePath(path)).exec(); + /* Update the database. */ + { + auto mc = startDbUpdate(); - txn.commit(); + pqxx::work txn(conn); + + /* Create failed build steps for every build that + depends on this, except when this step is cached + and is the top-level of that build (since then it's + redundant with the build's isCachedBuild field). */ + for (auto & build : indirect) { + if ((result.stepStatus == bsCachedFailure && build->drvPath == step->drvPath) || + ((result.stepStatus != bsCachedFailure && result.stepStatus != bsUnsupported) && buildId == build->id) || + build->finishedInDB) + continue; + createBuildStep(txn, + 0, build->id, step, machine ? machine->sshName : "", + result.stepStatus, result.errorMsg, buildId == build->id ? 0 : buildId); } - stepFinished = true; - - /* Remove the indirect dependencies from ‘builds’. This - will cause them to be destroyed. */ - for (auto & b : indirect) { - auto builds_(builds.lock()); - b->finishedInDB = true; - builds_->erase(b->id); - dependentIDs.push_back(b->id); - if (buildOne == b->id) quit = true; + /* Mark all builds that depend on this derivation as failed. */ + for (auto & build : indirect) { + if (build->finishedInDB) continue; + printMsg(lvlError, format("marking build %1% as failed") % build->id); + txn.parameterized + ("update Builds set finished = 1, buildStatus = $2, startTime = $3, stopTime = $4, isCachedBuild = $5, notificationPendingSince = $4 where id = $1 and finished = 0") + (build->id) + ((int) (build->drvPath != step->drvPath && result.buildStatus() == bsFailed ? bsDepFailed : result.buildStatus())) + (result.startTime) + (result.stopTime) + (result.stepStatus == bsCachedFailure ? 1 : 0).exec(); + nrBuildsDone++; } - } - /* Send notification about this build and its dependents. */ - { - pqxx::work txn(*conn); - notifyBuildFinished(txn, buildId, dependentIDs); + /* Remember failed paths in the database so that they + won't be built again. */ + if (result.stepStatus != bsCachedFailure && result.canCache) + for (auto & path : step->drv->outputPaths()) + txn.parameterized("insert into FailedPaths values ($1)")(localStore->printStorePath(path)).exec(); + txn.commit(); } - } - // FIXME: keep stats about aborted steps? - nrStepsDone++; - totalStepTime += stepStopTime - stepStartTime; - totalStepBuildTime += result.stopTime - result.startTime; - machine->state->nrStepsDone++; - machine->state->totalStepTime += stepStopTime - stepStartTime; - machine->state->totalStepBuildTime += result.stopTime - result.startTime; + stepFinished = true; - if (quit) exit(0); // testing hack; FIXME: this won't run plugins + /* Remove the indirect dependencies from ‘builds’. This + will cause them to be destroyed. */ + for (auto & b : indirect) { + auto builds_(builds.lock()); + b->finishedInDB = true; + builds_->erase(b->id); + dependentIDs.push_back(b->id); + if (buildOne == b->id) quit = true; + } + } - return sDone; + /* Send notification about this build and its dependents. */ + { + pqxx::work txn(conn); + notifyBuildFinished(txn, buildId, dependentIDs); + txn.commit(); + } } diff --git a/src/hydra-queue-runner/dispatcher.cc b/src/hydra-queue-runner/dispatcher.cc index 86596ff53..700fdd8e4 100644 --- a/src/hydra-queue-runner/dispatcher.cc +++ b/src/hydra-queue-runner/dispatcher.cc @@ -300,6 +300,8 @@ system_time State::doDispatch() } while (keepGoing); + abortUnsupported(); + return sleepUntil; } @@ -314,6 +316,89 @@ void State::wakeDispatcher() } +void State::abortUnsupported() +{ + /* Make a copy of 'runnable' and 'machines' so we don't block them + very long. */ + auto runnable2 = *runnable.lock(); + auto machines2 = *machines.lock(); + + system_time now = std::chrono::system_clock::now(); + auto now2 = time(0); + + std::unordered_set aborted; + + for (auto & wstep : runnable2) { + auto step(wstep.lock()); + if (!step) continue; + + bool supported = false; + for (auto & machine : machines2) { + if (machine.second->supportsStep(step)) { + step->state.lock()->lastSupported = now; + supported = true; + break; + } + } + + if (!supported + && std::chrono::duration_cast(now - step->state.lock()->lastSupported).count() >= maxUnsupportedTime) + { + printError("aborting unsupported build step '%s' (type '%s')", + localStore->printStorePath(step->drvPath), + step->systemType); + + aborted.insert(step); + + auto conn(dbPool.get()); + + std::set dependents; + std::set steps; + getDependents(step, dependents, steps); + + /* Maybe the step got cancelled. */ + if (dependents.empty()) continue; + + /* Find the build that has this step as the top-level (if + any). */ + Build::ptr build; + for (auto build2 : dependents) { + if (build2->drvPath == step->drvPath) + build = build2; + } + if (!build) build = *dependents.begin(); + + bool stepFinished = false; + bool quit = false; + + failStep( + *conn, step, build->id, + RemoteResult { + .stepStatus = bsUnsupported, + .errorMsg = fmt("unsupported system type '%s'", + step->systemType), + .startTime = now2, + .stopTime = now2, + }, + nullptr, stepFinished, quit); + + if (quit) exit(1); + } + } + + /* Clean up 'runnable'. */ + { + auto runnable_(runnable.lock()); + for (auto i = runnable_->begin(); i != runnable_->end(); ) { + if (aborted.count(i->lock())) + i = runnable_->erase(i); + else + ++i; + } + } +} + + void Jobset::addStep(time_t startTime, time_t duration) { auto steps_(steps.lock()); diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 3d5dad4d3..f7ecf268a 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -46,6 +46,7 @@ std::string getEnvOrDie(const std::string & key) State::State() : config(std::make_unique<::Config>()) + , maxUnsupportedTime(config->getIntOption("max_unsupported_time", 0)) , dbPool(config->getIntOption("max_db_connections", 128)) , memoryTokens(config->getIntOption("nar_buffer_size", getMemSize() / 2)) , maxOutputSize(config->getIntOption("max_output_size", 2ULL << 30)) diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index 180907e9f..4d87a3a73 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -68,7 +68,7 @@ struct RemoteResult std::unique_ptr tokens; std::shared_ptr accessor; - BuildStatus buildStatus() + BuildStatus buildStatus() const { return stepStatus == bsCachedFailure ? bsFailed : stepStatus; } @@ -198,6 +198,10 @@ struct Step /* The time at which this step became runnable. */ system_time runnableSince; + + /* The time that we last saw a machine that supports this + step. */ + system_time lastSupported = std::chrono::system_clock::now(); }; std::atomic_bool finished{false}; // debugging @@ -303,6 +307,9 @@ private: const float retryBackoff = 3.0; const unsigned int maxParallelCopyClosure = 4; + /* Time in seconds before unsupported build steps are aborted. */ + const unsigned int maxUnsupportedTime = 0; + nix::Path hydraData, logDir; bool useSubstitutes = false; @@ -483,6 +490,15 @@ private: Build::ptr referringBuild, Step::ptr referringStep, std::set & finishedDrvs, std::set & newSteps, std::set & newRunnable); + void failStep( + Connection & conn, + Step::ptr step, + BuildID buildId, + const RemoteResult & result, + Machine::ptr machine, + bool & stepFinished, + bool & quit); + Jobset::ptr createJobset(pqxx::work & txn, const std::string & projectName, const std::string & jobsetName); @@ -497,6 +513,8 @@ private: void wakeDispatcher(); + void abortUnsupported(); + void builder(MachineReservation::ptr reservation); /* Perform the given build step. Return true if the step is to be From 45ffe578b695f9de101b30d44d46f12aa0654f10 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 26 Mar 2020 15:26:12 +0100 Subject: [PATCH 65/73] Keep track of the number of unsupported steps --- src/hydra-queue-runner/dispatcher.cc | 7 +++++++ src/hydra-queue-runner/hydra-queue-runner.cc | 1 + src/hydra-queue-runner/state.hh | 1 + src/script/hydra-send-stats | 1 + 4 files changed, 10 insertions(+) diff --git a/src/hydra-queue-runner/dispatcher.cc b/src/hydra-queue-runner/dispatcher.cc index 700fdd8e4..6dc7f7002 100644 --- a/src/hydra-queue-runner/dispatcher.cc +++ b/src/hydra-queue-runner/dispatcher.cc @@ -328,6 +328,8 @@ void State::abortUnsupported() std::unordered_set aborted; + size_t count = 0; + for (auto & wstep : runnable2) { auto step(wstep.lock()); if (!step) continue; @@ -341,6 +343,9 @@ void State::abortUnsupported() } } + if (!supported) + count++; + if (!supported && std::chrono::duration_cast(now - step->state.lock()->lastSupported).count() >= maxUnsupportedTime) { @@ -396,6 +401,8 @@ void State::abortUnsupported() ++i; } } + + nrUnsupportedSteps = count; } diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index f7ecf268a..8d729bd10 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -523,6 +523,7 @@ void State::dumpStatus(Connection & conn, bool log) root.attr("nrStepsCopyingTo", nrStepsCopyingTo); root.attr("nrStepsCopyingFrom", nrStepsCopyingFrom); root.attr("nrStepsWaiting", nrStepsWaiting); + root.attr("nrUnsupportedSteps", nrUnsupportedSteps); root.attr("bytesSent", bytesSent); root.attr("bytesReceived", bytesReceived); root.attr("nrBuildsRead", nrBuildsRead); diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index 4d87a3a73..c702e8a33 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -355,6 +355,7 @@ private: counter nrStepsCopyingTo{0}; counter nrStepsCopyingFrom{0}; counter nrStepsWaiting{0}; + counter nrUnsupportedSteps{0}; counter nrRetries{0}; counter maxNrRetries{0}; counter totalStepTime{0}; // total time for steps, including closure copying diff --git a/src/script/hydra-send-stats b/src/script/hydra-send-stats index 2b8c550b9..cf653865b 100755 --- a/src/script/hydra-send-stats +++ b/src/script/hydra-send-stats @@ -34,6 +34,7 @@ sub sendQueueRunnerStats { gauge("hydra.queue.steps.unfinished", $json->{nrUnfinishedSteps}); gauge("hydra.queue.steps.finished", $json->{nrStepsDone}); gauge("hydra.queue.steps.retries", $json->{nrRetries}); + gauge("hydra.queue.steps.unsupported", $json->{nrUnsupportedSteps}); gauge("hydra.queue.steps.max_retries", $json->{maxNrRetries}); if ($json->{nrStepsDone}) { gauge("hydra.queue.steps.avg_total_time", $json->{avgStepTime}); From 15ae932488512ba235ed2f6f841cc5eb56ba9314 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 26 Mar 2020 15:30:37 +0100 Subject: [PATCH 66/73] Don't spam the journal with hydra-queue-runner status dumps --- src/hydra-queue-runner/hydra-queue-runner.cc | 13 ++++--------- src/hydra-queue-runner/state.hh | 5 +---- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 8d729bd10..65d93a041 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -491,7 +491,7 @@ std::shared_ptr State::acquireGlobalLock() } -void State::dumpStatus(Connection & conn, bool log) +void State::dumpStatus(Connection & conn) { std::ostringstream out; @@ -672,11 +672,6 @@ void State::dumpStatus(Connection & conn, bool log) } } - if (log && time(0) >= lastStatusLogged + statusLogInterval) { - printMsg(lvlInfo, format("status: %1%") % out.str()); - lastStatusLogged = time(0); - } - { auto mc = startDbUpdate(); pqxx::work txn(conn); @@ -785,7 +780,7 @@ void State::run(BuildID buildOne) { auto conn(dbPool.get()); clearBusy(*conn, 0); - dumpStatus(*conn, false); + dumpStatus(*conn); } std::thread(&State::monitorMachinesFile, this).detach(); @@ -848,8 +843,8 @@ void State::run(BuildID buildOne) auto conn(dbPool.get()); receiver dumpStatus_(*conn, "dump_status"); while (true) { - conn->await_notification(statusLogInterval / 2 + 1, 0); - dumpStatus(*conn, true); + conn->await_notification(); + dumpStatus(*conn); } } catch (std::exception & e) { printMsg(lvlError, format("main thread: %1%") % e.what()); diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index c702e8a33..31c7fddf5 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -420,9 +420,6 @@ private: size_t maxOutputSize; size_t maxLogSize; - time_t lastStatusLogged = 0; - const int statusLogInterval = 300; - /* Steps that were busy while we encounted a PostgreSQL error. These need to be cleared at a later time to prevent them from showing up as busy until the queue runner is restarted. */ @@ -546,7 +543,7 @@ private: has it. */ std::shared_ptr acquireGlobalLock(); - void dumpStatus(Connection & conn, bool log); + void dumpStatus(Connection & conn); void addRoot(const nix::StorePath & storePath); From 7889aef7927f61c1aaa54d1afcfe564b7bc69005 Mon Sep 17 00:00:00 2001 From: Samuel Dionne-Riel Date: Mon, 30 Mar 2020 14:36:16 -0400 Subject: [PATCH 67/73] hydra-eval-jobs: Mirror eval errors in STDERR Otherwise, errors will not be shown to end-users, which makes debugging long evals pretty much impossible. (cherry picked from commit 76299b9174d7cba06427220166dc4707b43d407d) --- src/hydra-eval-jobs/hydra-eval-jobs.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/hydra-eval-jobs/hydra-eval-jobs.cc b/src/hydra-eval-jobs/hydra-eval-jobs.cc index 4a4084e01..f3367e11a 100644 --- a/src/hydra-eval-jobs/hydra-eval-jobs.cc +++ b/src/hydra-eval-jobs/hydra-eval-jobs.cc @@ -238,7 +238,12 @@ static void worker( else throw TypeError("attribute '%s' is %s, which is not supported", attrPath, showType(*v)); } catch (EvalError & e) { + // Transmits the error we got from the previous evaluation + // in the JSON output. reply["error"] = filterANSIEscapes(e.msg(), true); + // Don't forget to print it into the STDERR log, this is + // what's shown in the Hydra UI. + printError("error: %s", reply["error"]); } writeLine(to.get(), reply.dump()); @@ -328,6 +333,9 @@ int main(int argc, char * * argv) nlohmann::json err; err["error"] = e.what(); writeLine(to->get(), err.dump()); + // Don't forget to print it into the STDERR log, this is + // what's shown in the Hydra UI. + printError("error: %s", err["error"]); } }, ProcessOptions { .allowVfork = false }); From 05bf2789712b739189932d71d181f137a0891731 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 31 Mar 2020 12:55:07 +0200 Subject: [PATCH 68/73] flake.lock: Update Flake input changes: * Updated 'nix': 'github:NixOS/nix/4ad5826a182f8eeb4cbc61e27d2128b08050b0d4' -> 'github:NixOS/nix/3e7aab81ce6787e01ea4ced5af1fc6a84e523762' * Updated 'nix/nixpkgs': 'github:edolstra/nixpkgs/43e283790fbe330b723ac44b584824cf6d85c1ee' -> 'github:NixOS/nixpkgs/b88ff468e9850410070d4e0ccd68c7011f15b2be' * Updated 'nixpkgs': 'github:edolstra/nixpkgs/d959778e0934c4c93896d509a65de798909c7cdd' -> 'github:NixOS/nixpkgs/b0c285807d6a9f1b7562ec417c24fa1a30ecc31a' --- flake.lock | 64 +++++++++++++++++++++++++++++------------------------- 1 file changed, 35 insertions(+), 29 deletions(-) diff --git a/flake.lock b/flake.lock index 16cc32b2d..b448bcae4 100644 --- a/flake.lock +++ b/flake.lock @@ -1,34 +1,17 @@ { - "inputs": { + "nodes": { "nix": { "info": { - "lastModified": 1582125169, - "narHash": "sha256-rgoyhvUVM78gLkvviTA/9f6esC7ZJtnDLt1hzAaq2Ds=" + "lastModified": 1585573619, + "narHash": "sha256-QbAxdnDkVvSVgkCNRbt3fUPUSNbeq9y3u7Vek/7Ib44=" }, "inputs": { - "nixpkgs": { - "info": { - "lastModified": 1580555706, - "narHash": "sha256-ly5Gxl9yZgW2w1F3aDU5bvg34Ct+juUaOg8vue+Mkbo=" - }, - "inputs": {}, - "locked": { - "owner": "edolstra", - "repo": "nixpkgs", - "rev": "43e283790fbe330b723ac44b584824cf6d85c1ee", - "type": "github" - }, - "original": { - "id": "nixpkgs", - "ref": "release-19.09", - "type": "indirect" - } - } + "nixpkgs": "nixpkgs" }, "locked": { "owner": "NixOS", "repo": "nix", - "rev": "4ad5826a182f8eeb4cbc61e27d2128b08050b0d4", + "rev": "3e7aab81ce6787e01ea4ced5af1fc6a84e523762", "type": "github" }, "original": { @@ -38,22 +21,45 @@ }, "nixpkgs": { "info": { - "lastModified": 1581348815, - "narHash": "sha256-ZYGphR59gsdFhtZYDoeuCtULm8ym/qJSHIslih9b1s8=" + "lastModified": 1585405475, + "narHash": "sha256-bESW0n4KgPmZ0luxvwJ+UyATrC6iIltVCsGdLiphVeE=" }, - "inputs": {}, "locked": { - "owner": "edolstra", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "b88ff468e9850410070d4e0ccd68c7011f15b2be", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "ref": "nixos-20.03-small", + "type": "indirect" + } + }, + "nixpkgs_2": { + "info": { + "lastModified": 1585388205, + "narHash": "sha256-lOXYmCE6FSikoOHr1HFIkNLnA0qdYhe8CxB8rE6+SnE=" + }, + "locked": { + "owner": "NixOS", "repo": "nixpkgs", - "rev": "d959778e0934c4c93896d509a65de798909c7cdd", + "rev": "b0c285807d6a9f1b7562ec417c24fa1a30ecc31a", "type": "github" }, "original": { "id": "nixpkgs", - "ref": "release-19.09", + "ref": "nixos-20.03", "type": "indirect" } + }, + "root": { + "inputs": { + "nix": "nix", + "nixpkgs": "nixpkgs_2" + } } }, - "version": 4 + "root": "root", + "version": 5 } From 810075cd3d9d9b8fcca1bb9e22c1b5c190f95473 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 31 Mar 2020 12:55:35 +0200 Subject: [PATCH 69/73] Update to 20.03 --- flake.nix | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 8acea7c33..edb8e4d69 100644 --- a/flake.nix +++ b/flake.nix @@ -3,7 +3,7 @@ edition = 201909; - inputs.nixpkgs.uri = "nixpkgs/release-19.09"; + inputs.nixpkgs.url = "nixpkgs/nixos-20.03"; outputs = { self, nixpkgs, nix }: let @@ -108,7 +108,9 @@ perlDeps perl final.nix postgresql95 # for running the tests boost - (nlohmann_json.override { multipleHeaders = true; }) + (if lib.versionAtLeast lib.version "20.03pre" + then nlohmann_json + else nlohmann_json.override { multipleHeaders = true; }) ]; hydraPath = lib.makeBinPath ( From 22a65d0269ce89f4962b781dbb78474ec2298c3c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 31 Mar 2020 12:58:24 +0200 Subject: [PATCH 70/73] Remove outdated email address --- configure.ac | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index f99d4c9da..baff26c21 100644 --- a/configure.ac +++ b/configure.ac @@ -1,5 +1,4 @@ -AC_INIT([Hydra], [m4_esyscmd([echo -n $(cat ./version)$VERSION_SUFFIX])], - [nix-dev@cs.uu.nl], [hydra], [http://nixos.org/hydra/]) +AC_INIT([Hydra], [m4_esyscmd([echo -n $(cat ./version)$VERSION_SUFFIX])]) AC_CONFIG_AUX_DIR(config) AM_INIT_AUTOMAKE([foreign serial-tests]) From f6119f2b193e984c74cd8cec90b67c510f013760 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 31 Mar 2020 13:19:36 +0200 Subject: [PATCH 71/73] perlDeps: Use lib.closePropagation Apparentely, buildEnv in 20.03 no longer respects propagated-build-inputs. Note that the use of a library function (closePropagation) seems fundamentally wrong to me - propagated-build-inputs should be used at runtime, not at evaluation time. --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index edb8e4d69..859695096 100644 --- a/flake.nix +++ b/flake.nix @@ -41,7 +41,7 @@ perlDeps = buildEnv { name = "hydra-perl-deps"; - paths = with perlPackages; + paths = with perlPackages; lib.closePropagation [ ModulePluggable CatalystActionREST CatalystAuthenticationStoreDBIxClass From 4009d4295e1e3c0e59e716b0b111f48c590a6020 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 31 Mar 2020 13:33:11 +0200 Subject: [PATCH 72/73] Fix Mercurial tests In Nixpkgs 20.03, Mercurial fails if PYTHONPATH is set: $ hg Traceback (most recent call last): File "/nix/store/q7s856v6nw4dffdrm9k3w38qs35i8kr3-mercurial-5.2.2/bin/..hg-wrapped-wrapped", line 37, in dispatch.run() File "/nix/store/bffdy7q3wi3qinflnvbdkigqj39zzynd-python3-3.7.6/lib/python3.7/importlib/util.py", line 245, in __getattribute__ self.__spec__.loader.exec_module(self) File "", line 728, in exec_module File "", line 219, in _call_with_frames_removed File "/nix/store/q7s856v6nw4dffdrm9k3w38qs35i8kr3-mercurial-5.2.2/lib/python3.7/site-packages/mercurial/dispatch.py", line 10, in import difflib File "/nix/store/bffdy7q3wi3qinflnvbdkigqj39zzynd-python3-3.7.6/lib/python3.7/difflib.py", line 1084, in import re File "/nix/store/bffdy7q3wi3qinflnvbdkigqj39zzynd-python3-3.7.6/lib/python3.7/re.py", line 143, in class RegexFlag(enum.IntFlag): AttributeError: module 'enum' has no attribute 'IntFlag' --- tests/Makefile.am | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/Makefile.am b/tests/Makefile.am index e56e528ef..d3153a55c 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -14,6 +14,7 @@ TESTS_ENVIRONMENT = \ NIX_BUILD_HOOK= \ PGHOST=/tmp \ PERL5LIB="$(srcdir):$(abs_top_srcdir)/src/lib:$$PERL5LIB" \ + PYTHONPATH= \ PATH=$(abs_top_srcdir)/src/hydra-evaluator:$(abs_top_srcdir)/src/script:$(abs_top_srcdir)/src/hydra-eval-jobs:$(abs_top_srcdir)/src/hydra-queue-runner:$$PATH \ perl -w From d1f367f65770a13b14874f7aabbd9e02b6334e16 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 31 Mar 2020 16:07:46 +0200 Subject: [PATCH 73/73] Add default.nix, shell.nix for compatibility --- default.nix | 3 +++ shell.nix | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 default.nix create mode 100644 shell.nix diff --git a/default.nix b/default.nix new file mode 100644 index 000000000..b22e926ea --- /dev/null +++ b/default.nix @@ -0,0 +1,3 @@ +(import (fetchTarball https://github.com/edolstra/flake-compat/archive/master.tar.gz) { + src = builtins.fetchGit ./.; +}).defaultNix diff --git a/shell.nix b/shell.nix new file mode 100644 index 000000000..db84e3d07 --- /dev/null +++ b/shell.nix @@ -0,0 +1,3 @@ +(import (fetchTarball https://github.com/edolstra/flake-compat/archive/master.tar.gz) { + src = builtins.fetchGit ./.; +}).shellNix