From 7e58f11aa78e9122be2ad45f2c4f8f846f464d00 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 18 Nov 2022 11:51:44 +0100 Subject: [PATCH 01/15] document "Import From Derivation" --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/glossary.md | 5 +- .../src/language/import-from-derivation.md | 86 +++++++++++++++++++ src/libexpr/eval-settings.hh | 8 +- 4 files changed, 94 insertions(+), 6 deletions(-) create mode 100644 doc/manual/src/language/import-from-derivation.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 6c599abcf4b..3c2c720b1de 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -33,6 +33,7 @@ - [Operators](language/operators.md) - [Derivations](language/derivations.md) - [Advanced Attributes](language/advanced-attributes.md) + - [Import From Derivation](language/import-from-derivation.md) - [Built-in Constants](language/builtin-constants.md) - [Built-in Functions](language/builtins.md) - [Advanced Topics](advanced-topics/advanced-topics.md) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index d950a4ca8d7..c44f93cc319 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -105,12 +105,15 @@ - [store object]{#gloss-store-object} - A store object consists of a [file system object], [reference]s to other store objects, and other metadata. It can be referred to by a [store path]. [store object]: #gloss-store-object +- [IFD]{#gloss-ifd} + + [Import From Derivation](./language/import-from-derivation.md) + - [input-addressed store object]{#gloss-input-addressed-store-object} A store object produced by building a diff --git a/doc/manual/src/language/import-from-derivation.md b/doc/manual/src/language/import-from-derivation.md new file mode 100644 index 00000000000..0296292b1c9 --- /dev/null +++ b/doc/manual/src/language/import-from-derivation.md @@ -0,0 +1,86 @@ +# Import From Derivation + +The value of a Nix expression can depend on the contents of a [store object] produced by a [derivation]. +In this case, when that store object is needed, evaluation will be paused, the store object [realised], and then evaluation resumed. + +[store object]: /glossary.md#gloss-store-object +[derivation]: /glossary.md#gloss-derivation +[realised]: /glossary.md#gloss-realise + +This has performance implications: +Since evaluation is sequential, each required store object that is not already in the store will also be realised sequentially. + +Passing a derivation `drv` to any built-in function that reads from the filesystem constitutes Import From Derivation: + +- [`import`](./language/builtins.md#builtins-import)` drv` +- [`builtins.readFile`](./language/builtins.md#builtins-readFile)` drv` +- [`builtins.readDir`](./language/builtins.md#builtins-readDir)` drv` +- [`builtins.pathExists`](./language/builtins.md#builtins-pathExists)` drv` +- [`builtins.filterSource`](./language/builtins.md#builtins-filterSource)` f drv` +- [`builtins.path`](./language/builtins.md#builtins-path)` { path = drv; }` +- [`builtins.hashFile`](./language/builtins.md#builtins-hashFile)` t drv` +- `builtins.scopedImport x drv` + +Building during evaluation can be disabled by setting [`allow-import-from-derivation`](../command-ref/conf-file.md#conf-allow-import-from-derivation) to `false`. + +## Example + +In the following Nix expression, the inner derivation `drv` produces a file containing `"hello"`. + +```nix +# IFD.nix +let + drv = derivation { + name = "hello"; + builder = /bin/sh; + args = [ "-c" ''echo \"hello\" > $out'' ]; + system = builtins.currentSystem; + }; +in "${import drv} world" +``` + +```shellSession +nix-instantiate IFD.nix --eval --read-write-mode +``` + +``` +building '/nix/store/348q1cal6sdgfxs8zqi9v8llrsn4kqkq-hello.drv'... +"hello world" +``` + +Since `"hello"` is a valid Nix expression, it can be [`import`](./builtins.md#builtins-import)ed. +That requires reading from the output [store path](@docroot@/glossary.md#gloss-store-path) of `drv`, which has to be [realised] before its contents can be read and evaluated. + +## Illustration + +The following diagram shows how evaluation is interrupted by a build, if the value of a Nix expression depends on realising a store object. + +``` ++----------------------+ +------------------------+ +| Nix language | | Nix store | +| .----------------. | | | +| | Nix expression | | | | +| '----------------' | | | +| | | | | +| evaluate | | | +| | | | | +| V | | | +| .------------. | | .------------------. | +| | derivation |----|-instantiate-|->| store derivation | | +| '------------' | | '------------------' | +| | | | | +| | | realise | +| | | | | +| | | V | +| .----------------. | | .--------------. | +| | Nix expression |<-|----read-----|----| store object | | +| '----------------' | | '--------------' | +| | | | | +| evaluate | | | +| | | | | +| V | | | +| .------------. | | | +| | value | | | | +| '------------' | | | ++----------------------+ +------------------------+ +``` diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index e6666061ae0..1b6d13bd69d 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -47,11 +47,9 @@ struct EvalSettings : Config Setting enableImportFromDerivation{ this, true, "allow-import-from-derivation", R"( - By default, Nix allows you to `import` from a derivation, allowing - building at evaluation time. With this option set to false, Nix will - throw an error when evaluating an expression that uses this feature, - allowing users to ensure their evaluation will not require any - builds to take place. + By default, Nix allows [Import From Derivation](@docroot@/language/import-from-derivation.md). + With this option set to `false`, Nix will throw an error when evaluating an expression that uses this feature, + ensuring that evaluation will not require any builds to take place. )"}; Setting allowedUris{this, {}, "allowed-uris", From fd93ec48a332337c8f4427d8b8a9d68415eb81c8 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 13 Jul 2023 23:09:04 +0200 Subject: [PATCH 02/15] fixup wording --- .../src/language/import-from-derivation.md | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/doc/manual/src/language/import-from-derivation.md b/doc/manual/src/language/import-from-derivation.md index 0296292b1c9..39c485cb0ed 100644 --- a/doc/manual/src/language/import-from-derivation.md +++ b/doc/manual/src/language/import-from-derivation.md @@ -1,6 +1,6 @@ # Import From Derivation -The value of a Nix expression can depend on the contents of a [store object] produced by a [derivation]. +The value of a Nix expression can depend on the contents of a [store object]. In this case, when that store object is needed, evaluation will be paused, the store object [realised], and then evaluation resumed. [store object]: /glossary.md#gloss-store-object @@ -10,18 +10,18 @@ In this case, when that store object is needed, evaluation will be paused, the s This has performance implications: Since evaluation is sequential, each required store object that is not already in the store will also be realised sequentially. -Passing a derivation `drv` to any built-in function that reads from the filesystem constitutes Import From Derivation: +Passing an expression `expr` which evaluates to a store path to any built-in function that reads from the filesystem constitutes Import From Derivation: -- [`import`](./language/builtins.md#builtins-import)` drv` -- [`builtins.readFile`](./language/builtins.md#builtins-readFile)` drv` -- [`builtins.readDir`](./language/builtins.md#builtins-readDir)` drv` -- [`builtins.pathExists`](./language/builtins.md#builtins-pathExists)` drv` -- [`builtins.filterSource`](./language/builtins.md#builtins-filterSource)` f drv` -- [`builtins.path`](./language/builtins.md#builtins-path)` { path = drv; }` -- [`builtins.hashFile`](./language/builtins.md#builtins-hashFile)` t drv` +- [`import`](./language/builtins.md#builtins-import)` expr` +- [`builtins.readFile`](./language/builtins.md#builtins-readFile)` expr` +- [`builtins.readDir`](./language/builtins.md#builtins-readDir)` expr` +- [`builtins.pathExists`](./language/builtins.md#builtins-pathExists)` expr` +- [`builtins.filterSource`](./language/builtins.md#builtins-filterSource)` f expr` +- [`builtins.path`](./language/builtins.md#builtins-path)` { path = expr; }` +- [`builtins.hashFile`](./language/builtins.md#builtins-hashFile)` t expr` - `builtins.scopedImport x drv` -Building during evaluation can be disabled by setting [`allow-import-from-derivation`](../command-ref/conf-file.md#conf-allow-import-from-derivation) to `false`. +Realising store objects during evaluation can be disabled by setting [`allow-import-from-derivation`](../command-ref/conf-file.md#conf-allow-import-from-derivation) to `false`. ## Example From 048e0bb83bda8b3a7d12eadc4854705460e69361 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 14 Jul 2023 10:29:17 +0200 Subject: [PATCH 03/15] fix links --- .../src/language/import-from-derivation.md | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/doc/manual/src/language/import-from-derivation.md b/doc/manual/src/language/import-from-derivation.md index 39c485cb0ed..0be9b1155f5 100644 --- a/doc/manual/src/language/import-from-derivation.md +++ b/doc/manual/src/language/import-from-derivation.md @@ -3,22 +3,22 @@ The value of a Nix expression can depend on the contents of a [store object]. In this case, when that store object is needed, evaluation will be paused, the store object [realised], and then evaluation resumed. -[store object]: /glossary.md#gloss-store-object -[derivation]: /glossary.md#gloss-derivation -[realised]: /glossary.md#gloss-realise +[store object]: @docroot@/glossary.md#gloss-store-object +[derivation]: @docroot@/glossary.md#gloss-derivation +[realised]: @docroot@/glossary.md#gloss-realise This has performance implications: Since evaluation is sequential, each required store object that is not already in the store will also be realised sequentially. Passing an expression `expr` which evaluates to a store path to any built-in function that reads from the filesystem constitutes Import From Derivation: -- [`import`](./language/builtins.md#builtins-import)` expr` -- [`builtins.readFile`](./language/builtins.md#builtins-readFile)` expr` -- [`builtins.readDir`](./language/builtins.md#builtins-readDir)` expr` -- [`builtins.pathExists`](./language/builtins.md#builtins-pathExists)` expr` -- [`builtins.filterSource`](./language/builtins.md#builtins-filterSource)` f expr` -- [`builtins.path`](./language/builtins.md#builtins-path)` { path = expr; }` -- [`builtins.hashFile`](./language/builtins.md#builtins-hashFile)` t expr` +- [`import`](./builtins.md#builtins-import)` expr` +- [`builtins.readFile`](./builtins.md#builtins-readFile)` expr` +- [`builtins.readDir`](./builtins.md#builtins-readDir)` expr` +- [`builtins.pathExists`](./builtins.md#builtins-pathExists)` expr` +- [`builtins.filterSource`](./builtins.md#builtins-filterSource)` f expr` +- [`builtins.path`](./builtins.md#builtins-path)` { path = expr; }` +- [`builtins.hashFile`](./builtins.md#builtins-hashFile)` t expr` - `builtins.scopedImport x drv` Realising store objects during evaluation can be disabled by setting [`allow-import-from-derivation`](../command-ref/conf-file.md#conf-allow-import-from-derivation) to `false`. From eec899867bc4efb546c0f3b50eeb95d17e6c0354 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 17 Jul 2023 23:34:00 +0200 Subject: [PATCH 04/15] add additional example Co-authored-by: Robert Hensing --- doc/manual/src/language/import-from-derivation.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manual/src/language/import-from-derivation.md b/doc/manual/src/language/import-from-derivation.md index 0be9b1155f5..bf100d02281 100644 --- a/doc/manual/src/language/import-from-derivation.md +++ b/doc/manual/src/language/import-from-derivation.md @@ -14,6 +14,7 @@ Passing an expression `expr` which evaluates to a store path to any built-in fun - [`import`](./builtins.md#builtins-import)` expr` - [`builtins.readFile`](./builtins.md#builtins-readFile)` expr` +- [`builtins.readFileType`](./builtins.md#builtins-readFileType)` expr` - [`builtins.readDir`](./builtins.md#builtins-readDir)` expr` - [`builtins.pathExists`](./builtins.md#builtins-pathExists)` expr` - [`builtins.filterSource`](./builtins.md#builtins-filterSource)` f expr` From 2f3dfcb47eae8f313d7b05bd3321b7016d25c184 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 17 Jul 2023 23:34:18 +0200 Subject: [PATCH 05/15] clarify wording Co-authored-by: Robert Hensing --- doc/manual/src/language/import-from-derivation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/language/import-from-derivation.md b/doc/manual/src/language/import-from-derivation.md index bf100d02281..1592b8a8e06 100644 --- a/doc/manual/src/language/import-from-derivation.md +++ b/doc/manual/src/language/import-from-derivation.md @@ -10,7 +10,7 @@ In this case, when that store object is needed, evaluation will be paused, the s This has performance implications: Since evaluation is sequential, each required store object that is not already in the store will also be realised sequentially. -Passing an expression `expr` which evaluates to a store path to any built-in function that reads from the filesystem constitutes Import From Derivation: +Passing a store path to any built-in function that reads from the filesystem constitutes Import From Derivation: - [`import`](./builtins.md#builtins-import)` expr` - [`builtins.readFile`](./builtins.md#builtins-readFile)` expr` From 9f2501810d85fda18438cbff4d77960a4b55a190 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 17 Jul 2023 23:36:18 +0200 Subject: [PATCH 06/15] clarify diagram label Co-authored-by: Robert Hensing --- doc/manual/src/language/import-from-derivation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/language/import-from-derivation.md b/doc/manual/src/language/import-from-derivation.md index 1592b8a8e06..cadc706cc99 100644 --- a/doc/manual/src/language/import-from-derivation.md +++ b/doc/manual/src/language/import-from-derivation.md @@ -58,7 +58,7 @@ The following diagram shows how evaluation is interrupted by a build, if the val ``` +----------------------+ +------------------------+ -| Nix language | | Nix store | +| Nix evaluator | | Nix store | | .----------------. | | | | | Nix expression | | | | | '----------------' | | | From dadb3b2d4eeece3b7eca2263c203fd81e7615caa Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 17 Jul 2023 23:40:47 +0200 Subject: [PATCH 07/15] clarify purpose of ifd option Co-authored-by: Robert Hensing --- src/libexpr/eval-settings.hh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index 1b6d13bd69d..fbeb0e659eb 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -47,9 +47,12 @@ struct EvalSettings : Config Setting enableImportFromDerivation{ this, true, "allow-import-from-derivation", R"( - By default, Nix allows [Import From Derivation](@docroot@/language/import-from-derivation.md). + By default, Nix allows [Import from Derivation](@docroot@/language/import-from-derivation.md). + With this option set to `false`, Nix will throw an error when evaluating an expression that uses this feature, - ensuring that evaluation will not require any builds to take place. + even when the required store object is readily available. + This ensures that evaluation will not require any builds to take place, + regardless of the state of the store. )"}; Setting allowedUris{this, {}, "allowed-uris", From ffd9f7aaba12a83f464294e7d100b57c91852828 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 17 Jul 2023 23:43:14 +0200 Subject: [PATCH 08/15] make example work without copying builder to the store Co-authored-by: Robert Hensing --- doc/manual/src/language/import-from-derivation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/language/import-from-derivation.md b/doc/manual/src/language/import-from-derivation.md index cadc706cc99..c8e1d95e59f 100644 --- a/doc/manual/src/language/import-from-derivation.md +++ b/doc/manual/src/language/import-from-derivation.md @@ -33,7 +33,7 @@ In the following Nix expression, the inner derivation `drv` produces a file cont let drv = derivation { name = "hello"; - builder = /bin/sh; + builder = "/bin/sh"; args = [ "-c" ''echo \"hello\" > $out'' ]; system = builtins.currentSystem; }; From 3951a2bc0299ddfdf360901356c6030b0b75c8b2 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 19 Jul 2023 12:28:00 +0200 Subject: [PATCH 09/15] explain `expr` sample --- doc/manual/src/language/import-from-derivation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/language/import-from-derivation.md b/doc/manual/src/language/import-from-derivation.md index c8e1d95e59f..08e51710fe1 100644 --- a/doc/manual/src/language/import-from-derivation.md +++ b/doc/manual/src/language/import-from-derivation.md @@ -10,7 +10,7 @@ In this case, when that store object is needed, evaluation will be paused, the s This has performance implications: Since evaluation is sequential, each required store object that is not already in the store will also be realised sequentially. -Passing a store path to any built-in function that reads from the filesystem constitutes Import From Derivation: +Passing an expression `expr` that evaluates to a store path to any built-in function which reads from the filesystem constitutes Import From Derivation: - [`import`](./builtins.md#builtins-import)` expr` - [`builtins.readFile`](./builtins.md#builtins-readFile)` expr` From 700c8e19af25b4901e2c9477d06861348ce7e3d5 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 20 Jul 2023 11:42:11 +0200 Subject: [PATCH 10/15] go into more detail with explanations --- .../src/language/import-from-derivation.md | 58 +++++++++++++++++-- 1 file changed, 54 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/language/import-from-derivation.md b/doc/manual/src/language/import-from-derivation.md index 08e51710fe1..8469d9ed990 100644 --- a/doc/manual/src/language/import-from-derivation.md +++ b/doc/manual/src/language/import-from-derivation.md @@ -26,7 +26,7 @@ Realising store objects during evaluation can be disabled by setting [`allow-imp ## Example -In the following Nix expression, the inner derivation `drv` produces a file containing `"hello"`. +In the following Nix expression, the inner derivation `drv` produces a file with contents `hello`. ```nix # IFD.nix @@ -34,10 +34,10 @@ let drv = derivation { name = "hello"; builder = "/bin/sh"; - args = [ "-c" ''echo \"hello\" > $out'' ]; + args = [ "-c" "echo -n hello > $out" ]; system = builtins.currentSystem; }; -in "${import drv} world" +in "${builtins.readFile drv} world" ``` ```shellSession @@ -54,7 +54,8 @@ That requires reading from the output [store path](@docroot@/glossary.md#gloss-s ## Illustration -The following diagram shows how evaluation is interrupted by a build, if the value of a Nix expression depends on realising a store object. +As a first approximation, the following data flow graph shows how evaluation and building are interleaved, if the value of a Nix expression depends on realising a store object. +Boxes are data structures, arrow labels are transformations. ``` +----------------------+ +------------------------+ @@ -85,3 +86,52 @@ The following diagram shows how evaluation is interrupted by a build, if the val | '------------' | | | +----------------------+ +------------------------+ ``` + +In more detail, the following sequence diagram shows how the expression is evaluated step by step, and where evaluation is blocked to wait for the build output to appear. + +``` +.-------. .-------------. .---------. +|Nix CLI| |Nix evaluator| |Nix store| +'-------' '-------------' '---------' + | | | + |evaluate IFD.nix| | + |--------------->| | + | | | + | evaluate `"${readFile drv} world"` | + | | | + | evaluate `readFile drv` | + | | | + | evaluate `drv` as string | + | | | + | |instantiate /nix/store/...-hello.drv| + | |----------------------------------->| + | : | + | : realise /nix/store/...-hello.drv | + | :----------------------------------->| + | : | + | |--------. + | : | | + | (evaluation blocked) | echo hello > $out + | : | | + | |<-------' + | : /nix/store/...-hello | + | |<-----------------------------------| + | | | + | resume `readFile /nix/store/...-hello` | + | | | + | | readFile /nix/store/...-hello | + | |----------------------------------->| + | | | + | | hello | + | |<-----------------------------------| + | | | + | resume `"${"hello"} world"` | + | | | + | resume `"hello world"` | + | | | + | "hello world" | | + |<---------------| | +.-------. .-------------. .---------. +|Nix CLI| |Nix evaluator| |Nix store| +'-------' '-------------' '---------' +``` From 7e2b9a4e74aaf92dec4362f881b33c97d6a85eb5 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 20 Jul 2023 11:50:16 +0200 Subject: [PATCH 11/15] add more performance considerations --- doc/manual/src/language/import-from-derivation.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manual/src/language/import-from-derivation.md b/doc/manual/src/language/import-from-derivation.md index 8469d9ed990..74977bc8a57 100644 --- a/doc/manual/src/language/import-from-derivation.md +++ b/doc/manual/src/language/import-from-derivation.md @@ -9,6 +9,7 @@ In this case, when that store object is needed, evaluation will be paused, the s This has performance implications: Since evaluation is sequential, each required store object that is not already in the store will also be realised sequentially. +Usually, if store objects are not already present, realisation is orders of magnitude slower than evaluation. Passing an expression `expr` that evaluates to a store path to any built-in function which reads from the filesystem constitutes Import From Derivation: @@ -23,6 +24,7 @@ Passing an expression `expr` that evaluates to a store path to any built-in func - `builtins.scopedImport x drv` Realising store objects during evaluation can be disabled by setting [`allow-import-from-derivation`](../command-ref/conf-file.md#conf-allow-import-from-derivation) to `false`. +Without IFD it is ensured that evaluation is complete and Nix can produce a build plan before starting any realisation. ## Example From b7553282903816eb19fcb116150fd0bca5914a16 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 20 Jul 2023 12:19:13 +0200 Subject: [PATCH 12/15] reword explanation of example --- doc/manual/src/language/import-from-derivation.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/language/import-from-derivation.md b/doc/manual/src/language/import-from-derivation.md index 74977bc8a57..976e8148b12 100644 --- a/doc/manual/src/language/import-from-derivation.md +++ b/doc/manual/src/language/import-from-derivation.md @@ -11,7 +11,7 @@ This has performance implications: Since evaluation is sequential, each required store object that is not already in the store will also be realised sequentially. Usually, if store objects are not already present, realisation is orders of magnitude slower than evaluation. -Passing an expression `expr` that evaluates to a store path to any built-in function which reads from the filesystem constitutes Import From Derivation: +Passing an expression `expr` that evaluates to a [store path](@docroot@/glossary.md#gloss-store-path) to any built-in function which reads from the filesystem constitutes Import From Derivation: - [`import`](./builtins.md#builtins-import)` expr` - [`builtins.readFile`](./builtins.md#builtins-readFile)` expr` @@ -51,8 +51,8 @@ building '/nix/store/348q1cal6sdgfxs8zqi9v8llrsn4kqkq-hello.drv'... "hello world" ``` -Since `"hello"` is a valid Nix expression, it can be [`import`](./builtins.md#builtins-import)ed. -That requires reading from the output [store path](@docroot@/glossary.md#gloss-store-path) of `drv`, which has to be [realised] before its contents can be read and evaluated. +The contents of the derivation's [output path](@docroot@/glossary.md#gloss-output-path) have to be [realised] before they can be read with [`readFile`](./builtins.md#builtins-readFile). +Only then evaluation can continue to produce the final result. ## Illustration From 16c7902978c2df6468ddce1cd355b51263f01083 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 26 Sep 2023 02:17:53 +0200 Subject: [PATCH 13/15] be more precise about sequentiality and parallelism --- doc/manual/src/language/import-from-derivation.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/language/import-from-derivation.md b/doc/manual/src/language/import-from-derivation.md index 976e8148b12..10ce7b9e88b 100644 --- a/doc/manual/src/language/import-from-derivation.md +++ b/doc/manual/src/language/import-from-derivation.md @@ -8,8 +8,11 @@ In this case, when that store object is needed, evaluation will be paused, the s [realised]: @docroot@/glossary.md#gloss-realise This has performance implications: -Since evaluation is sequential, each required store object that is not already in the store will also be realised sequentially. -Usually, if store objects are not already present, realisation is orders of magnitude slower than evaluation. +Evaluation can only finish when all required store objects are realised. +Since the Nix language evaluator is sequential, it only finds [store paths] to read from one at a time. +While realisation is always parallel, in this case it cannot be done for all required store paths at once, and is therefore much slower than otherwise. + +[store paths]: @docroot@/glossary.md#gloss-store-path Passing an expression `expr` that evaluates to a [store path](@docroot@/glossary.md#gloss-store-path) to any built-in function which reads from the filesystem constitutes Import From Derivation: From 1b97893dae61b926e60ce59f578e468436f9aa11 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 26 Sep 2023 02:20:53 +0200 Subject: [PATCH 14/15] avoid "output path" output path is not really a thing, since it's really just the store path to a derivation output. --- doc/manual/src/language/import-from-derivation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/language/import-from-derivation.md b/doc/manual/src/language/import-from-derivation.md index 10ce7b9e88b..64b2e15e92c 100644 --- a/doc/manual/src/language/import-from-derivation.md +++ b/doc/manual/src/language/import-from-derivation.md @@ -54,7 +54,7 @@ building '/nix/store/348q1cal6sdgfxs8zqi9v8llrsn4kqkq-hello.drv'... "hello world" ``` -The contents of the derivation's [output path](@docroot@/glossary.md#gloss-output-path) have to be [realised] before they can be read with [`readFile`](./builtins.md#builtins-readFile). +The contents of the derivation's output have to be [realised] before they can be read with [`readFile`](./builtins.md#builtins-readFile). Only then evaluation can continue to produce the final result. ## Illustration From a05b62f3d407c4cf794892439b195fb7834ed86e Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 26 Sep 2023 03:01:48 +0200 Subject: [PATCH 15/15] move definition to the beginning --- .../src/language/import-from-derivation.md | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/doc/manual/src/language/import-from-derivation.md b/doc/manual/src/language/import-from-derivation.md index 64b2e15e92c..03b3f9d91b1 100644 --- a/doc/manual/src/language/import-from-derivation.md +++ b/doc/manual/src/language/import-from-derivation.md @@ -1,20 +1,8 @@ # Import From Derivation -The value of a Nix expression can depend on the contents of a [store object]. -In this case, when that store object is needed, evaluation will be paused, the store object [realised], and then evaluation resumed. +The value of a Nix expression can depend on the contents of a [store object](@docroot@/glossary.md#gloss-store-object). -[store object]: @docroot@/glossary.md#gloss-store-object -[derivation]: @docroot@/glossary.md#gloss-derivation -[realised]: @docroot@/glossary.md#gloss-realise - -This has performance implications: -Evaluation can only finish when all required store objects are realised. -Since the Nix language evaluator is sequential, it only finds [store paths] to read from one at a time. -While realisation is always parallel, in this case it cannot be done for all required store paths at once, and is therefore much slower than otherwise. - -[store paths]: @docroot@/glossary.md#gloss-store-path - -Passing an expression `expr` that evaluates to a [store path](@docroot@/glossary.md#gloss-store-path) to any built-in function which reads from the filesystem constitutes Import From Derivation: +Passing an expression `expr` that evaluates to a [store path](@docroot@/glossary.md#gloss-store-path) to any built-in function which reads from the filesystem constitutes Import From Derivation (IFD): - [`import`](./builtins.md#builtins-import)` expr` - [`builtins.readFile`](./builtins.md#builtins-readFile)` expr` @@ -26,6 +14,15 @@ Passing an expression `expr` that evaluates to a [store path](@docroot@/glossary - [`builtins.hashFile`](./builtins.md#builtins-hashFile)` t expr` - `builtins.scopedImport x drv` +When the store path needs to be accessed, evaluation will be paused, the corresponding store object [realised], and then evaluation resumed. + +[realised]: @docroot@/glossary.md#gloss-realise + +This has performance implications: +Evaluation can only finish when all required store objects are realised. +Since the Nix language evaluator is sequential, it only finds store paths to read from one at a time. +While realisation is always parallel, in this case it cannot be done for all required store paths at once, and is therefore much slower than otherwise. + Realising store objects during evaluation can be disabled by setting [`allow-import-from-derivation`](../command-ref/conf-file.md#conf-allow-import-from-derivation) to `false`. Without IFD it is ensured that evaluation is complete and Nix can produce a build plan before starting any realisation. @@ -59,7 +56,7 @@ Only then evaluation can continue to produce the final result. ## Illustration -As a first approximation, the following data flow graph shows how evaluation and building are interleaved, if the value of a Nix expression depends on realising a store object. +As a first approximation, the following data flow graph shows how evaluation and building are interleaved, if the value of a Nix expression depends on realising a [store object]. Boxes are data structures, arrow labels are transformations. ```