diff --git a/.editorconfig b/.editorconfig index 887ecadba596..86360e6582d2 100644 --- a/.editorconfig +++ b/.editorconfig @@ -17,7 +17,7 @@ indent_style = space indent_size = 2 # Match c++/shell/perl, set indent to spaces with width of four -[*.{hpp,cc,hh,sh,pl}] +[*.{hpp,cc,hh,sh,pl,xs}] indent_style = space indent_size = 4 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ab59086491b2..39d595199d8f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -14,5 +14,12 @@ /doc @fricklerhandwerk *.md @fricklerhandwerk +# Documentation of built-in functions +src/libexpr/primops.cc @fricklerhandwerk @roberth +# Documentation on experimental features +src/libutil/experimental-features.cc @fricklerhandwerk +# Documentation on configuration settings +src/libstore/globals.hh @fricklerhandwerk + # Libstore layer /src/libstore @thufschmitt diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 4488c7b7df07..217b19108cfb 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -10,24 +10,6 @@ -# Checklist for maintainers - - - -Maintainers: tick if completed or explain if not relevant - - - [ ] agreed on idea - - [ ] agreed on implementation strategy - - [ ] tests, as appropriate - - functional tests - `tests/**.sh` - - unit tests - `src/*/tests` - - integration tests - `tests/nixos/*` - - [ ] documentation in the manual - - [ ] documentation in the internal API docs - - [ ] code and comments are self-explanatory - - [ ] commit message explains why the change was made - - [ ] new feature or incompatible change: updated release notes - # Priorities Add :+1: to [pull requests you find important](https://github.com/NixOS/nix/pulls?q=is%3Aopen+sort%3Areactions-%2B1-desc). diff --git a/.github/labeler.yml b/.github/labeler.yml index fce0d3aeb858..7544f07a6106 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -16,8 +16,8 @@ "new-cli": - src/nix/**/* -"tests": +"with-tests": # Unit tests - src/*/tests/**/* # Functional and integration tests - - tests/**/* + - tests/functional/**/* diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 816474ed5c32..975c90b9178e 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -14,14 +14,14 @@ jobs: if: github.repository_owner == 'NixOS' && github.event.pull_request.merged == true && (github.event_name != 'labeled' || startsWith('backport', github.event.label.name)) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} # required to find all branches fetch-depth: 0 - name: Create backport PRs # should be kept in sync with `version` - uses: zeebe-io/backport-action@v1.3.1 + uses: zeebe-io/backport-action@v2.1.1 with: # Config README: https://github.com/zeebe-io/backport-action#backport-action github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3a17d106b3f..afe4dc2e35e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,10 +17,10 @@ jobs: runs-on: ${{ matrix.os }} timeout-minutes: 60 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v22 + - uses: cachix/install-nix-action@v23 with: # The sandbox would otherwise be disabled by default on Darwin extra_nix_config: "sandbox = true" @@ -58,11 +58,11 @@ jobs: outputs: installerURL: ${{ steps.prepare-installer.outputs.installerURL }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v22 + - uses: cachix/install-nix-action@v23 with: install_url: https://releases.nixos.org/nix/nix-2.13.3/install - uses: cachix/cachix-action@v12 @@ -82,9 +82,9 @@ jobs: os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v22 + - uses: cachix/install-nix-action@v23 with: install_url: '${{needs.installer.outputs.installerURL}}' install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" @@ -101,6 +101,9 @@ jobs: docker_push_image: needs: [check_secrets, tests] + permissions: + contents: read + packages: write if: >- github.event_name == 'push' && github.ref_name == 'master' && @@ -108,10 +111,10 @@ jobs: needs.check_secrets.outputs.docker == 'true' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v22 + - uses: cachix/install-nix-action@v23 with: install_url: https://releases.nixos.org/nix/nix-2.13.3/install - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV @@ -126,10 +129,30 @@ jobs: - run: docker load -i ./result/image.tar.gz - run: docker tag nix:$NIX_VERSION nixos/nix:$NIX_VERSION - run: docker tag nix:$NIX_VERSION nixos/nix:master + # We'll deploy the newly built image to both Docker Hub and Github Container Registry. + # + # Push to Docker Hub first - name: Login to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - run: docker push nixos/nix:$NIX_VERSION - run: docker push nixos/nix:master + # Push to GitHub Container Registry as well + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Push image + run: | + IMAGE_ID=ghcr.io/${{ github.repository_owner }}/nix + # Change all uppercase to lowercase + IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + + docker tag nix:$NIX_VERSION $IMAGE_ID:$NIX_VERSION + docker tag nix:$NIX_VERSION $IMAGE_ID:master + docker push $IMAGE_ID:$NIX_VERSION + docker push $IMAGE_ID:master diff --git a/.github/workflows/hydra_status.yml b/.github/workflows/hydra_status.yml index 38a9c08776f7..2a75747479f2 100644 --- a/.github/workflows/hydra_status.yml +++ b/.github/workflows/hydra_status.yml @@ -13,7 +13,7 @@ jobs: if: github.repository_owner == 'NixOS' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - run: bash scripts/check-hydra-status.sh diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml index 5f949ddc56b1..d83cb4f18ac4 100644 --- a/.github/workflows/labels.yml +++ b/.github/workflows/labels.yml @@ -21,4 +21,4 @@ jobs: - uses: actions/labeler@v4 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - sync-labels: true + sync-labels: false diff --git a/.gitignore b/.gitignore index 969194650e7a..767a5d6ed8d5 100644 --- a/.gitignore +++ b/.gitignore @@ -21,12 +21,16 @@ perl/Makefile.config /doc/manual/language.json /doc/manual/xp-features.json /doc/manual/src/SUMMARY.md +/doc/manual/src/SUMMARY-rl-next.md +/doc/manual/src/store/types/* +!/doc/manual/src/store/types/index.md.in /doc/manual/src/command-ref/new-cli /doc/manual/src/command-ref/conf-file.md /doc/manual/src/command-ref/experimental-features-shortlist.md /doc/manual/src/contributing/experimental-feature-descriptions.md /doc/manual/src/language/builtins.md /doc/manual/src/language/builtin-constants.md +/doc/manual/src/release-notes/rl-next.md # /scripts/ /scripts/nix-profile.sh @@ -52,7 +56,7 @@ perl/Makefile.config /src/nix/nix -/src/nix/doc +/src/nix/generated-doc # /src/nix-env/ /src/nix-env/nix-env @@ -79,23 +83,24 @@ perl/Makefile.config /src/build-remote/build-remote -# /tests/ -/tests/test-tmp -/tests/common/vars-and-functions.sh -/tests/result* -/tests/restricted-innocent -/tests/shell -/tests/shell.drv -/tests/config.nix -/tests/ca/config.nix -/tests/dyn-drv/config.nix -/tests/repl-result-out -/tests/test-libstoreconsumer/test-libstoreconsumer - -# /tests/lang/ -/tests/lang/*.out -/tests/lang/*.out.xml -/tests/lang/*.ast +# /tests/functional/ +/tests/functional/test-tmp +/tests/functional/common/vars-and-functions.sh +/tests/functional/result* +/tests/functional/restricted-innocent +/tests/functional/shell +/tests/functional/shell.drv +/tests/functional/config.nix +/tests/functional/ca/config.nix +/tests/functional/dyn-drv/config.nix +/tests/functional/repl-result-out +/tests/functional/test-libstoreconsumer/test-libstoreconsumer + +# /tests/functional/lang/ +/tests/functional/lang/*.out +/tests/functional/lang/*.out.xml +/tests/functional/lang/*.err +/tests/functional/lang/*.ast /perl/lib/Nix/Config.pm /perl/lib/Nix/Store.cc @@ -137,4 +142,12 @@ nix-rust/target result +# IDE .vscode/ +.idea/ + +# clangd and possibly more +.cache/ + +# Mac OS +.DS_Store diff --git a/.version b/.version index d76bd2ba3eff..7329e21c3bd2 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.17.0 +2.20.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4a72a8eaca66..ffcc0268f564 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,30 +24,51 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). ## Making changes to Nix -1. Check for [pull requests](https://github.com/NixOS/nix/pulls) that might already cover the contribution you are about to make. - There are many open pull requests that might already do what you intent to work on. - You can use [labels](https://github.com/NixOS/nix/labels) to filter for relevant topics. +1. Search for related issues that cover what you're going to work on. + It could help to mention there that you will work on the issue. + + Issues labeled [good first issue](https://github.com/NixOS/nix/labels/good%20first%20issue) should be relatively easy to fix and are likely to get merged quickly. + Pull requests addressing issues labeled [idea approved](https://github.com/NixOS/nix/labels/idea%20approved) or [RFC](https://github.com/NixOS/nix/labels/RFC) are especially welcomed by maintainers and will receive prioritised review. -2. Search for related issues that cover what you're going to work on. It could help to mention there that you will work on the issue. + If you are proficient with C++, addressing one of the [popular issues](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc) will be highly appreciated by maintainers and Nix users all over the world. + For far-reaching changes, please investigate possible blockers and design implications, and coordinate with maintainers before investing too much time in writing code that may not end up getting merged. - Issues labeled ["good first issue"](https://github.com/NixOS/nix/labels/good-first-issue) should be relatively easy to fix and are likely to get merged quickly. - Pull requests addressing issues labeled ["idea approved"](https://github.com/NixOS/nix/labels/idea%20approved) are especially welcomed by maintainers and will receive prioritised review. + If there is no relevant issue yet and you're not sure whether your change is likely to be accepted, [open an issue](https://github.com/NixOS/nix/issues/new/choose) yourself. + +2. Check for [pull requests](https://github.com/NixOS/nix/pulls) that might already cover the contribution you are about to make. + There are many open pull requests that might already do what you intend to work on. + You can use [labels](https://github.com/NixOS/nix/labels) to filter for relevant topics. 3. Check the [Nix reference manual](https://nixos.org/manual/nix/unstable/contributing/hacking.html) for information on building Nix and running its tests. For contributions to the command line interface, please check the [CLI guidelines](https://nixos.org/manual/nix/unstable/contributing/cli-guideline.html). -4. Make your changes! +4. Make your change! 5. [Create a pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request) for your changes. - * [Mark the pull request as draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-stage-of-a-pull-request) if you're not done with the changes. + * Clearly explain the problem that you're solving. + + Link related issues to inform interested parties and future contributors about your change. + If your pull request closes one or multiple issues, mention that in the description using `Closes: #`, as it will then happen automatically when your change is merged. * Make sure to have [a clean history of commits on your branch by using rebase](https://www.digitalocean.com/community/tutorials/how-to-rebase-and-update-a-pull-request). - * Link related issues in your pull request to inform interested parties and future contributors about your change. - If your pull request closes one or multiple issues, note that in the description using `Closes: #`, as it will then happen automatically when your change is merged. + * [Mark the pull request as draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-stage-of-a-pull-request) if you're not done with the changes. 6. Do not expect your pull request to be reviewed immediately. Nix maintainers follow a [structured process for reviews and design decisions](https://github.com/NixOS/nix/tree/master/maintainers#project-board-protocol), which may or may not prioritise your work. + Following this checklist will make the process smoother for everyone: + + - [ ] Fixes an [idea approved](https://github.com/NixOS/nix/labels/idea%20approved) issue + - [ ] Tests, as appropriate: + - Functional tests – [`tests/functional/**.sh`](./tests/functional) + - Unit tests – [`src/*/tests`](./src/) + - Integration tests – [`tests/nixos/*`](./tests/nixos) + - [ ] User documentation in the [manual](..doc/manual/src) + - [ ] API documentation in header files + - [ ] Code and comments are self-explanatory + - [ ] Commit message explains **why** the change was made + - [ ] New feature or incompatible change: updated [release notes](./doc/manual/src/release-notes/rl-next.md) + 7. If you need additional feedback or help to getting pull request into shape, ask other contributors using [@mentions](https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax#mentioning-people-and-teams). ## Making changes to the Nix manual diff --git a/Makefile b/Makefile index c6220482adb7..92727bea5aaa 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,9 @@ +include mk/build-dir.mk + +-include $(buildprefix)Makefile.config +clean-files += $(buildprefix)Makefile.config + +ifeq ($(ENABLE_BUILD), yes) makefiles = \ mk/precompiled-headers.mk \ local.mk \ @@ -15,20 +21,24 @@ makefiles = \ misc/zsh/local.mk \ misc/systemd/local.mk \ misc/launchd/local.mk \ - misc/upstart/local.mk \ - doc/manual/local.mk \ - doc/internal-api/local.mk - --include Makefile.config + misc/upstart/local.mk +endif -ifeq ($(tests), yes) +ifeq ($(ENABLE_BUILD)_$(ENABLE_TESTS), yes_yes) +UNIT_TEST_ENV = _NIX_TEST_UNIT_DATA=unit-test-data makefiles += \ src/libutil/tests/local.mk \ src/libstore/tests/local.mk \ - src/libexpr/tests/local.mk \ - tests/local.mk \ - tests/test-libstoreconsumer/local.mk \ - tests/plugins/local.mk + src/libexpr/tests/local.mk +endif + +ifeq ($(ENABLE_TESTS), yes) +makefiles += \ + tests/functional/local.mk \ + tests/functional/ca/local.mk \ + tests/functional/dyn-drv/local.mk \ + tests/functional/test-libstoreconsumer/local.mk \ + tests/functional/plugins/local.mk else makefiles += \ mk/disable-tests.mk @@ -45,4 +55,11 @@ endif include mk/lib.mk +# Must be included after `mk/lib.mk` so rules refer to variables defined +# by the library. Rules are not "lazy" like variables, unfortunately. +ifeq ($(ENABLE_BUILD), yes) +$(eval $(call include-sub-makefile, doc/manual/local.mk)) +$(eval $(call include-sub-makefile, doc/internal-api/local.mk)) +endif + GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++2a -I src diff --git a/Makefile.config.in b/Makefile.config.in index 707cfe0e398b..c85e028c2eb6 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -8,15 +8,19 @@ CXX = @CXX@ CXXFLAGS = @CXXFLAGS@ CXXLTO = @CXXLTO@ EDITLINE_LIBS = @EDITLINE_LIBS@ +ENABLE_BUILD = @ENABLE_BUILD@ ENABLE_S3 = @ENABLE_S3@ +ENABLE_TESTS = @ENABLE_TESTS@ GTEST_LIBS = @GTEST_LIBS@ HAVE_LIBCPUID = @HAVE_LIBCPUID@ HAVE_SECCOMP = @HAVE_SECCOMP@ HOST_OS = @host_os@ +INSTALL_UNIT_TESTS = @INSTALL_UNIT_TESTS@ LDFLAGS = @LDFLAGS@ LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@ LIBBROTLI_LIBS = @LIBBROTLI_LIBS@ LIBCURL_LIBS = @LIBCURL_LIBS@ +LIBGIT2_LIBS = @LIBGIT2_LIBS@ LIBSECCOMP_LIBS = @LIBSECCOMP_LIBS@ LOWDOWN_LIBS = @LOWDOWN_LIBS@ OPENSSL_LIBS = @OPENSSL_LIBS@ @@ -28,6 +32,8 @@ SODIUM_LIBS = @SODIUM_LIBS@ SQLITE3_LIBS = @SQLITE3_LIBS@ bash = @bash@ bindir = @bindir@ +checkbindir = @checkbindir@ +checklibdir = @checklibdir@ datadir = @datadir@ datarootdir = @datarootdir@ doc_generate = @doc_generate@ @@ -35,6 +41,7 @@ docdir = @docdir@ embedded_sandbox_shell = @embedded_sandbox_shell@ exec_prefix = @exec_prefix@ includedir = @includedir@ +internal_api_docs = @internal_api_docs@ libdir = @libdir@ libexecdir = @libexecdir@ localstatedir = @localstatedir@ @@ -46,5 +53,3 @@ sandbox_shell = @sandbox_shell@ storedir = @storedir@ sysconfdir = @sysconfdir@ system = @system@ -tests = @tests@ -internal_api_docs = @internal_api_docs@ diff --git a/README.md b/README.md index 85b0902b107a..e1cace3b4d53 100644 --- a/README.md +++ b/README.md @@ -7,21 +7,20 @@ Nix is a powerful package manager for Linux and other Unix systems that makes pa management reliable and reproducible. Please refer to the [Nix manual](https://nixos.org/nix/manual) for more details. -## Installation +## Installation and first steps -On Linux and macOS the easiest way to install Nix is to run the following shell command -(as a user other than root): +Visit [nix.dev](https://nix.dev) for [installation instructions](https://nix.dev/tutorials/install-nix) and [beginner tutorials](https://nix.dev/tutorials/first-steps). -```console -$ curl -L https://nixos.org/nix/install | sh -``` - -Information on additional installation methods is available on the [Nix download page](https://nixos.org/download.html). +Full reference documentation can be found in the [Nix manual](https://nixos.org/nix/manual). ## Building And Developing See our [Hacking guide](https://nixos.org/manual/nix/unstable/contributing/hacking.html) in our manual for instruction on how to -to set up a development environment and build Nix from source. + set up a development environment and build Nix from source. + +## Contributing + +Check the [contributing guide](./CONTRIBUTING.md) if you want to get involved with developing Nix. ## Additional Resources @@ -29,7 +28,6 @@ to set up a development environment and build Nix from source. - [Nix jobsets on hydra.nixos.org](https://hydra.nixos.org/project/nix) - [NixOS Discourse](https://discourse.nixos.org/) - [Matrix - #nix:nixos.org](https://matrix.to/#/#nix:nixos.org) -- [IRC - #nixos on libera.chat](irc://irc.libera.chat/#nixos) ## License diff --git a/boehmgc-coroutine-sp-fallback.diff b/boehmgc-coroutine-sp-fallback.diff index 5066d8278ac2..2afbe96712bc 100644 --- a/boehmgc-coroutine-sp-fallback.diff +++ b/boehmgc-coroutine-sp-fallback.diff @@ -59,12 +59,18 @@ index b5d71e62..aed7b0bf 100644 GC_bool found_me = FALSE; size_t nthreads = 0; int i; -@@ -851,6 +853,31 @@ GC_INNER void GC_push_all_stacks(void) +@@ -851,6 +853,37 @@ GC_INNER void GC_push_all_stacks(void) hi = p->altstack + p->altstack_size; /* FIXME: Need to scan the normal stack too, but how ? */ /* FIXME: Assume stack grows down */ + } else { -+ if (pthread_getattr_np(p->id, &pattr)) { ++#ifdef HAVE_PTHREAD_ATTR_GET_NP ++ if (!pthread_attr_init(&pattr) ++ || !pthread_attr_get_np(p->id, &pattr)) ++#else /* HAVE_PTHREAD_GETATTR_NP */ ++ if (pthread_getattr_np(p->id, &pattr)) ++#endif ++ { + ABORT("GC_push_all_stacks: pthread_getattr_np failed!"); + } + if (pthread_attr_getstacksize(&pattr, &stack_limit)) { diff --git a/boehmgc-traceable_allocator-public.diff b/boehmgc-traceable_allocator-public.diff new file mode 100644 index 000000000000..903c707a6981 --- /dev/null +++ b/boehmgc-traceable_allocator-public.diff @@ -0,0 +1,12 @@ +diff --git a/include/gc_allocator.h b/include/gc_allocator.h +index 597c7f13..587286be 100644 +--- a/include/gc_allocator.h ++++ b/include/gc_allocator.h +@@ -312,6 +312,7 @@ public: + + template<> + class traceable_allocator { ++public: + typedef size_t size_type; + typedef ptrdiff_t difference_type; + typedef void* pointer; diff --git a/bootstrap.sh b/bootstrap.sh deleted file mode 100755 index e3e259351675..000000000000 --- a/bootstrap.sh +++ /dev/null @@ -1,4 +0,0 @@ -#! /bin/sh -e -rm -f aclocal.m4 -mkdir -p config -exec autoreconf -vfi diff --git a/configure.ac b/configure.ac index bb3f92e4d2c7..f8b937eb568d 100644 --- a/configure.ac +++ b/configure.ac @@ -5,7 +5,14 @@ AC_CONFIG_AUX_DIR(config) AC_PROG_SED -# Construct a Nix system name (like "i686-linux"). +# Construct a Nix system name (like "i686-linux"): +# https://www.gnu.org/software/autoconf/manual/html_node/Canonicalizing.html#index-AC_005fCANONICAL_005fHOST-1 +# The inital value is produced by the `config/config.guess` script: +# upstream: https://git.savannah.gnu.org/cgit/config.git/tree/config.guess +# It has the following form, which is not documented anywhere: +# --[][-] +# If `./configure` is passed any of the `--host`, `--build`, `--target` options, the value comes from `config/config.sub` instead: +# upstream: https://git.savannah.gnu.org/cgit/config.git/tree/config.sub AC_CANONICAL_HOST AC_MSG_CHECKING([for the canonical Nix system name]) @@ -61,6 +68,9 @@ case "$host_os" in esac +ENSURE_NO_GCC_BUG_80431 + + # Check for pubsetbuf. AC_MSG_CHECKING([for pubsetbuf]) AC_LANG_PUSH(C++) @@ -145,12 +155,29 @@ if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then LDFLAGS="-latomic $LDFLAGS" fi +# Running the functional tests without building Nix is useful for testing +# different pre-built versions of Nix against each other. +AC_ARG_ENABLE(build, AS_HELP_STRING([--disable-build],[Do not build nix]), + ENABLE_BUILD=$enableval, ENABLE_BUILD=yes) +AC_SUBST(ENABLE_BUILD) # Building without tests is useful for bootstrapping with a smaller footprint # or running the tests in a separate derivation. Otherwise, we do compile and # run them. AC_ARG_ENABLE(tests, AS_HELP_STRING([--disable-tests],[Do not build the tests]), - tests=$enableval, tests=yes) -AC_SUBST(tests) + ENABLE_TESTS=$enableval, ENABLE_TESTS=yes) +AC_SUBST(ENABLE_TESTS) + +AC_ARG_ENABLE(install-unit-tests, AS_HELP_STRING([--enable-install-unit-tests],[Install the unit tests for running later (default no)]), + INSTALL_UNIT_TESTS=$enableval, INSTALL_UNIT_TESTS=no) +AC_SUBST(INSTALL_UNIT_TESTS) + +AC_ARG_WITH(check-bin-dir, AS_HELP_STRING([--with-check-bin-dir=PATH],[path to install unit tests for running later (defaults to $libexecdir/nix)]), + checkbindir=$withval, checkbindir=$libexecdir/nix) +AC_SUBST(checkbindir) + +AC_ARG_WITH(check-lib-dir, AS_HELP_STRING([--with-check-lib-dir=PATH],[path to install unit tests for running later (defaults to $libdir)]), + checklibdir=$withval, checklibdir=$libdir) +AC_SUBST(checklibdir) # Building without API docs is the default as Nix' C++ interfaces are internal and unstable. AC_ARG_ENABLE(internal_api_docs, AS_HELP_STRING([--enable-internal-api-docs],[Build API docs for Nix's internal unstable C++ interfaces]), @@ -255,6 +282,8 @@ case "$host_os" in esac AC_SUBST(HAVE_SECCOMP, [$have_seccomp]) +# Optional dependencies for better normalizing file system data +AC_CHECK_HEADERS[sys/xattr.h] # Look for aws-cpp-sdk-s3. AC_LANG_PUSH(C++) @@ -282,7 +311,7 @@ if test "$gc" = yes; then fi -if test "$tests" = yes; then +if test "$ENABLE_TESTS" = yes; then # Look for gtest. PKG_CHECK_MODULES([GTEST], [gtest_main]) @@ -320,9 +349,15 @@ AC_ARG_ENABLE(doc-gen, AS_HELP_STRING([--disable-doc-gen],[disable documentation doc_generate=$enableval, doc_generate=yes) AC_SUBST(doc_generate) + # Look for lowdown library. PKG_CHECK_MODULES([LOWDOWN], [lowdown >= 0.9.0], [CXXFLAGS="$LOWDOWN_CFLAGS $CXXFLAGS"]) + +# Look for libgit2. +PKG_CHECK_MODULES([LIBGIT2], [libgit2]) + + # Setuid installations. AC_CHECK_FUNCS([setresuid setreuid lchown]) diff --git a/doc/internal-api/doxygen.cfg.in b/doc/internal-api/doxygen.cfg.in index 8f526536d53a..599be2470b8f 100644 --- a/doc/internal-api/doxygen.cfg.in +++ b/doc/internal-api/doxygen.cfg.in @@ -54,6 +54,23 @@ INPUT = \ src/nix-env \ src/nix-store +# If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names +# in the source code. If set to NO, only conditional compilation will be +# performed. Macro expansion can be done in a controlled way by setting +# EXPAND_ONLY_PREDEF to YES. +# The default value is: NO. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +MACRO_EXPANSION = YES + +# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then +# the macro expansion is limited to the macros specified with the PREDEFINED and +# EXPAND_AS_DEFINED tags. +# The default value is: NO. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +EXPAND_ONLY_PREDEF = YES + # The INCLUDE_PATH tag can be used to specify one or more directories that # contain include files that are not input files but should be processed by the # preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of @@ -61,3 +78,16 @@ INPUT = \ # This tag requires that the tag SEARCH_INCLUDES is set to YES. INCLUDE_PATH = @RAPIDCHECK_HEADERS@ + +# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this +# tag can be used to specify a list of macro names that should be expanded. The +# macro definition that is found in the sources will be used. Use the PREDEFINED +# tag if you want to use a different macro definition that overrules the +# definition found in the source code. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +EXPAND_AS_DEFINED = \ + DECLARE_COMMON_SERIALISER \ + DECLARE_WORKER_SERIALISER \ + DECLARE_SERVE_SERIALISER \ + LENGTH_PREFIXED_PROTO_HELPER diff --git a/doc/manual/_redirects b/doc/manual/_redirects new file mode 100644 index 000000000000..2038671d7b6c --- /dev/null +++ b/doc/manual/_redirects @@ -0,0 +1,40 @@ +# redirect rules for paths (server-side) to prevent link rot. +# see ./redirects.js for redirects based on URL fragments (client-side) +# +# concrete user story this supports: +# - user finds URL to the manual for Nix x.y +# - Nix x.z (z > y) is the most recent release +# - updating the version in the URL will show the right thing +# +# format documentation: +# - https://docs.netlify.com/routing/redirects/#syntax-for-the-redirects-file +# - https://docs.netlify.com/routing/redirects/redirect-options/ +# +# conventions: +# - always force (!) since this allows re-using file names +# - group related paths to ease readability +# - keep in alphabetical/wildcards-last order, which will reduce version control conflicts +# - redirects that should have been there but are missing can be inserted where they belong + +/advanced-topics/advanced-topics /advanced-topics 301! + +/command-ref/command-ref /command-ref 301! + +/contributing/contributing /contributing 301! + +/expressions/expression-language /language/ 301! +/expressions/language-constructs /language/constructs 301! +/expressions/language-operators /language/operators 301! +/expressions/language-values /language/values 301! +/expressions/* /language/:splat 301! + +/installation/installation /installation 301! + +/package-management/basic-package-mgmt /command-ref/nix-env 301! +/package-management/channels* /command-ref/nix-channel 301! +/package-management/package-management /package-management 301! +/package-management/s3-substituter* /command-ref/new-cli/nix3-help-stores#s3-binary-cache-store 301! + +/protocols/protocols /protocols 301! + +/release-notes/release-notes /release-notes 301! diff --git a/doc/manual/generate-builtin-constants.nix b/doc/manual/generate-builtin-constants.nix index 3fc1fae421d1..cccd1e279e62 100644 --- a/doc/manual/generate-builtin-constants.nix +++ b/doc/manual/generate-builtin-constants.nix @@ -1,6 +1,6 @@ let inherit (builtins) concatStringsSep attrValues mapAttrs; - inherit (import ./utils.nix) optionalString squash; + inherit (import ) optionalString squash; in builtinsInfo: @@ -10,12 +10,14 @@ let type' = optionalString (type != null) " (${type})"; impureNotice = optionalString impure-only '' - Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). + > **Note** + > + > Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). ''; in squash '' -
- ${name}${type'} +
+ ${name}${type'}
diff --git a/doc/manual/generate-builtins.nix b/doc/manual/generate-builtins.nix index 813a287f5213..05cae1c467fb 100644 --- a/doc/manual/generate-builtins.nix +++ b/doc/manual/generate-builtins.nix @@ -1,6 +1,6 @@ let inherit (builtins) concatStringsSep attrValues mapAttrs; - inherit (import ./utils.nix) optionalString squash; + inherit (import ) optionalString squash; in builtinsInfo: diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index fb34898f330d..365422af79fc 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -1,11 +1,32 @@ let inherit (builtins) - attrNames attrValues fromJSON listToAttrs mapAttrs - concatStringsSep concatMap length lessThan replaceStrings sort; - inherit (import ./utils.nix) concatStrings optionalString filterAttrs trim squash unique showSettings; + attrNames + attrValues + concatMap + concatStringsSep + fromJSON + groupBy + length + lessThan + listToAttrs + mapAttrs + match + replaceStrings + sort + ; + inherit (import ) + attrsToList + concatStrings + filterAttrs + optionalString + squash + trim + unique + ; + showStoreDocs = import ; in -commandDump: +inlineHTML: commandDump: let @@ -30,7 +51,7 @@ let ${maybeSubcommands} - ${maybeDocumentation} + ${maybeProse} ${maybeOptions} ''; @@ -40,15 +61,15 @@ let showArgument = arg: "*${arg.label}*" + optionalString (! arg ? arity) "..."; arguments = concatStringsSep " " (map showArgument args); in '' - `${command}` [*option*...] ${arguments} + `${command}` [*option*...] ${arguments} ''; maybeSubcommands = optionalString (details ? commands && details.commands != {}) - '' - where *subcommand* is one of the following: + '' + where *subcommand* is one of the following: - ${subcommands} - ''; + ${subcommands} + ''; subcommands = if length categories > 1 then listCategories @@ -70,40 +91,87 @@ let * [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description} ''; - maybeDocumentation = optionalString - (details ? doc) - (replaceStrings ["@stores@"] [storeDocs] details.doc); - - maybeOptions = optionalString (details.flags != {}) '' - # Options - - ${showOptions details.flags toplevel.flags} - ''; + maybeProse = + # FIXME: this is a horrible hack to keep `nix help-stores` working. + # the correct answer to this is to remove that command and replace it + # by statically generated manpages or the output of something like `nix + # store info `. + let + help-stores = '' + ${index} + + ${allStores} + ''; + index = replaceStrings + [ "@store-types@" ] [ storesOverview ] + details.doc; + storesOverview = + let + showEntry = store: + "- [${store.name}](#${store.slug})"; + in + concatStringsSep "\n" (map showEntry storesList) + "\n"; + allStores = concatStringsSep "\n" (attrValues storePages); + storePages = listToAttrs + (map (s: { name = s.filename; value = s.page; }) storesList); + storesList = showStoreDocs { + storeInfo = commandInfo.stores; + inherit inlineHTML; + }; + in + optionalString (details ? doc) ( + if match "@store-types@" details.doc != [ ] + then help-stores + else details.doc + ); + + maybeOptions = + let + allVisibleOptions = filterAttrs + (_: o: ! o.hiddenCategory) + (details.flags // toplevel.flags); + in + optionalString (allVisibleOptions != { }) '' + # Options + + ${showOptions inlineHTML allVisibleOptions} + + > **Note** + > + > See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags. + ''; - showOptions = options: commonOptions: + showOptions = inlineHTML: allOptions: let - allOptions = options // commonOptions; - showCategory = cat: '' - ${optionalString (cat != "") "**${cat}:**"} + showCategory = cat: opts: '' + ${optionalString (cat != "") "## ${cat}"} - ${listOptions (filterAttrs (n: v: v.category == cat) allOptions)} - ''; - listOptions = opts: concatStringsSep "\n" (attrValues (mapAttrs showOption opts)); + ${concatStringsSep "\n" (attrValues (mapAttrs showOption opts))} + ''; showOption = name: option: let + result = trim '' + - ${item} + + ${option.description} + ''; + item = if inlineHTML + then ''[`--${name}`](#opt-${name}) ${shortName} ${labels}'' + else "`--${name}` ${shortName} ${labels}"; shortName = optionalString (option ? shortName) ("/ `-${option.shortName}`"); labels = optionalString (option ? labels) (concatStringsSep " " (map (s: "*${s}*") option.labels)); - in trim '' - - `--${name}` ${shortName} ${labels} - - ${option.description} - ''; - categories = sort lessThan (unique (map (cmd: cmd.category) (attrValues allOptions))); - in concatStrings (map showCategory categories); + in result; + categories = mapAttrs + # Convert each group from a list of key-value pairs back to an attrset + (_: listToAttrs) + (groupBy + (cmd: cmd.value.category) + (attrsToList allOptions)); + in concatStrings (attrValues (mapAttrs showCategory categories)); in squash result; appendName = filename: name: (if filename == "nix" then "nix3" else filename) + "-" + name; @@ -135,18 +203,4 @@ let " - [${page.command}](command-ref/new-cli/${page.name})"; in concatStringsSep "\n" (map showEntry manpages) + "\n"; - storeDocs = - let - showStore = name: { settings, doc }: - '' - ## ${name} - - ${doc} - - **Settings**: - - ${showSettings { useAnchors = false; } settings} - ''; - in concatStrings (attrValues (mapAttrs showStore commandInfo.stores)); - in (listToAttrs manpages) // { "SUMMARY.md" = tableOfContents; } diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix new file mode 100644 index 000000000000..74446b70b301 --- /dev/null +++ b/doc/manual/generate-settings.nix @@ -0,0 +1,67 @@ +let + inherit (builtins) attrValues concatStringsSep isAttrs isBool mapAttrs; + inherit (import ) concatStrings indent optionalString squash; +in + +# `inlineHTML` is a hack to accommodate inconsistent output from `lowdown` +{ prefix, inlineHTML ? true }: settingsInfo: + +let + + showSetting = prefix: setting: { description, documentDefault, defaultValue, aliases, value, experimentalFeature }: + let + result = squash '' + - ${item} + + ${indent " " body} + ''; + item = if inlineHTML + then ''[`${setting}`](#${prefix}-${setting})'' + else "`${setting}`"; + # separate body to cleanly handle indentation + body = '' + ${description} + + ${experimentalFeatureNote} + + **Default:** ${showDefault documentDefault defaultValue} + + ${showAliases aliases} + ''; + + experimentalFeatureNote = optionalString (experimentalFeature != null) '' + > **Warning** + > + > This setting is part of an + > [experimental feature](@docroot@/contributing/experimental-features.md). + > + > To change this setting, make sure the + > [`${experimentalFeature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}) + > is enabled. + > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): + > + > ``` + > extra-experimental-features = ${experimentalFeature} + > ${setting} = ... + > ``` + ''; + + showDefault = documentDefault: defaultValue: + if documentDefault then + # a StringMap value type is specified as a string, but + # this shows the value type. The empty stringmap is `null` in + # JSON, but that converts to `{ }` here. + if defaultValue == "" || defaultValue == [] || isAttrs defaultValue + then "*empty*" + else if isBool defaultValue then + if defaultValue then "`true`" else "`false`" + else "`${toString defaultValue}`" + else "*machine-specific*"; + + showAliases = aliases: + optionalString (aliases != []) + "**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}"; + + in result; + +in concatStrings (attrValues (mapAttrs (showSetting prefix) settingsInfo)) diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix new file mode 100644 index 000000000000..57247a181b98 --- /dev/null +++ b/doc/manual/generate-store-info.nix @@ -0,0 +1,57 @@ +let + inherit (builtins) attrNames listToAttrs concatStringsSep readFile replaceStrings; + inherit (import ) optionalString filterAttrs trim squash toLower unique indent; + showSettings = import ; +in + +{ + # data structure describing all stores and their parameters + storeInfo, + # whether to add inline HTML tags + # `lowdown` does not eat those for one of the output modes + inlineHTML, +}: + +let + + showStore = { name, slug }: { settings, doc, experimentalFeature }: + let + result = squash '' + # ${name} + + ${doc} + + ${experimentalFeatureNote} + + ## Settings + + ${showSettings { prefix = "store-${slug}"; inherit inlineHTML; } settings} + ''; + + experimentalFeatureNote = optionalString (experimentalFeature != null) '' + > **Warning** + > + > This store is part of an + > [experimental feature](@docroot@/contributing/experimental-features.md). + > + > To use this store, make sure the + > [`${experimentalFeature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}) + > is enabled. + > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): + > + > ``` + > extra-experimental-features = ${experimentalFeature} + > ``` + ''; + in result; + + storesList = map + (name: rec { + inherit name; + slug = replaceStrings [ " " ] [ "-" ] (toLower name); + filename = "${slug}.md"; + page = showStore { inherit name slug; } storeInfo.${name}; + }) + (attrNames storeInfo); + +in storesList diff --git a/doc/manual/generate-store-types.nix b/doc/manual/generate-store-types.nix new file mode 100644 index 000000000000..3b78a0e1bb6f --- /dev/null +++ b/doc/manual/generate-store-types.nix @@ -0,0 +1,39 @@ +let + inherit (builtins) attrNames listToAttrs concatStringsSep readFile replaceStrings; + showSettings = import ; + showStoreDocs = import ; +in + +storeInfo: + +let + storesList = showStoreDocs { + inherit storeInfo; + inlineHTML = true; + }; + + index = + let + showEntry = store: + "- [${store.name}](./${store.filename})"; + in + concatStringsSep "\n" (map showEntry storesList); + + "index.md" = replaceStrings + [ "@store-types@" ] [ index ] + (readFile ./src/store/types/index.md.in); + + tableOfContents = + let + showEntry = store: + " - [${store.name}](store/types/${store.filename})"; + in + concatStringsSep "\n" (map showEntry storesList) + "\n"; + + "SUMMARY.md" = tableOfContents; + + storePages = listToAttrs + (map (s: { name = s.filename; value = s.page; }) storesList); + +in +storePages // { inherit "index.md" "SUMMARY.md"; } diff --git a/doc/manual/generate-xp-features-shortlist.nix b/doc/manual/generate-xp-features-shortlist.nix index 30e211c96b4e..ec09f4b75f76 100644 --- a/doc/manual/generate-xp-features-shortlist.nix +++ b/doc/manual/generate-xp-features-shortlist.nix @@ -1,5 +1,5 @@ with builtins; -with import ./utils.nix; +with import ; let showExperimentalFeature = name: doc: diff --git a/doc/manual/generate-xp-features.nix b/doc/manual/generate-xp-features.nix index adb94355c832..0eec0e1da237 100644 --- a/doc/manual/generate-xp-features.nix +++ b/doc/manual/generate-xp-features.nix @@ -1,5 +1,5 @@ with builtins; -with import ./utils.nix; +with import ; let showExperimentalFeature = name: doc: @@ -8,4 +8,6 @@ let ${doc} ''; -in xps: (concatStringsSep "\n" (attrValues (mapAttrs showExperimentalFeature xps))) +in + +xps: (concatStringsSep "\n" (attrValues (mapAttrs showExperimentalFeature xps))) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index abdfd6a622b4..456000d3d5bf 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -1,5 +1,10 @@ ifeq ($(doc_generate),yes) +# The version of Nix used to generate the doc. Can also be +# `$(nix_INSTALL_PATH)` or just `nix` (to grap ambient from the `PATH`), +# if one prefers. +doc_nix = $(nix_PATH) + MANUAL_SRCS := \ $(call rwildcard, $(d)/src, *.md) \ $(call rwildcard, $(d)/src, */*.md) @@ -24,7 +29,7 @@ man-pages += $(foreach subcommand, \ clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8 # Provide a dummy environment for nix, so that it will not access files outside the macOS sandbox. -# Set cores to 0 because otherwise nix show-config resolves the cores based on the current machine +# Set cores to 0 because otherwise `nix config show` resolves the cores based on the current machine dummy-env = env -i \ HOME=/dummy \ NIX_CONF_DIR=/dummy \ @@ -32,7 +37,7 @@ dummy-env = env -i \ NIX_STATE_DIR=/dummy \ NIX_CONFIG='cores = 0' -nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw +nix-eval = $(dummy-env) $(doc_nix) eval --experimental-features nix-command -I nix=doc/manual --store dummy:// --impure --raw # re-implement mdBook's include directive to make it usable for terminal output and for proper @docroot@ substitution define process-includes @@ -92,58 +97,83 @@ $(d)/nix-profiles.5: $(d)/src/command-ref/files/profiles.md $(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@ @rm $^.tmp -$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md +$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md @cp $< $@ @$(call process-includes,$@,$@) -$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(bindir)/nix +$(d)/src/store/types: $(d)/nix.json $(d)/utils.nix $(d)/generate-store-info.nix $(d)/generate-store-types.nix $(d)/src/store/types/index.md.in $(doc_nix) + @# FIXME: build out of tree! + @rm -rf $@.tmp + $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-store-types.nix (builtins.fromJSON (builtins.readFile $<)).stores' + @# do not destroy existing contents + @mv $@.tmp/* $@/ + +$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(doc_nix) @rm -rf $@ $@.tmp - $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix (builtins.readFile $<)' + $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix true (builtins.readFile $<)' @mv $@.tmp $@ -$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix +$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(doc_nix) @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) --expr '(import doc/manual/utils.nix).showSettings { useAnchors = true; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; + $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-settings.nix { prefix = "conf"; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; @mv $@.tmp $@ -$(d)/nix.json: $(bindir)/nix - $(trace-gen) $(dummy-env) $(bindir)/nix __dump-cli > $@.tmp +$(d)/nix.json: $(doc_nix) + $(trace-gen) $(dummy-env) $(doc_nix) __dump-cli > $@.tmp @mv $@.tmp $@ -$(d)/conf-file.json: $(bindir)/nix - $(trace-gen) $(dummy-env) $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp +$(d)/conf-file.json: $(doc_nix) + $(trace-gen) $(dummy-env) $(doc_nix) config show --json --experimental-features nix-command > $@.tmp @mv $@.tmp $@ -$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(bindir)/nix +$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(doc_nix) @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))' @mv $@.tmp $@ -$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(bindir)/nix +$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(doc_nix) @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features-shortlist.nix (builtins.fromJSON (builtins.readFile $<))' @mv $@.tmp $@ -$(d)/xp-features.json: $(bindir)/nix - $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-xp-features > $@.tmp +$(d)/xp-features.json: $(doc_nix) + $(trace-gen) $(dummy-env) $(doc_nix) __dump-xp-features > $@.tmp @mv $@.tmp $@ -$(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix +$(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(doc_nix) @cat doc/manual/src/language/builtins-prefix.md > $@.tmp $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<)).builtins' >> $@.tmp; @cat doc/manual/src/language/builtins-suffix.md >> $@.tmp @mv $@.tmp $@ -$(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin-constants.nix $(d)/src/language/builtin-constants-prefix.md $(bindir)/nix +$(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin-constants.nix $(d)/src/language/builtin-constants-prefix.md $(doc_nix) @cat doc/manual/src/language/builtin-constants-prefix.md > $@.tmp $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtin-constants.nix (builtins.fromJSON (builtins.readFile $<)).constants' >> $@.tmp; @cat doc/manual/src/language/builtin-constants-suffix.md >> $@.tmp @mv $@.tmp $@ -$(d)/language.json: $(bindir)/nix - $(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-language > $@.tmp +$(d)/language.json: $(doc_nix) + $(trace-gen) $(dummy-env) $(doc_nix) __dump-language > $@.tmp @mv $@.tmp $@ +# Generate "Upcoming release" notes (or clear it and remove from menu) +$(d)/src/release-notes/rl-next.md: $(d)/rl-next $(d)/rl-next/* + @if type -p changelog-d > /dev/null; then \ + echo " GEN " $@; \ + changelog-d doc/manual/rl-next > $@; \ + else \ + echo " NULL " $@; \ + true > $@; \ + fi + +$(d)/src/SUMMARY-rl-next.md: $(d)/src/release-notes/rl-next.md + $(trace-gen) true + @if [ -s $< ]; then \ + echo ' - [Upcoming release](release-notes/rl-next.md)' > $@; \ + else \ + true > $@; \ + fi + # Generate the HTML manual. .PHONY: manual-html manual-html: $(docdir)/manual/index.html @@ -173,14 +203,18 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli done @touch $@ -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md +# the `! -name 'contributing.md'` filter excludes the one place where +# `@docroot@` is to be preserved for documenting the mechanism +# FIXME: maybe contributing guides should live right next to the code +# instead of in the manual +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md $(trace-gen) \ tmp="$$(mktemp -d)"; \ cp -r doc/manual "$$tmp"; \ find "$$tmp" -name '*.md' | while read -r file; do \ $(call process-includes,$$file,$$file); \ done; \ - find "$$tmp" -name '*.md' | while read -r file; do \ + find "$$tmp" -name '*.md' ! -name 'documentation.md' | while read -r file; do \ docroot="$$(realpath --relative-to="$$(dirname "$$file")" $$tmp/manual/src)"; \ sed -i "s,@docroot@,$$docroot,g" "$$file"; \ done; \ diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index dcdb5d6e9688..3b507adf3418 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -1,7 +1,9 @@ -// redirect rules for anchors ensure backwards compatibility of URLs. -// this must be done on the client side, as web servers do not see the anchor part of the URL. +// redirect rules for URL fragments (client-side) to prevent link rot. +// this must be done on the client side, as web servers do not see the fragment part of the URL. +// it will only work with JavaScript enabled in the browser, but this is the best we can do here. +// see ./_redirects for path redirects (client-side) -// redirections are declared as follows: +// redirects are declared as follows: // each entry has as its key a path matching the requested URL path, relative to the mdBook document root. // // IMPORTANT: it must specify the full path with file name and suffix @@ -281,7 +283,7 @@ const redirects = { "chap-introduction": "introduction.html", "ch-basic-package-mgmt": "package-management/basic-package-mgmt.html", "ssec-binary-cache-substituter": "package-management/binary-cache-substituter.html", - "sec-channels": "package-management/channels.html", + "sec-channels": "command-ref/nix-channel.html", "ssec-copy-closure": "package-management/copy-closure.html", "sec-garbage-collection": "package-management/garbage-collection.html", "ssec-gc-roots": "package-management/garbage-collector-roots.html", @@ -336,14 +338,13 @@ const redirects = { "simple-values": "#primitives", "lists": "#list", "strings": "#string", - "lists": "#list", "attribute-sets": "#attribute-set", }, "installation/installing-binary.html": { "linux": "uninstall.html#linux", "macos": "uninstall.html#macos", "uninstalling": "uninstall.html", - } + }, "contributing/hacking.html": { "nix-with-flakes": "#building-nix-with-flakes", "classic-nix": "#building-nix", @@ -355,6 +356,7 @@ const redirects = { "installer-tests": "testing.html#installer-tests", "one-time-setup": "testing.html#one-time-setup", "using-the-ci-generated-installer-for-manual-testing": "testing.html#using-the-ci-generated-installer-for-manual-testing", + "characterization-testing": "#characterisation-testing-unit", } }; diff --git a/doc/manual/rl-next/config b/doc/manual/rl-next/config new file mode 100644 index 000000000000..b3c2e868f968 --- /dev/null +++ b/doc/manual/rl-next/config @@ -0,0 +1,2 @@ +organization: NixOS +repository: nix diff --git a/doc/manual/rl-next/mounted-ssh-store.md b/doc/manual/rl-next/mounted-ssh-store.md new file mode 100644 index 000000000000..39fac5283180 --- /dev/null +++ b/doc/manual/rl-next/mounted-ssh-store.md @@ -0,0 +1,9 @@ +synopsis: Mounted SSH Store +issues: #7890 +prs: #7912 +description: { + +Introduced the store [`mounted-ssh-ng://`](@docroot@/command-ref/new-cli/nix3-help-stores.md). +This store allows full access to a Nix store on a remote machine and additionally requires that the store be mounted in the local filesystem. + +} diff --git a/doc/manual/rl-next/nix-config-show.md b/doc/manual/rl-next/nix-config-show.md new file mode 100644 index 000000000000..08ad207cb29d --- /dev/null +++ b/doc/manual/rl-next/nix-config-show.md @@ -0,0 +1,8 @@ +synopsis: `nix config show` +issues: #7672 +prs: #9477 +description: { + +`nix show-config` was renamed to `nix config show` to be more consistent with the rest of the command-line interface. + +} diff --git a/doc/manual/rl-next/nix-env-json-drv-path.md b/doc/manual/rl-next/nix-env-json-drv-path.md new file mode 100644 index 000000000000..fbe2b67d888d --- /dev/null +++ b/doc/manual/rl-next/nix-env-json-drv-path.md @@ -0,0 +1,9 @@ +synopsis: Fix `nix-env --query --drv-path --json` +prs: #9257 +description: { + +Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set. + +} + + diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 1bd8fa774d69..c67ddc6cbbe8 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -2,7 +2,7 @@ - [Introduction](introduction.md) - [Quick Start](quick-start.md) -- [Installation](installation/installation.md) +- [Installation](installation/index.md) - [Supported Platforms](installation/supported-platforms.md) - [Installing a Binary Distribution](installation/installing-binary.md) - [Installing Nix from Source](installation/installing-source.md) @@ -16,32 +16,38 @@ - [Environment Variables](installation/env-variables.md) - [Upgrading Nix](installation/upgrading.md) - [Uninstalling Nix](installation/uninstall.md) -- [Package Management](package-management/package-management.md) - - [Basic Package Management](package-management/basic-package-mgmt.md) - - [Profiles](package-management/profiles.md) - - [Garbage Collection](package-management/garbage-collection.md) - - [Garbage Collector Roots](package-management/garbage-collector-roots.md) - - [Channels](package-management/channels.md) - - [Sharing Packages Between Machines](package-management/sharing-packages.md) - - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md) - - [Copying Closures via SSH](package-management/copy-closure.md) - - [Serving a Nix store via SSH](package-management/ssh-substituter.md) - - [Serving a Nix store via S3](package-management/s3-substituter.md) +- [Nix Store](store/index.md) + - [File System Object](store/file-system-object.md) + - [Store Object](store/store-object.md) + - [Store Path](store/store-path.md) + - [Store Types](store/types/index.md) +{{#include ./store/types/SUMMARY.md}} - [Nix Language](language/index.md) - [Data Types](language/values.md) - [Language Constructs](language/constructs.md) - [String interpolation](language/string-interpolation.md) + - [Lookup path](language/constructs/lookup-path.md) - [Operators](language/operators.md) - [Derivations](language/derivations.md) - [Advanced Attributes](language/advanced-attributes.md) + - [Import From Derivation](language/import-from-derivation.md) - [Built-in Constants](language/builtin-constants.md) - [Built-in Functions](language/builtins.md) -- [Advanced Topics](advanced-topics/advanced-topics.md) +- [Package Management](package-management/index.md) + - [Profiles](package-management/profiles.md) + - [Garbage Collection](package-management/garbage-collection.md) + - [Garbage Collector Roots](package-management/garbage-collector-roots.md) +- [Advanced Topics](advanced-topics/index.md) + - [Sharing Packages Between Machines](package-management/sharing-packages.md) + - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md) + - [Copying Closures via SSH](package-management/copy-closure.md) + - [Serving a Nix store via SSH](package-management/ssh-substituter.md) + - [Serving a Nix store via S3](package-management/s3-substituter.md) - [Remote Builds](advanced-topics/distributed-builds.md) - [Tuning Cores and Jobs](advanced-topics/cores-vs-jobs.md) - [Verifying Build Reproducibility](advanced-topics/diff-hook.md) - [Using the `post-build-hook`](advanced-topics/post-build-hook.md) -- [Command Reference](command-ref/command-ref.md) +- [Command Reference](command-ref/index.md) - [Common Options](command-ref/opt-common.md) - [Common Environment Variables](command-ref/env-common.md) - [Main Commands](command-ref/main-commands.md) @@ -98,18 +104,22 @@ - [Channels](command-ref/files/channels.md) - [Default Nix expression](command-ref/files/default-nix-expression.md) - [Architecture and Design](architecture/architecture.md) - - [File System Object](architecture/file-system-object.md) -- [Protocols](protocols/protocols.md) +- [Protocols](protocols/index.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md) + - [Derivation "ATerm" file format](protocols/derivation-aterm.md) - [Glossary](glossary.md) -- [Contributing](contributing/contributing.md) +- [Contributing](contributing/index.md) - [Hacking](contributing/hacking.md) - [Testing](contributing/testing.md) + - [Documentation](contributing/documentation.md) - [Experimental Features](contributing/experimental-features.md) - [CLI guideline](contributing/cli-guideline.md) - [C++ style guide](contributing/cxx.md) -- [Release Notes](release-notes/release-notes.md) - - [Release X.Y (202?-??-??)](release-notes/rl-next.md) +- [Release Notes](release-notes/index.md) +{{#include ./SUMMARY-rl-next.md}} + - [Release 2.19 (2023-11-17)](release-notes/rl-2.19.md) + - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md) + - [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md) - [Release 2.16 (2023-05-31)](release-notes/rl-2.16.md) - [Release 2.15 (2023-04-11)](release-notes/rl-2.15.md) - [Release 2.14 (2023-02-28)](release-notes/rl-2.14.md) diff --git a/doc/manual/src/advanced-topics/advanced-topics.md b/doc/manual/src/advanced-topics/advanced-topics.md deleted file mode 100644 index 8b137891791f..000000000000 --- a/doc/manual/src/advanced-topics/advanced-topics.md +++ /dev/null @@ -1 +0,0 @@ - diff --git a/doc/manual/src/advanced-topics/distributed-builds.md b/doc/manual/src/advanced-topics/distributed-builds.md index 73a113d352ba..507c5ecb7691 100644 --- a/doc/manual/src/advanced-topics/distributed-builds.md +++ b/doc/manual/src/advanced-topics/distributed-builds.md @@ -12,14 +12,14 @@ machine is accessible via SSH and that it has Nix installed. You can test whether connecting to the remote Nix instance works, e.g. ```console -$ nix store ping --store ssh://mac +$ nix store info --store ssh://mac ``` will try to connect to the machine named `mac`. It is possible to specify an SSH identity file as part of the remote store URI, e.g. ```console -$ nix store ping --store ssh://mac?ssh-key=/home/alice/my-key +$ nix store info --store ssh://mac?ssh-key=/home/alice/my-key ``` Since builds should be non-interactive, the key should not have a diff --git a/doc/manual/src/advanced-topics/index.md b/doc/manual/src/advanced-topics/index.md new file mode 100644 index 000000000000..9a4d12a335cf --- /dev/null +++ b/doc/manual/src/advanced-topics/index.md @@ -0,0 +1 @@ +This section lists advanced topics related to builds and builds performance diff --git a/doc/manual/src/advanced-topics/post-build-hook.md b/doc/manual/src/advanced-topics/post-build-hook.md index a251dec4886b..3c1cc9b369a0 100644 --- a/doc/manual/src/advanced-topics/post-build-hook.md +++ b/doc/manual/src/advanced-topics/post-build-hook.md @@ -17,9 +17,8 @@ the build loop. # Prerequisites -This tutorial assumes you have [configured an S3-compatible binary -cache](../package-management/s3-substituter.md), and that the `root` -user's default AWS profile can upload to the bucket. +This tutorial assumes you have configured an [S3-compatible binary cache](@docroot@/command-ref/new-cli/nix3-help-stores.md#s3-binary-cache-store) as a [substituter](../command-ref/conf-file.md#conf-substituters), +and that the `root` user's default AWS profile can upload to the bucket. # Set up a Signing Key @@ -69,6 +68,8 @@ exec nix copy --to "s3://example-nix-cache" $OUT_PATHS > store sign`. Nix guarantees the paths will not contain any spaces, > however a store path might contain glob characters. The `set -f` > disables globbing in the shell. +> If you want to upload the `.drv` file too, the `$DRV_PATH` variable +> is also defined for the script and works just like `$OUT_PATHS`. Then make sure the hook program is executable by the `root` user: diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 9e969972e7fb..2fec4ed20e18 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -52,17 +52,18 @@ The following [concept map] shows its main components (rectangles), the objects '---------------' ``` -At the top is the [command line interface](../command-ref/command-ref.md) that drives the underlying layers. +At the top is the [command line interface](../command-ref/index.md) that drives the underlying layers. The [Nix language](../language/index.md) evaluator transforms Nix expressions into self-contained *build plans*, which are used to derive *build results* from referenced *build inputs*. The command line interface and Nix expressions are what users deal with most. > **Note** +> > The Nix language itself does not have a notion of *packages* or *configurations*. > As far as we are concerned here, the inputs and results of a build plan are just data. -Underlying the command line interface and the Nix language evaluator is the [Nix store](../glossary.md#gloss-store), a mechanism to keep track of build plans, data, and references between them. +Underlying the command line interface and the Nix language evaluator is the [Nix store](../store/index.md), a mechanism to keep track of build plans, data, and references between them. It can also execute build plans to produce new data, which are made available to the operating system as files. A build plan itself is a series of *build tasks*, together with their build inputs. diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md index b4a9bb2a9757..34e0dbfbd91b 100644 --- a/doc/manual/src/command-ref/env-common.md +++ b/doc/manual/src/command-ref/env-common.md @@ -2,109 +2,124 @@ Most Nix commands interpret the following environment variables: - - [`IN_NIX_SHELL`](#env-IN_NIX_SHELL)\ - Indicator that tells if the current environment was set up by - `nix-shell`. It can have the values `pure` or `impure`. - - - [`NIX_PATH`](#env-NIX_PATH)\ - A colon-separated list of directories used to look up the location of Nix - expressions using [paths](@docroot@/language/values.md#type-path) - enclosed in angle brackets (i.e., ``), - e.g. `/home/eelco/Dev:/etc/nixos`. It can be extended using the - [`-I` option](@docroot@/command-ref/opt-common.md#opt-I). - - If `NIX_PATH` is not set at all, Nix will fall back to the following list in [impure](@docroot@/command-ref/conf-file.md#conf-pure-eval) and [unrestricted](@docroot@/command-ref/conf-file.md#conf-restrict-eval) evaluation mode: - - 1. `$HOME/.nix-defexpr/channels` - 2. `nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixpkgs` - 3. `/nix/var/nix/profiles/per-user/root/channels` - - If `NIX_PATH` is set to an empty string, resolving search paths will always fail. - For example, attempting to use `` will produce: - - error: file 'nixpkgs' was not found in the Nix search path - - - [`NIX_IGNORE_SYMLINK_STORE`](#env-NIX_IGNORE_SYMLINK_STORE)\ - Normally, the Nix store directory (typically `/nix/store`) is not - allowed to contain any symlink components. This is to prevent - “impure” builds. Builders sometimes “canonicalise” paths by - resolving all symlink components. Thus, builds on different machines - (with `/nix/store` resolving to different locations) could yield - different results. This is generally not a problem, except when - builds are deployed to machines where `/nix/store` resolves - differently. If you are sure that you’re not going to do that, you - can set `NIX_IGNORE_SYMLINK_STORE` to `1`. - - Note that if you’re symlinking the Nix store so that you can put it - on another file system than the root file system, on Linux you’re - better off using `bind` mount points, e.g., - - ```console - $ mkdir /nix - $ mount -o bind /mnt/otherdisk/nix /nix - ``` - - Consult the mount 8 manual page for details. - - - [`NIX_STORE_DIR`](#env-NIX_STORE_DIR)\ - Overrides the location of the Nix store (default `prefix/store`). - - - [`NIX_DATA_DIR`](#env-NIX_DATA_DIR)\ - Overrides the location of the Nix static data directory (default - `prefix/share`). - - - [`NIX_LOG_DIR`](#env-NIX_LOG_DIR)\ - Overrides the location of the Nix log directory (default - `prefix/var/log/nix`). - - - [`NIX_STATE_DIR`](#env-NIX_STATE_DIR)\ - Overrides the location of the Nix state directory (default - `prefix/var/nix`). - - - [`NIX_CONF_DIR`](#env-NIX_CONF_DIR)\ - Overrides the location of the system Nix configuration directory - (default `prefix/etc/nix`). - - - [`NIX_CONFIG`](#env-NIX_CONFIG)\ - Applies settings from Nix configuration from the environment. - The content is treated as if it was read from a Nix configuration file. - Settings are separated by the newline character. - - - [`NIX_USER_CONF_FILES`](#env-NIX_USER_CONF_FILES)\ - Overrides the location of the Nix user configuration files to load from. - - The default are the locations according to the [XDG Base Directory Specification]. - See the [XDG Base Directories](#xdg-base-directories) sub-section for details. - - The variable is treated as a list separated by the `:` token. - - - [`TMPDIR`](#env-TMPDIR)\ - Use the specified directory to store temporary files. In particular, - this includes temporary build directories; these can take up - substantial amounts of disk space. The default is `/tmp`. - - - [`NIX_REMOTE`](#env-NIX_REMOTE)\ - This variable should be set to `daemon` if you want to use the Nix - daemon to execute Nix operations. This is necessary in [multi-user - Nix installations](@docroot@/installation/multi-user.md). If the Nix - daemon's Unix socket is at some non-standard path, this variable - should be set to `unix://path/to/socket`. Otherwise, it should be - left unset. - - - [`NIX_SHOW_STATS`](#env-NIX_SHOW_STATS)\ - If set to `1`, Nix will print some evaluation statistics, such as - the number of values allocated. - - - [`NIX_COUNT_CALLS`](#env-NIX_COUNT_CALLS)\ - If set to `1`, Nix will print how often functions were called during - Nix expression evaluation. This is useful for profiling your Nix - expressions. - - - [`GC_INITIAL_HEAP_SIZE`](#env-GC_INITIAL_HEAP_SIZE)\ - If Nix has been configured to use the Boehm garbage collector, this - variable sets the initial size of the heap in bytes. It defaults to - 384 MiB. Setting it to a low value reduces memory consumption, but - will increase runtime due to the overhead of garbage collection. +- [`IN_NIX_SHELL`](#env-IN_NIX_SHELL) + + Indicator that tells if the current environment was set up by + `nix-shell`. It can have the values `pure` or `impure`. + +- [`NIX_PATH`](#env-NIX_PATH) + + A colon-separated list of directories used to look up the location of Nix + expressions using [paths](@docroot@/language/values.md#type-path) + enclosed in angle brackets (i.e., ``), + e.g. `/home/eelco/Dev:/etc/nixos`. It can be extended using the + [`-I` option](@docroot@/command-ref/opt-common.md#opt-I). + + If `NIX_PATH` is not set at all, Nix will fall back to the following list in [impure](@docroot@/command-ref/conf-file.md#conf-pure-eval) and [unrestricted](@docroot@/command-ref/conf-file.md#conf-restrict-eval) evaluation mode: + + 1. `$HOME/.nix-defexpr/channels` + 2. `nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixpkgs` + 3. `/nix/var/nix/profiles/per-user/root/channels` + + If `NIX_PATH` is set to an empty string, resolving search paths will always fail. + For example, attempting to use `` will produce: + + error: file 'nixpkgs' was not found in the Nix search path + +- [`NIX_IGNORE_SYMLINK_STORE`](#env-NIX_IGNORE_SYMLINK_STORE) + + Normally, the Nix store directory (typically `/nix/store`) is not + allowed to contain any symlink components. This is to prevent + “impure” builds. Builders sometimes “canonicalise” paths by + resolving all symlink components. Thus, builds on different machines + (with `/nix/store` resolving to different locations) could yield + different results. This is generally not a problem, except when + builds are deployed to machines where `/nix/store` resolves + differently. If you are sure that you’re not going to do that, you + can set `NIX_IGNORE_SYMLINK_STORE` to `1`. + + Note that if you’re symlinking the Nix store so that you can put it + on another file system than the root file system, on Linux you’re + better off using `bind` mount points, e.g., + + ```console + $ mkdir /nix + $ mount -o bind /mnt/otherdisk/nix /nix + ``` + + Consult the mount 8 manual page for details. + +- [`NIX_STORE_DIR`](#env-NIX_STORE_DIR) + + Overrides the location of the Nix store (default `prefix/store`). + +- [`NIX_DATA_DIR`](#env-NIX_DATA_DIR) + + Overrides the location of the Nix static data directory (default + `prefix/share`). + +- [`NIX_LOG_DIR`](#env-NIX_LOG_DIR) + + Overrides the location of the Nix log directory (default + `prefix/var/log/nix`). + +- [`NIX_STATE_DIR`](#env-NIX_STATE_DIR) + + Overrides the location of the Nix state directory (default + `prefix/var/nix`). + +- [`NIX_CONF_DIR`](#env-NIX_CONF_DIR) + + Overrides the location of the system Nix configuration directory + (default `prefix/etc/nix`). + +- [`NIX_CONFIG`](#env-NIX_CONFIG) + + Applies settings from Nix configuration from the environment. + The content is treated as if it was read from a Nix configuration file. + Settings are separated by the newline character. + +- [`NIX_USER_CONF_FILES`](#env-NIX_USER_CONF_FILES) + + Overrides the location of the Nix user configuration files to load from. + + The default are the locations according to the [XDG Base Directory Specification]. + See the [XDG Base Directories](#xdg-base-directories) sub-section for details. + + The variable is treated as a list separated by the `:` token. + +- [`TMPDIR`](#env-TMPDIR) + + Use the specified directory to store temporary files. In particular, + this includes temporary build directories; these can take up + substantial amounts of disk space. The default is `/tmp`. + +- [`NIX_REMOTE`](#env-NIX_REMOTE) + + This variable should be set to `daemon` if you want to use the Nix + daemon to execute Nix operations. This is necessary in [multi-user + Nix installations](@docroot@/installation/multi-user.md). If the Nix + daemon's Unix socket is at some non-standard path, this variable + should be set to `unix://path/to/socket`. Otherwise, it should be + left unset. + +- [`NIX_SHOW_STATS`](#env-NIX_SHOW_STATS) + + If set to `1`, Nix will print some evaluation statistics, such as + the number of values allocated. + +- [`NIX_COUNT_CALLS`](#env-NIX_COUNT_CALLS) + + If set to `1`, Nix will print how often functions were called during + Nix expression evaluation. This is useful for profiling your Nix + expressions. + +- [`GC_INITIAL_HEAP_SIZE`](#env-GC_INITIAL_HEAP_SIZE) + + If Nix has been configured to use the Boehm garbage collector, this + variable sets the initial size of the heap in bytes. It defaults to + 384 MiB. Setting it to a low value reduces memory consumption, but + will increase runtime due to the overhead of garbage collection. ## XDG Base Directories diff --git a/doc/manual/src/command-ref/command-ref.md b/doc/manual/src/command-ref/index.md similarity index 100% rename from doc/manual/src/command-ref/command-ref.md rename to doc/manual/src/command-ref/index.md diff --git a/doc/manual/src/command-ref/nix-channel.md b/doc/manual/src/command-ref/nix-channel.md index 025f758e72e0..cebbc7b00d0d 100644 --- a/doc/manual/src/command-ref/nix-channel.md +++ b/doc/manual/src/command-ref/nix-channel.md @@ -8,36 +8,46 @@ # Description -A Nix channel is a mechanism that allows you to automatically stay -up-to-date with a set of pre-built Nix expressions. A Nix channel is -just a URL that points to a place containing a set of Nix expressions. - -To see the list of official NixOS channels, visit -. +Channels are a mechanism for referencing remote Nix expressions and conveniently retrieving their latest version. + +The moving parts of channels are: +- The official channels listed at +- The user-specific list of [subscribed channels](#subscribed-channels) +- The [downloaded channel contents](#channels) +- The [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path), set with the [`-I` option](#opt-i) or the [`NIX_PATH` environment variable](#env-NIX_PATH) + +> **Note** +> +> The state of a subscribed channel is external to the Nix expressions relying on it. +> This may limit reproducibility. +> +> Dependencies on other Nix expressions can be declared explicitly with: +> - [`fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl), [`fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball), or [`fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) in Nix expressions +> - the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I) in command line invocations This command has the following operations: - `--add` *url* \[*name*\]\ - Adds a channel named *name* with URL *url* to the list of subscribed - channels. If *name* is omitted, it defaults to the last component of - *url*, with the suffixes `-stable` or `-unstable` removed. + Add a channel *name* located at *url* to the list of subscribed channels. + If *name* is omitted, default to the last component of *url*, with the suffixes `-stable` or `-unstable` removed. + + > **Note** + > + > `--add` does not automatically perform an update. + > Use `--update` explicitly. A channel URL must point to a directory containing a file `nixexprs.tar.gz`. At the top level, that tarball must contain a single directory with a `default.nix` file that serves as the channel’s entry point. - `--remove` *name*\ - Removes the channel named *name* from the list of subscribed - channels. + Remove the channel *name* from the list of subscribed channels. - `--list`\ - Prints the names and URLs of all subscribed channels on standard - output. + Print the names and URLs of all subscribed channels on standard output. - `--update` \[*names*…\]\ - Downloads the Nix expressions of all subscribed channels (or only - those included in *names* if specified) and makes them the default - for `nix-env` operations (by symlinking them from the directory - `~/.nix-defexpr`). + Download the Nix expressions of subscribed channels and create a new generation. + Update all channels if none is specified, and only those included in *names* otherwise. - `--list-generations`\ Prints a list of all the current existing generations for the @@ -49,13 +59,8 @@ This command has the following operations: ``` - `--rollback` \[*generation*\]\ - Reverts the previous call to `nix-channel - --update`. Optionally, you can specify a specific channel generation - number to restore. - -Note that `--add` does not automatically perform an update. - -The list of subscribed channels is stored in `~/.nix-channels`. + Revert channels to the state before the last call to `nix-channel --update`. + Optionally, you can specify a specific channel *generation* number to restore. {{#include ./opt-common.md}} @@ -69,23 +74,33 @@ The list of subscribed channels is stored in `~/.nix-channels`. # Examples -To subscribe to the Nixpkgs channel and install the GNU Hello package: +Subscribe to the Nixpkgs channel and run `hello` from the GNU Hello package: ```console $ nix-channel --add https://nixos.org/channels/nixpkgs-unstable +$ nix-channel --list +nixpkgs https://nixos.org/channels/nixpkgs $ nix-channel --update -$ nix-env --install --attr nixpkgs.hello +$ nix-shell -p hello --run hello +hello ``` -You can revert channel updates using `--rollback`: +Revert channel updates using `--rollback`: ```console -$ nix-instantiate --eval --expr '(import {}).lib.version' -"14.04.527.0e935f1" +$ nix-instantiate --eval '' --attr lib.version +"22.11pre296212.530a53dcbc9" $ nix-channel --rollback switching from generation 483 to 482 -$ nix-instantiate --eval --expr '(import {}).lib.version' -"14.04.526.dbadfad" +$ nix-instantiate --eval '' --attr lib.version +"22.11pre281526.d0419badfad" +``` + +Remove a channel: + +```console +$ nix-channel --remove nixpkgs +$ nix-channel --list ``` diff --git a/doc/manual/src/command-ref/nix-env/install.md b/doc/manual/src/command-ref/nix-env/install.md index ad179cbc7b68..c1fff50e80f7 100644 --- a/doc/manual/src/command-ref/nix-env/install.md +++ b/doc/manual/src/command-ref/nix-env/install.md @@ -14,23 +14,28 @@ # Description -The install operation creates a new user environment, based on the -current generation of the active profile, to which a set of store paths -described by *args* is added. The arguments *args* map to store paths in -a number of possible ways: +The install operation creates a new user environment. +It is based on the current generation of the active [profile](@docroot@/command-ref/files/profiles.md), to which a set of [store paths] described by *args* is added. - - By default, *args* is a set of derivation names denoting derivations - in the active Nix expression. These are realised, and the resulting - output paths are installed. Currently installed derivations with a - name equal to the name of a derivation being added are removed - unless the option `--preserve-installed` is specified. +[store paths]: @docroot@/glossary.md#gloss-store-path + +The arguments *args* map to store paths in a number of possible ways: + + + - By default, *args* is a set of [derivation] names denoting derivations in the [default Nix expression]. + These are [realised], and the resulting output paths are installed. + Currently installed derivations with a name equal to the name of a derivation being added are removed unless the option `--preserve-installed` is specified. + + [derivation]: @docroot@/glossary.md#gloss-derivation + [default Nix expression]: @docroot@/command-ref/files/default-nix-expression.md + [realised]: @docroot@/glossary.md#gloss-realise If there are multiple derivations matching a name in *args* that have the same name (e.g., `gcc-3.3.6` and `gcc-4.1.1`), then the derivation with the highest *priority* is used. A derivation can define a priority by declaring the `meta.priority` attribute. This attribute should be a number, with a higher value denoting a lower - priority. The default priority is `0`. + priority. The default priority is `5`. If there are multiple matching derivations with the same priority, then the derivation with the highest version will be installed. @@ -40,44 +45,90 @@ a number of possible ways: gcc-3.3.6 gcc-4.1.1` will install both version of GCC (and will probably cause a user environment conflict\!). - - If `--attr` (`-A`) is specified, the arguments are *attribute - paths* that select attributes from the top-level Nix - expression. This is faster than using derivation names and - unambiguous. To find out the attribute paths of available - packages, use `nix-env --query --available --attr-path `. + - If [`--attr`](#opt-attr) / `-A` is specified, the arguments are *attribute paths* that select attributes from the [default Nix expression]. + This is faster than using derivation names and unambiguous. + Show the attribute paths of available packages with [`nix-env --query`](./query.md): + + ```console + nix-env --query --available --attr-path` + ``` - If `--from-profile` *path* is given, *args* is a set of names - denoting installed store paths in the profile *path*. This is an + denoting installed [store paths] in the profile *path*. This is an easy way to copy user environment elements from one profile to another. - - If `--from-expression` is given, *args* are Nix - [functions](@docroot@/language/constructs.md#functions) - that are called with the active Nix expression as their single - argument. The derivations returned by those function calls are - installed. This allows derivations to be specified in an - unambiguous way, which is necessary if there are multiple - derivations with the same name. - - - If *args* are [store derivations](@docroot@/glossary.md#gloss-store-derivation), then these are - [realised](@docroot@/command-ref/nix-store/realise.md), and the resulting output paths - are installed. + - If `--from-expression` is given, *args* are [Nix language functions](@docroot@/language/constructs.md#functions) that are called with the [default Nix expression] as their single argument. + The derivations returned by those function calls are installed. + This allows derivations to be specified in an unambiguous way, which is necessary if there are multiple derivations with the same name. + + - If *args* are [store derivations](@docroot@/glossary.md#gloss-store-derivation), then these are [realised], and the resulting output paths are installed. + + - If *args* are [store paths] that are not store derivations, then these are [realised] and installed. + + - By default all [outputs](@docroot@/language/derivations.md#attr-outputs) are installed for each [derivation]. + This can be overridden by adding a `meta.outputsToInstall` attribute on the derivation listing a subset of the output names. + + Example: + + The file `example.nix` defines a derivation with two outputs `foo` and `bar`, each containing a file. + + ```nix + # example.nix + let + pkgs = import {}; + command = '' + ${pkgs.coreutils}/bin/mkdir -p $foo $bar + echo foo > $foo/foo-file + echo bar > $bar/bar-file + ''; + in + derivation { + name = "example"; + builder = "${pkgs.bash}/bin/bash"; + args = [ "-c" command ]; + outputs = [ "foo" "bar" ]; + system = builtins.currentSystem; + } + ``` + + Installing from this Nix expression will make files from both outputs appear in the current profile. + + ```console + $ nix-env --install --file example.nix + installing 'example' + $ ls ~/.nix-profile + foo-file + bar-file + manifest.nix + ``` + + Adding `meta.outputsToInstall` to that derivation will make `nix-env` only install files from the specified outputs. + + ```nix + # example-outputs.nix + import ./example.nix // { meta.outputsToInstall = [ "bar" ]; } + ``` + + ```console + $ nix-env --install --file example-outputs.nix + installing 'example' + $ ls ~/.nix-profile + bar-file + manifest.nix + ``` + +# Options + + - `--prebuilt-only` / `-b` - - If *args* are store paths that are not store derivations, then these - are [realised](@docroot@/command-ref/nix-store/realise.md) and installed. - - - By default all outputs are installed for each derivation. That can - be reduced by setting `meta.outputsToInstall`. - -# Flags - - - `--prebuilt-only` / `-b`\ Use only derivations for which a substitute is registered, i.e., there is a pre-built binary available that can be downloaded in lieu of building the derivation. Thus, no packages will be built from source. - - `--preserve-installed` / `-P`\ + - `--preserve-installed` / `-P` + Do not remove derivations with a name matching one of the derivations being installed. Usually, trying to have two versions of the same package installed in the same generation of a profile will @@ -85,7 +136,8 @@ a number of possible ways: clashes between the two versions. However, this is not the case for all packages. - - `--remove-all` / `-r`\ + - `--remove-all` / `-r` + Remove all previously installed packages first. This is equivalent to running `nix-env --uninstall '.*'` first, except that everything happens in a single transaction. diff --git a/doc/manual/src/command-ref/nix-prefetch-url.md b/doc/manual/src/command-ref/nix-prefetch-url.md index 3bcd209e275e..45ef01e02ccf 100644 --- a/doc/manual/src/command-ref/nix-prefetch-url.md +++ b/doc/manual/src/command-ref/nix-prefetch-url.md @@ -31,15 +31,18 @@ store already contains a file with the same hash and base name. Otherwise, the file is downloaded, and an error is signaled if the actual hash of the file does not match the specified hash. -This command prints the hash on standard output. Additionally, if the -option `--print-path` is used, the path of the downloaded file in the -Nix store is also printed. +This command prints the hash on standard output. +The hash is printed using base-32 unless `--type md5` is specified, +in which case it's printed using base-16. +Additionally, if the option `--print-path` is used, +the path of the downloaded file in the Nix store is also printed. # Options - `--type` *hashAlgo*\ - Use the specified cryptographic hash algorithm, which can be one of - `md5`, `sha1`, `sha256`, and `sha512`. + Use the specified cryptographic hash algorithm, + which can be one of `md5`, `sha1`, `sha256`, and `sha512`. + The default is `sha256`. - `--print-path`\ Print the store path of the downloaded file on standard output. diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index 195b72be50d8..1eaf3c36aa70 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -235,14 +235,14 @@ package like Terraform: ```bash #! /usr/bin/env nix-shell -#! nix-shell -i bash --packages "terraform.withPlugins (plugins: [ plugins.openstack ])" +#! nix-shell -i bash --packages 'terraform.withPlugins (plugins: [ plugins.openstack ])' terraform apply ``` > **Note** > -> You must use double quotes (`"`) when passing a simple Nix expression +> You must use single or double quotes (`'`, `"`) when passing a simple Nix expression > in a nix-shell shebang. Finally, using the merging of multiple nix-shell shebangs the following @@ -251,7 +251,7 @@ branch): ```haskell #! /usr/bin/env nix-shell -#! nix-shell -i runghc --packages "haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])" +#! nix-shell -i runghc --packages 'haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])' #! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-20.03.tar.gz import Network.Curl.Download diff --git a/doc/manual/src/command-ref/nix-store/query.md b/doc/manual/src/command-ref/nix-store/query.md index cd45a4932821..a158c76aae4f 100644 --- a/doc/manual/src/command-ref/nix-store/query.md +++ b/doc/manual/src/command-ref/nix-store/query.md @@ -5,8 +5,8 @@ # Synopsis `nix-store` {`--query` | `-q`} - {`--outputs` | `--requisites` | `-R` | `--references` | - `--referrers` | `--referrers-closure` | `--deriver` | `-d` | + {`--outputs` | `--requisites` | `-R` | `--references` | `--referrers` | + `--referrers-closure` | `--deriver` | `-d` | `--valid-derivers` | `--graph` | `--tree` | `--binding` *name* | `-b` *name* | `--hash` | `--size` | `--roots`} [`--use-output`] [`-u`] [`--force-realise`] [`-f`] @@ -82,13 +82,21 @@ symlink. in the Nix store that are dependent on *paths*. - `--deriver`; `-d`\ - Prints the [deriver] of the store paths *paths*. If + Prints the [deriver] that was used to build the store paths *paths*. If the path has no deriver (e.g., if it is a source file), or if the deriver is not known (e.g., in the case of a binary-only deployment), the string `unknown-deriver` is printed. + The returned deriver is not guaranteed to exist in the local store, for + example when *paths* were substituted from a binary cache. + Use `--valid-derivers` instead to obtain valid paths only. [deriver]: ../../glossary.md#gloss-deriver + - `--valid-derivers`\ + Prints a set of derivation files (`.drv`) which are supposed produce + said paths when realized. Might print nothing, for example for source paths + or paths subsituted from a binary cache. + - `--graph`\ Prints the references graph of the store paths *paths* in the format of the `dot` tool of AT\&T's [Graphviz diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index c19aea75e5f5..5428d57fa506 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -1,6 +1,6 @@ # Name -`nix-store --realise` - realise specified store paths +`nix-store --realise` - build or fetch store objects # Synopsis @@ -8,33 +8,39 @@ # Description -The operation `--realise` essentially “builds” the specified store -paths. Realisation is a somewhat overloaded term: - - - If the store path is a *derivation*, realisation ensures that the - output paths of the derivation are [valid] (i.e., - the output path and its closure exist in the file system). This - can be done in several ways. First, it is possible that the - outputs are already valid, in which case we are done - immediately. Otherwise, there may be [substitutes] - that produce the outputs (e.g., by downloading them). Finally, the - outputs can be produced by running the build task described - by the derivation. - - - If the store path is not a derivation, realisation ensures that the - specified path is valid (i.e., it and its closure exist in the file - system). If the path is already valid, we are done immediately. - Otherwise, the path and any missing paths in its closure may be - produced through substitutes. If there are no (successful) - substitutes, realisation fails. +Each of *paths* is processed as follows: + +- If the path leads to a [store derivation]: + 1. If it is not [valid], substitute the store derivation file itself. + 2. Realise its [output paths]: + - Try to fetch from [substituters] the [store objects] associated with the output paths in the store derivation's [closure]. + - With [content-addressed derivations] (experimental): + Determine the output paths to realise by querying content-addressed realisation entries in the [Nix database]. + - For any store paths that cannot be substituted, produce the required store objects: + 1. Realise all outputs of the derivation's dependencies + 2. Run the derivation's [`builder`](@docroot@/language/derivations.md#attr-builder) executable + +- Otherwise, and if the path is not already valid: Try to fetch the associated [store objects] in the path's [closure] from [substituters]. + +If no substitutes are available and no store derivation is given, realisation fails. + +[store paths]: @docroot@/glossary.md#gloss-store-path [valid]: @docroot@/glossary.md#gloss-validity -[substitutes]: @docroot@/glossary.md#gloss-substitute +[store derivation]: @docroot@/glossary.md#gloss-store-derivation +[output paths]: @docroot@/glossary.md#gloss-output-path +[store objects]: @docroot@/glossary.md#gloss-store-object +[closure]: @docroot@/glossary.md#gloss-closure +[substituters]: @docroot@/command-ref/conf-file.md#conf-substituters +[content-addressed derivations]: @docroot@/contributing/experimental-features.md#xp-feature-ca-derivations +[Nix database]: @docroot@/glossary.md#gloss-nix-database + +The resulting paths are printed on standard output. +For non-derivation arguments, the argument itself is printed. -The output path of each derivation is printed on standard output. (For -non-derivations argument, the argument itself is printed.) +{{#include ../status-build-failure.md}} -The following flags are available: +# Options - `--dry-run`\ Print on standard error a description of what packages would be @@ -54,8 +60,6 @@ The following flags are available: previous build, the new output path is left in `/nix/store/name.check.` -{{#include ../status-build-failure.md}} - {{#include ./opt-common.md}} {{#include ../opt-common.md}} @@ -67,8 +71,6 @@ The following flags are available: This operation is typically used to build [store derivation]s produced by [`nix-instantiate`](@docroot@/command-ref/nix-instantiate.md): -[store derivation]: @docroot@/glossary.md#gloss-store-derivation - ```console $ nix-store --realise $(nix-instantiate ./test.nix) /nix/store/31axcgrlbfsxzmfff1gyj1bf62hvkby2-aterm-2.3.1 diff --git a/doc/manual/src/command-ref/opt-common-syn.md b/doc/manual/src/command-ref/opt-common-syn.md deleted file mode 100644 index b66d318c24f9..000000000000 --- a/doc/manual/src/command-ref/opt-common-syn.md +++ /dev/null @@ -1,57 +0,0 @@ -\--help - -\--version - -\--verbose - -\-v - -\--quiet - -\--log-format - -format - -\--no-build-output - -\-Q - -\--max-jobs - -\-j - -number - -\--cores - -number - -\--max-silent-time - -number - -\--timeout - -number - -\--keep-going - -\-k - -\--keep-failed - -\-K - -\--fallback - -\--readonly-mode - -\-I - -path - -\--option - -name - -value diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index 54c0a1d0def5..114b292f9302 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -2,217 +2,208 @@ Most Nix commands accept the following command-line options: - - [`--help`](#opt-help)\ - Prints out a summary of the command syntax and exits. - - - [`--version`](#opt-version)\ - Prints out the Nix version number on standard output and exits. - - - [`--verbose`](#opt-verbose) / `-v`\ - Increases the level of verbosity of diagnostic messages printed on - standard error. For each Nix operation, the information printed on - standard output is well-defined; any diagnostic information is - printed on standard error, never on standard output. - - This option may be specified repeatedly. Currently, the following - verbosity levels exist: - - - 0\ - “Errors only”: only print messages explaining why the Nix - invocation failed. - - - 1\ - “Informational”: print *useful* messages about what Nix is - doing. This is the default. - - - 2\ - “Talkative”: print more informational messages. - - - 3\ - “Chatty”: print even more informational messages. - - - 4\ - “Debug”: print debug information. - - - 5\ - “Vomit”: print vast amounts of debug information. - - - [`--quiet`](#opt-quiet)\ - Decreases the level of verbosity of diagnostic messages printed on - standard error. This is the inverse option to `-v` / `--verbose`. - - This option may be specified repeatedly. See the previous verbosity - levels list. - - - [`--log-format`](#opt-log-format) *format*\ - This option can be used to change the output of the log format, with - *format* being one of: - - - raw\ - This is the raw format, as outputted by nix-build. - - - internal-json\ - Outputs the logs in a structured manner. - - > **Warning** - > - > While the schema itself is relatively stable, the format of - > the error-messages (namely of the `msg`-field) can change - > between releases. - - - bar\ - Only display a progress bar during the builds. - - - bar-with-logs\ - Display the raw logs, with the progress bar at the bottom. - - - [`--no-build-output`](#opt-no-build-output) / `-Q`\ - By default, output written by builders to standard output and - standard error is echoed to the Nix command's standard error. This - option suppresses this behaviour. Note that the builder's standard - output and error are always written to a log file in - `prefix/nix/var/log/nix`. - - - [`--max-jobs`](#opt-max-jobs) / `-j` *number*\ - Sets the maximum number of build jobs that Nix will perform in - parallel to the specified number. Specify `auto` to use the number - of CPUs in the system. The default is specified by the `max-jobs` - configuration setting, which itself defaults to `1`. A higher - value is useful on SMP systems or to exploit I/O latency. - - Setting it to `0` disallows building on the local machine, which is - useful when you want builds to happen only on remote builders. - - - [`--cores`](#opt-cores)\ - Sets the value of the `NIX_BUILD_CORES` environment variable in - the invocation of builders. Builders can use this variable at - their discretion to control the maximum amount of parallelism. For - instance, in Nixpkgs, if the derivation attribute - `enableParallelBuilding` is set to `true`, the builder passes the - `-jN` flag to GNU Make. It defaults to the value of the `cores` - configuration setting, if set, or `1` otherwise. The value `0` - means that the builder should use all available CPU cores in the - system. - - - [`--max-silent-time`](#opt-max-silent-time)\ - Sets the maximum number of seconds that a builder can go without - producing any data on standard output or standard error. The - default is specified by the `max-silent-time` configuration - setting. `0` means no time-out. - - - [`--timeout`](#opt-timeout)\ - Sets the maximum number of seconds that a builder can run. The - default is specified by the `timeout` configuration setting. `0` - means no timeout. - - - [`--keep-going`](#opt-keep-going) / `-k`\ - Keep going in case of failed builds, to the greatest extent - possible. That is, if building an input of some derivation fails, - Nix will still build the other inputs, but not the derivation - itself. Without this option, Nix stops if any build fails (except - for builds of substitutes), possibly killing builds in progress (in - case of parallel or distributed builds). - - - [`--keep-failed`](#opt-keep-failed) / `-K`\ - Specifies that in case of a build failure, the temporary directory - (usually in `/tmp`) in which the build takes place should not be - deleted. The path of the build directory is printed as an - informational message. - - - [`--fallback`](#opt-fallback)\ - Whenever Nix attempts to build a derivation for which substitutes - are known for each output path, but realising the output paths - through the substitutes fails, fall back on building the derivation. - - The most common scenario in which this is useful is when we have - registered substitutes in order to perform binary distribution from, - say, a network repository. If the repository is down, the - realisation of the derivation will fail. When this option is - specified, Nix will build the derivation instead. Thus, installation - from binaries falls back on installation from source. This option is - not the default since it is generally not desirable for a transient - failure in obtaining the substitutes to lead to a full build from - source (with the related consumption of resources). - - - [`--readonly-mode`](#opt-readonly-mode)\ - When this option is used, no attempt is made to open the Nix - database. Most Nix operations do need database access, so those - operations will fail. - - - [`--arg`](#opt-arg) *name* *value*\ - This option is accepted by `nix-env`, `nix-instantiate`, - `nix-shell` and `nix-build`. When evaluating Nix expressions, the - expression evaluator will automatically try to call functions that - it encounters. It can automatically call functions for which every - argument has a [default - value](@docroot@/language/constructs.md#functions) (e.g., - `{ argName ? defaultValue }: ...`). With `--arg`, you can also - call functions that have arguments without a default value (or - override a default value). That is, if the evaluator encounters a - function with an argument named *name*, it will call it with value - *value*. - - For instance, the top-level `default.nix` in Nixpkgs is actually a - function: - - ```nix - { # The system (e.g., `i686-linux') for which to build the packages. - system ? builtins.currentSystem - ... - }: ... - ``` - - So if you call this Nix expression (e.g., when you do `nix-env --install --attr - pkgname`), the function will be called automatically using the - value [`builtins.currentSystem`](@docroot@/language/builtins.md) for - the `system` argument. You can override this using `--arg`, e.g., - `nix-env --install --attr pkgname --arg system \"i686-freebsd\"`. (Note that - since the argument is a Nix string literal, you have to escape the - quotes.) - - - [`--argstr`](#opt-argstr) *name* *value*\ - This option is like `--arg`, only the value is not a Nix - expression but a string. So instead of `--arg system - \"i686-linux\"` (the outer quotes are to keep the shell happy) you - can say `--argstr system i686-linux`. - - - [`--attr`](#opt-attr) / `-A` *attrPath*\ - Select an attribute from the top-level Nix expression being - evaluated. (`nix-env`, `nix-instantiate`, `nix-build` and - `nix-shell` only.) The *attribute path* *attrPath* is a sequence - of attribute names separated by dots. For instance, given a - top-level Nix expression *e*, the attribute path `xorg.xorgserver` - would cause the expression `e.xorg.xorgserver` to be used. See - [`nix-env --install`](@docroot@/command-ref/nix-env/install.md) for some - concrete examples. - - In addition to attribute names, you can also specify array indices. - For instance, the attribute path `foo.3.bar` selects the `bar` - attribute of the fourth element of the array in the `foo` attribute - of the top-level expression. - - - [`--expr`](#opt-expr) / `-E`\ - Interpret the command line arguments as a list of Nix expressions to - be parsed and evaluated, rather than as a list of file names of Nix - expressions. (`nix-instantiate`, `nix-build` and `nix-shell` only.) - - For `nix-shell`, this option is commonly used to give you a shell in - which you can build the packages returned by the expression. If you - want to get a shell which contain the *built* packages ready for - use, give your expression to the `nix-shell --packages ` convenience flag - instead. - - - [`-I`](#opt-I) *path*\ - Add an entry to the [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path). - This option may be given multiple times. - Paths added through `-I` take precedence over [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH). - - - [`--option`](#opt-option) *name* *value*\ - Set the Nix configuration option *name* to *value*. This overrides - settings in the Nix configuration file (see nix.conf5). - - - [`--repair`](#opt-repair)\ - Fix corrupted or missing store paths by redownloading or rebuilding - them. Note that this is slow because it requires computing a - cryptographic hash of the contents of every path in the closure of - the build. Also note the warning under `nix-store --repair-path`. +- [`--help`](#opt-help) + + Prints out a summary of the command syntax and exits. + +- [`--version`](#opt-version) + + Prints out the Nix version number on standard output and exits. + +- [`--verbose`](#opt-verbose) / `-v` + + Increases the level of verbosity of diagnostic messages printed on standard error. + For each Nix operation, the information printed on standard output is well-defined; + any diagnostic information is printed on standard error, never on standard output. + + This option may be specified repeatedly. + Currently, the following verbosity levels exist: + + - `0` “Errors only” + + Only print messages explaining why the Nix invocation failed. + + - `1` “Informational” + + Print *useful* messages about what Nix is doing. + This is the default. + + - `2` “Talkative” + + Print more informational messages. + + - `3` “Chatty” + + Print even more informational messages. + + - `4` “Debug” + + Print debug information. + + - `5` “Vomit” + + Print vast amounts of debug information. + +- [`--quiet`](#opt-quiet) + + Decreases the level of verbosity of diagnostic messages printed on standard error. + This is the inverse option to `-v` / `--verbose`. + + This option may be specified repeatedly. + See the previous verbosity levels list. + +- [`--log-format`](#opt-log-format) *format* + + This option can be used to change the output of the log format, with *format* being one of: + + - `raw` + + This is the raw format, as outputted by nix-build. + + - `internal-json` + + Outputs the logs in a structured manner. + + > **Warning** + > + > While the schema itself is relatively stable, the format of + > the error-messages (namely of the `msg`-field) can change + > between releases. + + - `bar` + + Only display a progress bar during the builds. + + - `bar-with-logs` + + Display the raw logs, with the progress bar at the bottom. + +- [`--no-build-output`](#opt-no-build-output) / `-Q` + + By default, output written by builders to standard output and standard error is echoed to the Nix command's standard error. + This option suppresses this behaviour. + Note that the builder's standard output and error are always written to a log file in `prefix/nix/var/log/nix`. + +- [`--max-jobs`](#opt-max-jobs) / `-j` *number* + + Sets the maximum number of build jobs that Nix will perform in parallel to the specified number. + Specify `auto` to use the number of CPUs in the system. + The default is specified by the `max-jobs` configuration setting, which itself defaults to `1`. + A higher value is useful on SMP systems or to exploit I/O latency. + + Setting it to `0` disallows building on the local machine, which is useful when you want builds to happen only on remote builders. + +- [`--cores`](#opt-cores) + + Sets the value of the `NIX_BUILD_CORES` environment variable in the invocation of builders. + Builders can use this variable at their discretion to control the maximum amount of parallelism. + For instance, in Nixpkgs, if the derivation attribute `enableParallelBuilding` is set to `true`, the builder passes the `-jN` flag to GNU Make. + It defaults to the value of the `cores` configuration setting, if set, or `1` otherwise. + The value `0` means that the builder should use all available CPU cores in the system. + +- [`--max-silent-time`](#opt-max-silent-time) + + Sets the maximum number of seconds that a builder can go without producing any data on standard output or standard error. + The default is specified by the `max-silent-time` configuration setting. + `0` means no time-out. + +- [`--timeout`](#opt-timeout) + + Sets the maximum number of seconds that a builder can run. + The default is specified by the `timeout` configuration setting. + `0` means no timeout. + +- [`--keep-going`](#opt-keep-going) / `-k` + + Keep going in case of failed builds, to the greatest extent possible. + That is, if building an input of some derivation fails, Nix will still build the other inputs, but not the derivation itself. + Without this option, Nix stops if any build fails (except for builds of substitutes), possibly killing builds in progress (in case of parallel or distributed builds). + +- [`--keep-failed`](#opt-keep-failed) / `-K` + + Specifies that in case of a build failure, the temporary directory (usually in `/tmp`) in which the build takes place should not be deleted. + The path of the build directory is printed as an informational message. + +- [`--fallback`](#opt-fallback) + + Whenever Nix attempts to build a derivation for which substitutes are known for each output path, but realising the output paths through the substitutes fails, fall back on building the derivation. + + The most common scenario in which this is useful is when we have registered substitutes in order to perform binary distribution from, say, a network repository. + If the repository is down, the realisation of the derivation will fail. + When this option is specified, Nix will build the derivation instead. + Thus, installation from binaries falls back on installation from source. + This option is not the default since it is generally not desirable for a transient failure in obtaining the substitutes to lead to a full build from source (with the related consumption of resources). + +- [`--readonly-mode`](#opt-readonly-mode) + + When this option is used, no attempt is made to open the Nix database. + Most Nix operations do need database access, so those operations will fail. + +- [`--arg`](#opt-arg) *name* *value* + + This option is accepted by `nix-env`, `nix-instantiate`, `nix-shell` and `nix-build`. + When evaluating Nix expressions, the expression evaluator will automatically try to call functions that it encounters. + It can automatically call functions for which every argument has a [default value](@docroot@/language/constructs.md#functions) (e.g., `{ argName ? defaultValue }: ...`). + + With `--arg`, you can also call functions that have arguments without a default value (or override a default value). + That is, if the evaluator encounters a function with an argument named *name*, it will call it with value *value*. + + For instance, the top-level `default.nix` in Nixpkgs is actually a function: + + ```nix + { # The system (e.g., `i686-linux') for which to build the packages. + system ? builtins.currentSystem + ... + }: ... + ``` + + So if you call this Nix expression (e.g., when you do `nix-env --install --attr pkgname`), the function will be called automatically using the value [`builtins.currentSystem`](@docroot@/language/builtins.md) for the `system` argument. + You can override this using `--arg`, e.g., `nix-env --install --attr pkgname --arg system \"i686-freebsd\"`. + (Note that since the argument is a Nix string literal, you have to escape the quotes.) + +- [`--argstr`](#opt-argstr) *name* *value* + + This option is like `--arg`, only the value is not a Nix expression but a string. + So instead of `--arg system \"i686-linux\"` (the outer quotes are to keep the shell happy) you can say `--argstr system i686-linux`. + +- [`--attr`](#opt-attr) / `-A` *attrPath* + + Select an attribute from the top-level Nix expression being evaluated. + (`nix-env`, `nix-instantiate`, `nix-build` and `nix-shell` only.) + The *attribute path* *attrPath* is a sequence of attribute names separated by dots. + For instance, given a top-level Nix expression *e*, the attribute path `xorg.xorgserver` would cause the expression `e.xorg.xorgserver` to be used. + See [`nix-env --install`](@docroot@/command-ref/nix-env/install.md) for some concrete examples. + + In addition to attribute names, you can also specify array indices. + For instance, the attribute path `foo.3.bar` selects the `bar` + attribute of the fourth element of the array in the `foo` attribute + of the top-level expression. + +- [`--expr`](#opt-expr) / `-E` + + Interpret the command line arguments as a list of Nix expressions to be parsed and evaluated, rather than as a list of file names of Nix expressions. + (`nix-instantiate`, `nix-build` and `nix-shell` only.) + + For `nix-shell`, this option is commonly used to give you a shell in which you can build the packages returned by the expression. + If you want to get a shell which contain the *built* packages ready for use, give your expression to the `nix-shell --packages ` convenience flag instead. + +- [`-I`](#opt-I) *path* + + Add an entry to the [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path). + This option may be given multiple times. + Paths added through `-I` take precedence over [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH). + +- [`--option`](#opt-option) *name* *value* + + Set the Nix configuration option *name* to *value*. + This overrides settings in the Nix configuration file (see nix.conf5). + +- [`--repair`](#opt-repair) + + Fix corrupted or missing store paths by redownloading or rebuilding them. + Note that this is slow because it requires computing a cryptographic hash of the contents of every path in the closure of the build. + Also note the warning under `nix-store --repair-path`. + +> **Note** +> +> See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags. diff --git a/doc/manual/src/command-ref/opt-inst-syn.md b/doc/manual/src/command-ref/opt-inst-syn.md deleted file mode 100644 index 1703c40e39c2..000000000000 --- a/doc/manual/src/command-ref/opt-inst-syn.md +++ /dev/null @@ -1,15 +0,0 @@ -\--prebuilt-only - -\-b - -\--attr - -\-A - -\--from-expression - -\-E - -\--from-profile - -path diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/contributing/cli-guideline.md index e53d2d178dac..f7e24d96bbd5 100644 --- a/doc/manual/src/contributing/cli-guideline.md +++ b/doc/manual/src/contributing/cli-guideline.md @@ -426,7 +426,7 @@ This leads to the following guidelines: ### Examples -This is bad, because all keys must be assumed to be store implementations: +This is bad, because all keys must be assumed to be store types: ```json { diff --git a/doc/manual/src/contributing/contributing.md b/doc/manual/src/contributing/contributing.md deleted file mode 100644 index 854139a319a9..000000000000 --- a/doc/manual/src/contributing/contributing.md +++ /dev/null @@ -1 +0,0 @@ -# Contributing diff --git a/doc/manual/src/contributing/documentation.md b/doc/manual/src/contributing/documentation.md new file mode 100644 index 000000000000..75226cd1a0c3 --- /dev/null +++ b/doc/manual/src/contributing/documentation.md @@ -0,0 +1,210 @@ +# Contributing documentation + +Improvements to documentation are very much appreciated, and a good way to start out with contributing to Nix. + +This is how you can help: +- Address [open issues with documentation](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+label%3Adocumentation) +- Review [pull requests concerning documentation](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+label%3Adocumentation) + +Incremental refactorings of the documentation build setup to make it faster or easier to understand and maintain are also welcome. + +## Building the manual + +Build the manual from scratch: + +```console +nix-build $(nix-instantiate)'!doc' +``` + +or + +```console +nix build .#^doc +``` + +and open `./result-doc/share/doc/nix/manual/index.html`. + +To build the manual incrementally, [enter the development shell](./hacking.md) and run: + +```console +make manual-html -j $NIX_BUILD_CORES +``` + +and open `./outputs/out/share/doc/nix/manual/language/index.html`. + +In order to reflect changes to the [Makefile for the manual], clear all generated files before re-building: + +[Makefile for the manual]: https://github.com/NixOS/nix/blob/master/doc/manual/local.mk + +```console +rm $(git ls-files doc/manual/ -o | grep -F '.md') && rmdir doc/manual/src/command-ref/new-cli && make manual-html -j $NIX_BUILD_CORES +``` + +## Style guide + +The goal of this style guide is to make it such that +- The manual is easy to search and skim for relevant information +- Documentation sources are easy to edit +- Changes to documentation are easy to review + +You will notice that this is not implemented consistently yet. +Please follow the guide when making additions or changes to existing documentation. +Do not make sweeping changes, unless they are programmatic and can be validated easily. + +### Language + +This manual is [reference documentation](https://diataxis.fr/reference/). +The typical usage pattern is to look up isolated pieces of information. +It should therefore aim to be correct, consistent, complete, and easy to navigate at a glance. + +- Aim for clarity and brevity. + + Please take the time to read the [plain language guidelines](https://www.plainlanguage.gov/guidelines/) for details. + +- Describe the subject factually. + + In particular, do not make value judgements or recommendations. + Check the code or add tests if in doubt. + +- Provide complete, minimal examples, and explain them. + + Readers should be able to try examples verbatim and get the same results as shown in the manual. + Always describe in words what a given example does. + + Non-trivial examples may need additional explanation, especially if they use concepts from outside the given context. + +- Always explain code examples in the text. + + Use comments in code samples very sparingly, for instance to highlight a particular aspect. + Readers tend to glance over large amounts of code when scanning for information. + + Especially beginners will likely find reading more complex-looking code strenuous and may therefore avoid it altogether. + + If a code sample appears to require a lot of inline explanation, consider replacing it with a simpler one. + If that's not possible, break the example down into multiple parts, explain them separately, and then show the combined result at the end. + This should be a last resort, as that would amount to writing a [tutorial](https://diataxis.fr/tutorials/) on the given subject. + +- Use British English. + + This is a somewhat arbitrary choice to force consistency, and accounts for the fact that a majority of Nix users and developers are from Europe. + +### Links and anchors + +Reference documentation must be readable in arbitrary order. +Readers cannot be expected to have any particular prerequisite knowledge about Nix. +While the table of contents can provide guidance and full-text search can help, they are most likely to find what they need by following sensible cross-references. + +- Link to technical terms + + When mentioning Nix-specific concepts, commands, options, settings, etc., link to appropriate documentation. + Also link to external tools or concepts, especially if their meaning may be ambiguous. + You may also want to link to definitions of less common technical terms. + + Then readers won't have to actively search for definitions and are more likely to discover relevant information on their own. + + > **Note** + > + > `man` and `--help` pages don't display links. + > Use appropriate link texts such that readers of terminal output can infer search terms. + +- Do not break existing URLs between releases. + + There are countless links in the wild pointing to old versions of the manual. + We want people to find up-to-date documentation when following popular advice. + + - When moving files, update [redirects on nixos.org](https://github.com/NixOS/nixos-homepage/blob/master/netlify.toml). + + This is especially important when moving information out of the Nix manual to other resources. + + - When changing anchors, update [client-side redirects](https://github.com/NixOS/nix/blob/master/doc/manual/redirects.js) + + The current setup is cumbersome, and help making better automation is appreciated. + +The build checks for broken internal links with. +This happens late in the process, so [building the whole manual](#building-the-manual) is not suitable for iterating quickly. +[`mdbook-linkcheck`] does not implement checking [URI fragments] yet. + +[`mdbook-linkcheck`]: https://github.com/Michael-F-Bryan/mdbook-linkcheck +[URI fragments]: https://en.wikipedia.org/wiki/URI_fragment + +### Markdown conventions + +The manual is written in markdown, and rendered with [mdBook](https://github.com/rust-lang/mdBook) for the web and with [lowdown](https://github.com/kristapsdz/lowdown) for `man` pages and `--help` output. + +For supported markdown features, refer to: +- [mdBook documentation](https://rust-lang.github.io/mdBook/format/markdown.html) +- [lowdown documentation](https://kristaps.bsd.lv/lowdown/) + +Please observe these guidelines to ease reviews: + +- Write one sentence per line. + + This makes long sentences immediately visible, and makes it easier to review changes and make direct suggestions. + +- Use reference links – sparingly – to ease source readability. + Put definitions close to their first use. + + Example: + + ``` + A [store object] contains a [file system object] and [references] to other store objects. + + [store object]: @docroot@/glossary.md#gloss-store-object + [file system object]: @docroot@/architecture/file-system-object.md + [references]: @docroot@/glossary.md#gloss-reference + ``` + +- Use admonitions of the following form: + + ``` + > **Note** + > + > This is a note. + ``` + + Highlight examples as such: + + ```` + > **Example** + > + > ```console + > $ nix --version + > ``` + ```` + + Highlight syntax definiions as such, using [EBNF](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form) notation: + + ```` + > **Syntax** + > + > *attribute-set* = `{` [ *attribute-name* `=` *expression* `;` ... ] `}` + ```` + +### The `@docroot@` variable + +`@docroot@` provides a base path for links that occur in reusable snippets or other documentation that doesn't have a base path of its own. + +If a broken link occurs in a snippet that was inserted into multiple generated files in different directories, use `@docroot@` to reference the `doc/manual/src` directory. + +If the `@docroot@` literal appears in an error message from the [`mdbook-linkcheck`] tool, the `@docroot@` replacement needs to be applied to the generated source file that mentions it. +See existing `@docroot@` logic in the [Makefile for the manual]. +Regular markdown files used for the manual have a base path of their own and they can use relative paths instead of `@docroot@`. + +## API documentation + +[Doxygen API documentation] is available online. +You can also build and view it yourself: + +[Doxygen API documentation]: https://hydra.nixos.org/job/nix/master/internal-api-docs/latest/download-by-type/doc/internal-api-docs + +```console +# nix build .#hydraJobs.internal-api-docs +# xdg-open ./result/share/doc/nix/internal-api/html/index.html +``` + +or inside `nix-shell` or `nix develop`: + +``` +# make internal-api-html +# xdg-open ./outputs/doc/share/doc/nix/internal-api/html/index.html +``` diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 7b244097140e..9de5ad39b586 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -10,7 +10,7 @@ $ cd nix The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. If you don't have it installed, follow the [installation instructions]. -[installation instructions]: ../installation/installation.md +[installation instructions]: ../installation/index.md ## Building Nix with flakes @@ -42,8 +42,8 @@ $ nix develop .#native-clang11StdenvPackages To build Nix itself in this shell: ```console -[nix-shell]$ ./bootstrap.sh -[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out +[nix-shell]$ autoreconfPhase +[nix-shell]$ configurePhase [nix-shell]$ make -j $NIX_BUILD_CORES ``` @@ -86,7 +86,7 @@ $ nix-shell --attr devShells.x86_64-linux.native-clang11StdenvPackages To build Nix itself in this shell: ```console -[nix-shell]$ ./bootstrap.sh +[nix-shell]$ autoreconfPhase [nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out [nix-shell]$ make -j $NIX_BUILD_CORES ``` @@ -110,41 +110,97 @@ You can also build Nix for one of the [supported platforms](#platforms). ## Platforms -As specified in [`flake.nix`], Nix can be built for various platforms: - -- `aarch64-linux` -- `i686-linux` -- `x86_64-darwin` -- `x86_64-linux` +Nix can be built for various platforms, as specified in [`flake.nix`]: [`flake.nix`]: https://github.com/nixos/nix/blob/master/flake.nix +- `x86_64-linux` +- `x86_64-darwin` +- `i686-linux` +- `aarch64-linux` +- `aarch64-darwin` +- `armv6l-linux` +- `armv7l-linux` + In order to build Nix for a different platform than the one you're currently -on, you need to have some way for your system Nix to build code for that -platform. Common solutions include [remote builders] and [binfmt emulation] +on, you need a way for your current Nix installation to build code for that +platform. Common solutions include [remote builders] and [binary format emulation] (only supported on NixOS). [remote builders]: ../advanced-topics/distributed-builds.md -[binfmt emulation]: https://nixos.org/manual/nixos/stable/options.html#opt-boot.binfmt.emulatedSystems +[binary format emulation]: https://nixos.org/manual/nixos/stable/options.html#opt-boot.binfmt.emulatedSystems -These solutions let Nix perform builds as if you're on the native platform, so -executing the build is as simple as +Given such a setup, executing the build only requires selecting the respective attribute. +For example, to compile for `aarch64-linux`: ```console -$ nix build .#packages.aarch64-linux.default +$ nix-build --attr packages.aarch64-linux.default ``` -for flake-enabled Nix, or +or for Nix with the [`flakes`] and [`nix-command`] experimental features enabled: ```console -$ nix-build --attr packages.aarch64-linux.default +$ nix build .#packages.aarch64-linux.default ``` -for classic Nix. +Cross-compiled builds are available for ARMv6 (`armv6l-linux`) and ARMv7 (`armv7l-linux`). +Add more [system types](#system-type) to `crossSystems` in `flake.nix` to bootstrap Nix on unsupported platforms. + +### Building for multiple platforms at once + +It is useful to perform multiple cross and native builds on the same source tree, +for example to ensure that better support for one platform doesn't break the build for another. +In order to facilitate this, Nix has some support for being built out of tree – that is, placing build artefacts in a different directory than the source code: + +1. Create a directory for the build, e.g. -You can use any of the other supported platforms in place of `aarch64-linux`. + ```bash + mkdir build + ``` -Cross-compiled builds are available for ARMv6 and ARMv7, and Nix on unsupported platforms can be bootstrapped by adding more `crossSystems` in `flake.nix`. +2. Run the configure script from that directory, e.g. + + ```bash + cd build + ../configure + ``` + +3. Run make from the source directory, but with the build directory specified, e.g. + + ```bash + make builddir=build + ``` + +## System type + +Nix uses a string with he following format to identify the *system type* or *platform* it runs on: + +``` +-[-] +``` + +It is set when Nix is compiled for the given system, and based on the output of [`config.guess`](https://github.com/nixos/nix/blob/master/config/config.guess) ([upstream](https://git.savannah.gnu.org/cgit/config.git/tree/config.guess)): + +``` +--[][-] +``` + +When Nix is built such that `./configure` is passed any of the `--host`, `--build`, `--target` options, the value is based on the output of [`config.sub`](https://github.com/nixos/nix/blob/master/config/config.sub) ([upstream](https://git.savannah.gnu.org/cgit/config.git/tree/config.sub)): + +``` +-[-]- +``` + +For historic reasons and backward-compatibility, some CPU and OS identifiers are translated from the GNU Autotools naming convention in [`configure.ac`](https://github.com/nixos/nix/blob/master/configure.ac) as follows: + +| `config.guess` | Nix | +|----------------------------|---------------------| +| `amd64` | `x86_64` | +| `i*86` | `i686` | +| `arm6` | `arm6l` | +| `arm7` | `arm7l` | +| `linux-gnu*` | `linux` | +| `linux-musl*` | `linux` | ## Compilation environments @@ -179,7 +235,7 @@ See [supported compilation environments](#compilation-environments) and instruct To use the LSP with your editor, you first need to [set up `clangd`](https://clangd.llvm.org/installation#project-setup) by running: ```console -make clean && bear -- make -j$NIX_BUILD_CORES install +make clean && bear -- make -j$NIX_BUILD_CORES default check install ``` Configure your editor to use the `clangd` from the shell, either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration). @@ -190,67 +246,40 @@ Configure your editor to use the `clangd` from the shell, either by running it i > Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim). > Editor-specific setup is typically opinionated, so we will not cover it here in more detail. -### Checking links in the manual +## Add a release note -The build checks for broken internal links. -This happens late in the process, so `nix build` is not suitable for iterating. -To build the manual incrementally, run: +`doc/manual/rl-next` contains release notes entries for all unreleased changes. -```console -make html -j $NIX_BUILD_CORES -``` +User-visible changes should come with a release note. -In order to reflect changes to the [Makefile], clear all generated files before re-building: +### Add an entry -[Makefile]: https://github.com/NixOS/nix/blob/master/doc/manual/local.mk +Here's what a complete entry looks like. The file name is not incorporated in the document. -```console -rm $(git ls-files doc/manual/ -o | grep -F '.md') && rmdir doc/manual/src/command-ref/new-cli && make html -j $NIX_BUILD_CORES ``` +synopsis: Basically a title +issues: #1234 +prs: #1238 +description: { -[`mdbook-linkcheck`] does not implement checking [URI fragments] yet. - -[`mdbook-linkcheck`]: https://github.com/Michael-F-Bryan/mdbook-linkcheck -[URI fragments]: https://en.wikipedia.org/wiki/URI_fragment - -#### `@docroot@` variable - -`@docroot@` provides a base path for links that occur in reusable snippets or other documentation that doesn't have a base path of its own. - -If a broken link occurs in a snippet that was inserted into multiple generated files in different directories, use `@docroot@` to reference the `doc/manual/src` directory. +Here's one or more paragraphs that describe the change. -If the `@docroot@` literal appears in an error message from the `mdbook-linkcheck` tool, the `@docroot@` replacement needs to be applied to the generated source file that mentions it. -See existing `@docroot@` logic in the [Makefile]. -Regular markdown files used for the manual have a base path of their own and they can use relative paths instead of `@docroot@`. +- It's markdown +- Add references to the manual using @docroot@ -## API documentation - -Doxygen API documentation is [available -online](https://hydra.nixos.org/job/nix/master/internal-api-docs/latest/download-by-type/doc/internal-api-docs). You -can also build and view it yourself: - -```console -# nix build .#hydraJobs.internal-api-docs -# xdg-open ./result/share/doc/nix/internal-api/html/index.html +} ``` -or inside a `nix develop` shell by running: +Significant changes should add the following header, which moves them to the top. ``` -# make internal-api-html -# xdg-open ./outputs/doc/share/doc/nix/internal-api/html/index.html +significance: significant ``` -## Coverage analysis + +See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog). -A coverage analysis report is [available -online](https://hydra.nixos.org/job/nix/master/coverage/latest/download-by-type/report/coverage). You -can build it yourself: - -``` -# nix build .#hydraJobs.coverage -# xdg-open ./result/coverage/index.html -``` +### Build process -Metrics about the change in line/function coverage over time are also -[available](https://hydra.nixos.org/job/nix/master/coverage#tabs-charts). +Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. +Set `buildUnreleasedNotes = true;` in `flake.nix` to build the release notes on the fly. diff --git a/doc/manual/src/contributing/index.md b/doc/manual/src/contributing/index.md new file mode 100644 index 000000000000..4d55c17a46f0 --- /dev/null +++ b/doc/manual/src/contributing/index.md @@ -0,0 +1,8 @@ +# Development + +Nix is developed on GitHub. +Check the [contributing guide](https://github.com/NixOS/nix/blob/master/CONTRIBUTING.md) if you want to get involved. + +This chapter is a collection of guides for making changes to the code and documentation. + +If you're not sure where to start, try to [compile Nix from source](./hacking.md) and consider [making improvements to documentation](./documentation.md). diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index e5f80a928534..0b45b88a3239 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -1,49 +1,149 @@ # Running tests -## Unit-tests +## Coverage analysis + +A [coverage analysis report] is available online +You can build it yourself: + +[coverage analysis report]: https://hydra.nixos.org/job/nix/master/coverage/latest/download-by-type/report/coverage + +``` +# nix build .#hydraJobs.coverage +# xdg-open ./result/coverage/index.html +``` -The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined -under `src/{library_name}/tests` using the -[googletest](https://google.github.io/googletest/) and -[rapidcheck](https://github.com/emil-e/rapidcheck) frameworks. +[Extensive records of build metrics](https://hydra.nixos.org/job/nix/master/coverage#tabs-charts), such as test coverage over time, are also available online. -You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option. +## Unit-tests + +The unit tests are defined using the [googletest] and [rapidcheck] frameworks. + +[googletest]: https://google.github.io/googletest/ +[rapidcheck]: https://github.com/emil-e/rapidcheck + +### Source and header layout + +> An example of some files, demonstrating much of what is described below +> +> ``` +> src +> ├── libexpr +> │ ├── value/context.hh +> │ ├── value/context.cc +> │ │ +> │ … +> └── tests +> │ ├── value/context.hh +> │ ├── value/context.cc +> │ │ +> │ … +> │ +> ├── unit-test-data +> │ ├── libstore +> │ │ ├── worker-protocol/content-address.bin +> │ │ … +> │ … +> … +> ``` + +The unit tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `src/${library_shortname}/tests` within the directory for the library (`src/${library_shortname}`). + +The data is in `unit-test-data`, with one subdir per library, with the same name as where the code goes. +For example, `libnixstore` code is in `src/libstore`, and its test data is in `unit-test-data/libstore`. +The path to the `unit-test-data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. + +> **Note** +> Due to the way googletest works, downstream unit test executables will actually include and re-run upstream library tests. +> Therefore it is important that the same value for `_NIX_TEST_UNIT_DATA` be used with the tests for each library. +> That is why we have the test data nested within a single `unit-test-data` directory. + +### Running tests + +You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. +Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option, or the `GTEST_FILTER` environment variable. + +### Characterisation testing { #characaterisation-testing-unit } + +See [functional characterisation testing](#characterisation-testing-functional) for a broader discussion of characterisation testing. + +Like with the functional characterisation, `_NIX_TEST_ACCEPT=1` is also used. +For example: +```shell-session +$ _NIX_TEST_ACCEPT=1 make libstore-tests-exe_RUN +... +[ SKIPPED ] WorkerProtoTest.string_read +[ SKIPPED ] WorkerProtoTest.string_write +[ SKIPPED ] WorkerProtoTest.storePath_read +[ SKIPPED ] WorkerProtoTest.storePath_write +... +``` +will regenerate the "golden master" expected result for the `libnixstore` characterisation tests. +The characterisation tests will mark themselves "skipped" since they regenerated the expected result instead of actually testing anything. ## Functional tests -The functional tests reside under the `tests` directory and are listed in `tests/local.mk`. +The functional tests reside under the `tests/functional` directory and are listed in `tests/functional/local.mk`. Each test is a bash script. +### Running the whole test suite + The whole test suite can be run with: ```shell-session $ make install && make installcheck -ran test tests/foo.sh... [PASS] -ran test tests/bar.sh... [PASS] +ran test tests/functional/foo.sh... [PASS] +ran test tests/functional/bar.sh... [PASS] ... ``` +### Grouping tests + +Sometimes it is useful to group related tests so they can be easily run together without running the entire test suite. +Each test group is in a subdirectory of `tests`. +For example, `tests/functional/ca/local.mk` defines a `ca` test group for content-addressed derivation outputs. + +That test group can be run like this: + +```shell-session +$ make ca.test-group -j50 +ran test tests/functional/ca/nix-run.sh... [PASS] +ran test tests/functional/ca/import-derivation.sh... [PASS] +... +``` + +The test group is defined in Make like this: +```makefile +$(test-group-name)-tests := \ + $(d)/test0.sh \ + $(d)/test1.sh \ + ... + +install-tests-groups += $(test-group-name) +``` + +### Running individual tests + Individual tests can be run with `make`: ```shell-session -$ make tests/${testName}.sh.test -ran test tests/${testName}.sh... [PASS] +$ make tests/functional/${testName}.sh.test +ran test tests/functional/${testName}.sh... [PASS] ``` or without `make`: ```shell-session -$ ./mk/run-test.sh tests/${testName}.sh -ran test tests/${testName}.sh... [PASS] +$ ./mk/run-test.sh tests/functional/${testName}.sh tests/functional/init.sh +ran test tests/functional/${testName}.sh... [PASS] ``` To see the complete output, one can also run: ```shell-session -$ ./mk/debug-test.sh tests/${testName}.sh -+ foo +$ ./mk/debug-test.sh tests/functional/${testName}.sh tests/functional/init.sh ++(${testName}.sh:1) foo output from foo -+ bar ++(${testName}.sh:2) bar output from bar ... ``` @@ -75,7 +175,7 @@ edit it like so: Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point: ```shell-session -$ ./mk/debug-test.sh tests/${testName}.sh +$ ./mk/debug-test.sh tests/functional/${testName}.sh tests/functional/init.sh ... + gdb blash blub GNU gdb (GDB) 12.1 @@ -86,6 +186,43 @@ GNU gdb (GDB) 12.1 One can debug the Nix invocation in all the usual ways. For example, enter `run` to start the Nix invocation. +### Troubleshooting + +Sometimes running tests in the development shell may leave artefacts in the local repository. +To remove any traces of that: + +```console +git clean -x --force tests +``` + +### Characterisation testing { #characterisation-testing-functional } + +Occasionally, Nix utilizes a technique called [Characterisation Testing](https://en.wikipedia.org/wiki/Characterization_test) as part of the functional tests. +This technique is to include the exact output/behavior of a former version of Nix in a test in order to check that Nix continues to produce the same behavior going forward. + +For example, this technique is used for the language tests, to check both the printed final value if evaluation was successful, and any errors and warnings encountered. + +It is frequently useful to regenerate the expected output. +To do that, rerun the failed test(s) with `_NIX_TEST_ACCEPT=1`. +For example: +```bash +_NIX_TEST_ACCEPT=1 make tests/functional/lang.sh.test +``` +This convention is shared with the [characterisation unit tests](#characterisation-testing-unit) too. + +An interesting situation to document is the case when these tests are "overfitted". +The language tests are, again, an example of this. +The expected successful output of evaluation is supposed to be highly stable – we do not intend to make breaking changes to (the stable parts of) the Nix language. +However, the errors and warnings during evaluation (successful or not) are not stable in this way. +We are free to change how they are displayed at any time. + +It may be surprising that we would test non-normative behavior like diagnostic outputs. +Diagnostic outputs are indeed not a stable interface, but they still are important to users. +By recording the expected output, the test suite guards against accidental changes, and ensure the *result* (not just the code that implements it) of the diagnostic code paths are under code review. +Regressions are caught, and improvements always show up in code review. + +To ensure that characterisation testing doesn't make it harder to intentionally change these interfaces, there always must be an easy way to regenerate the expected output, as we do with `_NIX_TEST_ACCEPT=1`. + ## Integration tests The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. @@ -98,7 +235,7 @@ You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix- After a one-time setup, the Nix repository's GitHub Actions continuous integration (CI) workflow can test the installer each time you push to a branch. -Creating a Cachix cache for your installer tests and adding its authorization token to GitHub enables [two installer-specific jobs in the CI workflow](https://github.com/NixOS/nix/blob/88a45d6149c0e304f6eb2efcc2d7a4d0d569f8af/.github/workflows/ci.yml#L50-L91): +Creating a Cachix cache for your installer tests and adding its authorisation token to GitHub enables [two installer-specific jobs in the CI workflow](https://github.com/NixOS/nix/blob/88a45d6149c0e304f6eb2efcc2d7a4d0d569f8af/.github/workflows/ci.yml#L50-L91): - The `installer` job generates installers for the platforms below and uploads them to your Cachix cache: - `x86_64-linux` diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index ac0bb3c2f682..07891175a4e4 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -1,236 +1,284 @@ # Glossary - - [derivation]{#gloss-derivation}\ - A description of a build task. The result of a derivation is a - store object. Derivations are typically specified in Nix expressions - using the [`derivation` primitive](./language/derivations.md). These are - translated into low-level *store derivations* (implicitly by - `nix-env` and `nix-build`, or explicitly by `nix-instantiate`). +- [derivation]{#gloss-derivation} - [derivation]: #gloss-derivation + A description of a build task. The result of a derivation is a + store object. Derivations are typically specified in Nix expressions + using the [`derivation` primitive](./language/derivations.md). These are + translated into low-level *store derivations* (implicitly by + `nix-env` and `nix-build`, or explicitly by `nix-instantiate`). - - [store derivation]{#gloss-store-derivation}\ - A [derivation] represented as a `.drv` file in the [store]. - It has a [store path], like any [store object]. + [derivation]: #gloss-derivation - Example: `/nix/store/g946hcz4c8mdvq2g8vxx42z51qb71rvp-git-2.38.1.drv` +- [store derivation]{#gloss-store-derivation} - See [`nix derivation show`](./command-ref/new-cli/nix3-derivation-show.md) (experimental) for displaying the contents of store derivations. + A [derivation] represented as a `.drv` file in the [store]. + It has a [store path], like any [store object]. - [store derivation]: #gloss-store-derivation + Example: `/nix/store/g946hcz4c8mdvq2g8vxx42z51qb71rvp-git-2.38.1.drv` - - [instantiate]{#gloss-instantiate}, instantiation\ - Translate a [derivation] into a [store derivation]. + See [`nix derivation show`](./command-ref/new-cli/nix3-derivation-show.md) (experimental) for displaying the contents of store derivations. - See [`nix-instantiate`](./command-ref/nix-instantiate.md). + [store derivation]: #gloss-store-derivation - [instantiate]: #gloss-instantiate +- [instantiate]{#gloss-instantiate}, instantiation - - [realise]{#gloss-realise}, realisation\ - Ensure a [store path] is [valid][validity]. + Translate a [derivation] into a [store derivation]. - This means either running the `builder` executable as specified in the corresponding [derivation] or fetching a pre-built [store object] from a [substituter]. + See [`nix-instantiate`](./command-ref/nix-instantiate.md). - See [`nix-build`](./command-ref/nix-build.md) and [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md). + [instantiate]: #gloss-instantiate - See [`nix build`](./command-ref/new-cli/nix3-build.md) (experimental). +- [realise]{#gloss-realise}, realisation - [realise]: #gloss-realise + Ensure a [store path] is [valid][validity]. - - [content-addressed derivation]{#gloss-content-addressed-derivation}\ - A derivation which has the - [`__contentAddressed`](./language/advanced-attributes.md#adv-attr-__contentAddressed) - attribute set to `true`. + This can be achieved by: + - Fetching a pre-built [store object] from a [substituter] + - Running the [`builder`](@docroot@/language/derivations.md#attr-builder) executable as specified in the corresponding [derivation] + - Delegating to a [remote builder](@docroot@/advanced-topics/distributed-builds.html) and retrieving the outputs + - - [fixed-output derivation]{#gloss-fixed-output-derivation}\ - A derivation which includes the - [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute. + See [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md) for a detailed description of the algorithm. - - [store]{#gloss-store}\ - The location in the file system where store objects live. Typically - `/nix/store`. + See also [`nix-build`](./command-ref/nix-build.md) and [`nix build`](./command-ref/new-cli/nix3-build.md) (experimental). - From the perspective of the location where Nix is - invoked, the Nix store can be referred to - as a "_local_" or a "_remote_" one: + [realise]: #gloss-realise - + A [local store]{#gloss-local-store} exists on the filesystem of - the machine where Nix is invoked. You can use other - local stores by passing the `--store` flag to the - `nix` command. Local stores can be used for building derivations. +- [content-addressed derivation]{#gloss-content-addressed-derivation} - + A *remote store* exists anywhere other than the - local filesystem. One example is the `/nix/store` - directory on another machine, accessed via `ssh` or - served by the `nix-serve` Perl script. + A derivation which has the + [`__contentAddressed`](./language/advanced-attributes.md#adv-attr-__contentAddressed) + attribute set to `true`. - [store]: #gloss-store - [local store]: #gloss-local-store +- [fixed-output derivation]{#gloss-fixed-output-derivation} - - [chroot store]{#gloss-chroot-store}\ - A [local store] whose canonical path is anything other than `/nix/store`. + A derivation which includes the + [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute. - - [binary cache]{#gloss-binary-cache}\ - A *binary cache* is a Nix store which uses a different format: its - metadata and signatures are kept in `.narinfo` files rather than in a - [Nix database]. This different format simplifies serving store objects - over the network, but cannot host builds. Examples of binary caches - include S3 buckets and the [NixOS binary cache](https://cache.nixos.org). +- [store]{#gloss-store} - - [store path]{#gloss-store-path}\ - The location of a [store object] in the file system, i.e., an - immediate child of the Nix store directory. + A collection of store objects, with operations to manipulate that collection. + See [Nix store](./store/index.md) for details. - Example: `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1` + There are many types of stores. + See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md) for a complete list. - [store path]: #gloss-store-path + From the perspective of the location where Nix is invoked, the Nix store can be referred to _local_ or _remote_. + Only a [local store]{#gloss-local-store} exposes a location in the file system of the machine where Nix is invoked that allows access to store objects, typically `/nix/store`. + Local stores can be used for building [derivations](#derivation). + See [Local Store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) for details. - - [file system object]{#gloss-store-object}\ - The Nix data model for representing simplified file system data. + [store]: #gloss-store + [local store]: #gloss-local-store - See [File System Object](@docroot@/architecture/file-system-object.md) for details. +- [chroot store]{#gloss-chroot-store} - [file system object]: #gloss-file-system-object + A [local store] whose canonical path is anything other than `/nix/store`. - - [store object]{#gloss-store-object}\ +- [binary cache]{#gloss-binary-cache} - A store object consists of a [file system object], [reference]s to other store objects, and other metadata. - It can be referred to by a [store path]. + A *binary cache* is a Nix store which uses a different format: its + metadata and signatures are kept in `.narinfo` files rather than in a + [Nix database]. This different format simplifies serving store objects + over the network, but cannot host builds. Examples of binary caches + include S3 buckets and the [NixOS binary cache](https://cache.nixos.org). - [store object]: #gloss-store-object +- [store path]{#gloss-store-path} - - [input-addressed store object]{#gloss-input-addressed-store-object}\ - A store object produced by building a - non-[content-addressed](#gloss-content-addressed-derivation), - non-[fixed-output](#gloss-fixed-output-derivation) - derivation. + The location of a [store object](@docroot@/store/index.md#store-object) in the file system, i.e., an immediate child of the Nix store directory. - - [output-addressed store object]{#gloss-output-addressed-store-object}\ - A [store object] whose [store path] is determined by its contents. - This includes derivations, the outputs of [content-addressed derivations](#gloss-content-addressed-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation). + > **Example** + > + > `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1` - - [substitute]{#gloss-substitute}\ - A substitute is a command invocation stored in the [Nix database] that - describes how to build a store object, bypassing the normal build - mechanism (i.e., derivations). Typically, the substitute builds the - store object by downloading a pre-built version of the store object - from some server. + See [Store Path](@docroot@/store/store-path.md) for details. - - [substituter]{#gloss-substituter}\ - An additional [store]{#gloss-store} from which Nix can obtain store objects instead of building them. - Often the substituter is a [binary cache](#gloss-binary-cache), but any store can serve as substituter. + [store path]: #gloss-store-path - See the [`substituters` configuration option](./command-ref/conf-file.md#conf-substituters) for details. +- [file system object]{#gloss-store-object} - [substituter]: #gloss-substituter + The Nix data model for representing simplified file system data. - - [purity]{#gloss-purity}\ - The assumption that equal Nix derivations when run always produce - the same output. This cannot be guaranteed in general (e.g., a - builder can rely on external inputs such as the network or the - system time) but the Nix model assumes it. + See [File System Object](@docroot@/store/file-system-object.md) for details. - - [Nix database]{#gloss-nix-database}\ - An SQlite database to track [reference]s between [store object]s. - This is an implementation detail of the [local store]. + [file system object]: #gloss-file-system-object - Default location: `/nix/var/nix/db`. +- [store object]{#gloss-store-object} - [Nix database]: #gloss-nix-database + Part of the contents of a [store]. - - [Nix expression]{#gloss-nix-expression}\ - A high-level description of software packages and compositions - thereof. Deploying software using Nix entails writing Nix - expressions for your packages. Nix expressions are translated to - derivations that are stored in the Nix store. These derivations can - then be built. + A store object consists of a [file system object], [references][reference] to other store objects, and other metadata. + It can be referred to by a [store path]. - - [reference]{#gloss-reference}\ - A [store object] `O` is said to have a *reference* to a store object `P` if a [store path] to `P` appears in the contents of `O`. + See [Store Object](@docroot@/store/index.md#store-object) for details. - Store objects can refer to both other store objects and themselves. - References from a store object to itself are called *self-references*. - References other than a self-reference must not form a cycle. + [store object]: #gloss-store-object - [reference]: #gloss-reference +- [IFD]{#gloss-ifd} - - [reachable]{#gloss-reachable}\ - A store path `Q` is reachable from another store path `P` if `Q` - is in the *closure* of the *references* relation. + [Import From Derivation](./language/import-from-derivation.md) - - [closure]{#gloss-closure}\ - The closure of a store path is the set of store paths that are - directly or indirectly “reachable” from that store path; that is, - it’s the closure of the path under the *references* relation. For - a package, the closure of its derivation is equivalent to the - build-time dependencies, while the closure of its output path is - equivalent to its runtime dependencies. For correct deployment it - is necessary to deploy whole closures, since otherwise at runtime - files could be missing. The command `nix-store --query --requisites ` prints out - closures of store paths. +- [input-addressed store object]{#gloss-input-addressed-store-object} - As an example, if the [store object] at path `P` contains a [reference] - to a store object at path `Q`, then `Q` is in the closure of `P`. Further, if `Q` - references `R` then `R` is also in the closure of `P`. + A store object produced by building a + non-[content-addressed](#gloss-content-addressed-derivation), + non-[fixed-output](#gloss-fixed-output-derivation) + derivation. - [closure]: #gloss-closure +- [output-addressed store object]{#gloss-output-addressed-store-object} - - [output path]{#gloss-output-path}\ - A [store path] produced by a [derivation]. + A [store object] whose [store path] is determined by its contents. + This includes derivations, the outputs of [content-addressed derivations](#gloss-content-addressed-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation). - [output path]: #gloss-output-path +- [substitute]{#gloss-substitute} - - [deriver]{#gloss-deriver}\ - The [store derivation] that produced an [output path]. + A substitute is a command invocation stored in the [Nix database] that + describes how to build a store object, bypassing the normal build + mechanism (i.e., derivations). Typically, the substitute builds the + store object by downloading a pre-built version of the store object + from some server. - - [validity]{#gloss-validity}\ - A store path is valid if all [store object]s in its [closure] can be read from the [store]. +- [substituter]{#gloss-substituter} - For a [local store], this means: - - The store path leads to an existing [store object] in that [store]. - - The store path is listed in the [Nix database] as being valid. - - All paths in the store path's [closure] are valid. + An additional [store]{#gloss-store} from which Nix can obtain store objects instead of building them. + Often the substituter is a [binary cache](#gloss-binary-cache), but any store can serve as substituter. - [validity]: #gloss-validity + See the [`substituters` configuration option](./command-ref/conf-file.md#conf-substituters) for details. - - [user environment]{#gloss-user-env}\ - An automatically generated store object that consists of a set of - symlinks to “active” applications, i.e., other store paths. These - are generated automatically by - [`nix-env`](./command-ref/nix-env.md). See *profiles*. + [substituter]: #gloss-substituter - - [profile]{#gloss-profile}\ - A symlink to the current *user environment* of a user, e.g., - `/nix/var/nix/profiles/default`. +- [purity]{#gloss-purity} - - [installable]{#gloss-installable}\ - Something that can be realised in the Nix store. + The assumption that equal Nix derivations when run always produce + the same output. This cannot be guaranteed in general (e.g., a + builder can rely on external inputs such as the network or the + system time) but the Nix model assumes it. - See [installables](./command-ref/new-cli/nix.md#installables) for [`nix` commands](./command-ref/new-cli/nix.md) (experimental) for details. +- [Nix database]{#gloss-nix-database} - - [NAR]{#gloss-nar}\ - A *N*ix *AR*chive. This is a serialisation of a path in the Nix - store. It can contain regular files, directories and symbolic - links. NARs are generated and unpacked using `nix-store --dump` - and `nix-store --restore`. + An SQlite database to track [reference]s between [store object]s. + This is an implementation detail of the [local store]. - - [`∅`]{#gloss-emtpy-set}\ - The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile. + Default location: `/nix/var/nix/db`. - - [`ε`]{#gloss-epsilon}\ - The epsilon symbol. In the context of a package, this means the version is empty. More precisely, the derivation does not have a version attribute. + [Nix database]: #gloss-nix-database - - [string interpolation]{#gloss-string-interpolation}\ - Expanding expressions enclosed in `${ }` within a [string], [path], or [attribute name]. +- [Nix expression]{#gloss-nix-expression} - See [String interpolation](./language/string-interpolation.md) for details. + A high-level description of software packages and compositions + thereof. Deploying software using Nix entails writing Nix + expressions for your packages. Nix expressions are translated to + derivations that are stored in the Nix store. These derivations can + then be built. - [string]: ./language/values.md#type-string - [path]: ./language/values.md#type-path - [attribute name]: ./language/values.md#attribute-set +- [reference]{#gloss-reference} - - [experimental feature]{#gloss-experimental-feature}\ - Not yet stabilized functionality guarded by named experimental feature flags. - These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. + A [store object] `O` is said to have a *reference* to a store object `P` if a [store path] to `P` appears in the contents of `O`. - See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/contributing/experimental-features.md). + Store objects can refer to both other store objects and themselves. + References from a store object to itself are called *self-references*. + References other than a self-reference must not form a cycle. + + [reference]: #gloss-reference + +- [reachable]{#gloss-reachable} + + A store path `Q` is reachable from another store path `P` if `Q` + is in the *closure* of the *references* relation. + +- [closure]{#gloss-closure} + + The closure of a store path is the set of store paths that are + directly or indirectly “reachable” from that store path; that is, + it’s the closure of the path under the *references* relation. For + a package, the closure of its derivation is equivalent to the + build-time dependencies, while the closure of its output path is + equivalent to its runtime dependencies. For correct deployment it + is necessary to deploy whole closures, since otherwise at runtime + files could be missing. The command `nix-store --query --requisites ` prints out + closures of store paths. + + As an example, if the [store object] at path `P` contains a [reference] + to a store object at path `Q`, then `Q` is in the closure of `P`. Further, if `Q` + references `R` then `R` is also in the closure of `P`. + + [closure]: #gloss-closure + +- [output]{#gloss-output} + + A [store object] produced by a [derivation]. + See [the `outputs` argument to the `derivation` function](@docroot@/language/derivations.md#attr-outputs) for details. + + [output]: #gloss-output + +- [output path]{#gloss-output-path} + + The [store path] to the [output] of a [derivation]. + + [output path]: #gloss-output-path + +- [deriver]{#gloss-deriver} + + The [store derivation] that produced an [output path]. + +- [validity]{#gloss-validity} + + A store path is valid if all [store object]s in its [closure] can be read from the [store]. + + For a [local store], this means: + - The store path leads to an existing [store object] in that [store]. + - The store path is listed in the [Nix database] as being valid. + - All paths in the store path's [closure] are valid. + + [validity]: #gloss-validity + +- [user environment]{#gloss-user-env} + + An automatically generated store object that consists of a set of + symlinks to “active” applications, i.e., other store paths. These + are generated automatically by + [`nix-env`](./command-ref/nix-env.md). See *profiles*. + +- [profile]{#gloss-profile} + + A symlink to the current *user environment* of a user, e.g., + `/nix/var/nix/profiles/default`. + +- [installable]{#gloss-installable} + + Something that can be realised in the Nix store. + + See [installables](./command-ref/new-cli/nix.md#installables) for [`nix` commands](./command-ref/new-cli/nix.md) (experimental) for details. + +- [NAR]{#gloss-nar} + + A *N*ix *AR*chive. This is a serialisation of a path in the Nix + store. It can contain regular files, directories and symbolic + links. NARs are generated and unpacked using `nix-store --dump` + and `nix-store --restore`. + +- [`∅`]{#gloss-emtpy-set} + + The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile. + +- [`ε`]{#gloss-epsilon} + + The epsilon symbol. In the context of a package, this means the version is empty. More precisely, the derivation does not have a version attribute. + +- [string interpolation]{#gloss-string-interpolation} + + Expanding expressions enclosed in `${ }` within a [string], [path], or [attribute name]. + + See [String interpolation](./language/string-interpolation.md) for details. + + [string]: ./language/values.md#type-string + [path]: ./language/values.md#type-path + [attribute name]: ./language/values.md#attribute-set + +- [experimental feature]{#gloss-experimental-feature} + + Not yet stabilized functionality guarded by named experimental feature flags. + These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. + + See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/contributing/experimental-features.md). diff --git a/doc/manual/src/installation/building-source.md b/doc/manual/src/installation/building-source.md index ed1efffd8eb7..7dad9805a23d 100644 --- a/doc/manual/src/installation/building-source.md +++ b/doc/manual/src/installation/building-source.md @@ -3,7 +3,7 @@ After cloning Nix's Git repository, issue the following commands: ```console -$ ./bootstrap.sh +$ autoreconf -vfi $ ./configure options... $ make $ make install diff --git a/doc/manual/src/installation/installation.md b/doc/manual/src/installation/index.md similarity index 100% rename from doc/manual/src/installation/installation.md rename to doc/manual/src/installation/index.md diff --git a/doc/manual/src/installation/installing-docker.md b/doc/manual/src/installation/installing-docker.md index 9d6d8f2d9453..6f77d6a57083 100644 --- a/doc/manual/src/installation/installing-docker.md +++ b/doc/manual/src/installation/installing-docker.md @@ -3,14 +3,14 @@ To run the latest stable release of Nix with Docker run the following command: ```console -$ docker run -ti nixos/nix -Unable to find image 'nixos/nix:latest' locally -latest: Pulling from nixos/nix +$ docker run -ti ghcr.io/nixos/nix +Unable to find image 'ghcr.io/nixos/nix:latest' locally +latest: Pulling from ghcr.io/nixos/nix 5843afab3874: Pull complete b52bf13f109c: Pull complete 1e2415612aa3: Pull complete Digest: sha256:27f6e7f60227e959ee7ece361f75d4844a40e1cc6878b6868fe30140420031ff -Status: Downloaded newer image for nixos/nix:latest +Status: Downloaded newer image for ghcr.io/nixos/nix:latest 35ca4ada6e96:/# nix --version nix (Nix) 2.3.12 35ca4ada6e96:/# exit diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 3079714347ca..282b75af2cff 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -112,6 +112,13 @@ Derivations can declare some infrequently used optional attributes. > environmental variables come from the environment of the > `nix-build`. + If the [`configurable-impure-env` experimental + feature](@docroot@/contributing/experimental-features.md#xp-feature-configurable-impure-env) + is enabled, these environment variables can also be controlled + through the + [`impure-env`](@docroot@/command-ref/conf-file.md#conf-impure-env) + configuration setting. + - [`outputHash`]{#adv-attr-outputHash}; [`outputHashAlgo`]{#adv-attr-outputHashAlgo}; [`outputHashMode`]{#adv-attr-outputHashMode}\ These attributes declare that the derivation is a so-called *fixed-output derivation*, which means that a cryptographic hash of @@ -229,6 +236,8 @@ Derivations can declare some infrequently used optional attributes. [`outputHashAlgo`](#adv-attr-outputHashAlgo) like for *fixed-output derivations* (see above). + It also implicitly requires that the machine to build the derivation must have the `ca-derivations` [system feature](@docroot@/command-ref/conf-file.md#conf-system-features). + - [`passAsFile`]{#adv-attr-passAsFile}\ A list of names of attributes that should be passed via files rather than environment variables. For example, if you have @@ -261,6 +270,9 @@ Derivations can declare some infrequently used optional attributes. useful for very trivial derivations (such as `writeText` in Nixpkgs) that are cheaper to build than to substitute from a binary cache. + You may disable the effects of this attibute by enabling the + `always-allow-substitutes` configuration option in Nix. + > **Note** > > You need to have a builder configured which satisfies the @@ -271,18 +283,21 @@ Derivations can declare some infrequently used optional attributes. - [`__structuredAttrs`]{#adv-attr-structuredAttrs}\ If the special attribute `__structuredAttrs` is set to `true`, the other derivation - attributes are serialised in JSON format and made available to the - builder via the file `.attrs.json` in the builder’s temporary - directory. This obviates the need for [`passAsFile`](#adv-attr-passAsFile) since JSON files - have no size restrictions, unlike process environments. + attributes are serialised into a file in JSON format. The environment variable + `NIX_ATTRS_JSON_FILE` points to the exact location of that file both in a build + and a [`nix-shell`](../command-ref/nix-shell.md). This obviates the need for + [`passAsFile`](#adv-attr-passAsFile) since JSON files have no size restrictions, + unlike process environments. It also makes it possible to tweak derivation settings in a structured way; see [`outputChecks`](#adv-attr-outputChecks) for example. As a convenience to Bash builders, - Nix writes a script named `.attrs.sh` to the builder’s directory - that initialises shell variables corresponding to all attributes - that are representable in Bash. This includes non-nested + Nix writes a script that initialises shell variables + corresponding to all attributes that are representable in Bash. The + environment variable `NIX_ATTRS_SH_FILE` points to the exact + location of the script, both in a build and a + [`nix-shell`](../command-ref/nix-shell.md). This includes non-nested (associative) arrays. For example, the attribute `hardening.format = true` ends up as the Bash associative array element `${hardening[format]}`. @@ -320,16 +335,6 @@ Derivations can declare some infrequently used optional attributes. ``` - [`unsafeDiscardReferences`]{#adv-attr-unsafeDiscardReferences}\ - > **Warning** - > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md). - > - > To use this attribute, you must enable the - > [`discard-references`](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) experimental feature. - > For example, in [nix.conf](../command-ref/conf-file.md) you could add: - > - > ``` - > extra-experimental-features = discard-references - > ``` When using [structured attributes](#adv-attr-structuredAttrs), the attribute `unsafeDiscardReferences` is an attribute set with a boolean value for each output name. @@ -345,3 +350,15 @@ Derivations can declare some infrequently used optional attributes. This is useful, for example, when generating self-contained filesystem images with their own embedded Nix store: hashes found inside such an image refer to the embedded store and not to the host's Nix store. + +- [`requiredSystemFeatures`]{#adv-attr-requiredSystemFeatures}\ + + If a derivation has the `requiredSystemFeatures` attribute, then Nix will only build it on a machine that has the corresponding features set in its [`system-features` configuration](@docroot@/command-ref/conf-file.md#conf-system-features). + + For example, setting + + ```nix + requiredSystemFeatures = [ "kvm" ]; + ``` + + ensures that the derivation can only be built on a machine with the `kvm` feature. diff --git a/doc/manual/src/language/builtins-prefix.md b/doc/manual/src/language/builtins-prefix.md index 35e3dccc3df9..7b2321466ddc 100644 --- a/doc/manual/src/language/builtins-prefix.md +++ b/doc/manual/src/language/builtins-prefix.md @@ -3,7 +3,7 @@ This section lists the functions built into the Nix language evaluator. All built-in functions are available through the global [`builtins`](./builtin-constants.md#builtins-builtins) constant. -For convenience, some built-ins are can be accessed directly: +For convenience, some built-ins can be accessed directly: - [`derivation`](#builtins-derivation) - [`import`](#builtins-import) diff --git a/doc/manual/src/language/constructs.md b/doc/manual/src/language/constructs.md index c53eb88897d1..a82ec5960a8d 100644 --- a/doc/manual/src/language/constructs.md +++ b/doc/manual/src/language/constructs.md @@ -92,10 +92,10 @@ In this fragment from `all-packages.nix`, ```nix graphviz = (import ../tools/graphics/graphviz) { inherit fetchurl stdenv libpng libjpeg expat x11 yacc; - inherit (xlibs) libXaw; + inherit (xorg) libXaw; }; -xlibs = { +xorg = { libX11 = ...; libXaw = ...; ... @@ -109,7 +109,7 @@ libjpg = ...; the set used in the function call to the function defined in `../tools/graphics/graphviz` inherits a number of variables from the surrounding scope (`fetchurl` ... `yacc`), but also inherits `libXaw` -(the X Athena Widgets) from the `xlibs` (X11 client-side libraries) set. +(the X Athena Widgets) from the `xorg` set. Summarizing the fragment @@ -132,6 +132,32 @@ a = src-set.a; b = src-set.b; c = src-set.c; when used while defining local variables in a let-expression or while defining a set. +In a `let` expression, `inherit` can be used to selectively bring specific attributes of a set into scope. For example + + +```nix +let + x = { a = 1; b = 2; }; + inherit (builtins) attrNames; +in +{ + names = attrNames x; +} +``` + +is equivalent to + +```nix +let + x = { a = 1; b = 2; }; +in +{ + names = builtins.attrNames x; +} +``` + +both evaluate to `{ names = [ "a" "b" ]; }`. + ## Functions Functions have the following form: @@ -146,92 +172,103 @@ three kinds of patterns: - If a pattern is a single identifier, then the function matches any argument. Example: - + ```nix let negate = x: !x; concat = x: y: x + y; in if negate true then concat "foo" "bar" else "" ``` - + Note that `concat` is a function that takes one argument and returns a function that takes another argument. This allows partial parameterisation (i.e., only filling some of the arguments of a function); e.g., - + ```nix map (concat "foo") [ "bar" "bla" "abc" ] ``` - + evaluates to `[ "foobar" "foobla" "fooabc" ]`. - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a set containing the listed attributes, and binds the values of those attributes to variables in the function body. For example, the function - + ```nix { x, y, z }: z + y + x ``` - + can only be called with a set containing exactly the attributes `x`, `y` and `z`. No other attributes are allowed. If you want to allow additional arguments, you can use an ellipsis (`...`): - + ```nix { x, y, z, ... }: z + y + x ``` - + This works on any set that contains at least the three named attributes. - + It is possible to provide *default values* for attributes, in which case they are allowed to be missing. A default value is specified by writing `name ? e`, where *e* is an arbitrary expression. For example, - + ```nix { x, y ? "foo", z ? "bar" }: z + y + x ``` - + specifies a function that only requires an attribute named `x`, but optionally accepts `y` and `z`. - An `@`-pattern provides a means of referring to the whole value being matched: - + ```nix args@{ x, y, z, ... }: z + y + x + args.a ``` - + but can also be written as: - + ```nix { x, y, z, ... } @ args: z + y + x + args.a ``` - - Here `args` is bound to the entire argument, which is further - matched against the pattern `{ x, y, z, - ... }`. `@`-pattern makes mainly sense with an ellipsis(`...`) as + + Here `args` is bound to the argument *as passed*, which is further + matched against the pattern `{ x, y, z, ... }`. + The `@`-pattern makes mainly sense with an ellipsis(`...`) as you can access attribute names as `a`, using `args.a`, which was given as an additional attribute to the function. - + > **Warning** - > - > The `args@` expression is bound to the argument passed to the - > function which means that attributes with defaults that aren't - > explicitly specified in the function call won't cause an - > evaluation error, but won't exist in `args`. - > + > + > `args@` binds the name `args` to the attribute set that is passed to the function. + > In particular, `args` does *not* include any default values specified with `?` in the function's set pattern. + > > For instance - > + > > ```nix > let - > function = args@{ a ? 23, ... }: args; + > f = args@{ a ? 23, ... }: [ a args ]; > in - > function {} - > ```` - > - > will evaluate to an empty attribute set. + > f {} + > ``` + > + > is equivalent to + > + > ```nix + > let + > f = args @ { ... }: [ (args.a or 23) args ]; + > in + > f {} + > ``` + > + > and both expressions will evaluate to: + > + > ```nix + > [ 23 {} ] + > ``` Note that functions do not have names. If you want to give them a name, you can bind them to an attribute, e.g., diff --git a/doc/manual/src/language/constructs/lookup-path.md b/doc/manual/src/language/constructs/lookup-path.md new file mode 100644 index 000000000000..e87d2922bd31 --- /dev/null +++ b/doc/manual/src/language/constructs/lookup-path.md @@ -0,0 +1,27 @@ +# Lookup path + +> **Syntax** +> +> *lookup-path* = `<` *identifier* [ `/` *identifier* ]... `>` + +A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/values.md#type-path) if the identifier matches a search path entry. + +The value of a lookup path is determined by [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath). + +See [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile) for details on lookup path resolution. + +> **Example** +> +> ```nix +> +>``` +> +> /nix/var/nix/profiles/per-user/root/channels/nixpkgs + +> **Example** +> +> ```nix +> +>``` +> +> /nix/var/nix/profiles/per-user/root/channels/nixpkgs/nixos diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 043a38191630..2aded5527b9c 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -1,161 +1,315 @@ # Derivations -The most important built-in function is `derivation`, which is used to -describe a single derivation (a build task). It takes as input a set, -the attributes of which specify the inputs of the build. - - - There must be an attribute named [`system`]{#attr-system} whose value must be a - string specifying a Nix system type, such as `"i686-linux"` or - `"x86_64-darwin"`. (To figure out your system type, run `nix -vv - --version`.) The build can only be performed on a machine and - operating system matching the system type. (Nix can automatically - [forward builds for other - platforms](../advanced-topics/distributed-builds.md) by forwarding - them to other machines.) - - - There must be an attribute named `name` whose value must be a - string. This is used as a symbolic name for the package by - `nix-env`, and it is appended to the output paths of the derivation. - - - There must be an attribute named `builder` that identifies the - program that is executed to perform the build. It can be either a - derivation or a source (a local file reference, e.g., - `./builder.sh`). - - - Every attribute is passed as an environment variable to the builder. - Attribute values are translated to environment variables as follows: - - - Strings and numbers are just passed verbatim. - - - A *path* (e.g., `../foo/sources.tar`) causes the referenced file - to be copied to the store; its location in the store is put in - the environment variable. The idea is that all sources should - reside in the Nix store, since all inputs to a derivation should - reside in the Nix store. - - - A *derivation* causes that derivation to be built prior to the - present derivation; its default output path is put in the - environment variable. - - - Lists of the previous types are also allowed. They are simply - concatenated, separated by spaces. - - - `true` is passed as the string `1`, `false` and `null` are - passed as an empty string. - - - The optional attribute `args` specifies command-line arguments to be - passed to the builder. It should be a list. - - - The optional attribute `outputs` specifies a list of symbolic - outputs of the derivation. By default, a derivation produces a - single output path, denoted as `out`. However, derivations can - produce multiple output paths. This is useful because it allows - outputs to be downloaded or garbage-collected separately. For - instance, imagine a library package that provides a dynamic library, - header files, and documentation. A program that links against the - library doesn’t need the header files and documentation at runtime, - and it doesn’t need the documentation at build time. Thus, the - library package could specify: - - ```nix - outputs = [ "lib" "headers" "doc" ]; - ``` - - This will cause Nix to pass environment variables `lib`, `headers` - and `doc` to the builder containing the intended store paths of each - output. The builder would typically do something like - - ```bash - ./configure \ - --libdir=$lib/lib \ - --includedir=$headers/include \ - --docdir=$doc/share/doc - ``` - - for an Autoconf-style package. You can refer to each output of a - derivation by selecting it as an attribute, e.g. - - ```nix - buildInputs = [ pkg.lib pkg.headers ]; - ``` - - The first element of `outputs` determines the *default output*. - Thus, you could also write - - ```nix - buildInputs = [ pkg pkg.headers ]; - ``` - - since `pkg` is equivalent to `pkg.lib`. - -The function `mkDerivation` in the Nixpkgs standard environment is a -wrapper around `derivation` that adds a default value for `system` and -always uses Bash as the builder, to which the supplied builder is passed -as a command-line argument. See the Nixpkgs manual for details. - -The builder is executed as follows: - - - A temporary directory is created under the directory specified by - `TMPDIR` (default `/tmp`) where the build will take place. The - current directory is changed to this directory. - - - The environment is cleared and set to the derivation attributes, as - specified above. - - - In addition, the following variables are set: - - - `NIX_BUILD_TOP` contains the path of the temporary directory for - this build. - - - Also, `TMPDIR`, `TEMPDIR`, `TMP`, `TEMP` are set to point to the - temporary directory. This is to prevent the builder from - accidentally writing temporary files anywhere else. Doing so - might cause interference by other processes. - - - `PATH` is set to `/path-not-set` to prevent shells from - initialising it to their built-in default value. - - - `HOME` is set to `/homeless-shelter` to prevent programs from - using `/etc/passwd` or the like to find the user's home - directory, which could cause impurity. Usually, when `HOME` is - set, it is used as the location of the home directory, even if - it points to a non-existent path. - - - `NIX_STORE` is set to the path of the top-level Nix store - directory (typically, `/nix/store`). - - - For each output declared in `outputs`, the corresponding - environment variable is set to point to the intended path in the - Nix store for that output. Each output path is a concatenation - of the cryptographic hash of all build inputs, the `name` - attribute and the output name. (The output name is omitted if - it’s `out`.) - - - If an output path already exists, it is removed. Also, locks are - acquired to prevent multiple Nix instances from performing the same - build at the same time. - - - A log of the combined standard output and error is written to - `/nix/var/log/nix`. - - - The builder is executed with the arguments specified by the - attribute `args`. If it exits with exit code 0, it is considered to - have succeeded. - - - The temporary directory is removed (unless the `-K` option was - specified). - - - If the build was successful, Nix scans each output path for - references to input paths by looking for the hash parts of the input - paths. Since these are potential runtime dependencies, Nix registers - them as dependencies of the output paths. - - - After the build, Nix sets the last-modified timestamp on all files - in the build result to 1 (00:00:01 1/1/1970 UTC), sets the group to - the default group, and sets the mode of the file to 0444 or 0555 - (i.e., read-only, with execute permission enabled if the file was - originally executable). Note that possible `setuid` and `setgid` - bits are cleared. Setuid and setgid programs are not currently - supported by Nix. This is because the Nix archives used in - deployment have no concept of ownership information, and because it - makes the build result dependent on the user performing the build. +The most important built-in function is `derivation`, which is used to describe a single derivation: +a specification for running an executable on precisely defined input files to repeatably produce output files at uniquely determined file system paths. + +It takes as input an attribute set, the attributes of which specify the inputs to the process. +It outputs an attribute set, and produces a [store derivation] as a side effect of evaluation. + +[store derivation]: @docroot@/glossary.md#gloss-store-derivation + +## Input attributes + +### Required + +- [`name`]{#attr-name} ([String](@docroot@/language/values.md#type-string)) + + A symbolic name for the derivation. + It is added to the [store path] of the corresponding [store derivation] as well as to its [output paths](@docroot@/glossary.md#gloss-output-path). + + [store path]: @docroot@/glossary.md#gloss-store-path + + > **Example** + > + > ```nix + > derivation { + > name = "hello"; + > # ... + > } + > ``` + > + > The store derivation's path will be `/nix/store/-hello.drv`. + > The [output](#attr-outputs) paths will be of the form `/nix/store/-hello[-]` + +- [`system`]{#attr-system} ([String](@docroot@/language/values.md#type-string)) + + The system type on which the [`builder`](#attr-builder) executable is meant to be run. + + A necessary condition for Nix to build derivations locally is that the `system` attribute matches the current [`system` configuration option]. + It can automatically [build on other platforms](../advanced-topics/distributed-builds.md) by forwarding build requests to other machines. + + [`system` configuration option]: @docroot@/command-ref/conf-file.md#conf-system + + > **Example** + > + > Declare a derivation to be built on a specific system type: + > + > ```nix + > derivation { + > # ... + > system = "x86_64-linux"; + > # ... + > } + > ``` + + > **Example** + > + > Declare a derivation to be built on the system type that evaluates the expression: + > + > ```nix + > derivation { + > # ... + > system = builtins.currentSystem; + > # ... + > } + > ``` + > + > [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation. + +- [`builder`]{#attr-builder} ([Path](@docroot@/language/values.md#type-path) | [String](@docroot@/language/values.md#type-string)) + + Path to an executable that will perform the build. + + > **Example** + > + > Use the file located at `/bin/bash` as the builder executable: + > + > ```nix + > derivation { + > # ... + > builder = "/bin/bash"; + > # ... + > }; + > ``` + + + + > **Example** + > + > Copy a local file to the Nix store for use as the builder executable: + > + > ```nix + > derivation { + > # ... + > builder = ./builder.sh; + > # ... + > }; + > ``` + + + + > **Example** + > + > Use a file from another derivation as the builder executable: + > + > ```nix + > let pkgs = import {}; in + > derivation { + > # ... + > builder = "${pkgs.python}/bin/python"; + > # ... + > }; + > ``` + +### Optional + +- [`args`]{#attr-args} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) + + Default: `[ ]` + + Command-line arguments to be passed to the [`builder`](#attr-builder) executable. + + > **Example** + > + > Pass arguments to Bash to interpret a shell command: + > + > ```nix + > derivation { + > # ... + > builder = "/bin/bash"; + > args = [ "-c" "echo hello world > $out" ]; + > # ... + > }; + > ``` + +- [`outputs`]{#attr-outputs} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) + + Default: `[ "out" ]` + + Symbolic outputs of the derivation. + Each output name is passed to the [`builder`](#attr-builder) executable as an environment variable with its value set to the corresponding [store path]. + + By default, a derivation produces a single output called `out`. + However, derivations can produce multiple outputs. + This allows the associated [store objects](@docroot@/glossary.md#gloss-store-object) and their [closures](@docroot@/glossary.md#gloss-closure) to be copied or garbage-collected separately. + + > **Example** + > + > Imagine a library package that provides a dynamic library, header files, and documentation. + > A program that links against such a library doesn’t need the header files and documentation at runtime, and it doesn’t need the documentation at build time. + > Thus, the library package could specify: + > + > ```nix + > derivation { + > # ... + > outputs = [ "lib" "dev" "doc" ]; + > # ... + > } + > ``` + > + > This will cause Nix to pass environment variables `lib`, `dev`, and `doc` to the builder containing the intended store paths of each output. + > The builder would typically do something like + > + > ```bash + > ./configure \ + > --libdir=$lib/lib \ + > --includedir=$dev/include \ + > --docdir=$doc/share/doc + > ``` + > + > for an Autoconf-style package. + + The name of an output is combined with the name of the derivation to create the name part of the output's store path, unless it is `out`, in which case just the name of the derivation is used. + + > **Example** + > + > + > ```nix + > derivation { + > name = "example"; + > outputs = [ "lib" "dev" "doc" "out" ]; + > # ... + > } + > ``` + > + > The store derivation path will be `/nix/store/-example.drv`. + > The output paths will be + > - `/nix/store/-example-lib` + > - `/nix/store/-example-dev` + > - `/nix/store/-example-doc` + > - `/nix/store/-example` + + You can refer to each output of a derivation by selecting it as an attribute. + The first element of `outputs` determines the *default output* and ends up at the top-level. + + > **Example** + > + > Select an output by attribute name: + > + > ```nix + > let + > myPackage = derivation { + > name = "example"; + > outputs = [ "lib" "dev" "doc" "out" ]; + > # ... + > }; + > in myPackage.dev + > ``` + > + > Since `lib` is the first output, `myPackage` is equivalent to `myPackage.lib`. + + + +- See [Advanced Attributes](./advanced-attributes.md) for more, infrequently used, optional attributes. + + + +- Every other attribute is passed as an environment variable to the builder. + Attribute values are translated to environment variables as follows: + + - Strings are passed unchanged. + + - Integral numbers are converted to decimal notation. + + - Floating point numbers are converted to simple decimal or scientific notation with a preset precision. + + - A *path* (e.g., `../foo/sources.tar`) causes the referenced file + to be copied to the store; its location in the store is put in + the environment variable. The idea is that all sources should + reside in the Nix store, since all inputs to a derivation should + reside in the Nix store. + + - A *derivation* causes that derivation to be built prior to the + present derivation. The environment variable is set to the [store path] of the derivation's default [output](#attr-outputs). + + - Lists of the previous types are also allowed. They are simply + concatenated, separated by spaces. + + - `true` is passed as the string `1`, `false` and `null` are + passed as an empty string. + + + +## Builder execution + +The [`builder`](#attr-builder) is executed as follows: + +- A temporary directory is created under the directory specified by + `TMPDIR` (default `/tmp`) where the build will take place. The + current directory is changed to this directory. + +- The environment is cleared and set to the derivation attributes, as + specified above. + +- In addition, the following variables are set: + + - `NIX_BUILD_TOP` contains the path of the temporary directory for + this build. + + - Also, `TMPDIR`, `TEMPDIR`, `TMP`, `TEMP` are set to point to the + temporary directory. This is to prevent the builder from + accidentally writing temporary files anywhere else. Doing so + might cause interference by other processes. + + - `PATH` is set to `/path-not-set` to prevent shells from + initialising it to their built-in default value. + + - `HOME` is set to `/homeless-shelter` to prevent programs from + using `/etc/passwd` or the like to find the user's home + directory, which could cause impurity. Usually, when `HOME` is + set, it is used as the location of the home directory, even if + it points to a non-existent path. + + - `NIX_STORE` is set to the path of the top-level Nix store + directory (typically, `/nix/store`). + + - `NIX_ATTRS_JSON_FILE` & `NIX_ATTRS_SH_FILE` if `__structuredAttrs` + is set to `true` for the dervation. A detailed explanation of this + behavior can be found in the + [section about structured attrs](./advanced-attributes.md#adv-attr-structuredAttrs). + + - For each output declared in `outputs`, the corresponding + environment variable is set to point to the intended path in the + Nix store for that output. Each output path is a concatenation + of the cryptographic hash of all build inputs, the `name` + attribute and the output name. (The output name is omitted if + it’s `out`.) + +- If an output path already exists, it is removed. Also, locks are + acquired to prevent multiple Nix instances from performing the same + build at the same time. + +- A log of the combined standard output and error is written to + `/nix/var/log/nix`. + +- The builder is executed with the arguments specified by the + attribute `args`. If it exits with exit code 0, it is considered to + have succeeded. + +- The temporary directory is removed (unless the `-K` option was + specified). + +- If the build was successful, Nix scans each output path for + references to input paths by looking for the hash parts of the input + paths. Since these are potential runtime dependencies, Nix registers + them as dependencies of the output paths. + +- After the build, Nix sets the last-modified timestamp on all files + in the build result to 1 (00:00:01 1/1/1970 UTC), sets the group to + the default group, and sets the mode of the file to 0444 or 0555 + (i.e., read-only, with execute permission enabled if the file was + originally executable). Note that possible `setuid` and `setgid` + bits are cleared. Setuid and setgid programs are not currently + supported by Nix. This is because the Nix archives used in + deployment have no concept of ownership information, and because it + makes the build result dependent on the user performing the build. diff --git a/doc/manual/src/language/import-from-derivation.md b/doc/manual/src/language/import-from-derivation.md new file mode 100644 index 000000000000..03b3f9d91b10 --- /dev/null +++ b/doc/manual/src/language/import-from-derivation.md @@ -0,0 +1,139 @@ +# Import From Derivation + +The value of a Nix expression can depend on the contents of a [store object](@docroot@/glossary.md#gloss-store-object). + +Passing an expression `expr` that evaluates to a [store path](@docroot@/glossary.md#gloss-store-path) to any built-in function which reads from the filesystem constitutes Import From Derivation (IFD): + +- [`import`](./builtins.md#builtins-import)` expr` +- [`builtins.readFile`](./builtins.md#builtins-readFile)` expr` +- [`builtins.readFileType`](./builtins.md#builtins-readFileType)` expr` +- [`builtins.readDir`](./builtins.md#builtins-readDir)` expr` +- [`builtins.pathExists`](./builtins.md#builtins-pathExists)` expr` +- [`builtins.filterSource`](./builtins.md#builtins-filterSource)` f expr` +- [`builtins.path`](./builtins.md#builtins-path)` { path = expr; }` +- [`builtins.hashFile`](./builtins.md#builtins-hashFile)` t expr` +- `builtins.scopedImport x drv` + +When the store path needs to be accessed, evaluation will be paused, the corresponding store object [realised], and then evaluation resumed. + +[realised]: @docroot@/glossary.md#gloss-realise + +This has performance implications: +Evaluation can only finish when all required store objects are realised. +Since the Nix language evaluator is sequential, it only finds store paths to read from one at a time. +While realisation is always parallel, in this case it cannot be done for all required store paths at once, and is therefore much slower than otherwise. + +Realising store objects during evaluation can be disabled by setting [`allow-import-from-derivation`](../command-ref/conf-file.md#conf-allow-import-from-derivation) to `false`. +Without IFD it is ensured that evaluation is complete and Nix can produce a build plan before starting any realisation. + +## Example + +In the following Nix expression, the inner derivation `drv` produces a file with contents `hello`. + +```nix +# IFD.nix +let + drv = derivation { + name = "hello"; + builder = "/bin/sh"; + args = [ "-c" "echo -n hello > $out" ]; + system = builtins.currentSystem; + }; +in "${builtins.readFile drv} world" +``` + +```shellSession +nix-instantiate IFD.nix --eval --read-write-mode +``` + +``` +building '/nix/store/348q1cal6sdgfxs8zqi9v8llrsn4kqkq-hello.drv'... +"hello world" +``` + +The contents of the derivation's output have to be [realised] before they can be read with [`readFile`](./builtins.md#builtins-readFile). +Only then evaluation can continue to produce the final result. + +## Illustration + +As a first approximation, the following data flow graph shows how evaluation and building are interleaved, if the value of a Nix expression depends on realising a [store object]. +Boxes are data structures, arrow labels are transformations. + +``` ++----------------------+ +------------------------+ +| Nix evaluator | | Nix store | +| .----------------. | | | +| | Nix expression | | | | +| '----------------' | | | +| | | | | +| evaluate | | | +| | | | | +| V | | | +| .------------. | | .------------------. | +| | derivation |----|-instantiate-|->| store derivation | | +| '------------' | | '------------------' | +| | | | | +| | | realise | +| | | | | +| | | V | +| .----------------. | | .--------------. | +| | Nix expression |<-|----read-----|----| store object | | +| '----------------' | | '--------------' | +| | | | | +| evaluate | | | +| | | | | +| V | | | +| .------------. | | | +| | value | | | | +| '------------' | | | ++----------------------+ +------------------------+ +``` + +In more detail, the following sequence diagram shows how the expression is evaluated step by step, and where evaluation is blocked to wait for the build output to appear. + +``` +.-------. .-------------. .---------. +|Nix CLI| |Nix evaluator| |Nix store| +'-------' '-------------' '---------' + | | | + |evaluate IFD.nix| | + |--------------->| | + | | | + | evaluate `"${readFile drv} world"` | + | | | + | evaluate `readFile drv` | + | | | + | evaluate `drv` as string | + | | | + | |instantiate /nix/store/...-hello.drv| + | |----------------------------------->| + | : | + | : realise /nix/store/...-hello.drv | + | :----------------------------------->| + | : | + | |--------. + | : | | + | (evaluation blocked) | echo hello > $out + | : | | + | |<-------' + | : /nix/store/...-hello | + | |<-----------------------------------| + | | | + | resume `readFile /nix/store/...-hello` | + | | | + | | readFile /nix/store/...-hello | + | |----------------------------------->| + | | | + | | hello | + | |<-----------------------------------| + | | | + | resume `"${"hello"} world"` | + | | | + | resume `"hello world"` | + | | | + | "hello world" | | + |<---------------| | +.-------. .-------------. .---------. +|Nix CLI| |Nix evaluator| |Nix store| +'-------' '-------------' '---------' +``` diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md index 29950a52d38f..a26e43a05daa 100644 --- a/doc/manual/src/language/index.md +++ b/doc/manual/src/language/index.md @@ -83,7 +83,8 @@ This is an incomplete overview of language features, by example. - A multi-line string. Strips common prefixed whitespace. Evaluates to `"multi\n line\n string"`. + + A multi-line string. Strips common prefixed whitespace. Evaluates to `"multi\n line\n  string"`. diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index f8382ae196eb..e9cbb5c92427 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -25,7 +25,7 @@ | Inequality | *expr* `!=` *expr* | none | 11 | | Logical conjunction (`AND`) | *bool* `&&` *bool* | left | 12 | | Logical disjunction (`OR`) | *bool* \|\| *bool* | left | 13 | -| [Logical implication] | *bool* `->` *bool* | none | 14 | +| [Logical implication] | *bool* `->` *bool* | right | 14 | [string]: ./values.md#type-string [path]: ./values.md#type-path @@ -35,6 +35,8 @@ ## Attribute selection +> **Syntax** +> > *attrset* `.` *attrpath* \[ `or` *expr* \] Select the attribute denoted by attribute path *attrpath* from [attribute set] *attrset*. @@ -42,21 +44,29 @@ If the attribute doesn’t exist, return the *expr* after `or` if provided, othe An attribute path is a dot-separated list of [attribute names](./values.md#attribute-set). +> **Syntax** +> > *attrpath* = *name* [ `.` *name* ]... [Attribute selection]: #attribute-selection ## Has attribute +> **Syntax** +> > *attrset* `?` *attrpath* Test whether [attribute set] *attrset* contains the attribute denoted by *attrpath*. The result is a [Boolean] value. +See also: [`builtins.hasAttr`](@docroot@/language/builtins.md#builtins-hasAttr) + [Boolean]: ./values.md#type-boolean [Has attribute]: #has-attribute +After evaluating *attrset* and *attrpath*, the computational complexity is O(log(*n*)) for *n* attributes in the *attrset* + ## Arithmetic Numbers are type-compatible: @@ -70,6 +80,8 @@ The `+` operator is overloaded to also work on strings and paths. ## String concatenation +> **Syntax** +> > *string* `+` *string* Concatenate two [string]s and merge their string contexts. @@ -78,6 +90,8 @@ Concatenate two [string]s and merge their string contexts. ## Path concatenation +> **Syntax** +> > *path* `+` *path* Concatenate two [path]s. @@ -87,6 +101,8 @@ The result is a path. ## Path and string concatenation +> **Syntax** +> > *path* + *string* Concatenate *[path]* with *[string]*. @@ -100,6 +116,8 @@ The result is a path. ## String and path concatenation +> **Syntax** +> > *string* + *path* Concatenate *[string]* with *[path]*. @@ -117,6 +135,8 @@ The result is a string. ## Update +> **Syntax** +> > *attrset1* // *attrset2* Update [attribute set] *attrset1* with names and values from *attrset2*. diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md index ddc6b8230718..e999b287b9c6 100644 --- a/doc/manual/src/language/string-interpolation.md +++ b/doc/manual/src/language/string-interpolation.md @@ -1,19 +1,12 @@ # String interpolation -String interpolation is a language feature where a [string], [path], or [attribute name] can contain expressions enclosed in `${ }` (dollar-sign with curly brackets). +String interpolation is a language feature where a [string], [path], or [attribute name][attribute set] can contain expressions enclosed in `${ }` (dollar-sign with curly brackets). -Such a string is an *interpolated string*, and an expression inside is an *interpolated expression*. - -Interpolated expressions must evaluate to one of the following: - -- a [string] -- a [path] -- a [derivation] +Such a construct is called *interpolated string*, and the expression inside is an [interpolated expression](#interpolated-expression). [string]: ./values.md#type-string [path]: ./values.md#type-path -[attribute name]: ./values.md#attribute-set -[derivation]: ../glossary.md#gloss-derivation +[attribute set]: ./values.md#attribute-set ## Examples @@ -70,13 +63,136 @@ you can instead write ### Attribute name -Attribute names can be created dynamically with string interpolation: + -```nix -let name = "foo"; in -{ - ${name} = "bar"; -} -``` +Attribute names can be interpolated strings. + +> **Example** +> +> ```nix +> let name = "foo"; in +> { ${name} = 123; } +> ``` +> +> { foo = 123; } - { foo = "bar"; } +Attributes can be selected with interpolated strings. + +> **Example** +> +> ```nix +> let name = "foo"; in +> { foo = 123; }.${name} +> ``` +> +> 123 + +# Interpolated expression + +An expression that is interpolated must evaluate to one of the following: + +- a [string] +- a [path] +- an [attribute set] that has a `__toString` attribute or an `outPath` attribute + + - `__toString` must be a function that takes the attribute set itself and returns a string + - `outPath` must be a string + + This includes [derivations](./derivations.md) or [flake inputs](@docroot@/command-ref/new-cli/nix3-flake.md#flake-inputs) (experimental). + +A string interpolates to itself. + +A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](../glossary.md#gloss-store-object). + +[store path]: ../glossary.md#gloss-store-path + +> **Example** +> +> ```console +> $ mkdir foo +> ``` +> +> Reference the empty directory in an interpolated expression: +> +> ```nix +> "${./foo}" +> ``` +> +> "/nix/store/2hhl2nz5v0khbn06ys82nrk99aa1xxdw-foo" + +A derivation interpolates to the [store path] of its first [output](./derivations.md#attr-outputs). + +> **Example** +> +> ```nix +> let +> pkgs = import {}; +> in +> "${pkgs.hello}" +> ``` +> +> "/nix/store/4xpfqf29z4m8vbhrqcz064wfmb46w5r7-hello-2.12.1" + +An attribute set interpolates to the return value of the function in the `__toString` applied to the attribute set itself. + +> **Example** +> +> ```nix +> let +> a = { +> value = 1; +> __toString = self: toString (self.value + 1); +> }; +> in +> "${a}" +> ``` +> +> "2" + +An attribute set also interpolates to the value of its `outPath` attribute. + +> **Example** +> +> ```nix +> let +> a = { outPath = "foo"; }; +> in +> "${a}" +> ``` +> +> "foo" + +If both `__toString` and `outPath` are present in an attribute set, `__toString` takes precedence. + +> **Example** +> +> ```nix +> let +> a = { __toString = _: "yes"; outPath = throw "no"; }; +> in +> "${a}" +> ``` +> +> "yes" + +If neither is present, an error is thrown. + +> **Example** +> +> ```nix +> let +> a = {}; +> in +> "${a}" +> ``` +> +> error: cannot coerce a set to a string +> +> at «string»:4:2: +> +> 3| in +> 4| "${a}" +> | ^ diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md index 2ae3e143a16d..aea68a441709 100644 --- a/doc/manual/src/language/values.md +++ b/doc/manual/src/language/values.md @@ -107,29 +107,25 @@ e.g. `~/foo` would be equivalent to `/home/edolstra/foo` for a user whose home directory is `/home/edolstra`. - Paths can also be specified between angle brackets, e.g. - ``. This means that the directories listed in the - environment variable `NIX_PATH` will be searched for the given file - or directory name. - - When an [interpolated string][string interpolation] evaluates to a path, the path is first copied into the Nix store and the resulting string is the [store path] of the newly created [store object]. - - [store path]: ../glossary.md#gloss-store-path - [store object]: ../glossary.md#gloss-store-object - For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` in the current directory to be copied into the Nix store and result in the string `"/nix/store/-foo.txt"`. Note that the Nix language assumes that all input files will remain _unchanged_ while evaluating a Nix expression. For example, assume you used a file path in an interpolated string during a `nix repl` session. - Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new store path, since Nix might not re-read the file contents. + Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. - Paths themselves, except those in angle brackets (`< >`), support [string interpolation]. + [store path]: ../glossary.md#gloss-store-path + + Paths can include [string interpolation] and can themselves be [interpolated in other expressions]. + + [interpolated in other expressions]: ./string-interpolation.md#interpolated-expressions At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. `a.${foo}/b.${bar}` is a syntactically valid division operation. `./a.${foo}/b.${bar}` is a path. + [Lookup paths](./constructs/lookup-path.md) such as `` resolve to path values. + - Boolean *Booleans* with values `true` and `false`. @@ -167,13 +163,17 @@ An attribute set is a collection of name-value-pairs (called *attributes*) enclo An attribute name can be an identifier or a [string](#string). An identifier must start with a letter (`a-z`, `A-Z`) or underscore (`_`), and can otherwise contain letters (`a-z`, `A-Z`), numbers (`0-9`), underscores (`_`), apostrophes (`'`), or dashes (`-`). +> **Syntax** +> > *name* = *identifier* | *string* \ > *identifier* ~ `[a-zA-Z_][a-zA-Z0-9_'-]*` Names and values are separated by an equal sign (`=`). Each value is an arbitrary expression terminated by a semicolon (`;`). -> *attrset* = `{` [ *name* `=` *expr* `;` `]`... `}` +> **Syntax** +> +> *attrset* = `{` [ *name* `=` *expr* `;` ]... `}` Attributes can appear in any order. An attribute name may only occur once. diff --git a/doc/manual/src/package-management/basic-package-mgmt.md b/doc/manual/src/package-management/basic-package-mgmt.md deleted file mode 100644 index 6b86e763e43d..000000000000 --- a/doc/manual/src/package-management/basic-package-mgmt.md +++ /dev/null @@ -1,179 +0,0 @@ -# Basic Package Management - -The main command for package management is -[`nix-env`](../command-ref/nix-env.md). You can use it to install, -upgrade, and erase packages, and to query what packages are installed -or are available for installation. - -In Nix, different users can have different “views” on the set of -installed applications. That is, there might be lots of applications -present on the system (possibly in many different versions), but users -can have a specific selection of those active — where “active” just -means that it appears in a directory in the user’s `PATH`. Such a view -on the set of installed applications is called a *user environment*, -which is just a directory tree consisting of symlinks to the files of -the active applications. - -Components are installed from a set of *Nix expressions* that tell Nix -how to build those packages, including, if necessary, their -dependencies. There is a collection of Nix expressions called the -Nixpkgs package collection that contains packages ranging from basic -development stuff such as GCC and Glibc, to end-user applications like -Mozilla Firefox. (Nix is however not tied to the Nixpkgs package -collection; you could write your own Nix expressions based on Nixpkgs, -or completely new ones.) - -You can manually download the latest version of Nixpkgs from -. However, it’s much more -convenient to use the Nixpkgs [*channel*](channels.md), since it makes -it easy to stay up to date with new versions of Nixpkgs. Nixpkgs is -automatically added to your list of “subscribed” channels when you -install Nix. If this is not the case for some reason, you can add it -as follows: - -```console -$ nix-channel --add https://nixos.org/channels/nixpkgs-unstable -$ nix-channel --update -``` - -> **Note** -> -> On NixOS, you’re automatically subscribed to a NixOS channel -> corresponding to your NixOS major release (e.g. -> ). A NixOS channel is identical -> to the Nixpkgs channel, except that it contains only Linux binaries -> and is updated only if a set of regression tests succeed. - -You can view the set of available packages in Nixpkgs: - -```console -$ nix-env --query --available --attr-path -nixpkgs.aterm aterm-2.2 -nixpkgs.bash bash-3.0 -nixpkgs.binutils binutils-2.15 -nixpkgs.bison bison-1.875d -nixpkgs.blackdown blackdown-1.4.2 -nixpkgs.bzip2 bzip2-1.0.2 -… -``` - -The flag `-q` specifies a query operation, `-a` means that you want -to show the “available” (i.e., installable) packages, as opposed to the -installed packages, and `-P` prints the attribute paths that can be used -to unambiguously select a package for installation (listed in the first column). -If you downloaded Nixpkgs yourself, or if you checked it out from GitHub, -then you need to pass the path to your Nixpkgs tree using the `-f` flag: - -```console -$ nix-env --query --available --attr-path --file /path/to/nixpkgs -aterm aterm-2.2 -bash bash-3.0 -… -``` - -where */path/to/nixpkgs* is where you’ve unpacked or checked out -Nixpkgs. - -You can filter the packages by name: - -```console -$ nix-env --query --available --attr-path firefox -nixpkgs.firefox-esr firefox-91.3.0esr -nixpkgs.firefox firefox-94.0.1 -``` - -and using regular expressions: - -```console -$ nix-env --query --available --attr-path 'firefox.*' -``` - -It is also possible to see the *status* of available packages, i.e., -whether they are installed into the user environment and/or present in -the system: - -```console -$ nix-env --query --available --attr-path --status -… --PS nixpkgs.bash bash-3.0 ---S nixpkgs.binutils binutils-2.15 -IPS nixpkgs.bison bison-1.875d -… -``` - -The first character (`I`) indicates whether the package is installed in -your current user environment. The second (`P`) indicates whether it is -present on your system (in which case installing it into your user -environment would be a very quick operation). The last one (`S`) -indicates whether there is a so-called *substitute* for the package, -which is Nix’s mechanism for doing binary deployment. It just means that -Nix knows that it can fetch a pre-built package from somewhere -(typically a network server) instead of building it locally. - -You can install a package using `nix-env --install --attr `. For instance, - -```console -$ nix-env --install --attr nixpkgs.subversion -``` - -will install the package called `subversion` from `nixpkgs` channel (which is, of course, the -[Subversion version management system](http://subversion.tigris.org/)). - -> **Note** -> -> When you ask Nix to install a package, it will first try to get it in -> pre-compiled form from a *binary cache*. By default, Nix will use the -> binary cache ; it contains binaries for most -> packages in Nixpkgs. Only if no binary is available in the binary -> cache, Nix will build the package from source. So if `nix-env -> -iA nixpkgs.subversion` results in Nix building stuff from source, then either -> the package is not built for your platform by the Nixpkgs build -> servers, or your version of Nixpkgs is too old or too new. For -> instance, if you have a very recent checkout of Nixpkgs, then the -> Nixpkgs build servers may not have had a chance to build everything -> and upload the resulting binaries to . The -> Nixpkgs channel is only updated after all binaries have been uploaded -> to the cache, so if you stick to the Nixpkgs channel (rather than -> using a Git checkout of the Nixpkgs tree), you will get binaries for -> most packages. - -Naturally, packages can also be uninstalled. Unlike when installing, you will -need to use the derivation name (though the version part can be omitted), -instead of the attribute path, as `nix-env` does not record which attribute -was used for installing: - -```console -$ nix-env --uninstall subversion -``` - -Upgrading to a new version is just as easy. If you have a new release of -Nix Packages, you can do: - -```console -$ nix-env --upgrade --attr nixpkgs.subversion -``` - -This will *only* upgrade Subversion if there is a “newer” version in the -new set of Nix expressions, as defined by some pretty arbitrary rules -regarding ordering of version numbers (which generally do what you’d -expect of them). To just unconditionally replace Subversion with -whatever version is in the Nix expressions, use `-i` instead of `-u`; -`-i` will remove whatever version is already installed. - -You can also upgrade all packages for which there are newer versions: - -```console -$ nix-env --upgrade -``` - -Sometimes it’s useful to be able to ask what `nix-env` would do, without -actually doing it. For instance, to find out what packages would be -upgraded by `nix-env --upgrade `, you can do - -```console -$ nix-env --upgrade --dry-run -(dry run; not doing anything) -upgrading `libxslt-1.1.0' to `libxslt-1.1.10' -upgrading `graphviz-1.10' to `graphviz-1.12' -upgrading `coreutils-5.0' to `coreutils-5.2.1' -``` diff --git a/doc/manual/src/package-management/channels.md b/doc/manual/src/package-management/channels.md deleted file mode 100644 index 8e4da180bbc7..000000000000 --- a/doc/manual/src/package-management/channels.md +++ /dev/null @@ -1,50 +0,0 @@ -# Channels - -If you want to stay up to date with a set of packages, it’s not very -convenient to manually download the latest set of Nix expressions for -those packages and upgrade using `nix-env`. Fortunately, there’s a -better way: *Nix channels*. - -A Nix channel is just a URL that points to a place that contains a set -of Nix expressions and a manifest. Using the command -[`nix-channel`](../command-ref/nix-channel.md) you can automatically -stay up to date with whatever is available at that URL. - -To see the list of official NixOS channels, visit -. - -You can “subscribe” to a channel using `nix-channel --add`, e.g., - -```console -$ nix-channel --add https://nixos.org/channels/nixpkgs-unstable -``` - -subscribes you to a channel that always contains that latest version of -the Nix Packages collection. (Subscribing really just means that the URL -is added to the file `~/.nix-channels`, where it is read by subsequent -calls to `nix-channel ---update`.) You can “unsubscribe” using `nix-channel ---remove`: - -```console -$ nix-channel --remove nixpkgs -``` - -To obtain the latest Nix expressions available in a channel, do - -```console -$ nix-channel --update -``` - -This downloads and unpacks the Nix expressions in every channel -(downloaded from `url/nixexprs.tar.bz2`). It also makes the union of -each channel’s Nix expressions available by default to `nix-env` -operations (via the symlink `~/.nix-defexpr/channels`). Consequently, -you can then say - -```console -$ nix-env --upgrade -``` - -to upgrade all packages in your profile to the latest versions available -in the subscribed channels. diff --git a/doc/manual/src/package-management/package-management.md b/doc/manual/src/package-management/index.md similarity index 100% rename from doc/manual/src/package-management/package-management.md rename to doc/manual/src/package-management/index.md diff --git a/doc/manual/src/package-management/s3-substituter.md b/doc/manual/src/package-management/s3-substituter.md deleted file mode 100644 index d8a1d9105b44..000000000000 --- a/doc/manual/src/package-management/s3-substituter.md +++ /dev/null @@ -1,115 +0,0 @@ -# Serving a Nix store via S3 - -Nix has [built-in support](@docroot@/command-ref/new-cli/nix3-help-stores.md#s3-binary-cache-store) -for storing and fetching store paths from -Amazon S3 and S3-compatible services. This uses the same *binary* -cache mechanism that Nix usually uses to fetch prebuilt binaries from -[cache.nixos.org](https://cache.nixos.org/). - -In this example we will use the bucket named `example-nix-cache`. - -## Anonymous Reads to your S3-compatible binary cache - -If your binary cache is publicly accessible and does not require -authentication, the simplest and easiest way to use Nix with your S3 -compatible binary cache is to use the HTTP URL for that cache. - -For AWS S3 the binary cache URL for example bucket will be exactly - or -. For S3 compatible binary caches, consult that -cache's documentation. - -Your bucket will need the following bucket policy: - -```json -{ - "Id": "DirectReads", - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "AllowDirectReads", - "Action": [ - "s3:GetObject", - "s3:GetBucketLocation" - ], - "Effect": "Allow", - "Resource": [ - "arn:aws:s3:::example-nix-cache", - "arn:aws:s3:::example-nix-cache/*" - ], - "Principal": "*" - } - ] -} -``` - -## Authenticated Reads to your S3 binary cache - -For AWS S3 the binary cache URL for example bucket will be exactly -. - -Nix will use the [default credential provider -chain](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html) -for authenticating requests to Amazon S3. - -Nix supports authenticated reads from Amazon S3 and S3 compatible binary -caches. - -Your bucket will need a bucket policy allowing the desired users to -perform the `s3:GetObject` and `s3:GetBucketLocation` action on all -objects in the bucket. The [anonymous policy given -above](#anonymous-reads-to-your-s3-compatible-binary-cache) can be -updated to have a restricted `Principal` to support this. - -## Authenticated Writes to your S3-compatible binary cache - -Nix support fully supports writing to Amazon S3 and S3 compatible -buckets. The binary cache URL for our example bucket will be -. - -Nix will use the [default credential provider -chain](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html) -for authenticating requests to Amazon S3. - -Your account will need the following IAM policy to upload to the cache: - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "UploadToCache", - "Effect": "Allow", - "Action": [ - "s3:AbortMultipartUpload", - "s3:GetBucketLocation", - "s3:GetObject", - "s3:ListBucket", - "s3:ListBucketMultipartUploads", - "s3:ListMultipartUploadParts", - "s3:PutObject" - ], - "Resource": [ - "arn:aws:s3:::example-nix-cache", - "arn:aws:s3:::example-nix-cache/*" - ] - } - ] -} -``` - -## Examples - -To upload with a specific credential profile for Amazon S3: - -```console -$ nix copy nixpkgs.hello \ - --to 's3://example-nix-cache?profile=cache-upload®ion=eu-west-2' -``` - -To upload to an S3-compatible binary cache: - -```console -$ nix copy nixpkgs.hello --to \ - 's3://example-nix-cache?profile=cache-upload&scheme=https&endpoint=minio.example.com' -``` diff --git a/doc/manual/src/protocols/derivation-aterm.md b/doc/manual/src/protocols/derivation-aterm.md new file mode 100644 index 000000000000..e58b602a3495 --- /dev/null +++ b/doc/manual/src/protocols/derivation-aterm.md @@ -0,0 +1,19 @@ +# Derivation "ATerm" file format + +For historical reasons, [derivations](@docroot@/glossary.md#gloss-store-derivation) are stored on-disk in [ATerm](https://homepages.cwi.nl/~daybuild/daily-books/technology/aterm-guide/aterm-guide.html) format. + +Derivations are serialised in one of the following formats: + +- ``` + Derive(...) + ``` + + For all stable derivations. + +- ``` + DrvWithVersion(, ...) + ``` + + The only `version-string`s that are in use today are for [experimental features](@docroot@/contributing/experimental-features.md): + + - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. diff --git a/doc/manual/src/protocols/protocols.md b/doc/manual/src/protocols/index.md similarity index 100% rename from doc/manual/src/protocols/protocols.md rename to doc/manual/src/protocols/index.md diff --git a/doc/manual/src/protocols/tarball-fetcher.md b/doc/manual/src/protocols/tarball-fetcher.md index 0d32123031a1..274fa6d63c8d 100644 --- a/doc/manual/src/protocols/tarball-fetcher.md +++ b/doc/manual/src/protocols/tarball-fetcher.md @@ -20,8 +20,8 @@ Link: ; rel="immutable" (Note the required `<` and `>` characters around *flakeref*.) -*flakeref* must be a tarball flakeref. It can contain flake attributes -such as `narHash`, `rev` and `revCount`. If `narHash` is included, its +*flakeref* must be a tarball flakeref. It can contain the tarball flake attributes +`narHash`, `rev`, `revCount` and `lastModified`. If `narHash` is included, its value must be the NAR hash of the unpacked tarball (as computed via `nix hash path`). Nix checks the contents of the returned tarball against the `narHash` attribute. The `rev` and `revCount` attributes diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md index 1d2688ede0e1..5f54abbde394 100644 --- a/doc/manual/src/quick-start.md +++ b/doc/manual/src/quick-start.md @@ -13,7 +13,7 @@ to subsequent chapters. The install script will use `sudo`, so make sure you have sufficient rights. On Linux, `--daemon` can be omitted for a single-user install. - For other installation methods, see [here](installation/installation.md). + For other installation methods, see [here](installation/index.md). 1. See what installable packages are currently available in the channel: diff --git a/doc/manual/src/release-notes/index.md b/doc/manual/src/release-notes/index.md new file mode 100644 index 000000000000..cc805e63116b --- /dev/null +++ b/doc/manual/src/release-notes/index.md @@ -0,0 +1,12 @@ +# Nix Release Notes + +Nix has a release cycle of roughly 6 weeks. +Notable changes and additions are announced in the release notes for each version. + +Bugfixes can be backported on request to previous Nix releases. +We typically backport only as far back as the Nix version used in the latest NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes). + +Backports never skip releases. +If a feature is backported to version `x.y`, it must also be available in version `x.(y+1)`. +This ensures that upgrading from an older version with backports is still safe and no backported functionality will go missing. + diff --git a/doc/manual/src/release-notes/release-notes.md b/doc/manual/src/release-notes/release-notes.md deleted file mode 100644 index b05d5ee0a2da..000000000000 --- a/doc/manual/src/release-notes/release-notes.md +++ /dev/null @@ -1 +0,0 @@ -# Nix Release Notes diff --git a/doc/manual/src/release-notes/rl-2.12.md b/doc/manual/src/release-notes/rl-2.12.md index e2045d7bf5b8..e1e3efe1a8a0 100644 --- a/doc/manual/src/release-notes/rl-2.12.md +++ b/doc/manual/src/release-notes/rl-2.12.md @@ -2,7 +2,6 @@ * On Linux, Nix can now run builds in a user namespace where they run as root (UID 0) and have 65,536 UIDs available. - This is primarily useful for running containers such as `systemd-nspawn` inside a Nix build. For an example, see [`tests/systemd-nspawn/nix`][nspawn]. diff --git a/doc/manual/src/release-notes/rl-2.17.md b/doc/manual/src/release-notes/rl-2.17.md new file mode 100644 index 000000000000..0b861aecca68 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.17.md @@ -0,0 +1,42 @@ +# Release 2.17 (2023-07-24) + +* [`nix-channel`](../command-ref/nix-channel.md) now supports a `--list-generations` subcommand. + +* The function [`builtins.fetchClosure`](../language/builtins.md#builtins-fetchClosure) can now fetch input-addressed paths in [pure evaluation mode](../command-ref/conf-file.md#conf-pure-eval), as those are not impure. + +* Nix now allows unprivileged/[`allowed-users`](../command-ref/conf-file.md#conf-allowed-users) to sign paths. + Previously, only [`trusted-users`](../command-ref/conf-file.md#conf-trusted-users) users could sign paths. + +* Nested dynamic attributes are now merged correctly by the parser. For example: + + ```nix + { + nested = { + foo = 1; + }; + nested = { + ${"ba" + "r"} = 2; + }; + } + ``` + + This used to silently discard `nested.bar`, but now behaves as one would expect and evaluates to: + + ```nix + { nested = { bar = 2; foo = 1; }; } + ``` + + Note that the feature of merging multiple *full declarations* of attribute sets like `nested` in the example is of questionable value. + It allows writing expressions that are very hard to read, for instance when there are many lines of code between two declarations of the same attribute. + This has been around for a long time and is therefore supported for backwards compatibility, but should not be relied upon. + + Instead, consider using the *nested attribute path* syntax: + + ```nix + { + nested.foo = 1; + nested.${"ba" + "r"} = 2; + } + ``` + +* Tarball flakes can now redirect to an "immutable" URL that will be recorded in lock files. This allows the use of "mutable" tarball URLs like `https://example.org/hello/latest.tar.gz` in flakes. See the [tarball fetcher](../protocols/tarball-fetcher.md) for details. diff --git a/doc/manual/src/release-notes/rl-2.18.md b/doc/manual/src/release-notes/rl-2.18.md new file mode 100644 index 000000000000..4bbc52b508fd --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.18.md @@ -0,0 +1,28 @@ +# Release 2.18 (2023-09-20) + +- Two new builtin functions, + [`builtins.parseFlakeRef`](@docroot@/language/builtins.md#builtins-parseFlakeRef) + and + [`builtins.flakeRefToString`](@docroot@/language/builtins.md#builtins-flakeRefToString), + have been added. + These functions are useful for converting between flake references encoded as attribute sets and URLs. + +- [`builtins.toJSON`](@docroot@/language/builtins.md#builtins-parseFlakeRef) now prints [--show-trace](@docroot@/command-ref/conf-file.html#conf-show-trace) items for the path in which it finds an evaluation error. + +- Error messages regarding malformed input to [`nix derivation add`](@docroot@/command-ref/new-cli/nix3-derivation-add.md) are now clearer and more detailed. + +- The `discard-references` feature has been stabilized. + This means that the + [unsafeDiscardReferences](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) + attribute is no longer guarded by an experimental flag and can be used + freely. + +- The JSON output for derived paths which are store paths is now a string, not an object with a single `path` field. + This only affects `nix-build --json` when "building" non-derivation things like fetched sources, which is a no-op. + +- A new builtin [`outputOf`](@docroot@/language/builtins.md#builtins-outputOf) has been added. + It is part of the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + +- Flake follow paths at depths greater than 2 are now handled correctly, preventing "follows a non-existent input" errors. + +- [`nix-store --query`](@docroot@/command-ref/nix-store/query.md) gained a new type of query: `--valid-derivers`. It returns all `.drv` files in the local store that *can be* used to build the output passed in argument. This is in contrast to `--deriver`, which returns the single `.drv` file that *was actually* used to build the output passed in argument. In case the output was substituted from a binary cache, this `.drv` file may only exist on said binary cache and not locally. diff --git a/doc/manual/src/release-notes/rl-2.19.md b/doc/manual/src/release-notes/rl-2.19.md new file mode 100644 index 000000000000..4eecaf929f75 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.19.md @@ -0,0 +1,77 @@ +# Release 2.19 (2023-11-17) + +- The experimental `nix` command can now act as a [shebang interpreter](@docroot@/command-ref/new-cli/nix.md#shebang-interpreter) + by appending the contents of any `#! nix` lines and the script's location into a single call. + +- [URL flake references](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) now support [percent-encoded](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1) characters. + +- [Path-like flake references](@docroot@/command-ref/new-cli/nix3-flake.md#path-like-syntax) now accept arbitrary unicode characters (except `#` and `?`). + +- The experimental feature `repl-flake` is no longer needed, as its functionality is now part of the `flakes` experimental feature. To get the previous behavior, use the `--file/--expr` flags accordingly. + +- There is a new flake installable syntax `flakeref#.attrPath` where the "." prefix specifies that `attrPath` is interpreted from the root of the flake outputs, with no searching of default attribute prefixes like `packages.` or `legacyPackages.`. + +- Nix adds `apple-virt` to the default system features on macOS systems that support virtualization. This is similar to what's done for the `kvm` system feature on Linux hosts. + +- Add a new built-in function [`builtins.convertHash`](@docroot@/language/builtins.md#builtins-convertHash). + +- `nix-shell` shebang lines now support single-quoted arguments. + +- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). + As described in the documentation for that feature, this is because we anticipate polishing it and then stabilizing it before the rest of flakes. + +- The interface for creating and updating lock files has been overhauled: + + - [`nix flake lock`](@docroot@/command-ref/new-cli/nix3-flake-lock.md) only creates lock files and adds missing inputs now. + It will *never* update existing inputs. + + - [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) does the same, but *will* update inputs. + - Passing no arguments will update all inputs of the current flake, just like it already did. + - Passing input names as arguments will ensure only those are updated. This replaces the functionality of `nix flake lock --update-input` + - To operate on a flake outside the current directory, you must now pass `--flake path/to/flake`. + + - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables. + They are superceded by `nix flake update`. + +- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + +- [`nix path-info --json`](@docroot@/command-ref/new-cli/nix3-path-info.md) + (experimental) now returns a JSON map rather than JSON list. + The `path` field of each object has instead become the key in the outer map, since it is unique. + The `valid` field also goes away because we just use `null` instead. + + - Old way: + + ```json5 + [ + { + "path": "/nix/store/8fv91097mbh5049i9rglc73dx6kjg3qk-bash-5.2-p15", + "valid": true, + // ... + }, + { + "path": "/nix/store/wffw7l0alvs3iw94cbgi1gmmbmw99sqb-home-manager-path", + "valid": false + } + ] + ``` + + - New way + + ```json5 + { + "/nix/store/8fv91097mbh5049i9rglc73dx6kjg3qk-bash-5.2-p15": { + // ... + }, + "/nix/store/wffw7l0alvs3iw94cbgi1gmmbmw99sqb-home-manager-path": null, + } + ``` + + This makes it match `nix derivation show`, which also maps store paths to information. + +- When Nix is installed using the [binary installer](@docroot@/installation/installing-binary.md), in supported shells (Bash, Zsh, Fish) + [`XDG_DATA_DIRS`](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html#variables) is now populated with the path to the `/share` subdirectory of the current profile. + This means that command completion scripts, `.desktop` files, and similar artifacts installed via [`nix-env`](@docroot@/command-ref/nix-env.md) or [`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md) + (experimental) can be found by any program that follows the [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html). + +- A new command `nix store add` has been added. It replaces `nix store add-file` and `nix store add-path` which are now deprecated. diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md deleted file mode 100644 index 8479b166ace5..000000000000 --- a/doc/manual/src/release-notes/rl-next.md +++ /dev/null @@ -1,6 +0,0 @@ -# Release X.Y (202?-??-??) - -- [`nix-channel`](../command-ref/nix-channel.md) now supports a `--list-generations` subcommand - -- Nix now allows unprivileged/[`allowed-users`](../command-ref/conf-file.md#conf-allowed-users) to sign paths. - Previously, only [`trusted-users`](../command-ref/conf-file.md#conf-trusted-users) users could sign paths. diff --git a/doc/manual/src/architecture/file-system-object.md b/doc/manual/src/store/file-system-object.md similarity index 100% rename from doc/manual/src/architecture/file-system-object.md rename to doc/manual/src/store/file-system-object.md diff --git a/doc/manual/src/store/index.md b/doc/manual/src/store/index.md new file mode 100644 index 000000000000..f1e8f1402988 --- /dev/null +++ b/doc/manual/src/store/index.md @@ -0,0 +1,5 @@ +# Nix Store + +The *Nix store* is an abstraction to store immutable file system data (such as software packages) that can have dependencies on other such data. + +There are [multiple types of Nix stores](./types/index.md) with different capabilities, such as the default one on the [local filesystem](./types/local-store.md) (`/nix/store`) or [binary caches](./types/http-binary-cache-store.md). diff --git a/doc/manual/src/store/store-object.md b/doc/manual/src/store/store-object.md new file mode 100644 index 000000000000..caf5657d1f0a --- /dev/null +++ b/doc/manual/src/store/store-object.md @@ -0,0 +1,10 @@ +## Store Object + +A Nix store is a collection of *store objects* with *references* between them. +A store object consists of + + - A [file system object](./file-system-object.md) as data + - A set of [store paths](./store-path.md) as references to other store objects + +Store objects are [immutable](https://en.wikipedia.org/wiki/Immutable_object): +Once created, they do not change until they are deleted. diff --git a/doc/manual/src/store/store-path.md b/doc/manual/src/store/store-path.md new file mode 100644 index 000000000000..b5ad0c654f47 --- /dev/null +++ b/doc/manual/src/store/store-path.md @@ -0,0 +1,69 @@ +# Store Path + +Nix implements references to [store objects](./index.md#store-object) as *store paths*. + +Think of a store path as an [opaque], [unique identifier]: +The only way to obtain store path is by adding or building store objects. +A store path will always reference exactly one store object. + +[opaque]: https://en.m.wikipedia.org/wiki/Opaque_data_type +[unique identifier]: https://en.m.wikipedia.org/wiki/Unique_identifier + +Store paths are pairs of + +- A 20-byte digest for identification +- A symbolic name for people to read + +> **Example** +> +> - Digest: `b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z` +> - Name: `firefox-33.1` + +To make store objects accessible to operating system processes, stores have to expose store objects through the file system. + +A store path is rendered to a file system path as the concatenation of + +- [Store directory](#store-directory) (typically `/nix/store`) +- Path separator (`/`) +- Digest rendered in a custom variant of [Base32](https://en.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters) +- Hyphen (`-`) +- Name + +> **Example** +> +> ``` +> /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1 +> |--------| |------------------------------| |----------| +> store directory digest name +> ``` + +## Store Directory + +Every [Nix store](./index.md) has a store directory. + +Not every store can be accessed through the file system. +But if the store has a file system representation, the store directory contains the store’s [file system objects], which can be addressed by [store paths](#store-path). + +[file system objects]: ./file-system-object.md + +This means a store path is not just derived from the referenced store object itself, but depends on the store the store object is in. + +> **Note** +> +> The store directory defaults to `/nix/store`, but is in principle arbitrary. + +It is important which store a given store object belongs to: +Files in the store object can contain store paths, and processes may read these paths. +Nix can only guarantee referential integrity if store paths do not cross store boundaries. + +Therefore one can only copy store objects to a different store if + +- The source and target stores' directories match + + or + +- The store object in question has no references, that is, contains no store paths + +One cannot copy a store object to a store with a different store directory. +Instead, it has to be rebuilt, together with all its dependencies. +It is in general not enough to replace the store directory string in file contents, as this may render executables unusable by invalidating their internal offsets or checksums. diff --git a/src/nix/help-stores.md b/doc/manual/src/store/types/index.md.in similarity index 93% rename from src/nix/help-stores.md rename to doc/manual/src/store/types/index.md.in index 47ba9b94d89f..b4db553a2360 100644 --- a/src/nix/help-stores.md +++ b/doc/manual/src/store/types/index.md.in @@ -1,6 +1,6 @@ -R"( +Nix supports different types of stores: -Nix supports different types of stores. These are described below. +@store-types@ ## Store URL format @@ -41,6 +41,3 @@ store as follows: * Otherwise, use the [local store](#local-store) `/nix/store`. -@stores@ - -)" diff --git a/doc/manual/utils.nix b/doc/manual/utils.nix index 9043dd8cdba4..19ff49b64d96 100644 --- a/doc/manual/utils.nix +++ b/doc/manual/utils.nix @@ -1,5 +1,11 @@ with builtins; +let + lowerChars = stringToCharacters "abcdefghijklmnopqrstuvwxyz"; + upperChars = stringToCharacters "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + stringToCharacters = s: genList (p: substring p 1 s) (stringLength s); +in + rec { splitLines = s: filter (x: !isList x) (split "\n" s); @@ -18,6 +24,8 @@ rec { in if replaced == string then string else replaceStringsRec from to replaced; + toLower = replaceStrings upperChars lowerChars; + squash = replaceStringsRec "\n\n\n" "\n\n"; trim = string: @@ -44,63 +52,6 @@ rec { optionalString = cond: string: if cond then string else ""; - showSetting = { useAnchors }: name: { description, documentDefault, defaultValue, aliases, value, experimentalFeature }: - let - result = squash '' - - ${if useAnchors - then ''[`${name}`](#conf-${name})'' - else ''`${name}`''} - - ${indent " " body} - ''; - - experimentalFeatureNote = optionalString (experimentalFeature != null) '' - > **Warning** - > This setting is part of an - > [experimental feature](@docroot@/contributing/experimental-features.md). - - To change this setting, you need to make sure the corresponding experimental feature, - [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}), - is enabled. - For example, include the following in [`nix.conf`](#): - - ``` - extra-experimental-features = ${experimentalFeature} - ${name} = ... - ``` - ''; - - # separate body to cleanly handle indentation - body = '' - ${description} - - ${experimentalFeatureNote} - - **Default:** ${showDefault documentDefault defaultValue} - - ${showAliases aliases} - ''; - - showDefault = documentDefault: defaultValue: - if documentDefault then - # a StringMap value type is specified as a string, but - # this shows the value type. The empty stringmap is `null` in - # JSON, but that converts to `{ }` here. - if defaultValue == "" || defaultValue == [] || isAttrs defaultValue - then "*empty*" - else if isBool defaultValue then - if defaultValue then "`true`" else "`false`" - else "`${toString defaultValue}`" - else "*machine-specific*"; - - showAliases = aliases: - optionalString (aliases != []) - "**Deprecated alias:** ${(concatStringsSep ", " (map (s: "`${s}`") aliases))}"; - - in result; - indent = prefix: s: concatStringsSep "\n" (map (x: if x == "" then x else "${prefix}${x}") (splitLines s)); - - showSettings = args: settingsInfo: concatStrings (attrValues (mapAttrs (showSetting args) settingsInfo)); } diff --git a/flake.lock b/flake.lock index 1d2aab5ed586..f120d3b5f24f 100644 --- a/flake.lock +++ b/flake.lock @@ -16,6 +16,22 @@ "type": "github" } }, + "libgit2": { + "flake": false, + "locked": { + "lastModified": 1697646580, + "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=", + "owner": "libgit2", + "repo": "libgit2", + "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5", + "type": "github" + }, + "original": { + "owner": "libgit2", + "repo": "libgit2", + "type": "github" + } + }, "lowdown-src": { "flake": false, "locked": { @@ -34,16 +50,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1670461440, - "narHash": "sha256-jy1LB8HOMKGJEGXgzFRLDU1CBGL0/LlkolgnqIsF0D8=", + "lastModified": 1700748986, + "narHash": "sha256-/nqLrNU297h3PCw4QyDpZKZEUHmialJdZW2ceYFobds=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "04a75b2eecc0acf6239acf9dd04485ff8d14f425", + "rev": "9ba29e2346bc542e9909d1021e8fd7d4b3f64db0", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-22.11-small", + "ref": "nixos-23.05-small", "repo": "nixpkgs", "type": "github" } @@ -67,6 +83,7 @@ "root": { "inputs": { "flake-compat": "flake-compat", + "libgit2": "libgit2", "lowdown-src": "lowdown-src", "nixpkgs": "nixpkgs", "nixpkgs-regression": "nixpkgs-regression" diff --git a/flake.nix b/flake.nix index 694bb15bdad5..2d3d52b34e57 100644 --- a/flake.nix +++ b/flake.nix @@ -1,29 +1,38 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-22.11-small"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; + inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; }; - outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, flake-compat }: + outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, flake-compat, libgit2 }: let inherit (nixpkgs) lib; officialRelease = false; + # Set to true to build the release notes for the next release. + buildUnreleasedNotes = false; + version = lib.fileContents ./.version + versionSuffix; versionSuffix = if officialRelease then "" else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; + linux32BitSystems = [ "i686-linux" ]; linux64BitSystems = [ "x86_64-linux" "aarch64-linux" ]; - linuxSystems = linux64BitSystems ++ [ "i686-linux" ]; - systems = linuxSystems ++ [ "x86_64-darwin" "aarch64-darwin" ]; + linuxSystems = linux32BitSystems ++ linux64BitSystems; + darwinSystems = [ "x86_64-darwin" "aarch64-darwin" ]; + systems = linuxSystems ++ darwinSystems; - crossSystems = [ "armv6l-linux" "armv7l-linux" ]; + crossSystems = [ + "armv6l-linux" "armv7l-linux" + "x86_64-freebsd13" "x86_64-netbsd" + ]; stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" "libcxxStdenv" "ccacheStdenv" ]; @@ -40,12 +49,69 @@ }) stdenvs); + # Experimental fileset library: https://github.com/NixOS/nixpkgs/pull/222981 + # Not an "idiomatic" flake input because: + # - Propagation to dependent locks: https://github.com/NixOS/nix/issues/7730 + # - Subflake would download redundant and huge parent flake + # - No git tree hash support: https://github.com/NixOS/nix/issues/6044 + inherit (import (builtins.fetchTarball { url = "https://github.com/NixOS/nix/archive/1bdcd7fc8a6a40b2e805bad759b36e64e911036b.tar.gz"; sha256 = "sha256:14ljlpdsp4x7h1fkhbmc4bd3vsqnx8zdql4h3037wh09ad6a0893"; })) + fileset; + + baseFiles = + # .gitignore has already been processed, so any changes in it are irrelevant + # at this point. It is not represented verbatim for test purposes because + # that would interfere with repo semantics. + fileset.fileFilter (f: f.name != ".gitignore") ./.; + + configureFiles = fileset.unions [ + ./.version + ./configure.ac + ./m4 + # TODO: do we really need README.md? It doesn't seem used in the build. + ./README.md + ]; + + topLevelBuildFiles = fileset.unions [ + ./local.mk + ./Makefile + ./Makefile.config.in + ./mk + ]; + + functionalTestFiles = fileset.unions [ + ./tests/functional + (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) + ]; + + nixSrc = fileset.toSource { + root = ./.; + fileset = fileset.intersect baseFiles (fileset.unions [ + configureFiles + topLevelBuildFiles + ./boehmgc-coroutine-sp-fallback.diff + ./doc + ./misc + ./precompiled-headers.h + ./src + ./unit-test-data + ./COPYING + ./scripts/local.mk + functionalTestFiles + ]); + }; # Memoize nixpkgs for different platforms for efficiency. nixpkgsFor = forAllSystems (system: let make-pkgs = crossSystem: stdenv: import nixpkgs { - inherit system crossSystem; + localSystem = { + inherit system; + }; + crossSystem = if crossSystem == null then null else { + system = crossSystem; + } // lib.optionalAttrs (crossSystem == "x86_64-freebsd13") { + useLLVM = true; + }; overlays = [ (overlayFor (p: p.${stdenv})) ]; @@ -100,12 +166,18 @@ testConfigureFlags = [ "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include" + ] ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ + "--enable-install-unit-tests" + "--with-check-bin-dir=${builtins.placeholder "check"}/bin" + "--with-check-lib-dir=${builtins.placeholder "check"}/lib" ]; internalApiDocsConfigureFlags = [ "--enable-internal-api-docs" ]; + changelog-d = pkgs.buildPackages.changelog-d; + nativeBuildDeps = [ buildPackages.bison @@ -121,19 +193,28 @@ buildPackages.git buildPackages.mercurial # FIXME: remove? only needed for tests buildPackages.jq # Also for custom mdBook preprocessor. + buildPackages.openssh # only needed for tests (ssh-keygen) ] - ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)]; + ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)] + # Official releases don't have rl-next, so we don't need to compile a changelog + ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d + ; buildDeps = [ curl bzip2 xz brotli editline openssl sqlite libarchive + (pkgs.libgit2.overrideAttrs (attrs: { + src = libgit2; + version = libgit2.lastModifiedDate; + cmakeFlags = (attrs.cmakeFlags or []) ++ ["-DUSE_SSH=exec"]; + })) boost lowdown-nix + libsodium ] ++ lib.optionals stdenv.isLinux [libseccomp acl] - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid; checkDeps = [ @@ -157,6 +238,9 @@ }).overrideAttrs(o: { patches = (o.patches or []) ++ [ ./boehmgc-coroutine-sp-fallback.diff + + # https://github.com/ivmai/bdwgc/pull/586 + ./boehmgc-traceable_allocator-public.diff ]; }) ) @@ -209,7 +293,14 @@ "-${client.version}-against-${daemon.version}"; inherit version; - src = self; + src = fileset.toSource { + root = ./.; + fileset = fileset.intersect baseFiles (fileset.unions [ + configureFiles + topLevelBuildFiles + functionalTestFiles + ]); + }; VERSION_SUFFIX = versionSuffix; @@ -219,7 +310,9 @@ enableParallelBuilding = true; - configureFlags = testConfigureFlags; # otherwise configure fails + configureFlags = + testConfigureFlags # otherwise configure fails + ++ [ "--disable-build" ]; dontBuild = true; doInstallCheck = true; @@ -227,7 +320,10 @@ mkdir -p $out ''; - installCheckPhase = "make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES"; + installCheckPhase = '' + mkdir -p src/nix-channel + make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES + ''; }; binaryTarball = nix: pkgs: @@ -320,21 +416,15 @@ }; let canRunInstalled = currentStdenv.buildPlatform.canExecute currentStdenv.hostPlatform; - - sourceByRegexInverted = rxs: origSrc: final.lib.cleanSourceWith { - filter = (path: type: - let relPath = final.lib.removePrefix (toString origSrc + "/") (toString path); - in ! lib.any (re: builtins.match re relPath != null) rxs); - src = origSrc; - }; in currentStdenv.mkDerivation (finalAttrs: { name = "nix-${version}"; inherit version; - src = sourceByRegexInverted [ "tests/nixos/.*" "tests/installer/.*" ] self; + src = nixSrc; VERSION_SUFFIX = versionSuffix; - outputs = [ "out" "dev" "doc" ]; + outputs = [ "out" "dev" "doc" ] + ++ lib.optional (currentStdenv.hostPlatform != currentStdenv.buildPlatform) "check"; nativeBuildInputs = nativeBuildDeps; buildInputs = buildDeps @@ -407,40 +497,13 @@ hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - passthru.perl-bindings = with final; perl.pkgs.toPerlModule (currentStdenv.mkDerivation { - name = "nix-perl-${version}"; - - src = self; - - nativeBuildInputs = - [ buildPackages.autoconf-archive - buildPackages.autoreconfHook - buildPackages.pkg-config - ]; - - buildInputs = - [ nix - curl - bzip2 - xz - pkgs.perl - boost - ] - ++ lib.optional stdenv.isLinux acl - ++ lib.optional (currentStdenv.isLinux || currentStdenv.isDarwin) libsodium - ++ lib.optional currentStdenv.isDarwin darwin.apple_sdk.frameworks.Security; - - configureFlags = [ - "--with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}" - "--with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix}" - ]; - - enableParallelBuilding = true; - - postUnpack = "sourceRoot=$sourceRoot/perl"; - }); + passthru.perl-bindings = final.callPackage ./perl { + inherit fileset; + stdenv = currentStdenv; + }; meta.platforms = lib.platforms.unix; + meta.mainProgram = "nix"; }); lowdown-nix = with final; currentStdenv.mkDerivation rec { @@ -461,18 +524,6 @@ }; }; - nixos-lib = import (nixpkgs + "/nixos/lib") { }; - - # https://nixos.org/manual/nixos/unstable/index.html#sec-calling-nixos-tests - runNixOSTestFor = system: test: nixos-lib.runTest { - imports = [ test ]; - hostPkgs = nixpkgsFor.${system}.native; - defaults = { - nixpkgs.pkgs = nixpkgsFor.${system}.native; - }; - _module.args.nixpkgs = nixpkgs; - }; - in { # A Nixpkgs overlay that overrides the 'nix' and # 'nix.perl-bindings' packages. @@ -530,7 +581,7 @@ releaseTools.coverageAnalysis { name = "nix-coverage-${version}"; - src = self; + src = nixSrc; configureFlags = testConfigureFlags; @@ -547,6 +598,8 @@ lcovFilter = [ "*/boost/*" "*-tab.*" ]; hardeningDisable = ["fortify"]; + + NIX_CFLAGS_COMPILE = "-DCOVERAGE=1"; }; # API docs for Nix's unstable internal C++ interfaces. @@ -558,7 +611,7 @@ pname = "nix-internal-api-docs"; inherit version; - src = self; + src = nixSrc; configureFlags = testConfigureFlags ++ internalApiDocsConfigureFlags; @@ -577,49 +630,29 @@ }; # System tests. - tests.acls = runNixOSTestFor "x86_64-linux" ./tests/nixos/acls.nix; + tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // { - tests.authorization = runNixOSTestFor "x86_64-linux" ./tests/nixos/authorization.nix; - - tests.remoteBuilds = runNixOSTestFor "x86_64-linux" ./tests/nixos/remote-builds.nix; - - tests.nix-copy-closure = runNixOSTestFor "x86_64-linux" ./tests/nixos/nix-copy-closure.nix; - - tests.nix-copy = runNixOSTestFor "x86_64-linux" ./tests/nixos/nix-copy.nix; - - tests.nssPreload = runNixOSTestFor "x86_64-linux" ./tests/nixos/nss-preload.nix; - - tests.githubFlakes = runNixOSTestFor "x86_64-linux" ./tests/nixos/github-flakes.nix; - - tests.sourcehutFlakes = runNixOSTestFor "x86_64-linux" ./tests/nixos/sourcehut-flakes.nix; - - tests.tarballFlakes = runNixOSTestFor "x86_64-linux" ./tests/nixos/tarball-flakes.nix; - - tests.containers = runNixOSTestFor "x86_64-linux" ./tests/nixos/containers/containers.nix; - - tests.setuid = lib.genAttrs - ["i686-linux" "x86_64-linux"] - (system: runNixOSTestFor system ./tests/nixos/setuid.nix); - - - # Make sure that nix-env still produces the exact same result - # on a particular version of Nixpkgs. - tests.evalNixpkgs = - with nixpkgsFor.x86_64-linux.native; - runCommand "eval-nixos" { buildInputs = [ nix ]; } - '' - type -p nix-env - # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593. - time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages - [[ $(sha1sum < packages | cut -c1-40) = ff451c521e61e4fe72bdbe2d0ca5d1809affa733 ]] - mkdir $out - ''; + # Make sure that nix-env still produces the exact same result + # on a particular version of Nixpkgs. + evalNixpkgs = + with nixpkgsFor.x86_64-linux.native; + runCommand "eval-nixos" { buildInputs = [ nix ]; } + '' + type -p nix-env + # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593. + time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages + [[ $(sha1sum < packages | cut -c1-40) = ff451c521e61e4fe72bdbe2d0ca5d1809affa733 ]] + mkdir $out + ''; - tests.nixpkgsLibTests = - forAllSystems (system: - import (nixpkgs + "/lib/tests/release.nix") - { pkgs = nixpkgsFor.${system}.native; } - ); + nixpkgsLibTests = + forAllSystems (system: + import (nixpkgs + "/lib/tests/release.nix") + { pkgs = nixpkgsFor.${system}.native; + nixVersions = [ self.packages.${system}.nix ]; + } + ); + }; metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { pkgs = nixpkgsFor.x86_64-linux.native; @@ -690,20 +723,31 @@ devShells = let makeShell = pkgs: stdenv: + let + canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform; + in with commonDeps { inherit pkgs; }; stdenv.mkDerivation { name = "nix"; - outputs = [ "out" "dev" "doc" ]; + outputs = [ "out" "dev" "doc" ] + ++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) "check"; nativeBuildInputs = nativeBuildDeps - ++ (lib.optionals stdenv.cc.isClang [ pkgs.bear pkgs.clang-tools ]); + ++ lib.optional stdenv.cc.isClang pkgs.buildPackages.bear + ++ lib.optional + (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) + pkgs.buildPackages.clang-tools + # We want changelog-d in the shell even if the current build doesn't need it + ++ lib.optional (officialRelease || ! buildUnreleasedNotes) changelog-d + ; buildInputs = buildDeps ++ propagatedDeps ++ awsDeps ++ checkDeps ++ internalApiDocsDeps; configureFlags = configureFlags - ++ testConfigureFlags ++ internalApiDocsConfigureFlags; + ++ testConfigureFlags ++ internalApiDocsConfigureFlags + ++ lib.optional (!canRunInstalled) "--disable-doc-gen"; enableParallelBuilding = true; diff --git a/local.mk b/local.mk index 6951c179e9d1..3f3abb9f0d69 100644 --- a/local.mk +++ b/local.mk @@ -1,5 +1,3 @@ -clean-files += Makefile.config - GLOBAL_CXXFLAGS += -Wno-deprecated-declarations -Werror=switch # Allow switch-enum to be overridden for files that do not support it, usually because of dependency headers. ERROR_SWITCH_ENUM = -Werror=switch-enum diff --git a/m4/gcc_bug_80431.m4 b/m4/gcc_bug_80431.m4 new file mode 100644 index 000000000000..e42f0195614f --- /dev/null +++ b/m4/gcc_bug_80431.m4 @@ -0,0 +1,64 @@ +# Ensure that this bug is not present in the C++ toolchain we are using. +# +# URL for bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431 +# +# The test program is from that issue, with only a slight modification +# to set an exit status instead of printing strings. +AC_DEFUN([ENSURE_NO_GCC_BUG_80431], +[ + AC_MSG_CHECKING([that GCC bug 80431 is fixed]) + AC_LANG_PUSH(C++) + AC_RUN_IFELSE( + [AC_LANG_PROGRAM( + [[ + #include + + static bool a = true; + static bool b = true; + + struct Options { }; + + struct Option + { + Option(Options * options) + { + a = false; + } + + ~Option() + { + b = false; + } + }; + + struct MyOptions : Options { }; + + struct MyOptions2 : virtual MyOptions + { + Option foo{this}; + }; + ]], + [[ + { + MyOptions2 opts; + } + return (a << 1) | b; + ]])], + [status_80431=0], + [status_80431=$?], + [ + # Assume we're bug-free when cross-compiling + ]) + AC_LANG_POP(C++) + AS_CASE([$status_80431], + [0],[ + AC_MSG_RESULT(yes) + ], + [2],[ + AC_MSG_RESULT(no) + AC_MSG_ERROR(Cannot build Nix with C++ compiler with this bug) + ], + [ + AC_MSG_RESULT(unexpected result $status_80431: not expected failure with bug, ignoring) + ]) +]) diff --git a/maintainers/README.md b/maintainers/README.md index 0d520cb0cc5b..ee97c1195d68 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -2,7 +2,7 @@ ## Motivation -The team's main responsibility is to set a direction for the development of Nix and ensure that the code is in good shape. +The team's main responsibility is to guide and direct the development of Nix and ensure that the code is in good shape. We aim to achieve this by improving the contributor experience and attracting more maintainers – that is, by helping other people contributing to Nix and eventually taking responsibility – in order to scale the development process to match users' needs. @@ -50,7 +50,9 @@ The team meets twice a week: 1. Code review on pull requests from [In review](#in-review). 2. Other chores and tasks. -Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.us/Cv7FpYx-Ri-4VjUykQOLAw), and published on Discourse under the [Nix category](https://discourse.nixos.org/c/dev/nix/50). +Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.us/Cv7FpYx-Ri-4VjUykQOLAw). +Notes on issues and pull requests are posted as comments and linked from the meeting notes, so they are easy to find from both places. +[All meeting notes](https://discourse.nixos.org/search?expanded=true&q=Nix%20team%20meeting%20minutes%20%23%20%23dev%3Anix%20in%3Atitle%20order%3Alatest_topic) are published on Discourse under the [Nix category](https://discourse.nixos.org/c/dev/nix/50). ## Project board protocol @@ -96,8 +98,10 @@ What constitutes a trivial pull request is up to maintainers' judgement. Pull requests and issues that are deemed important and controversial are discussed by the team during discussion meetings. This may be where the merit of the change itself or the implementation strategy is contested by a team member. +Whenever the discussion opens up questions about the process or this team's goals, this may indicate that the change is too large in scope. +In that case it is taken off the board to be reconsidered by the author or broken down into smaller pieces that are less far-reaching and can be reviewed independently. -As a general guideline, the order of items is determined as follows: +As a general guideline, the order of items to discuss is determined as follows: - Prioritise pull requests over issues diff --git a/maintainers/release-notes b/maintainers/release-notes new file mode 100755 index 000000000000..34cd85a56921 --- /dev/null +++ b/maintainers/release-notes @@ -0,0 +1,179 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash ../shell.nix -I nixpkgs=channel:nixos-unstable-small +# ^^^^^^^ +# Only used for bash. shell.nix goes to the flake. + +# --- CONFIGURATION --- + +# This does double duty for +# - including rl-next +# - marking where to insert new links (right after) +SUMMARY_MARKER_LINE='{{#include ./SUMMARY-rl-next.md}}' + +# --- LIB --- + +log() { + echo 1>&2 "release-notes:" "$@" +} +logcmd() { + local cmd="$1" + shift + logcmd2 "$cmd" "${*@Q}" "$cmd" "$@" +} +logcmd2() { + local fakecmd="$1" + local fakeargs="$2" + shift + shift + printf 1>&2 "release-notes: \033[34;1m$fakecmd\033[0m " + echo "$fakeargs" 1>&2 + "$@" +} +die() { + # ANSI red + printf 1>&2 "release-notes: \033[31;1merror:\033[0m" + echo 1>&2 "" "$@" + exit 1 +} +confirm() { + local answer + echo 1>&2 "$@" "[y/n]" + read -r answer + case "$answer" in + y|Y|yes|Yes|YES) + return 0 + ;; + n|N|no|No|NO) + return 1 + ;; + *) + echo 1>&2 "please answer y or n" + confirm "$@" + ;; + esac +} +report_done() { + logcmd2 "git" "show" git -c pager.show=false show + printf 1>&2 "release-notes: \033[32;1mdone\033[0m\n" +} + +# --- PARSE ARGS --- + +if [[ $# -gt 0 ]]; then + die "Release notes takes no arguments, but make sure to set VERSION." +fi + +# --- CHECKS --- + +if [[ ! -e flake.nix ]] || [[ ! -e .git ]]; then + die "must run in repo root" + exit 1 +fi + +# repo must be clean +if ! git diff --quiet; then + die "repo is dirty, please commit or stash changes" +fi + +if ! git diff --quiet --cached; then + die "repo has staged changes, please commit or stash them" +fi + +if ! grep "$SUMMARY_MARKER_LINE" doc/manual/src/SUMMARY.md.in >/dev/null; then + # would have been nice to catch this early, but won't be worth the extra infra + die "SUMMARY.md.in is missing the marker line '$SUMMARY_MARKER_LINE', which would be used for inserting a new release notes page. Please fix the script." +fi + +if [[ ! -n "${VERSION:-}" ]]; then + die "please set the VERSION environment variable before invoking this script" + exit 1 +fi + +# version_major_minor: MAJOR.MINOR +# version_full: MAJOR.MINOR.PATCH +# IS_PATCH: true if this is a patch release; append instead of create +if grep -E '^[0-9]+\.[0-9]+$' <<< "$VERSION" >/dev/null; then + log 'is minor' + IS_PATCH=false + version_full="$VERSION.0" + version_major_minor="$VERSION" +elif grep -E '^[0-9]+\.[0-9]+\.0$' <<< "$VERSION" >/dev/null; then + log 'is minor (.0)' + IS_PATCH=false + version_full="$VERSION" + version_major_minor="$(echo "$VERSION" | sed -e 's/\.0$//')" +elif grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' <<< "$VERSION" >/dev/null; then + log 'is patch' + IS_PATCH=true + version_full="$VERSION" + version_major_minor="$(echo "$VERSION" | sed -e 's/\.[0-9]*$//')" +else + die "VERSION must be MAJOR.MINOR[.PATCH], where each is a number, e.g. 2.20 or 2.20.1 (VERSION was set to $VERSION)" +fi + +unset VERSION + +log "version_major_minor=$version_major_minor" +log "version_full=$version_full" +log "IS_PATCH=$IS_PATCH" + +basename=rl-${version_major_minor}.md +file=doc/manual/src/release-notes/$basename + +if ! $IS_PATCH; then + if [[ -e $file ]]; then + die "release notes file $file already exists. If you'd like to make a minor release, pass a patch version, e.g. 2.20.1" + fi +fi + +# --- DEFAULTS --- + +if [[ ! -n "${DATE:-}" ]]; then + DATE="$(date +%Y-%m-%d)" + log "DATE not set, using $DATE" +fi + +case "$DATE" in + [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]) + ;; + *) + die "DATE must be YYYY-MM-DD, e.g. 2021-12-31 (DATE was set to $DATE)" + ;; +esac + +# --- DO THE WORK --- + +# menu +title="Release $version_major_minor ($DATE)" +# section on page +section_title="Release $version_full ($DATE)" + +( + # TODO add minor number, and append? + echo "# $section_title" + echo + changelog-d doc/manual/rl-next | sed -e 's/ *$//' +) | tee -a $file + +log "Wrote $file" + +if ! $IS_PATCH; then + NEW_SUMMARY_LINE=" - [$title](release-notes/$basename)" + + # find the marker line, insert new link after it + escaped_marker="$(echo "$SUMMARY_MARKER_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')" + escaped_line="$(echo "$NEW_SUMMARY_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')" + logcmd sed -i -e "/$escaped_marker/a $escaped_line" doc/manual/src/SUMMARY.md.in +fi + +for f in doc/manual/rl-next/*.md; do + if [[ config != "$(basename $f)" ]]; then + logcmd git rm $f + fi +done + +logcmd git add $file doc/manual/src/SUMMARY.md.in +logcmd git status +logcmd git commit -m "release notes: $version_full" + +report_done diff --git a/maintainers/release-process.md b/maintainers/release-process.md index d85266b81610..db8b064a514c 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -24,34 +24,23 @@ release: * In a checkout of the Nix repo, make sure you're on `master` and run `git pull`. -* Move the contents of `doc/manual/src/release-notes/rl-next.md` - (except the first line) to - `doc/manual/src/release-notes/rl-$VERSION.md` (where `$VERSION` is - the contents of `.version` *without* the patch level, e.g. `2.12` - rather than `2.12.0`). +* Compile the release notes by running -* Add a header to `doc/manual/src/release-notes/rl-$VERSION.md` like - - ``` - # Release 2.12 (2022-12-06) + ```console + $ git checkout -b release-notes + $ VERSION=X.YY ./maintainers/release-notes ``` -* Proof-read / edit / rearrange the release notes. Breaking changes - and highlights should go to the top. + where `X.YY` is *without* the patch level, e.g. `2.12` rather than ~~`2.12.0`~~. -* Add a link to the release notes to `doc/manual/src/SUMMARY.md.in` - (*not* `SUMMARY.md`), e.g. + A commit is created. - ``` - - [Release 2.12 (2022-12-06)](release-notes/rl-2.12.md) - ``` +* Proof-read / edit / rearrange the release notes if needed. Breaking changes + and highlights should go to the top. -* Run +* Push. ```console - $ git checkout -b release-notes - $ git add doc/manual/src/release-notes/rl-$VERSION.md - $ git commit -a -m 'Release notes' $ git push --set-upstream $REMOTE release-notes ``` @@ -67,15 +56,17 @@ release: $ git checkout -b $VERSION-maintenance ``` -* Mark the release as stable: +* Mark the release as official: ```console - $ git cherry-pick f673551e71942a52b6d7ae66af8b67140904a76a + $ sed -e 's/officialRelease = false;/officialRelease = true;/' -i flake.nix ``` This removes the link to `rl-next.md` from the manual and sets `officialRelease = true` in `flake.nix`. +* Commit + * Push the release branch: ```console @@ -159,6 +150,30 @@ release: ## Creating a point release +* Checkout. + + ```console + $ git checkout XX.YY-maintenance + ``` + +* Determine the next patch version. + + ```console + $ export VERSION=XX.YY.ZZ + ``` + +* Update release notes. + + ```console + $ ./maintainers/release-notes + ``` + +* Push. + + ```console + $ git push + ``` + * Wait for the desired evaluation of the maintenance jobset to finish building. diff --git a/misc/systemd/nix-daemon.service.in b/misc/systemd/nix-daemon.service.in index f46413630ae2..45fbea02c479 100644 --- a/misc/systemd/nix-daemon.service.in +++ b/misc/systemd/nix-daemon.service.in @@ -10,6 +10,7 @@ ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket ExecStart=@@bindir@/nix-daemon nix-daemon --daemon KillMode=process LimitNOFILE=1048576 +TasksMax=1048576 [Install] WantedBy=multi-user.target diff --git a/mk/build-dir.mk b/mk/build-dir.mk new file mode 100644 index 000000000000..02f4cae60f6a --- /dev/null +++ b/mk/build-dir.mk @@ -0,0 +1,10 @@ +# Initialise support for build directories. +builddir ?= + +ifdef builddir + buildprefix = $(builddir)/ + buildprefixrel = $(builddir) +else + buildprefix = + buildprefixrel = . +endif diff --git a/mk/common-test.sh b/mk/common-test.sh index 0a2e4c1c2e29..00ccd1584239 100644 --- a/mk/common-test.sh +++ b/mk/common-test.sh @@ -1,11 +1,27 @@ -TESTS_ENVIRONMENT=("TEST_NAME=${test%.*}" 'NIX_REMOTE=') +# Remove overall test dir (at most one of the two should match) and +# remove file extension. +test_name=$(echo -n "$test" | sed \ + -e "s|^unit-test-data/||" \ + -e "s|^tests/functional/||" \ + -e "s|\.sh$||" \ + ) + +TESTS_ENVIRONMENT=( + "TEST_NAME=$test_name" + 'NIX_REMOTE=' + 'PS4=+(${BASH_SOURCE[0]-$0}:$LINENO) ' +) : ${BASH:=/usr/bin/env bash} +run () { + cd "$(dirname $1)" && env "${TESTS_ENVIRONMENT[@]}" $BASH -x -e -u -o pipefail $(basename $1) +} + init_test () { - cd tests && env "${TESTS_ENVIRONMENT[@]}" $BASH -e init.sh 2>/dev/null > /dev/null + run "$init" 2>/dev/null > /dev/null } run_test_proper () { - cd $(dirname $test) && env "${TESTS_ENVIRONMENT[@]}" $BASH -e $(basename $test) + run "$test" } diff --git a/mk/debug-test.sh b/mk/debug-test.sh index b5b628ecd689..52482c01e47c 100755 --- a/mk/debug-test.sh +++ b/mk/debug-test.sh @@ -3,9 +3,12 @@ set -eu -o pipefail test=$1 +init=${2-} dir="$(dirname "${BASH_SOURCE[0]}")" source "$dir/common-test.sh" -(init_test) +if [ -n "$init" ]; then + (init_test) +fi run_test_proper diff --git a/mk/install-dirs.mk b/mk/install-dirs.mk new file mode 100644 index 000000000000..732b0d6fc852 --- /dev/null +++ b/mk/install-dirs.mk @@ -0,0 +1,11 @@ +# Default installation paths. +prefix ?= /usr/local +libdir ?= $(prefix)/lib +bindir ?= $(prefix)/bin +libexecdir ?= $(prefix)/libexec +datadir ?= $(prefix)/share +localstatedir ?= $(prefix)/var +sysconfdir ?= $(prefix)/etc +mandir ?= $(prefix)/share/man + +DESTDIR ?= diff --git a/mk/lib.mk b/mk/lib.mk index 34fa624d8d53..3d503364f147 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -10,6 +10,7 @@ bin-scripts := noinst-scripts := man-pages := install-tests := +install-tests-groups := ifdef HOST_OS HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS))) @@ -42,27 +43,6 @@ define newline endef -# Default installation paths. -prefix ?= /usr/local -libdir ?= $(prefix)/lib -bindir ?= $(prefix)/bin -libexecdir ?= $(prefix)/libexec -datadir ?= $(prefix)/share -localstatedir ?= $(prefix)/var -sysconfdir ?= $(prefix)/etc -mandir ?= $(prefix)/share/man - - -# Initialise support for build directories. -builddir ?= - -ifdef builddir - buildprefix = $(builddir)/ -else - buildprefix = -endif - - # Pass -fPIC if we're building dynamic libraries. BUILD_SHARED_LIBS ?= 1 @@ -93,6 +73,8 @@ ifeq ($(BUILD_DEBUG), 1) endif +include mk/build-dir.mk +include mk/install-dirs.mk include mk/functions.mk include mk/tracing.mk include mk/clean.mk @@ -111,7 +93,7 @@ define include-sub-makefile include $(1) endef -$(foreach mf, $(makefiles), $(eval $(call include-sub-makefile, $(mf)))) +$(foreach mf, $(makefiles), $(eval $(call include-sub-makefile,$(mf)))) # Instantiate stuff. @@ -121,7 +103,17 @@ $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(b $(foreach script, $(bin-scripts), $(eval programs-list += $(script))) $(foreach script, $(noinst-scripts), $(eval programs-list += $(script))) $(foreach template, $(template-files), $(eval $(call instantiate-template,$(template)))) -$(foreach test, $(install-tests), $(eval $(call run-install-test,$(test)))) +install_test_init=tests/functional/init.sh +$(foreach test, $(install-tests), \ + $(eval $(call run-test,$(test),$(install_test_init))) \ + $(eval installcheck: $(test).test)) +$(foreach test-group, $(install-tests-groups), \ + $(eval $(call run-test-group,$(test-group),$(install_test_init))) \ + $(eval installcheck: $(test-group).test-group) \ + $(foreach test, $($(test-group)-tests), \ + $(eval $(call run-test,$(test),$(install_test_init))) \ + $(eval $(test-group).test-group: $(test).test))) + $(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/man$(patsubst .%,%,$(suffix $(file)))))) @@ -151,6 +143,14 @@ ifdef libs-list @echo "The following libraries can be built:" @echo "" @for i in $(libs-list); do echo " $$i"; done +endif +ifdef install-tests-groups + @echo "" + @echo "The following groups of functional tests can be run:" + @echo "" + @for i in $(install-tests-groups); do echo " $$i.test-group"; done + @echo "" + @echo "(installcheck includes tests in test groups too.)" endif @echo "" @echo "The following variables control the build:" diff --git a/mk/programs.mk b/mk/programs.mk index 1ee1d3fa5de8..a88d9d949849 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -87,6 +87,6 @@ define build-program # Phony target to run this program (typically as a dependency of 'check'). .PHONY: $(1)_RUN $(1)_RUN: $$($(1)_PATH) - $(trace-test) $$($(1)_PATH) + $(trace-test) $$(UNIT_TEST_ENV) $$($(1)_PATH) endef diff --git a/mk/run-test.sh b/mk/run-test.sh index 1a1d659304e6..da9c5a473b40 100755 --- a/mk/run-test.sh +++ b/mk/run-test.sh @@ -8,6 +8,7 @@ yellow="" normal="" test=$1 +init=${2-} dir="$(dirname "${BASH_SOURCE[0]}")" source "$dir/common-test.sh" @@ -21,7 +22,9 @@ if [ -t 1 ]; then fi run_test () { - (init_test 2>/dev/null > /dev/null) + if [ -n "$init" ]; then + (init_test 2>/dev/null > /dev/null) + fi log="$(run_test_proper 2>&1)" && status=0 || status=$? } diff --git a/mk/templates.mk b/mk/templates.mk index c7ac7afbff27..866bdc17f8b5 100644 --- a/mk/templates.mk +++ b/mk/templates.mk @@ -10,10 +10,10 @@ endef ifneq ($(MAKECMDGOALS), clean) -%.h: %.h.in - $(trace-gen) rm -f $@ && ./config.status --quiet --header=$@ +$(buildprefix)%.h: %.h.in + $(trace-gen) rm -f $@ && cd $(buildprefixrel) && ./config.status --quiet --header=$(@:$(buildprefix)%=%) -%: %.in - $(trace-gen) rm -f $@ && ./config.status --quiet --file=$@ +$(buildprefix)%: %.in + $(trace-gen) rm -f $@ && cd $(buildprefixrel) && ./config.status --quiet --file=$(@:$(buildprefix)%=%) endif diff --git a/mk/tests.mk b/mk/tests.mk index 3ebbd86e3192..bac9b704ad10 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -2,17 +2,24 @@ test-deps = -define run-install-test +define run-bash - installcheck: $1.test + .PHONY: $1 + $1: $2 + @env BASH=$(bash) $(bash) $3 < /dev/null - .PHONY: $1.test - $1.test: $1 $(test-deps) - @env BASH=$(bash) $(bash) mk/run-test.sh $1 < /dev/null +endef + +define run-test + + $(eval $(call run-bash,$1.test,$1 $(test-deps),mk/run-test.sh $1 $2)) + $(eval $(call run-bash,$1.test-debug,$1 $(test-deps),mk/debug-test.sh $1 $2)) + +endef + +define run-test-group - .PHONY: $1.test-debug - $1.test-debug: $1 $(test-deps) - @env BASH=$(bash) $(bash) mk/debug-test.sh $1 < /dev/null + .PHONY: $1.test-group endef diff --git a/perl/Makefile b/perl/Makefile index c2c95f255030..832668dd1559 100644 --- a/perl/Makefile +++ b/perl/Makefile @@ -1,6 +1,12 @@ makefiles = local.mk -GLOBAL_CXXFLAGS += -g -Wall -std=c++2a -I ../src +GLOBAL_CXXFLAGS += -g -Wall -std=c++2a + +# A convenience for concurrent development of Nix and its Perl bindings. +# Not needed in a standalone build of the Perl bindings. +ifneq ("$(wildcard ../src)", "") + GLOBAL_CXXFLAGS += -I ../src +endif -include Makefile.config diff --git a/perl/default.nix b/perl/default.nix new file mode 100644 index 000000000000..0fa57f7815ea --- /dev/null +++ b/perl/default.nix @@ -0,0 +1,52 @@ +{ lib, fileset +, stdenv +, perl, perlPackages +, autoconf-archive, autoreconfHook, pkg-config +, nix, curl, bzip2, xz, boost, libsodium, darwin +}: + +perl.pkgs.toPerlModule (stdenv.mkDerivation { + name = "nix-perl-${nix.version}"; + + src = fileset.toSource { + root = ../.; + fileset = fileset.unions [ + ../.version + ../m4 + ../mk + ./MANIFEST + ./Makefile + ./Makefile.config.in + ./configure.ac + ./lib + ./local.mk + ]; + }; + + nativeBuildInputs = + [ autoconf-archive + autoreconfHook + pkg-config + ]; + + buildInputs = + [ nix + curl + bzip2 + xz + perl + boost + ] + ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium + ++ lib.optional stdenv.isLinux acl + ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security; + + configureFlags = [ + "--with-dbi=${perlPackages.DBI}/${perl.libPrefix}" + "--with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}" + ]; + + enableParallelBuilding = true; + + postUnpack = "sourceRoot=$sourceRoot/perl"; +}) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 41ecbbeb4794..f89ac4077d0d 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -11,7 +11,6 @@ #include "derivations.hh" #include "globals.hh" #include "store-api.hh" -#include "util.hh" #include "crypto.hh" #include @@ -78,7 +77,7 @@ SV * queryReferences(char * path) SV * queryPathHash(char * path) PPCODE: try { - auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true); + auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(HashFormat::Base32, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -104,7 +103,7 @@ SV * queryPathInfo(char * path, int base32) XPUSHs(&PL_sv_undef); else XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0))); - auto s = info->narHash.to_string(base32 ? Base32 : Base16, true); + auto s = info->narHash.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); @@ -206,7 +205,7 @@ SV * hashPath(char * algo, int base32, char * path) PPCODE: try { Hash h = hashPath(parseHashType(algo), path).first; - auto s = h.to_string(base32 ? Base32 : Base16, false); + auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -217,7 +216,7 @@ SV * hashFile(char * algo, int base32, char * path) PPCODE: try { Hash h = hashFile(parseHashType(algo), path); - auto s = h.to_string(base32 ? Base32 : Base16, false); + auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -228,7 +227,7 @@ SV * hashString(char * algo, int base32, char * s) PPCODE: try { Hash h = hashString(parseHashType(algo), s); - auto s = h.to_string(base32 ? Base32 : Base16, false); + auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -239,7 +238,7 @@ SV * convertHash(char * algo, char * s, int toBase32) PPCODE: try { auto h = Hash::parseAny(s, parseHashType(algo)); - auto s = h.to_string(toBase32 ? Base32 : Base16, false); + auto s = h.to_string(toBase32 ? HashFormat::Base32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -294,10 +293,8 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) auto h = Hash::parseAny(hash, parseHashType(algo)); auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; auto path = store()->makeFixedOutputPath(name, FixedOutputInfo { - .hash = { - .method = method, - .hash = h, - }, + .method = method, + .hash = h, .references = {}, }); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); @@ -326,7 +323,7 @@ SV * derivationFromPath(char * drvPath) hv_stores(hash, "outputs", newRV((SV *) outputs)); AV * inputDrvs = newAV(); - for (auto & i : drv.inputDrvs) + for (auto & i : drv.inputDrvs.map) av_push(inputDrvs, newSVpv(store()->printStorePath(i.first).c_str(), 0)); // !!! ignores i->second hv_stores(hash, "inputDrvs", newRV((SV *) inputDrvs)); diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 656769d84d44..a08f62333dc2 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -452,6 +452,14 @@ EOF # a row for different files. if [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then # this backup process first released in Nix 2.1 + + if diff -q "$profile_target$PROFILE_BACKUP_SUFFIX" "$profile_target" > /dev/null; then + # a backup file for the rc-file exist, but they are identical, + # so we can safely ignore it and overwrite it with the same + # content later + continue + fi + failure <parseStorePathSet(inputs); optResult = sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv); auto & result = *optResult; @@ -322,7 +322,12 @@ static int main_build_remote(int argc, char * * argv) throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg); } else { copyClosure(*store, *sshStore, StorePathSet {*drvPath}, NoRepair, NoCheckSigs, substitute); - auto res = sshStore->buildPathsWithResults({ DerivedPath::Built { *drvPath, OutputsSpec::All {} } }); + auto res = sshStore->buildPathsWithResults({ + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(*drvPath), + .outputs = OutputsSpec::All {}, + } + }); // One path to build should produce exactly one build result assert(res.size() == 1); optResult = std::move(res[0]); diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc new file mode 100644 index 000000000000..9a2dce806ce0 --- /dev/null +++ b/src/libcmd/built-path.cc @@ -0,0 +1,149 @@ +#include "built-path.hh" +#include "derivations.hh" +#include "store-api.hh" + +#include + +#include + +namespace nix { + +#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \ + bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ + { \ + const MY_TYPE* me = this; \ + auto fields1 = std::make_tuple(*me->drvPath, me->FIELD); \ + me = &other; \ + auto fields2 = std::make_tuple(*me->drvPath, me->FIELD); \ + return fields1 COMPARATOR fields2; \ + } +#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <) + +#define FIELD_TYPE std::pair +CMP(SingleBuiltPath, SingleBuiltPathBuilt, output) +#undef FIELD_TYPE + +#define FIELD_TYPE std::map +CMP(SingleBuiltPath, BuiltPathBuilt, outputs) +#undef FIELD_TYPE + +#undef CMP +#undef CMP_ONE + +StorePath SingleBuiltPath::outPath() const +{ + return std::visit( + overloaded{ + [](const SingleBuiltPath::Opaque & p) { return p.path; }, + [](const SingleBuiltPath::Built & b) { return b.output.second; }, + }, raw() + ); +} + +StorePathSet BuiltPath::outPaths() const +{ + return std::visit( + overloaded{ + [](const BuiltPath::Opaque & p) { return StorePathSet{p.path}; }, + [](const BuiltPath::Built & b) { + StorePathSet res; + for (auto & [_, path] : b.outputs) + res.insert(path); + return res; + }, + }, raw() + ); +} + +SingleDerivedPath::Built SingleBuiltPath::Built::discardOutputPath() const +{ + return SingleDerivedPath::Built { + .drvPath = make_ref(drvPath->discardOutputPath()), + .output = output.first, + }; +} + +SingleDerivedPath SingleBuiltPath::discardOutputPath() const +{ + return std::visit( + overloaded{ + [](const SingleBuiltPath::Opaque & p) -> SingleDerivedPath { + return p; + }, + [](const SingleBuiltPath::Built & b) -> SingleDerivedPath { + return b.discardOutputPath(); + }, + }, raw() + ); +} + +nlohmann::json BuiltPath::Built::toJSON(const Store & store) const +{ + nlohmann::json res; + res["drvPath"] = drvPath->toJSON(store); + for (const auto & [outputName, outputPath] : outputs) { + res["outputs"][outputName] = store.printStorePath(outputPath); + } + return res; +} + +nlohmann::json SingleBuiltPath::Built::toJSON(const Store & store) const +{ + nlohmann::json res; + res["drvPath"] = drvPath->toJSON(store); + auto & [outputName, outputPath] = output; + res["output"] = outputName; + res["outputPath"] = store.printStorePath(outputPath); + return res; +} + +nlohmann::json SingleBuiltPath::toJSON(const Store & store) const +{ + return std::visit([&](const auto & buildable) { + return buildable.toJSON(store); + }, raw()); +} + +nlohmann::json BuiltPath::toJSON(const Store & store) const +{ + return std::visit([&](const auto & buildable) { + return buildable.toJSON(store); + }, raw()); +} + +RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const +{ + RealisedPath::Set res; + std::visit( + overloaded{ + [&](const BuiltPath::Opaque & p) { res.insert(p.path); }, + [&](const BuiltPath::Built & p) { + auto drvHashes = + staticOutputHashes(store, store.readDerivation(p.drvPath->outPath())); + for (auto& [outputName, outputPath] : p.outputs) { + if (experimentalFeatureSettings.isEnabled( + Xp::CaDerivations)) { + auto drvOutput = get(drvHashes, outputName); + if (!drvOutput) + throw Error( + "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)", + store.printStorePath(p.drvPath->outPath()), outputName); + auto thisRealisation = store.queryRealisation( + DrvOutput{*drvOutput, outputName}); + assert(thisRealisation); // We’ve built it, so we must + // have the realisation + res.insert(*thisRealisation); + } else { + res.insert(outputPath); + } + } + }, + }, + raw()); + return res; +} + +} diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh new file mode 100644 index 000000000000..7154cc504f8b --- /dev/null +++ b/src/libcmd/built-path.hh @@ -0,0 +1,97 @@ +#pragma once +///@file + +#include "derived-path.hh" +#include "realisation.hh" + +namespace nix { + +struct SingleBuiltPath; + +struct SingleBuiltPathBuilt { + ref drvPath; + std::pair output; + + SingleDerivedPathBuilt discardOutputPath() const; + + std::string to_string(const Store & store) const; + static SingleBuiltPathBuilt parse(const Store & store, std::string_view, std::string_view); + nlohmann::json toJSON(const Store & store) const; + + DECLARE_CMP(SingleBuiltPathBuilt); +}; + +using _SingleBuiltPathRaw = std::variant< + DerivedPathOpaque, + SingleBuiltPathBuilt +>; + +struct SingleBuiltPath : _SingleBuiltPathRaw { + using Raw = _SingleBuiltPathRaw; + using Raw::Raw; + + using Opaque = DerivedPathOpaque; + using Built = SingleBuiltPathBuilt; + + inline const Raw & raw() const { + return static_cast(*this); + } + + StorePath outPath() const; + + SingleDerivedPath discardOutputPath() const; + + static SingleBuiltPath parse(const Store & store, std::string_view); + nlohmann::json toJSON(const Store & store) const; +}; + +static inline ref staticDrv(StorePath drvPath) +{ + return make_ref(SingleBuiltPath::Opaque { drvPath }); +} + +/** + * A built derived path with hints in the form of optional concrete output paths. + * + * See 'BuiltPath' for more an explanation. + */ +struct BuiltPathBuilt { + ref drvPath; + std::map outputs; + + std::string to_string(const Store & store) const; + static BuiltPathBuilt parse(const Store & store, std::string_view, std::string_view); + nlohmann::json toJSON(const Store & store) const; + + DECLARE_CMP(BuiltPathBuilt); +}; + +using _BuiltPathRaw = std::variant< + DerivedPath::Opaque, + BuiltPathBuilt +>; + +/** + * A built path. Similar to a DerivedPath, but enriched with the corresponding + * output path(s). + */ +struct BuiltPath : _BuiltPathRaw { + using Raw = _BuiltPathRaw; + using Raw::Raw; + + using Opaque = DerivedPathOpaque; + using Built = BuiltPathBuilt; + + inline const Raw & raw() const { + return static_cast(*this); + } + + StorePathSet outPaths() const; + RealisedPath::Set toRealisedPaths(Store & store) const; + + nlohmann::json toJSON(const Store & store) const; +}; + +typedef std::vector BuiltPaths; + +} diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 4fc1979563d5..de9f546fc701 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -98,7 +98,7 @@ EvalCommand::EvalCommand() EvalCommand::~EvalCommand() { if (evalState) - evalState->printStats(); + evalState->maybePrintStats(); } ref EvalCommand::getEvalStore() @@ -175,7 +175,7 @@ void BuiltPathsCommand::run(ref store, Installables && installables) throw UsageError("'--all' does not expect arguments"); // XXX: Only uses opaque paths, ignores all the realisations for (auto & p : store->queryAllValidPaths()) - paths.push_back(BuiltPath::Opaque{p}); + paths.emplace_back(BuiltPath::Opaque{p}); } else { paths = Installable::toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables); if (recursive) { @@ -188,7 +188,7 @@ void BuiltPathsCommand::run(ref store, Installables && installables) } store->computeFSClosure(pathsRoots, pathsClosure); for (auto & path : pathsClosure) - paths.push_back(BuiltPath::Opaque{path}); + paths.emplace_back(BuiltPath::Opaque{path}); } } diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 96236b987b09..120c832ac23b 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -34,21 +34,28 @@ struct NixMultiCommand : virtual MultiCommand, virtual Command // For the overloaded run methods #pragma GCC diagnostic ignored "-Woverloaded-virtual" -/* A command that requires a Nix store. */ +/** + * A command that requires a \ref Store "Nix store". + */ struct StoreCommand : virtual Command { StoreCommand(); void run() override; ref getStore(); virtual ref createStore(); + /** + * Main entry point, with a `Store` provided + */ virtual void run(ref) = 0; private: std::shared_ptr _store; }; -/* A command that copies something between `--from` and `--to` - stores. */ +/** + * A command that copies something between `--from` and `--to` \ref + * Store stores. + */ struct CopyCommand : virtual StoreCommand { std::string srcUri, dstUri; @@ -60,6 +67,9 @@ struct CopyCommand : virtual StoreCommand ref getDstStore(); }; +/** + * A command that needs to evaluate Nix language expressions. + */ struct EvalCommand : virtual StoreCommand, MixEvalArgs { bool startReplOnEvalErrors = false; @@ -79,20 +89,26 @@ private: std::shared_ptr evalState; }; +/** + * A mixin class for commands that process flakes, adding a few standard + * flake-related options/flags. + */ struct MixFlakeOptions : virtual Args, EvalCommand { flake::LockFlags lockFlags; - std::optional needsFlakeInputCompletion = {}; - MixFlakeOptions(); - virtual std::vector getFlakesForCompletion() + /** + * The completion for some of these flags depends on the flake(s) in + * question. + * + * This method should be implemented to gather all flakerefs the + * command is operating with (presumably specified via some other + * arguments) so that the completions for these flags can use them. + */ + virtual std::vector getFlakeRefsForCompletion() { return {}; } - - void completeFlakeInput(std::string_view prefix); - - void completionHook() override; }; struct SourceExprCommand : virtual Args, MixFlakeOptions @@ -112,15 +128,35 @@ struct SourceExprCommand : virtual Args, MixFlakeOptions virtual Strings getDefaultFlakeAttrPathPrefixes(); - void completeInstallable(std::string_view prefix); + /** + * Complete an installable from the given prefix. + */ + void completeInstallable(AddCompletions & completions, std::string_view prefix); + + /** + * Convenience wrapper around the underlying function to make setting the + * callback easier. + */ + CompleterClosure getCompleteInstallable(); }; +/** + * A mixin class for commands that need a read-only flag. + * + * What exactly is "read-only" is unspecified, but it will usually be + * the \ref Store "Nix store". + */ struct MixReadOnlyOption : virtual Args { MixReadOnlyOption(); }; -/* Like InstallablesCommand but the installables are not loaded */ +/** + * Like InstallablesCommand but the installables are not loaded. + * + * This is needed by `CmdRepl` which wants to load (and reload) the + * installables itself. + */ struct RawInstallablesCommand : virtual Args, SourceExprCommand { RawInstallablesCommand(); @@ -129,19 +165,22 @@ struct RawInstallablesCommand : virtual Args, SourceExprCommand void run(ref store) override; - // FIXME make const after CmdRepl's override is fixed up + // FIXME make const after `CmdRepl`'s override is fixed up virtual void applyDefaultInstallables(std::vector & rawInstallables); bool readFromStdIn = false; - std::vector getFlakesForCompletion() override; + std::vector getFlakeRefsForCompletion() override; private: std::vector rawInstallables; }; -/* A command that operates on a list of "installables", which can be - store paths, attribute paths, Nix expressions, etc. */ + +/** + * A command that operates on a list of "installables", which can be + * store paths, attribute paths, Nix expressions, etc. + */ struct InstallablesCommand : RawInstallablesCommand { virtual void run(ref store, Installables && installables) = 0; @@ -149,7 +188,9 @@ struct InstallablesCommand : RawInstallablesCommand void run(ref store, std::vector && rawInstallables) override; }; -/* A command that operates on exactly one "installable" */ +/** + * A command that operates on exactly one "installable". + */ struct InstallableCommand : virtual Args, SourceExprCommand { InstallableCommand(); @@ -158,10 +199,7 @@ struct InstallableCommand : virtual Args, SourceExprCommand void run(ref store) override; - std::vector getFlakesForCompletion() override - { - return {_installable}; - } + std::vector getFlakeRefsForCompletion() override; private: @@ -175,7 +213,12 @@ struct MixOperateOnOptions : virtual Args MixOperateOnOptions(); }; -/* A command that operates on zero or more store paths. */ +/** + * A command that operates on zero or more extant store paths. + * + * If the argument the user passes is a some sort of recipe for a path + * not yet built, it must be built first. + */ struct BuiltPathsCommand : InstallablesCommand, virtual MixOperateOnOptions { private: @@ -207,7 +250,9 @@ struct StorePathsCommand : public BuiltPathsCommand void run(ref store, BuiltPaths && paths) override; }; -/* A command that operates on exactly one store path. */ +/** + * A command that operates on exactly one store path. + */ struct StorePathCommand : public StorePathsCommand { virtual void run(ref store, const StorePath & storePath) = 0; @@ -215,7 +260,9 @@ struct StorePathCommand : public StorePathsCommand void run(ref store, StorePaths && storePaths) override; }; -/* A helper class for registering commands globally. */ +/** + * A helper class for registering \ref Command commands globally. + */ struct RegisterCommand { typedef std::map, std::function()>> Commands; @@ -271,13 +318,24 @@ struct MixEnvironment : virtual Args { MixEnvironment(); - /* Modify global environ based on ignoreEnvironment, keep, and unset. It's expected that exec will be called before this class goes out of scope, otherwise environ will become invalid. */ + /*** + * Modify global environ based on `ignoreEnvironment`, `keep`, and + * `unset`. It's expected that exec will be called before this class + * goes out of scope, otherwise `environ` will become invalid. + */ void setEnviron(); }; -void completeFlakeRef(ref store, std::string_view prefix); +void completeFlakeInputPath( + AddCompletions & completions, + ref evalState, + const std::vector & flakeRefs, + std::string_view prefix); + +void completeFlakeRef(AddCompletions & completions, ref store, std::string_view prefix); void completeFlakeRefWithFragment( + AddCompletions & completions, ref evalState, flake::LockFlags lockFlags, Strings attrPathPrefixes, diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 7f97364a1a66..193972272d5d 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -1,13 +1,14 @@ +#include "eval-settings.hh" #include "common-eval-args.hh" #include "shared.hh" #include "filetransfer.hh" -#include "util.hh" #include "eval.hh" #include "fetchers.hh" #include "registry.hh" #include "flake/flakeref.hh" #include "store-api.hh" #include "command.hh" +#include "tarball.hh" namespace nix { @@ -105,7 +106,9 @@ MixEvalArgs::MixEvalArgs() )", .category = category, .labels = {"path"}, - .handler = {[&](std::string s) { searchPath.push_back(s); }} + .handler = {[&](std::string s) { + searchPath.elements.emplace_back(SearchPath::Elem::parse(s)); + }} }); addFlag({ @@ -129,8 +132,8 @@ MixEvalArgs::MixEvalArgs() if (to.subdir != "") extraAttrs["dir"] = to.subdir; fetchers::overrideRegistry(from.input, to.input, extraAttrs); }}, - .completer = {[&](size_t, std::string_view prefix) { - completeFlakeRef(openStore(), prefix); + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, openStore(), prefix); }} }); @@ -138,7 +141,7 @@ MixEvalArgs::MixEvalArgs() .longName = "eval-store", .description = R"( - The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + The [URL of the Nix store](@docroot@/store/types/index.md#store-url-format) to use for evaluation, i.e. to store derivations (`.drv` files) and inputs referenced by them. )", .category = category, @@ -161,18 +164,18 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) return res.finish(); } -SourcePath lookupFileArg(EvalState & state, std::string_view s) +SourcePath lookupFileArg(EvalState & state, std::string_view s, CanonPath baseDir) { if (EvalSettings::isPseudoUrl(s)) { auto storePath = fetchers::downloadTarball( - state.store, EvalSettings::resolvePseudoUrl(s), "source", false).tree.storePath; + state.store, EvalSettings::resolvePseudoUrl(s), "source", false).storePath; return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } else if (hasPrefix(s, "flake:")) { experimentalFeatureSettings.require(Xp::Flakes); auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false); - auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first.storePath; + auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first; return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } @@ -182,7 +185,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s) } else - return state.rootPath(CanonPath::fromCwd(s)); + return state.rootPath(CanonPath(s, baseDir)); } } diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index b65cb5b20a79..4b403d936740 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -2,7 +2,9 @@ ///@file #include "args.hh" +#include "canon-path.hh" #include "common-args.hh" +#include "search-path.hh" namespace nix { @@ -19,7 +21,7 @@ struct MixEvalArgs : virtual Args, virtual MixRepair Bindings * getAutoArgs(EvalState & state); - Strings searchPath; + SearchPath searchPath; std::optional evalStoreUrl; @@ -27,6 +29,6 @@ private: std::map autoArgs; }; -SourcePath lookupFileArg(EvalState & state, std::string_view s); +SourcePath lookupFileArg(EvalState & state, std::string_view s, CanonPath baseDir = CanonPath::fromCwd()); } diff --git a/src/libcmd/editor-for.cc b/src/libcmd/editor-for.cc index a17c6f12a7fb..619d3673f39d 100644 --- a/src/libcmd/editor-for.cc +++ b/src/libcmd/editor-for.cc @@ -1,5 +1,5 @@ -#include "util.hh" #include "editor-for.hh" +#include "environment-variables.hh" namespace nix { diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc index b35ca2910b2e..06e507872b51 100644 --- a/src/libcmd/installable-attr-path.cc +++ b/src/libcmd/installable-attr-path.cc @@ -80,7 +80,7 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec { return e; }, - }, extendedOutputsSpec.raw()); + }, extendedOutputsSpec.raw); auto [iter, didInsert] = byDrvPath.emplace(*drvPath, newOutputs); @@ -92,10 +92,11 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() for (auto & [drvPath, outputs] : byDrvPath) res.push_back({ .path = DerivedPath::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .outputs = outputs, }, .info = make_ref(ExtraPathInfoValue::Value { + .extendedOutputsSpec = outputs, /* FIXME: reconsider backwards compatibility above so we can fill in this info. */ }), @@ -114,7 +115,7 @@ InstallableAttrPath InstallableAttrPath::parse( return { state, cmd, v, prefix == "." ? "" : std::string { prefix }, - extendedOutputsSpec + std::move(extendedOutputsSpec), }; } diff --git a/src/libcmd/installable-attr-path.hh b/src/libcmd/installable-attr-path.hh index e9f0c33da813..86c2f82192c8 100644 --- a/src/libcmd/installable-attr-path.hh +++ b/src/libcmd/installable-attr-path.hh @@ -4,7 +4,6 @@ #include "globals.hh" #include "installable-value.hh" #include "outputs-spec.hh" -#include "util.hh" #include "command.hh" #include "attr-path.hh" #include "common-eval-args.hh" diff --git a/src/libcmd/installable-derived-path.cc b/src/libcmd/installable-derived-path.cc index 6ecf54b7ce81..4d1f83a1c9e0 100644 --- a/src/libcmd/installable-derived-path.cc +++ b/src/libcmd/installable-derived-path.cc @@ -18,14 +18,7 @@ DerivedPathsWithInfo InstallableDerivedPath::toDerivedPaths() std::optional InstallableDerivedPath::getStorePath() { - return std::visit(overloaded { - [&](const DerivedPath::Built & bfd) { - return bfd.drvPath; - }, - [&](const DerivedPath::Opaque & bo) { - return bo.path; - }, - }, derivedPath.raw()); + return derivedPath.getBaseStorePath(); } InstallableDerivedPath InstallableDerivedPath::parse( @@ -42,7 +35,7 @@ InstallableDerivedPath InstallableDerivedPath::parse( // Remove this prior to stabilizing the new CLI. if (storePath.isDerivation()) { auto oldDerivedPath = DerivedPath::Built { - .drvPath = storePath, + .drvPath = makeConstantStorePathRef(storePath), .outputs = OutputsSpec::All { }, }; warn( @@ -55,12 +48,14 @@ InstallableDerivedPath InstallableDerivedPath::parse( }, // If the user did use ^, we just do exactly what is written. [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> DerivedPath { + auto drv = make_ref(SingleDerivedPath::parse(*store, prefix)); + drvRequireExperiment(*drv); return DerivedPath::Built { - .drvPath = store->parseStorePath(prefix), + .drvPath = std::move(drv), .outputs = outputSpec, }; }, - }, extendedOutputsSpec.raw()); + }, extendedOutputsSpec.raw); return InstallableDerivedPath { store, std::move(derivedPath), diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index eb944240be9b..2f428cb7e4ba 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -28,6 +28,11 @@ namespace nix { std::vector InstallableFlake::getActualAttrPaths() { std::vector res; + if (attrPaths.size() == 1 && attrPaths.front().starts_with(".")){ + attrPaths.front().erase(0,1); + res.push_back(attrPaths.front()); + return res; + } for (auto & prefix : prefixes) res.push_back(prefix + *attrPaths.begin()); @@ -118,7 +123,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() return {{ .path = DerivedPath::Built { - .drvPath = std::move(drvPath), + .drvPath = makeConstantStorePathRef(std::move(drvPath)), .outputs = std::visit(overloaded { [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec { std::set outputsToInstall; @@ -141,7 +146,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec { return e; }, - }, extendedOutputsSpec.raw()), + }, extendedOutputsSpec.raw), }, .info = make_ref( ExtraPathInfoValue::Value { @@ -151,7 +156,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() }, ExtraPathInfoFlake::Flake { .originalRef = flakeRef, - .resolvedRef = getLockedFlake()->flake.lockedRef, + .lockedRef = getLockedFlake()->flake.lockedRef, }), }}; } diff --git a/src/libcmd/installable-flake.hh b/src/libcmd/installable-flake.hh index 7ac4358d2a32..314918c140d2 100644 --- a/src/libcmd/installable-flake.hh +++ b/src/libcmd/installable-flake.hh @@ -19,7 +19,7 @@ struct ExtraPathInfoFlake : ExtraPathInfoValue */ struct Flake { FlakeRef originalRef; - FlakeRef resolvedRef; + FlakeRef lockedRef; }; Flake flake; diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc index 1eff293cc1a6..08ad35105bcf 100644 --- a/src/libcmd/installable-value.cc +++ b/src/libcmd/installable-value.cc @@ -55,7 +55,8 @@ std::optional InstallableValue::trySinglePathToDerivedPaths else if (v.type() == nString) { return {{ - .path = state->coerceToDerivedPath(pos, v, errorCtx), + .path = DerivedPath::fromSingle( + state->coerceToSingleDerivedPath(pos, v, errorCtx)), .info = make_ref(), }}; } diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index d9a1bef6365c..940b02bca3e3 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -5,6 +5,7 @@ #include "installable-attr-path.hh" #include "installable-flake.hh" #include "outputs-spec.hh" +#include "users.hh" #include "util.hh" #include "command.hh" #include "attr-path.hh" @@ -12,6 +13,7 @@ #include "derivations.hh" #include "eval-inline.hh" #include "eval.hh" +#include "eval-settings.hh" #include "get-drvs.hh" #include "store-api.hh" #include "shared.hh" @@ -30,15 +32,38 @@ namespace nix { +void completeFlakeInputPath( + AddCompletions & completions, + ref evalState, + const std::vector & flakeRefs, + std::string_view prefix) +{ + for (auto & flakeRef : flakeRefs) { + auto flake = flake::getFlake(*evalState, flakeRef, true); + for (auto & input : flake.inputs) + if (hasPrefix(input.first, prefix)) + completions.add(input.first); + } +} + MixFlakeOptions::MixFlakeOptions() { auto category = "Common flake-related options"; addFlag({ .longName = "recreate-lock-file", - .description = "Recreate the flake's lock file from scratch.", + .description = R"( + Recreate the flake's lock file from scratch. + + > **DEPRECATED** + > + > Use [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) instead. + )", .category = category, - .handler = {&lockFlags.recreateLockFile, true} + .handler = {[&]() { + lockFlags.recreateLockFile = true; + warn("'--recreate-lock-file' is deprecated and will be removed in a future version; use 'nix flake update' instead."); + }} }); addFlag({ @@ -57,8 +82,13 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "no-registries", - .description = - "Don't allow lookups in the flake registries. This option is deprecated; use `--no-use-registries`.", + .description = R"( + Don't allow lookups in the flake registries. + + > **DEPRECATED** + > + > Use [`--no-use-registries`](#opt-no-use-registries) instead. + )", .category = category, .handler = {[&]() { lockFlags.useRegistries = false; @@ -75,14 +105,21 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "update-input", - .description = "Update a specific flake input (ignoring its previous entry in the lock file).", + .description = R"( + Update a specific flake input (ignoring its previous entry in the lock file). + + > **DEPRECATED** + > + > Use [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) instead. + )", .category = category, .labels = {"input-path"}, .handler = {[&](std::string s) { + warn("'--update-input' is a deprecated alias for 'flake update' and will be removed in a future version."); lockFlags.inputUpdates.insert(flake::parseInputPath(s)); }}, - .completer = {[&](size_t, std::string_view prefix) { - needsFlakeInputCompletion = {std::string(prefix)}; + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix); }} }); @@ -95,13 +132,14 @@ MixFlakeOptions::MixFlakeOptions() lockFlags.writeLockFile = false; lockFlags.inputOverrides.insert_or_assign( flake::parseInputPath(inputPath), - parseFlakeRef(flakeRef, absPath("."), true)); + parseFlakeRef(flakeRef, absPath(getCommandBaseDir()), true)); }}, - .completer = {[&](size_t n, std::string_view prefix) { - if (n == 0) - needsFlakeInputCompletion = {std::string(prefix)}; - else if (n == 1) - completeFlakeRef(getEvalState()->store, prefix); + .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) { + if (n == 0) { + completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix); + } else if (n == 1) { + completeFlakeRef(completions, getEvalState()->store, prefix); + } }} }); @@ -136,7 +174,7 @@ MixFlakeOptions::MixFlakeOptions() auto evalState = getEvalState(); auto flake = flake::lockFlake( *evalState, - parseFlakeRef(flakeRef, absPath(".")), + parseFlakeRef(flakeRef, absPath(getCommandBaseDir())), { .writeLockFile = false }); for (auto & [inputName, input] : flake.lockFile.root->inputs) { auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes @@ -148,30 +186,12 @@ MixFlakeOptions::MixFlakeOptions() } } }}, - .completer = {[&](size_t, std::string_view prefix) { - completeFlakeRef(getEvalState()->store, prefix); + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getEvalState()->store, prefix); }} }); } -void MixFlakeOptions::completeFlakeInput(std::string_view prefix) -{ - auto evalState = getEvalState(); - for (auto & flakeRefS : getFlakesForCompletion()) { - auto flakeRef = parseFlakeRefWithFragment(expandTilde(flakeRefS), absPath(".")).first; - auto flake = flake::getFlake(*evalState, flakeRef, true); - for (auto & input : flake.inputs) - if (hasPrefix(input.first, prefix)) - completions->add(input.first); - } -} - -void MixFlakeOptions::completionHook() -{ - if (auto & prefix = needsFlakeInputCompletion) - completeFlakeInput(*prefix); -} - SourceExprCommand::SourceExprCommand() { addFlag({ @@ -228,11 +248,18 @@ Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes() }; } -void SourceExprCommand::completeInstallable(std::string_view prefix) +Args::CompleterClosure SourceExprCommand::getCompleteInstallable() +{ + return [this](AddCompletions & completions, size_t, std::string_view prefix) { + completeInstallable(completions, prefix); + }; +} + +void SourceExprCommand::completeInstallable(AddCompletions & completions, std::string_view prefix) { try { if (file) { - completionType = ctAttrs; + completions.setType(AddCompletions::Type::Attrs); evalSettings.pureEval = false; auto state = getEvalState(); @@ -267,14 +294,15 @@ void SourceExprCommand::completeInstallable(std::string_view prefix) std::string name = state->symbols[i.name]; if (name.find(searchWord) == 0) { if (prefix_ == "") - completions->add(name); + completions.add(name); else - completions->add(prefix_ + "." + name); + completions.add(prefix_ + "." + name); } } } } else { completeFlakeRefWithFragment( + completions, getEvalState(), lockFlags, getDefaultFlakeAttrPathPrefixes(), @@ -287,6 +315,7 @@ void SourceExprCommand::completeInstallable(std::string_view prefix) } void completeFlakeRefWithFragment( + AddCompletions & completions, ref evalState, flake::LockFlags lockFlags, Strings attrPathPrefixes, @@ -298,12 +327,19 @@ void completeFlakeRefWithFragment( try { auto hash = prefix.find('#'); if (hash == std::string::npos) { - completeFlakeRef(evalState->store, prefix); + completeFlakeRef(completions, evalState->store, prefix); } else { - completionType = ctAttrs; + completions.setType(AddCompletions::Type::Attrs); auto fragment = prefix.substr(hash + 1); + std::string prefixRoot = ""; + if (fragment.starts_with(".")){ + fragment = fragment.substr(1); + prefixRoot = "."; + } auto flakeRefS = std::string(prefix.substr(0, hash)); + + // TODO: ideally this would use the command base directory instead of assuming ".". auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(".")); auto evalCache = openEvalCache(*evalState, @@ -311,6 +347,9 @@ void completeFlakeRefWithFragment( auto root = evalCache->getRoot(); + if (prefixRoot == "."){ + attrPathPrefixes.clear(); + } /* Complete 'fragment' relative to all the attrpath prefixes as well as the root of the flake. */ @@ -335,7 +374,7 @@ void completeFlakeRefWithFragment( auto attrPath2 = (*attr)->getAttrPath(attr2); /* Strip the attrpath prefix. */ attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size()); - completions->add(flakeRefS + "#" + concatStringsSep(".", evalState->symbols.resolve(attrPath2))); + completions.add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2))); } } } @@ -346,7 +385,7 @@ void completeFlakeRefWithFragment( for (auto & attrPath : defaultFlakeAttrPaths) { auto attr = root->findAlongAttrPath(parseAttrPath(*evalState, attrPath)); if (!attr) continue; - completions->add(flakeRefS + "#"); + completions.add(flakeRefS + "#" + prefixRoot); } } } @@ -355,15 +394,15 @@ void completeFlakeRefWithFragment( } } -void completeFlakeRef(ref store, std::string_view prefix) +void completeFlakeRef(AddCompletions & completions, ref store, std::string_view prefix) { if (!experimentalFeatureSettings.isEnabled(Xp::Flakes)) return; if (prefix == "") - completions->add("."); + completions.add("."); - completeDir(0, prefix); + Args::completeDir(completions, 0, prefix); /* Look for registry entries that match the prefix. */ for (auto & registry : fetchers::getRegistries(store)) { @@ -372,10 +411,10 @@ void completeFlakeRef(ref store, std::string_view prefix) if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) { std::string from2(from, 6); if (hasPrefix(from2, prefix)) - completions->add(from2); + completions.add(from2); } else { if (hasPrefix(from, prefix)) - completions->add(from); + completions.add(from); } } } @@ -449,10 +488,12 @@ Installables SourceExprCommand::parseInstallables( auto e = state->parseStdin(); state->eval(e, *vFile); } - else if (file) - state->evalFile(lookupFileArg(*state, *file), *vFile); + else if (file) { + state->evalFile(lookupFileArg(*state, *file, CanonPath::fromCwd(getCommandBaseDir())), *vFile); + } else { - auto e = state->parseExprFromString(*expr, state->rootPath(CanonPath::fromCwd())); + CanonPath dir(CanonPath::fromCwd(getCommandBaseDir())); + auto e = state->parseExprFromString(*expr, state->rootPath(dir)); state->eval(e, *vFile); } @@ -461,7 +502,7 @@ Installables SourceExprCommand::parseInstallables( result.push_back( make_ref( InstallableAttrPath::parse( - state, *this, vFile, prefix, extendedOutputsSpec))); + state, *this, vFile, std::move(prefix), std::move(extendedOutputsSpec)))); } } else { @@ -477,7 +518,7 @@ Installables SourceExprCommand::parseInstallables( if (prefix.find('/') != std::string::npos) { try { result.push_back(make_ref( - InstallableDerivedPath::parse(store, prefix, extendedOutputsSpec))); + InstallableDerivedPath::parse(store, prefix, extendedOutputsSpec.raw))); continue; } catch (BadStorePath &) { } catch (...) { @@ -487,13 +528,13 @@ Installables SourceExprCommand::parseInstallables( } try { - auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(".")); + auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(getCommandBaseDir())); result.push_back(make_ref( this, getEvalState(), std::move(flakeRef), fragment, - extendedOutputsSpec, + std::move(extendedOutputsSpec), getDefaultFlakeAttrPaths(), getDefaultFlakeAttrPathPrefixes(), lockFlags)); @@ -517,6 +558,30 @@ ref SourceExprCommand::parseInstallable( return installables.front(); } +static SingleBuiltPath getBuiltPath(ref evalStore, ref store, const SingleDerivedPath & b) +{ + return std::visit( + overloaded{ + [&](const SingleDerivedPath::Opaque & bo) -> SingleBuiltPath { + return SingleBuiltPath::Opaque { bo.path }; + }, + [&](const SingleDerivedPath::Built & bfd) -> SingleBuiltPath { + auto drvPath = getBuiltPath(evalStore, store, *bfd.drvPath); + // Resolving this instead of `bfd` will yield the same result, but avoid duplicative work. + SingleDerivedPath::Built truncatedBfd { + .drvPath = makeConstantStorePathRef(drvPath.outPath()), + .output = bfd.output, + }; + auto outputPath = resolveDerivedPath(*store, truncatedBfd, &*evalStore); + return SingleBuiltPath::Built { + .drvPath = make_ref(std::move(drvPath)), + .output = { bfd.output, outputPath }, + }; + }, + }, + b.raw()); +} + std::vector Installable::build( ref evalStore, ref store, @@ -583,7 +648,10 @@ std::vector, BuiltPathWithResult>> Installable::build [&](const DerivedPath::Built & bfd) { auto outputs = resolveDerivedPath(*store, bfd, &*evalStore); res.push_back({aux.installable, { - .path = BuiltPath::Built { bfd.drvPath, outputs }, + .path = BuiltPath::Built { + .drvPath = make_ref(getBuiltPath(evalStore, store, *bfd.drvPath)), + .outputs = outputs, + }, .info = aux.info}}); }, [&](const DerivedPath::Opaque & bo) { @@ -612,7 +680,10 @@ std::vector, BuiltPathWithResult>> Installable::build for (auto & [outputName, realisation] : buildResult.builtOutputs) outputs.emplace(outputName, realisation.outPath); res.push_back({aux.installable, { - .path = BuiltPath::Built { bfd.drvPath, outputs }, + .path = BuiltPath::Built { + .drvPath = make_ref(getBuiltPath(evalStore, store, *bfd.drvPath)), + .outputs = outputs, + }, .info = aux.info, .result = buildResult}}); }, @@ -654,7 +725,7 @@ BuiltPaths Installable::toBuiltPaths( BuiltPaths res; for (auto & drvPath : Installable::toDerivations(store, installables, true)) - res.push_back(BuiltPath::Opaque{drvPath}); + res.emplace_back(BuiltPath::Opaque{drvPath}); return res; } } @@ -706,7 +777,7 @@ StorePathSet Installable::toDerivations( : throw Error("argument '%s' did not evaluate to a derivation", i->what())); }, [&](const DerivedPath::Built & bfd) { - drvPaths.insert(bfd.drvPath); + drvPaths.insert(resolveDerivedPath(*store, *bfd.drvPath)); }, }, b.path.raw()); @@ -724,9 +795,7 @@ RawInstallablesCommand::RawInstallablesCommand() expectArgs({ .label = "installables", .handler = {&rawInstallables}, - .completer = {[&](size_t, std::string_view prefix) { - completeInstallable(prefix); - }} + .completer = getCompleteInstallable(), }); } @@ -739,6 +808,17 @@ void RawInstallablesCommand::applyDefaultInstallables(std::vector & } } +std::vector RawInstallablesCommand::getFlakeRefsForCompletion() +{ + applyDefaultInstallables(rawInstallables); + std::vector res; + for (auto i : rawInstallables) + res.push_back(parseFlakeRefWithFragment( + expandTilde(i), + absPath(getCommandBaseDir())).first); + return res; +} + void RawInstallablesCommand::run(ref store) { if (readFromStdIn && !isatty(STDIN_FILENO)) { @@ -752,10 +832,13 @@ void RawInstallablesCommand::run(ref store) run(store, std::move(rawInstallables)); } -std::vector RawInstallablesCommand::getFlakesForCompletion() +std::vector InstallableCommand::getFlakeRefsForCompletion() { - applyDefaultInstallables(rawInstallables); - return rawInstallables; + return { + parseFlakeRefWithFragment( + expandTilde(_installable), + absPath(getCommandBaseDir())).first + }; } void InstallablesCommand::run(ref store, std::vector && rawInstallables) @@ -771,9 +854,7 @@ InstallableCommand::InstallableCommand() .label = "installable", .optional = true, .handler = {&_installable}, - .completer = {[&](size_t, std::string_view prefix) { - completeInstallable(prefix); - }} + .completer = getCompleteInstallable(), }); } diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index 87a623b79893..db411145cd9e 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -1,10 +1,10 @@ #pragma once ///@file -#include "util.hh" #include "path.hh" #include "outputs-spec.hh" #include "derived-path.hh" +#include "built-path.hh" #include "store-api.hh" #include "build-result.hh" diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk index 541a7d2bac71..afd35af08422 100644 --- a/src/libcmd/local.mk +++ b/src/libcmd/local.mk @@ -12,4 +12,4 @@ libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) -pthread libcmd_LIBS = libstore libutil libexpr libmain libfetchers -$(eval $(call install-file-in, $(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644)) +$(eval $(call install-file-in, $(buildprefix)$(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644)) diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index 668a077632c9..8b3bbc1b5e86 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -1,6 +1,7 @@ #include "markdown.hh" #include "util.hh" #include "finally.hh" +#include "terminal.hh" #include #include diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 4b160a100ce5..0986296ad451 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -22,10 +22,12 @@ extern "C" { #include "repl.hh" #include "ansicolor.hh" +#include "signals.hh" #include "shared.hh" #include "eval.hh" #include "eval-cache.hh" #include "eval-inline.hh" +#include "eval-settings.hh" #include "attr-path.hh" #include "store-api.hh" #include "log-store.hh" @@ -35,11 +37,12 @@ extern "C" { #include "globals.hh" #include "flake/flake.hh" #include "flake/lockfile.hh" +#include "users.hh" +#include "terminal.hh" #include "editor-for.hh" #include "finally.hh" #include "markdown.hh" #include "local-fs-store.hh" -#include "progress-bar.hh" #include "print.hh" #if HAVE_BOEHMGC @@ -68,7 +71,7 @@ struct NixRepl const Path historyFile; - NixRepl(const Strings & searchPath, nix::ref store,ref state, + NixRepl(const SearchPath & searchPath, nix::ref store,ref state, std::function getValues); virtual ~NixRepl(); @@ -104,7 +107,7 @@ std::string removeWhitespace(std::string s) } -NixRepl::NixRepl(const Strings & searchPath, nix::ref store, ref state, +NixRepl::NixRepl(const SearchPath & searchPath, nix::ref store, ref state, std::function getValues) : AbstractNixRepl(state) , debugTraceIndex(0) @@ -258,13 +261,11 @@ void NixRepl::mainLoop() rl_set_list_possib_func(listPossibleCallback); #endif - /* Stop the progress bar because it interferes with the display of - the repl. */ - stopProgressBar(); - std::string input; while (true) { + // Hide the progress bar while waiting for user input, so that it won't interfere. + logger->pause(); // When continuing input from previous lines, don't print a prompt, just align to the same // number of chars as the prompt. if (!getLine(input, input.empty() ? "nix-repl> " : " ")) { @@ -274,6 +275,7 @@ void NixRepl::mainLoop() logger->cout(""); break; } + logger->resume(); try { if (!removeWhitespace(input).empty() && !processLine(input)) return; } catch (ParseError & e) { @@ -487,35 +489,40 @@ bool NixRepl::processLine(std::string line) std::cout << "The following commands are available:\n" << "\n" - << " Evaluate and print expression\n" - << " = Bind expression to variable\n" - << " :a Add attributes from resulting set to scope\n" - << " :b Build a derivation\n" - << " :bl Build a derivation, creating GC roots in the working directory\n" - << " :e Open package or function in $EDITOR\n" - << " :i Build derivation, then install result into current profile\n" - << " :l Load Nix expression and add it to scope\n" - << " :lf Load Nix flake and add it to scope\n" - << " :p Evaluate and print expression recursively\n" - << " :q Exit nix-repl\n" - << " :r Reload all files\n" - << " :sh Build dependencies of derivation, then start nix-shell\n" - << " :t Describe result of evaluation\n" - << " :u Build derivation, then start nix-shell\n" - << " :doc Show documentation of a builtin function\n" - << " :log Show logs for a derivation\n" - << " :te [bool] Enable, disable or toggle showing traces for errors\n" + << " Evaluate and print expression\n" + << " = Bind expression to variable\n" + << " :a, :add Add attributes from resulting set to scope\n" + << " :b Build a derivation\n" + << " :bl Build a derivation, creating GC roots in the\n" + << " working directory\n" + << " :e, :edit Open package or function in $EDITOR\n" + << " :i Build derivation, then install result into\n" + << " current profile\n" + << " :l, :load Load Nix expression and add it to scope\n" + << " :lf, :load-flake Load Nix flake and add it to scope\n" + << " :p, :print Evaluate and print expression recursively\n" + << " :q, :quit Exit nix-repl\n" + << " :r, :reload Reload all files\n" + << " :sh Build dependencies of derivation, then start\n" + << " nix-shell\n" + << " :t Describe result of evaluation\n" + << " :u Build derivation, then start nix-shell\n" + << " :doc Show documentation of a builtin function\n" + << " :log Show logs for a derivation\n" + << " :te, :trace-enable [bool] Enable, disable or toggle showing traces for\n" + << " errors\n" + << " :?, :help Brings up this help menu\n" ; if (state->debugRepl) { std::cout << "\n" << " Debug mode commands\n" - << " :env Show env stack\n" - << " :bt Show trace stack\n" - << " :st Show current trace\n" - << " :st Change to another trace in the stack\n" - << " :c Go until end of program, exception, or builtins.break\n" - << " :s Go one step\n" + << " :env Show env stack\n" + << " :bt, :backtrace Show trace stack\n" + << " :st Show current trace\n" + << " :st Change to another trace in the stack\n" + << " :c, :continue Go until end of program, exception, or builtins.break\n" + << " :s, :step Go one step\n" ; } @@ -647,7 +654,7 @@ bool NixRepl::processLine(std::string line) if (command == ":b" || command == ":bl") { state->store->buildPaths({ DerivedPath::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All { }, }, }); @@ -916,7 +923,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m case nString: str << ANSI_WARNING; - printLiteralString(str, v.string.s); + printLiteralString(str, v.string_view()); str << ANSI_NORMAL; break; @@ -1024,7 +1031,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m std::unique_ptr AbstractNixRepl::create( - const Strings & searchPath, nix::ref store, ref state, + const SearchPath & searchPath, nix::ref store, ref state, std::function getValues) { return std::make_unique( @@ -1044,7 +1051,7 @@ void AbstractNixRepl::runSimple( NixRepl::AnnotatedValues values; return values; }; - const Strings & searchPath = {}; + SearchPath searchPath = {}; auto repl = std::make_unique( searchPath, openStore(), diff --git a/src/libcmd/repl.hh b/src/libcmd/repl.hh index 731c8e6db605..6d88883fefaf 100644 --- a/src/libcmd/repl.hh +++ b/src/libcmd/repl.hh @@ -25,7 +25,7 @@ struct AbstractNixRepl typedef std::vector> AnnotatedValues; static std::unique_ptr create( - const Strings & searchPath, nix::ref store, ref state, + const SearchPath & searchPath, nix::ref store, ref state, std::function getValues); static void runSimple( diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index ab654c1b0f2f..7481a2232da3 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -1,6 +1,5 @@ #include "attr-path.hh" #include "eval-inline.hh" -#include "util.hh" namespace nix { @@ -132,7 +131,7 @@ std::pair findPackageFilename(EvalState & state, Value & v if (colon == std::string::npos) fail(); std::string filename(fn, 0, colon); auto lineno = std::stoi(std::string(fn, colon + 1, std::string::npos)); - return {CanonPath(fn.substr(0, colon)), lineno}; + return {SourcePath{path.accessor, CanonPath(fn.substr(0, colon))}, lineno}; } catch (std::invalid_argument & e) { fail(); abort(); diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 9e734e654b2b..6c0e33709f72 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -1,3 +1,4 @@ +#include "users.hh" #include "eval-cache.hh" #include "sqlite.hh" #include "eval.hh" @@ -50,7 +51,7 @@ struct AttrDb Path cacheDir = getCacheDir() + "/nix/eval-cache-v5"; createDirs(cacheDir); - Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite"; + Path dbPath = cacheDir + "/" + fingerprint.to_string(HashFormat::Base16, false) + ".sqlite"; state->db = SQLite(dbPath); state->db.isCache(); @@ -440,8 +441,8 @@ Value & AttrCursor::forceValue() if (root->db && (!cachedValue || std::get_if(&cachedValue->second))) { if (v.type() == nString) - cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), - string_t{v.string.s, {}}}; + cachedValue = {root->db->setString(getKey(), v.c_str(), v.context()), + string_t{v.c_str(), {}}}; else if (v.type() == nPath) { auto path = v.path().path; cachedValue = {root->db->setString(getKey(), path.abs()), string_t{path.abs(), {}}}; @@ -582,7 +583,7 @@ std::string AttrCursor::getString() if (v.type() != nString && v.type() != nPath) root->state.error("'%s' is not a string but %s", getAttrPathStr()).debugThrow(); - return v.type() == nString ? v.string.s : v.path().to_string(); + return v.type() == nString ? v.c_str() : v.path().to_string(); } string_t AttrCursor::getStringWithContext() @@ -599,12 +600,12 @@ string_t AttrCursor::getStringWithContext() return d.drvPath; }, [&](const NixStringContextElem::Built & b) -> const StorePath & { - return b.drvPath; + return b.drvPath->getBaseStorePath(); }, [&](const NixStringContextElem::Opaque & o) -> const StorePath & { return o.path; }, - }, c.raw()); + }, c.raw); if (!root->state.store->isValidPath(path)) { valid = false; break; @@ -624,7 +625,7 @@ string_t AttrCursor::getStringWithContext() if (v.type() == nString) { NixStringContext context; copyContext(v, context); - return {v.string.s, std::move(context)}; + return {v.c_str(), std::move(context)}; } else if (v.type() == nPath) return {v.path().to_string(), {}}; diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc new file mode 100644 index 000000000000..444a7d7d6dc2 --- /dev/null +++ b/src/libexpr/eval-settings.cc @@ -0,0 +1,103 @@ +#include "users.hh" +#include "globals.hh" +#include "profiles.hh" +#include "eval.hh" +#include "eval-settings.hh" + +namespace nix { + +/* Very hacky way to parse $NIX_PATH, which is colon-separated, but + can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */ +static Strings parseNixPath(const std::string & s) +{ + Strings res; + + auto p = s.begin(); + + while (p != s.end()) { + auto start = p; + auto start2 = p; + + while (p != s.end() && *p != ':') { + if (*p == '=') start2 = p + 1; + ++p; + } + + if (p == s.end()) { + if (p != start) res.push_back(std::string(start, p)); + break; + } + + if (*p == ':') { + auto prefix = std::string(start2, s.end()); + if (EvalSettings::isPseudoUrl(prefix) || hasPrefix(prefix, "flake:")) { + ++p; + while (p != s.end() && *p != ':') ++p; + } + res.push_back(std::string(start, p)); + if (p == s.end()) break; + } + + ++p; + } + + return res; +} + +EvalSettings::EvalSettings() +{ + auto var = getEnv("NIX_PATH"); + if (var) nixPath = parseNixPath(*var); +} + +Strings EvalSettings::getDefaultNixPath() +{ + Strings res; + auto add = [&](const Path & p, const std::string & s = std::string()) { + if (pathAccessible(p)) { + if (s.empty()) { + res.push_back(p); + } else { + res.push_back(s + "=" + p); + } + } + }; + + if (!evalSettings.restrictEval && !evalSettings.pureEval) { + add(getNixDefExpr() + "/channels"); + add(rootChannelsDir() + "/nixpkgs", "nixpkgs"); + add(rootChannelsDir()); + } + + return res; +} + +bool EvalSettings::isPseudoUrl(std::string_view s) +{ + if (s.compare(0, 8, "channel:") == 0) return true; + size_t pos = s.find("://"); + if (pos == std::string::npos) return false; + std::string scheme(s, 0, pos); + return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh"; +} + +std::string EvalSettings::resolvePseudoUrl(std::string_view url) +{ + if (hasPrefix(url, "channel:")) + return "https://nixos.org/channels/" + std::string(url.substr(8)) + "/nixexprs.tar.xz"; + else + return std::string(url); +} + +EvalSettings evalSettings; + +static GlobalConfig::Register rEvalSettings(&evalSettings); + +Path getNixDefExpr() +{ + return settings.useXDGBaseDirectories + ? getStateDir() + "/nix/defexpr" + : getHome() + "/.nix-defexpr"; +} + +} diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh new file mode 100644 index 000000000000..db2971acb77d --- /dev/null +++ b/src/libexpr/eval-settings.hh @@ -0,0 +1,111 @@ +#pragma once +///@file + +#include "config.hh" + +namespace nix { + +struct EvalSettings : Config +{ + EvalSettings(); + + static Strings getDefaultNixPath(); + + static bool isPseudoUrl(std::string_view s); + + static std::string resolvePseudoUrl(std::string_view url); + + Setting enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", + "Whether builtin functions that allow executing native code should be enabled."}; + + Setting nixPath{ + this, getDefaultNixPath(), "nix-path", + R"( + List of directories to be searched for `<...>` file references + + In particular, outside of [pure evaluation mode](#conf-pure-eval), this determines the value of + [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath). + )"}; + + Setting restrictEval{ + this, false, "restrict-eval", + R"( + If set to `true`, the Nix evaluator will not allow access to any + files outside of + [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath), + or to URIs outside of + [`allowed-uris`](@docroot@/command-ref/conf-file.md#conf-allowed-uris). + + Also the default value for [`nix-path`](#conf-nix-path) is ignored, such that only explicitly set search path entries are taken into account. + )"}; + + Setting pureEval{this, false, "pure-eval", + R"( + Pure evaluation mode ensures that the result of Nix expressions is fully determined by explicitly declared inputs, and not influenced by external state: + + - Restrict file system and network access to files specified by cryptographic hash + - Disable impure constants: + - [`bultins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) + - [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime) + - [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath) + )" + }; + + Setting enableImportFromDerivation{ + this, true, "allow-import-from-derivation", + R"( + By default, Nix allows [Import from Derivation](@docroot@/language/import-from-derivation.md). + + With this option set to `false`, Nix will throw an error when evaluating an expression that uses this feature, + even when the required store object is readily available. + This ensures that evaluation will not require any builds to take place, + regardless of the state of the store. + )"}; + + Setting allowedUris{this, {}, "allowed-uris", + R"( + A list of URI prefixes to which access is allowed in restricted + evaluation mode. For example, when set to + `https://github.com/NixOS`, builtin functions such as `fetchGit` are + allowed to access `https://github.com/NixOS/patchelf.git`. + )"}; + + Setting traceFunctionCalls{this, false, "trace-function-calls", + R"( + If set to `true`, the Nix evaluator will trace every function call. + Nix will print a log message at the "vomit" level for every function + entrance and function exit. + + function-trace entered undefined position at 1565795816999559622 + function-trace exited undefined position at 1565795816999581277 + function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150 + function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684 + + The `undefined position` means the function call is a builtin. + + Use the `contrib/stack-collapse.py` script distributed with the Nix + source code to convert the trace logs in to a format suitable for + `flamegraph.pl`. + )"}; + + Setting useEvalCache{this, true, "eval-cache", + "Whether to use the flake evaluation cache."}; + + Setting ignoreExceptionsDuringTry{this, false, "ignore-try", + R"( + If set to true, ignore exceptions inside 'tryEval' calls when evaluating nix expressions in + debug mode (using the --debugger flag). By default the debugger will pause on all exceptions. + )"}; + + Setting traceVerbose{this, false, "trace-verbose", + "Whether `builtins.traceVerbose` should trace its first argument when evaluated."}; +}; + +extern EvalSettings evalSettings; + +/** + * Conventionally part of the default nix path in impure mode. + */ +Path getNixDefExpr(); + +} diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 69f9481c4470..10cd17da1d24 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1,5 +1,7 @@ #include "eval.hh" +#include "eval-settings.hh" #include "hash.hh" +#include "primops.hh" #include "types.hh" #include "util.hh" #include "store-api.hh" @@ -11,6 +13,10 @@ #include "function-trace.hh" #include "profiles.hh" #include "print.hh" +#include "fs-input-accessor.hh" +#include "memory-input-accessor.hh" +#include "signals.hh" +#include "gc-small-vector.hh" #include #include @@ -26,6 +32,7 @@ #include #include +#include #if HAVE_BOEHMGC @@ -113,7 +120,7 @@ void Value::print(const SymbolTable &symbols, std::ostream &str, printLiteralBool(str, boolean); break; case tString: - printLiteralString(str, string.s); + printLiteralString(str, string_view()); break; case tPath: str << path().to_string(); // !!! escaping? @@ -338,7 +345,7 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env) Value nameValue; name.expr->eval(state, env, nameValue); state.forceStringNoCtx(nameValue, noPos, "while evaluating an attribute name"); - return state.symbols.create(nameValue.string.s); + return state.symbols.create(nameValue.string_view()); } } @@ -420,44 +427,6 @@ void initGC() } -/* Very hacky way to parse $NIX_PATH, which is colon-separated, but - can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */ -static Strings parseNixPath(const std::string & s) -{ - Strings res; - - auto p = s.begin(); - - while (p != s.end()) { - auto start = p; - auto start2 = p; - - while (p != s.end() && *p != ':') { - if (*p == '=') start2 = p + 1; - ++p; - } - - if (p == s.end()) { - if (p != start) res.push_back(std::string(start, p)); - break; - } - - if (*p == ':') { - auto prefix = std::string(start2, s.end()); - if (EvalSettings::isPseudoUrl(prefix) || hasPrefix(prefix, "flake:")) { - ++p; - while (p != s.end() && *p != ':') ++p; - } - res.push_back(std::string(start, p)); - if (p == s.end()) break; - } - - ++p; - } - - return res; -} - ErrorBuilder & ErrorBuilder::atPos(PosIdx pos) { info.errPos = state.positions[pos]; @@ -498,7 +467,7 @@ ErrorBuilder & ErrorBuilder::withFrame(const Env & env, const Expr & expr) EvalState::EvalState( - const Strings & _searchPath, + const SearchPath & _searchPath, ref store, std::shared_ptr buildStore) : sWith(symbols.create("")) @@ -546,7 +515,17 @@ EvalState::EvalState( , sOutputSpecified(symbols.create("outputSpecified")) , repair(NoRepair) , emptyBindings(0) - , derivationInternal(rootPath(CanonPath("/builtin/derivation.nix"))) + , rootFS(makeFSInputAccessor(CanonPath::root)) + , corepkgsFS(makeMemoryInputAccessor()) + , internalFS(makeMemoryInputAccessor()) + , derivationInternal{corepkgsFS->addFile( + CanonPath("derivation-internal.nix"), + #include "primops/derivation.nix.gen.hh" + )} + , callFlakeInternal{internalFS->addFile( + CanonPath("call-flake.nix"), + #include "flake/call-flake.nix.gen.hh" + )} , store(store) , buildStore(buildStore ? buildStore : store) , debugRepl(nullptr) @@ -561,6 +540,9 @@ EvalState::EvalState( , baseEnv(allocEnv(128)) , staticBaseEnv{std::make_shared(false, nullptr)} { + corepkgsFS->setPathDisplay(""); + internalFS->setPathDisplay("«nix-internal»", ""); + countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0"; assert(gcInitialised); @@ -569,33 +551,40 @@ EvalState::EvalState( /* Initialise the Nix expression search path. */ if (!evalSettings.pureEval) { - for (auto & i : _searchPath) addToSearchPath(i); - for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i); + for (auto & i : _searchPath.elements) + searchPath.elements.emplace_back(SearchPath::Elem {i}); + for (auto & i : evalSettings.nixPath.get()) + searchPath.elements.emplace_back(SearchPath::Elem::parse(i)); } if (evalSettings.restrictEval || evalSettings.pureEval) { allowedPaths = PathSet(); - for (auto & i : searchPath) { - auto r = resolveSearchPathElem(i); - if (!r.first) continue; + for (auto & i : searchPath.elements) { + auto r = resolveSearchPathPath(i.path); + if (!r) continue; - auto path = r.second; + auto path = std::move(*r); - if (store->isInStore(r.second)) { + if (store->isInStore(path)) { try { StorePathSet closure; - store->computeFSClosure(store->toStorePath(r.second).first, closure); + store->computeFSClosure(store->toStorePath(path).first, closure); for (auto & path : closure) allowPath(path); } catch (InvalidPath &) { - allowPath(r.second); + allowPath(path); } } else - allowPath(r.second); + allowPath(path); } } + corepkgsFS->addFile( + CanonPath("fetchurl.nix"), + #include "fetchurl.nix.gen.hh" + ); + createBaseEnv(); } @@ -626,6 +615,9 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & SourcePath EvalState::checkSourcePath(const SourcePath & path_) { + // Don't check non-rootFS accessors, they're in a different namespace. + if (path_.accessor != ref(rootFS)) return path_; + if (!allowedPaths) return path_; auto i = resolvedPaths.find(path_.path.abs()); @@ -640,8 +632,6 @@ SourcePath EvalState::checkSourcePath(const SourcePath & path_) */ Path abspath = canonPath(path_.path.abs()); - if (hasPrefix(abspath, corepkgsPrefix)) return CanonPath(abspath); - for (auto & i : *allowedPaths) { if (isDirOrInDir(abspath, i)) { found = true; @@ -658,7 +648,7 @@ SourcePath EvalState::checkSourcePath(const SourcePath & path_) /* Resolve symlinks. */ debug("checking access to '%s'", abspath); - SourcePath path = CanonPath(canonPath(abspath, true)); + SourcePath path = rootPath(CanonPath(canonPath(abspath, true))); for (auto & i : *allowedPaths) { if (isDirOrInDir(path.path.abs(), i)) { @@ -690,12 +680,12 @@ void EvalState::checkURI(const std::string & uri) /* If the URI is a path, then check it against allowedPaths as well. */ if (hasPrefix(uri, "/")) { - checkSourcePath(CanonPath(uri)); + checkSourcePath(rootPath(CanonPath(uri))); return; } if (hasPrefix(uri, "file://")) { - checkSourcePath(CanonPath(std::string(uri, 7))); + checkSourcePath(rootPath(CanonPath(std::string(uri, 7)))); return; } @@ -744,6 +734,23 @@ void EvalState::addConstant(const std::string & name, Value * v, Constant info) } +void PrimOp::check() +{ + if (arity > maxPrimOpArity) { + throw Error("primop arity must not exceed %1%", maxPrimOpArity); + } +} + + +void Value::mkPrimOp(PrimOp * p) +{ + p->check(); + clearValue(); + internalType = tPrimOp; + primOp = p; +} + + Value * EvalState::addPrimOp(PrimOp && primOp) { /* Hack to make constants lazy: turn them into a application of @@ -991,7 +998,7 @@ void Value::mkStringMove(const char * s, const NixStringContext & context) void Value::mkPath(const SourcePath & path) { - mkPath(makeImmutableString(path.path.abs())); + mkPath(&*path.accessor, makeImmutableString(path.path.abs())); } @@ -1068,23 +1075,67 @@ void EvalState::mkStorePathString(const StorePath & p, Value & v) } +std::string EvalState::mkOutputStringRaw( + const SingleDerivedPath::Built & b, + std::optional optStaticOutputPath, + const ExperimentalFeatureSettings & xpSettings) +{ + /* In practice, this is testing for the case of CA derivations, or + dynamic derivations. */ + return optStaticOutputPath + ? store->printStorePath(std::move(*optStaticOutputPath)) + /* Downstream we would substitute this for an actual path once + we build the floating CA derivation */ + : DownstreamPlaceholder::fromSingleDerivedPathBuilt(b, xpSettings).render(); +} + + void EvalState::mkOutputString( Value & value, - const StorePath & drvPath, - const std::string outputName, - std::optional optOutputPath) + const SingleDerivedPath::Built & b, + std::optional optStaticOutputPath, + const ExperimentalFeatureSettings & xpSettings) { value.mkString( - optOutputPath - ? store->printStorePath(*std::move(optOutputPath)) - /* Downstream we would substitute this for an actual path once - we build the floating CA derivation */ - : DownstreamPlaceholder::unknownCaOutput(drvPath, outputName).render(), + mkOutputStringRaw(b, optStaticOutputPath, xpSettings), + NixStringContext { b }); +} + + +std::string EvalState::mkSingleDerivedPathStringRaw( + const SingleDerivedPath & p) +{ + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) { + return store->printStorePath(o.path); + }, + [&](const SingleDerivedPath::Built & b) { + auto optStaticOutputPath = std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) { + auto drv = store->readDerivation(o.path); + auto i = drv.outputs.find(b.output); + if (i == drv.outputs.end()) + throw Error("derivation '%s' does not have output '%s'", b.drvPath->to_string(*store), b.output); + return i->second.path(*store, drv.name, b.output); + }, + [&](const SingleDerivedPath::Built & o) -> std::optional { + return std::nullopt; + }, + }, b.drvPath->raw()); + return mkOutputStringRaw(b, optStaticOutputPath); + } + }, p.raw()); +} + + +void EvalState::mkSingleDerivedPathString( + const SingleDerivedPath & p, + Value & v) +{ + v.mkString( + mkSingleDerivedPathStringRaw(p), NixStringContext { - NixStringContextElem::Built { - .drvPath = drvPath, - .output = outputName, - } + std::visit([](auto && v) -> NixStringContextElem { return v; }, p), }); } @@ -1162,24 +1213,6 @@ void EvalState::evalFile(const SourcePath & path_, Value & v, bool mustBeTrivial if (!e) e = parseExprFromFile(checkSourcePath(resolvedPath)); - cacheFile(path, resolvedPath, e, v, mustBeTrivial); -} - - -void EvalState::resetFileCache() -{ - fileEvalCache.clear(); - fileParseCache.clear(); -} - - -void EvalState::cacheFile( - const SourcePath & path, - const SourcePath & resolvedPath, - Expr * e, - Value & v, - bool mustBeTrivial) -{ fileParseCache[resolvedPath] = e; try { @@ -1208,6 +1241,13 @@ void EvalState::cacheFile( } +void EvalState::resetFileCache() +{ + fileEvalCache.clear(); + fileParseCache.clear(); +} + + void EvalState::eval(Expr * e, Value & v) { e->eval(*this, baseEnv, v); @@ -1340,7 +1380,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) if (nameVal.type() == nNull) continue; state.forceStringNoCtx(nameVal, i.pos, "while evaluating the name of a dynamic attribute"); - auto nameSym = state.symbols.create(nameVal.string.s); + auto nameSym = state.symbols.create(nameVal.string_view()); Bindings::iterator j = v.attrs->find(nameSym); if (j != v.attrs->end()) state.error("dynamic attribute '%1%' already defined at %2%", state.symbols[nameSym], state.positions[j->pos]).atPos(i.pos).withFrame(env, *this).debugThrow(); @@ -1680,7 +1720,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & /* We have all the arguments, so call the primop with the previous and new arguments. */ - Value * vArgs[arity]; + Value * vArgs[maxPrimOpArity]; auto n = argsDone; for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp.left) vArgs[--n] = arg->primOpApp.right; @@ -1737,11 +1777,17 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v) Value vFun; fun->eval(state, env, vFun); - Value * vArgs[args.size()]; + // Empirical arity of Nixpkgs lambdas by regex e.g. ([a-zA-Z]+:(\s|(/\*.*\/)|(#.*\n))*){5} + // 2: over 4000 + // 3: about 300 + // 4: about 60 + // 5: under 10 + // This excluded attrset lambdas (`{...}:`). Contributions of mixed lambdas appears insignificant at ~150 total. + SmallValueVector<4> vArgs(args.size()); for (size_t i = 0; i < args.size(); ++i) vArgs[i] = args[i]->maybeThunk(state, env); - state.callFunction(vFun, args.size(), vArgs, v, pos); + state.callFunction(vFun, args.size(), vArgs.data(), v, pos); } @@ -1980,8 +2026,9 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) return result; }; - Value values[es->size()]; - Value * vTmpP = values; + // List of returned strings. References to these Values must NOT be persisted. + SmallTemporaryValueVector values(es->size()); + Value * vTmpP = values.data(); for (auto & [i_pos, i] : *es) { Value & vTmp = *vTmpP++; @@ -2034,7 +2081,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) else if (firstType == nPath) { if (!context.empty()) state.error("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow(); - v.mkPath(CanonPath(canonPath(str()))); + v.mkPath(state.rootPath(CanonPath(canonPath(str())))); } else v.mkStringMove(c_str(), context); } @@ -2152,7 +2199,7 @@ std::string_view EvalState::forceString(Value & v, const PosIdx pos, std::string forceValue(v, pos); if (v.type() != nString) error("value is %1% while a string was expected", showType(v)).debugThrow(); - return v.string.s; + return v.string_view(); } catch (Error & e) { e.addTrace(positions[pos], errorCtx); throw; @@ -2179,8 +2226,8 @@ std::string_view EvalState::forceString(Value & v, NixStringContext & context, c std::string_view EvalState::forceStringNoCtx(Value & v, const PosIdx pos, std::string_view errorCtx) { auto s = forceString(v, pos, errorCtx); - if (v.string.context) { - error("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string.s, v.string.context[0]).withTrace(pos, errorCtx).debugThrow(); + if (v.context()) { + error("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string_view(), v.context()[0]).withTrace(pos, errorCtx).debugThrow(); } return s; } @@ -2193,7 +2240,7 @@ bool EvalState::isDerivation(Value & v) if (i == v.attrs->end()) return false; forceValue(*i->value, i->pos); if (i->value->type() != nString) return false; - return strcmp(i->value->string.s, "derivation") == 0; + return i->value->string_view().compare("derivation") == 0; } @@ -2225,7 +2272,7 @@ BackedStringView EvalState::coerceToString( if (v.type() == nString) { copyContext(v, context); - return std::string_view(v.string.s); + return v.string_view(); } if (v.type() == nPath) { @@ -2233,7 +2280,7 @@ BackedStringView EvalState::coerceToString( !canonicalizePath && !copyToStore ? // FIXME: hack to preserve path literals that end in a // slash, as in /foo/${x}. - v._path + v._path.path : copyToStore ? store->printStorePath(copyPathToStore(context, v.path())) : std::string(v.path().path.abs()); @@ -2287,7 +2334,7 @@ BackedStringView EvalState::coerceToString( && (!v2->isList() || v2->listSize() != 0)) result += " "; } - return std::move(result); + return result; } } @@ -2307,7 +2354,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat auto dstPath = i != srcToStore.end() ? i->second : [&]() { - auto dstPath = path.fetchToStore(store, path.baseName(), nullptr, repair); + auto dstPath = path.fetchToStore(store, path.baseName(), FileIngestionMethod::Recursive, nullptr, repair); allowPath(dstPath); srcToStore.insert_or_assign(path, dstPath); printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath)); @@ -2323,10 +2370,34 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx) { + try { + forceValue(v, pos); + } catch (Error & e) { + e.addTrace(positions[pos], errorCtx); + throw; + } + + /* Handle path values directly, without coercing to a string. */ + if (v.type() == nPath) + return v.path(); + + /* Similarly, handle __toString where the result may be a path + value. */ + if (v.type() == nAttrs) { + auto i = v.attrs->find(sToString); + if (i != v.attrs->end()) { + Value v1; + callFunction(*i->value, v, v1, pos); + return coerceToPath(pos, v1, context, errorCtx); + } + } + + /* Any other value should be coercable to a string, interpreted + relative to the root filesystem. */ auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); if (path == "" || path[0] != '/') error("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow(); - return CanonPath(path); + return rootPath(CanonPath(path)); } @@ -2339,7 +2410,7 @@ StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringCon } -std::pair EvalState::coerceToDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx) +std::pair EvalState::coerceToSingleDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx) { NixStringContext context; auto s = forceString(v, context, pos, errorCtx); @@ -2350,23 +2421,18 @@ std::pair EvalState::coerceToDerivedPathUnchecked s, csize) .withTrace(pos, errorCtx).debugThrow(); auto derivedPath = std::visit(overloaded { - [&](NixStringContextElem::Opaque && o) -> DerivedPath { - return DerivedPath::Opaque { - .path = std::move(o.path), - }; + [&](NixStringContextElem::Opaque && o) -> SingleDerivedPath { + return std::move(o); }, - [&](NixStringContextElem::DrvDeep &&) -> DerivedPath { + [&](NixStringContextElem::DrvDeep &&) -> SingleDerivedPath { error( "string '%s' has a context which refers to a complete source and binary closure. This is not supported at this time", s).withTrace(pos, errorCtx).debugThrow(); }, - [&](NixStringContextElem::Built && b) -> DerivedPath { - return DerivedPath::Built { - .drvPath = std::move(b.drvPath), - .outputs = OutputsSpec::Names { std::move(b.output) }, - }; + [&](NixStringContextElem::Built && b) -> SingleDerivedPath { + return std::move(b); }, - }, ((NixStringContextElem &&) *context.begin()).raw()); + }, ((NixStringContextElem &&) *context.begin()).raw); return { std::move(derivedPath), std::move(s), @@ -2374,41 +2440,29 @@ std::pair EvalState::coerceToDerivedPathUnchecked } -DerivedPath EvalState::coerceToDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx) +SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx) { - auto [derivedPath, s_] = coerceToDerivedPathUnchecked(pos, v, errorCtx); + auto [derivedPath, s_] = coerceToSingleDerivedPathUnchecked(pos, v, errorCtx); auto s = s_; - std::visit(overloaded { - [&](const DerivedPath::Opaque & o) { - auto sExpected = store->printStorePath(o.path); - if (s != sExpected) + auto sExpected = mkSingleDerivedPathStringRaw(derivedPath); + if (s != sExpected) { + /* `std::visit` is used here just to provide a more precise + error message. */ + std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) { error( "path string '%s' has context with the different path '%s'", s, sExpected) .withTrace(pos, errorCtx).debugThrow(); - }, - [&](const DerivedPath::Built & b) { - // TODO need derived path with single output to make this - // total. Will add as part of RFC 92 work and then this is - // cleaned up. - auto output = *std::get(b.outputs).begin(); - - auto drv = store->readDerivation(b.drvPath); - auto i = drv.outputs.find(output); - if (i == drv.outputs.end()) - throw Error("derivation '%s' does not have output '%s'", store->printStorePath(b.drvPath), output); - auto optOutputPath = i->second.path(*store, drv.name, output); - // This is testing for the case of CA derivations - auto sExpected = optOutputPath - ? store->printStorePath(*optOutputPath) - : DownstreamPlaceholder::unknownCaOutput(b.drvPath, output).render(); - if (s != sExpected) + }, + [&](const SingleDerivedPath::Built & b) { error( "string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'", - s, output, store->printStorePath(b.drvPath), sExpected) + s, b.output, b.drvPath->to_string(*store), sExpected) .withTrace(pos, errorCtx).debugThrow(); - } - }, derivedPath.raw()); + } + }, derivedPath.raw()); + } return derivedPath; } @@ -2440,10 +2494,13 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v return v1.boolean == v2.boolean; case nString: - return strcmp(v1.string.s, v2.string.s) == 0; + return v1.string_view().compare(v2.string_view()) == 0; case nPath: - return strcmp(v1._path, v2._path) == 0; + return + // FIXME: compare accessors by their fingerprint. + v1._path.accessor == v2._path.accessor + && strcmp(v1._path.path, v2._path.path) == 0; case nNull: return true; @@ -2491,10 +2548,37 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v } } -void EvalState::printStats() +bool EvalState::fullGC() { +#if HAVE_BOEHMGC + GC_gcollect(); + // Check that it ran. We might replace this with a version that uses more + // of the boehm API to get this reliably, at a maintenance cost. + // We use a 1K margin because technically this has a race condtion, but we + // probably won't encounter it in practice, because the CLI isn't concurrent + // like that. + return GC_get_bytes_since_gc() < 1024; +#else + return false; +#endif +} + +void EvalState::maybePrintStats() { bool showStats = getEnv("NIX_SHOW_STATS").value_or("0") != "0"; + if (showStats) { + // Make the final heap size more deterministic. +#if HAVE_BOEHMGC + if (!fullGC()) { + warn("failed to perform a full GC before reporting stats"); + } +#endif + printStatistics(); + } +} + +void EvalState::printStatistics() +{ struct rusage buf; getrusage(RUSAGE_SELF, &buf); float cpuTime = buf.ru_utime.tv_sec + ((float) buf.ru_utime.tv_usec / 1000000); @@ -2508,105 +2592,105 @@ void EvalState::printStats() GC_word heapSize, totalBytes; GC_get_heap_usage_safe(&heapSize, 0, 0, 0, &totalBytes); #endif - if (showStats) { - auto outPath = getEnv("NIX_SHOW_STATS_PATH").value_or("-"); - std::fstream fs; - if (outPath != "-") - fs.open(outPath, std::fstream::out); - json topObj = json::object(); - topObj["cpuTime"] = cpuTime; - topObj["envs"] = { - {"number", nrEnvs}, - {"elements", nrValuesInEnvs}, - {"bytes", bEnvs}, - }; - topObj["list"] = { - {"elements", nrListElems}, - {"bytes", bLists}, - {"concats", nrListConcats}, - }; - topObj["values"] = { - {"number", nrValues}, - {"bytes", bValues}, - }; - topObj["symbols"] = { - {"number", symbols.size()}, - {"bytes", symbols.totalSize()}, - }; - topObj["sets"] = { - {"number", nrAttrsets}, - {"bytes", bAttrsets}, - {"elements", nrAttrsInAttrsets}, - }; - topObj["sizes"] = { - {"Env", sizeof(Env)}, - {"Value", sizeof(Value)}, - {"Bindings", sizeof(Bindings)}, - {"Attr", sizeof(Attr)}, - }; - topObj["nrOpUpdates"] = nrOpUpdates; - topObj["nrOpUpdateValuesCopied"] = nrOpUpdateValuesCopied; - topObj["nrThunks"] = nrThunks; - topObj["nrAvoided"] = nrAvoided; - topObj["nrLookups"] = nrLookups; - topObj["nrPrimOpCalls"] = nrPrimOpCalls; - topObj["nrFunctionCalls"] = nrFunctionCalls; + + auto outPath = getEnv("NIX_SHOW_STATS_PATH").value_or("-"); + std::fstream fs; + if (outPath != "-") + fs.open(outPath, std::fstream::out); + json topObj = json::object(); + topObj["cpuTime"] = cpuTime; + topObj["envs"] = { + {"number", nrEnvs}, + {"elements", nrValuesInEnvs}, + {"bytes", bEnvs}, + }; + topObj["nrExprs"] = Expr::nrExprs; + topObj["list"] = { + {"elements", nrListElems}, + {"bytes", bLists}, + {"concats", nrListConcats}, + }; + topObj["values"] = { + {"number", nrValues}, + {"bytes", bValues}, + }; + topObj["symbols"] = { + {"number", symbols.size()}, + {"bytes", symbols.totalSize()}, + }; + topObj["sets"] = { + {"number", nrAttrsets}, + {"bytes", bAttrsets}, + {"elements", nrAttrsInAttrsets}, + }; + topObj["sizes"] = { + {"Env", sizeof(Env)}, + {"Value", sizeof(Value)}, + {"Bindings", sizeof(Bindings)}, + {"Attr", sizeof(Attr)}, + }; + topObj["nrOpUpdates"] = nrOpUpdates; + topObj["nrOpUpdateValuesCopied"] = nrOpUpdateValuesCopied; + topObj["nrThunks"] = nrThunks; + topObj["nrAvoided"] = nrAvoided; + topObj["nrLookups"] = nrLookups; + topObj["nrPrimOpCalls"] = nrPrimOpCalls; + topObj["nrFunctionCalls"] = nrFunctionCalls; #if HAVE_BOEHMGC - topObj["gc"] = { - {"heapSize", heapSize}, - {"totalBytes", totalBytes}, - }; + topObj["gc"] = { + {"heapSize", heapSize}, + {"totalBytes", totalBytes}, + }; #endif - if (countCalls) { - topObj["primops"] = primOpCalls; - { - auto& list = topObj["functions"]; - list = json::array(); - for (auto & [fun, count] : functionCalls) { - json obj = json::object(); - if (fun->name) - obj["name"] = (std::string_view) symbols[fun->name]; - else - obj["name"] = nullptr; - if (auto pos = positions[fun->pos]) { - if (auto path = std::get_if(&pos.origin)) - obj["file"] = path->to_string(); - obj["line"] = pos.line; - obj["column"] = pos.column; - } - obj["count"] = count; - list.push_back(obj); + if (countCalls) { + topObj["primops"] = primOpCalls; + { + auto& list = topObj["functions"]; + list = json::array(); + for (auto & [fun, count] : functionCalls) { + json obj = json::object(); + if (fun->name) + obj["name"] = (std::string_view) symbols[fun->name]; + else + obj["name"] = nullptr; + if (auto pos = positions[fun->pos]) { + if (auto path = std::get_if(&pos.origin)) + obj["file"] = path->to_string(); + obj["line"] = pos.line; + obj["column"] = pos.column; } + obj["count"] = count; + list.push_back(obj); } - { - auto list = topObj["attributes"]; - list = json::array(); - for (auto & i : attrSelects) { - json obj = json::object(); - if (auto pos = positions[i.first]) { - if (auto path = std::get_if(&pos.origin)) - obj["file"] = path->to_string(); - obj["line"] = pos.line; - obj["column"] = pos.column; - } - obj["count"] = i.second; - list.push_back(obj); + } + { + auto list = topObj["attributes"]; + list = json::array(); + for (auto & i : attrSelects) { + json obj = json::object(); + if (auto pos = positions[i.first]) { + if (auto path = std::get_if(&pos.origin)) + obj["file"] = path->to_string(); + obj["line"] = pos.line; + obj["column"] = pos.column; } + obj["count"] = i.second; + list.push_back(obj); } } + } - if (getEnv("NIX_SHOW_SYMBOLS").value_or("0") != "0") { - // XXX: overrides earlier assignment - topObj["symbols"] = json::array(); - auto &list = topObj["symbols"]; - symbols.dump([&](const std::string & s) { list.emplace_back(s); }); - } - if (outPath == "-") { - std::cerr << topObj.dump(2) << std::endl; - } else { - fs << topObj.dump(2) << std::endl; - } + if (getEnv("NIX_SHOW_SYMBOLS").value_or("0") != "0") { + // XXX: overrides earlier assignment + topObj["symbols"] = json::array(); + auto &list = topObj["symbols"]; + symbols.dump([&](const std::string & s) { list.emplace_back(s); }); + } + if (outPath == "-") { + std::cerr << topObj.dump(2) << std::endl; + } else { + fs << topObj.dump(2) << std::endl; } } @@ -2630,54 +2714,4 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) { } -EvalSettings::EvalSettings() -{ - auto var = getEnv("NIX_PATH"); - if (var) nixPath = parseNixPath(*var); -} - -Strings EvalSettings::getDefaultNixPath() -{ - Strings res; - auto add = [&](const Path & p, const std::string & s = std::string()) { - if (pathAccessible(p)) { - if (s.empty()) { - res.push_back(p); - } else { - res.push_back(s + "=" + p); - } - } - }; - - if (!evalSettings.restrictEval && !evalSettings.pureEval) { - add(settings.useXDGBaseDirectories ? getStateDir() + "/nix/defexpr/channels" : getHome() + "/.nix-defexpr/channels"); - add(rootChannelsDir() + "/nixpkgs", "nixpkgs"); - add(rootChannelsDir()); - } - - return res; -} - -bool EvalSettings::isPseudoUrl(std::string_view s) -{ - if (s.compare(0, 8, "channel:") == 0) return true; - size_t pos = s.find("://"); - if (pos == std::string::npos) return false; - std::string scheme(s, 0, pos); - return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh"; -} - -std::string EvalSettings::resolvePseudoUrl(std::string_view url) -{ - if (hasPrefix(url, "channel:")) - return "https://nixos.org/channels/" + std::string(url.substr(8)) + "/nixexprs.tar.xz"; - else - return std::string(url); -} - -EvalSettings evalSettings; - -static GlobalConfig::Register rEvalSettings(&evalSettings); - - } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 3aaec5ee1679..be4c4ab22922 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -9,6 +9,7 @@ #include "config.hh" #include "experimental-features.hh" #include "input-accessor.hh" +#include "search-path.hh" #include #include @@ -17,12 +18,20 @@ namespace nix { +/** + * We put a limit on primop arity because it lets us use a fixed size array on + * the stack. 8 is already an impractical number of arguments. Use an attrset + * argument for such overly complicated functions. + */ +constexpr size_t maxPrimOpArity = 8; class Store; class EvalState; class StorePath; -struct DerivedPath; +struct SingleDerivedPath; enum RepairFlag : bool; +struct FSInputAccessor; +struct MemoryInputAccessor; /** @@ -68,6 +77,12 @@ struct PrimOp * Optional experimental for this to be gated on. */ std::optional experimentalFeature; + + /** + * Validity check to be performed by functions that introduce primops, + * such as RegisterPrimOp() and Value::mkPrimOp(). + */ + void check(); }; /** @@ -122,15 +137,6 @@ std::string printValue(const EvalState & state, const Value & v); std::ostream & operator << (std::ostream & os, const ValueType t); -struct SearchPathElem -{ - std::string prefix; - // FIXME: maybe change this to an std::variant. - std::string path; -}; -typedef std::list SearchPath; - - /** * Initialise the Boehm GC, if applicable. */ @@ -220,8 +226,26 @@ public: Bindings emptyBindings; + /** + * The accessor for the root filesystem. + */ + const ref rootFS; + + /** + * The in-memory filesystem for paths. + */ + const ref corepkgsFS; + + /** + * In-memory filesystem for internal, non-user-callable Nix + * expressions like call-flake.nix. + */ + const ref internalFS; + const SourcePath derivationInternal; + const SourcePath callFlakeInternal; + /** * Store used to materialise .drv files. */ @@ -232,7 +256,6 @@ public: */ const ref buildStore; - RootValue vCallFlake = nullptr; RootValue vImportedDrvToDerivation = nullptr; /** @@ -318,7 +341,7 @@ private: SearchPath searchPath; - std::map> searchPathResolved; + std::map> searchPathResolved; /** * Cache used by checkSourcePath(). @@ -345,13 +368,11 @@ private: public: EvalState( - const Strings & _searchPath, + const SearchPath & _searchPath, ref store, std::shared_ptr buildStore = nullptr); ~EvalState(); - void addToSearchPath(const std::string & s); - SearchPath getSearchPath() { return searchPath; } /** @@ -416,28 +437,22 @@ public: */ void evalFile(const SourcePath & path, Value & v, bool mustBeTrivial = false); - /** - * Like `evalFile`, but with an already parsed expression. - */ - void cacheFile( - const SourcePath & path, - const SourcePath & resolvedPath, - Expr * e, - Value & v, - bool mustBeTrivial = false); - void resetFileCache(); /** * Look up a file in the search path. */ SourcePath findFile(const std::string_view path); - SourcePath findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); + SourcePath findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); /** + * Try to resolve a search path value (not the optional key part) + * * If the specified search path element is a URI, download it. + * + * If it is not found, return `std::nullopt` */ - std::pair resolveSearchPathElem(const SearchPathElem & elem); + std::optional resolveSearchPathPath(const SearchPath::Path & path); /** * Evaluate an expression to normal form @@ -537,12 +552,12 @@ public: StorePath coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx); /** - * Part of `coerceToDerivedPath()` without any store IO which is exposed for unit testing only. + * Part of `coerceToSingleDerivedPath()` without any store IO which is exposed for unit testing only. */ - std::pair coerceToDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx); + std::pair coerceToSingleDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx); /** - * Coerce to `DerivedPath`. + * Coerce to `SingleDerivedPath`. * * Must be a string which is either a literal store path or a * "placeholder (see `DownstreamPlaceholder`). @@ -556,7 +571,7 @@ public: * source of truth, and ultimately tells us what we want, and then * we ensure the string corresponds to it. */ - DerivedPath coerceToDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx); + SingleDerivedPath coerceToSingleDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx); public: @@ -673,40 +688,68 @@ public: /** * Create a string representing a store path. * - * The string is the printed store path with a context containing a single - * `NixStringContextElem::Opaque` element of that store path. + * The string is the printed store path with a context containing a + * single `NixStringContextElem::Opaque` element of that store path. */ void mkStorePathString(const StorePath & storePath, Value & v); /** - * Create a string representing a `DerivedPath::Built`. + * Create a string representing a `SingleDerivedPath::Built`. * - * The string is the printed store path with a context containing a single - * `NixStringContextElem::Built` element of the drv path and output name. + * The string is the printed store path with a context containing a + * single `NixStringContextElem::Built` element of the drv path and + * output name. * * @param value Value we are settings * - * @param drvPath Path the drv whose output we are making a string for + * @param b the drv whose output we are making a string for, and the + * output * - * @param outputName Name of the output + * @param optStaticOutputPath Optional output path for that string. + * Must be passed if and only if output store object is + * input-addressed or fixed output. Will be printed to form string + * if passed, otherwise a placeholder will be used (see + * `DownstreamPlaceholder`). * - * @param optOutputPath Optional output path for that string. Must - * be passed if and only if output store object is input-addressed. - * Will be printed to form string if passed, otherwise a placeholder - * will be used (see `DownstreamPlaceholder`). + * @param xpSettings Stop-gap to avoid globals during unit tests. */ void mkOutputString( Value & value, - const StorePath & drvPath, - const std::string outputName, - std::optional optOutputPath); + const SingleDerivedPath::Built & b, + std::optional optStaticOutputPath, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + + /** + * Create a string representing a `SingleDerivedPath`. + * + * A combination of `mkStorePathString` and `mkOutputString`. + */ + void mkSingleDerivedPathString( + const SingleDerivedPath & p, + Value & v); void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx); /** - * Print statistics. + * Print statistics, if enabled. + * + * Performs a full memory GC before printing the statistics, so that the + * GC statistics are more accurate. + */ + void maybePrintStats(); + + /** + * Print statistics, unconditionally, cheaply, without performing a GC first. + */ + void printStatistics(); + + /** + * Perform a full memory garbage collection - not incremental. + * + * @return true if Nix was built with GC and a GC was performed, false if not. + * The return value is currently not thread safe - just the return value. */ - void printStats(); + bool fullGC(); /** * Realise the given context, and return a mapping from the placeholders @@ -716,6 +759,22 @@ public: private: + /** + * Like `mkOutputString` but just creates a raw string, not an + * string Value, which would also have a string context. + */ + std::string mkOutputStringRaw( + const SingleDerivedPath::Built & b, + std::optional optStaticOutputPath, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + + /** + * Like `mkSingleDerivedPathStringRaw` but just creates a raw string + * Value, which would also have a string context. + */ + std::string mkSingleDerivedPathStringRaw( + const SingleDerivedPath & p); + unsigned long nrEnvs = 0; unsigned long nrValuesInEnvs = 0; unsigned long nrValues = 0; @@ -781,7 +840,7 @@ std::string showType(const Value & v); /** * If `path` refers to a directory, then append "/default.nix". */ -SourcePath resolveExprPath(const SourcePath & path); +SourcePath resolveExprPath(SourcePath path); struct InvalidPathError : EvalError { @@ -792,100 +851,6 @@ struct InvalidPathError : EvalError #endif }; -struct EvalSettings : Config -{ - EvalSettings(); - - static Strings getDefaultNixPath(); - - static bool isPseudoUrl(std::string_view s); - - static std::string resolvePseudoUrl(std::string_view url); - - Setting enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", - "Whether builtin functions that allow executing native code should be enabled."}; - - Setting nixPath{ - this, getDefaultNixPath(), "nix-path", - R"( - List of directories to be searched for `<...>` file references - - In particular, outside of [pure evaluation mode](#conf-pure-evaluation), this determines the value of - [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtin-constants-nixPath). - )"}; - - Setting restrictEval{ - this, false, "restrict-eval", - R"( - If set to `true`, the Nix evaluator will not allow access to any - files outside of the Nix search path (as set via the `NIX_PATH` - environment variable or the `-I` option), or to URIs outside of - [`allowed-uris`](../command-ref/conf-file.md#conf-allowed-uris). - The default is `false`. - )"}; - - Setting pureEval{this, false, "pure-eval", - R"( - Pure evaluation mode ensures that the result of Nix expressions is fully determined by explicitly declared inputs, and not influenced by external state: - - - Restrict file system and network access to files specified by cryptographic hash - - Disable [`bultins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) and [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime) - )" - }; - - Setting enableImportFromDerivation{ - this, true, "allow-import-from-derivation", - R"( - By default, Nix allows you to `import` from a derivation, allowing - building at evaluation time. With this option set to false, Nix will - throw an error when evaluating an expression that uses this feature, - allowing users to ensure their evaluation will not require any - builds to take place. - )"}; - - Setting allowedUris{this, {}, "allowed-uris", - R"( - A list of URI prefixes to which access is allowed in restricted - evaluation mode. For example, when set to - `https://github.com/NixOS`, builtin functions such as `fetchGit` are - allowed to access `https://github.com/NixOS/patchelf.git`. - )"}; - - Setting traceFunctionCalls{this, false, "trace-function-calls", - R"( - If set to `true`, the Nix evaluator will trace every function call. - Nix will print a log message at the "vomit" level for every function - entrance and function exit. - - function-trace entered undefined position at 1565795816999559622 - function-trace exited undefined position at 1565795816999581277 - function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150 - function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684 - - The `undefined position` means the function call is a builtin. - - Use the `contrib/stack-collapse.py` script distributed with the Nix - source code to convert the trace logs in to a format suitable for - `flamegraph.pl`. - )"}; - - Setting useEvalCache{this, true, "eval-cache", - "Whether to use the flake evaluation cache."}; - - Setting ignoreExceptionsDuringTry{this, false, "ignore-try", - R"( - If set to true, ignore exceptions inside 'tryEval' calls when evaluating nix expressions in - debug mode (using the --debugger flag). By default the debugger will pause on all exceptions. - )"}; - - Setting traceVerbose{this, false, "trace-verbose", - "Whether `builtins.traceVerbose` should trace its first argument when evaluated."}; -}; - -extern EvalSettings evalSettings; - -static const std::string corepkgsPrefix{"/__corepkgs__/"}; - template void ErrorBuilder::debugThrow() { diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc index e890148621eb..3c7ed5d8a5b3 100644 --- a/src/libexpr/flake/config.cc +++ b/src/libexpr/flake/config.cc @@ -1,6 +1,7 @@ -#include "flake.hh" +#include "users.hh" #include "globals.hh" #include "fetch-settings.hh" +#include "flake.hh" #include diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 5aa44d6a1ba0..8f8fc64f09e7 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -1,5 +1,7 @@ +#include "terminal.hh" #include "flake.hh" #include "eval.hh" +#include "eval-settings.hh" #include "lockfile.hh" #include "primops.hh" #include "eval-inline.hh" @@ -7,6 +9,7 @@ #include "fetchers.hh" #include "finally.hh" #include "fetch-settings.hh" +#include "value-to-json.hh" namespace nix { @@ -14,7 +17,7 @@ using namespace flake; namespace flake { -typedef std::pair FetchedFlake; +typedef std::pair FetchedFlake; typedef std::vector> FlakeCache; static std::optional lookupInFlakeCache( @@ -33,7 +36,7 @@ static std::optional lookupInFlakeCache( return std::nullopt; } -static std::tuple fetchOrSubstituteTree( +static std::tuple fetchOrSubstituteTree( EvalState & state, const FlakeRef & originalRef, bool allowLookup, @@ -60,16 +63,16 @@ static std::tuple fetchOrSubstituteTree( flakeCache.push_back({originalRef, *fetched}); } - auto [tree, lockedRef] = *fetched; + auto [storePath, lockedRef] = *fetched; debug("got tree '%s' from '%s'", - state.store->printStorePath(tree.storePath), lockedRef); + state.store->printStorePath(storePath), lockedRef); - state.allowPath(tree.storePath); + state.allowPath(storePath); - assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store)); + assert(!originalRef.input.getNarHash() || storePath == originalRef.input.computeStorePath(*state.store)); - return {std::move(tree), resolvedRef, lockedRef}; + return {std::move(storePath), resolvedRef, lockedRef}; } static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos) @@ -112,7 +115,7 @@ static FlakeInput parseFlakeInput(EvalState & state, try { if (attr.name == sUrl) { expectType(state, nString, *attr.value, attr.pos); - url = attr.value->string.s; + url = attr.value->string_view(); attrs.emplace("url", *url); } else if (attr.name == sFlake) { expectType(state, nBool, *attr.value, attr.pos); @@ -121,7 +124,7 @@ static FlakeInput parseFlakeInput(EvalState & state, input.overrides = parseFlakeInputs(state, attr.value, attr.pos, baseDir, lockRootPath); } else if (attr.name == sFollows) { expectType(state, nString, *attr.value, attr.pos); - auto follows(parseInputPath(attr.value->string.s)); + auto follows(parseInputPath(attr.value->c_str())); follows.insert(follows.begin(), lockRootPath.begin(), lockRootPath.end()); input.follows = follows; } else { @@ -130,7 +133,7 @@ static FlakeInput parseFlakeInput(EvalState & state, #pragma GCC diagnostic ignored "-Wswitch-enum" switch (attr.value->type()) { case nString: - attrs.emplace(state.symbols[attr.name], attr.value->string.s); + attrs.emplace(state.symbols[attr.name], attr.value->c_str()); break; case nBool: attrs.emplace(state.symbols[attr.name], Explicit { attr.value->boolean }); @@ -139,8 +142,13 @@ static FlakeInput parseFlakeInput(EvalState & state, attrs.emplace(state.symbols[attr.name], (long unsigned int)attr.value->integer); break; default: - throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected", - state.symbols[attr.name], showType(*attr.value)); + if (attr.name == state.symbols.create("publicKeys")) { + experimentalFeatureSettings.require(Xp::VerifiedFetches); + NixStringContext emptyContext = {}; + attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, emptyContext).dump()); + } else + throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected", + state.symbols[attr.name], showType(*attr.value)); } #pragma GCC diagnostic pop } @@ -201,34 +209,39 @@ static Flake getFlake( FlakeCache & flakeCache, InputPath lockRootPath) { - auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( + auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree( state, originalRef, allowLookup, flakeCache); + // We need to guard against symlink attacks, but before we start doing + // filesystem operations we should make sure there's a flake.nix in the + // first place. + auto unsafeFlakeDir = state.store->toRealPath(storePath) + "/" + lockedRef.subdir; + auto unsafeFlakeFile = unsafeFlakeDir + "/flake.nix"; + if (!pathExists(unsafeFlakeFile)) + throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir); + // Guard against symlink attacks. - auto flakeDir = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir, true); + auto flakeDir = canonPath(unsafeFlakeDir, true); auto flakeFile = canonPath(flakeDir + "/flake.nix", true); - if (!isInDir(flakeFile, sourceInfo.actualPath)) + if (!isInDir(flakeFile, state.store->toRealPath(storePath))) throw Error("'flake.nix' file of flake '%s' escapes from '%s'", - lockedRef, state.store->printStorePath(sourceInfo.storePath)); + lockedRef, state.store->printStorePath(storePath)); Flake flake { .originalRef = originalRef, .resolvedRef = resolvedRef, .lockedRef = lockedRef, - .sourceInfo = std::make_shared(std::move(sourceInfo)) + .storePath = storePath, }; - if (!pathExists(flakeFile)) - throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir); - Value vInfo; - state.evalFile(CanonPath(flakeFile), vInfo, true); // FIXME: symlink attack + state.evalFile(state.rootPath(CanonPath(flakeFile)), vInfo, true); // FIXME: symlink attack - expectType(state, nAttrs, vInfo, state.positions.add({CanonPath(flakeFile)}, 1, 1)); + expectType(state, nAttrs, vInfo, state.positions.add({state.rootPath(CanonPath(flakeFile))}, 1, 1)); if (auto description = vInfo.attrs->get(state.sDescription)) { expectType(state, nString, *description->value, description->pos); - flake.description = description->value->string.s; + flake.description = description->value->c_str(); } auto sInputs = state.symbols.create("inputs"); @@ -345,15 +358,18 @@ LockedFlake lockFlake( // FIXME: symlink attack auto oldLockFile = LockFile::read( lockFlags.referenceLockFilePath.value_or( - flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock")); + state.store->toRealPath(flake.storePath) + "/" + flake.lockedRef.subdir + "/flake.lock")); debug("old lock file: %s", oldLockFile); std::map overrides; + std::set explicitCliOverrides; std::set overridesUsed, updatesUsed; - for (auto & i : lockFlags.inputOverrides) + for (auto & i : lockFlags.inputOverrides) { overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second }); + explicitCliOverrides.insert(i.first); + } LockFile newLockFile; @@ -424,6 +440,7 @@ LockedFlake lockFlake( ancestors? */ auto i = overrides.find(inputPath); bool hasOverride = i != overrides.end(); + bool hasCliOverride = explicitCliOverrides.contains(inputPath); if (hasOverride) { overridesUsed.insert(inputPath); // Respect the “flakeness” of the input even if we @@ -446,8 +463,8 @@ LockedFlake lockFlake( assert(input.ref); - /* Do we have an entry in the existing lock file? And we - don't have a --update-input flag for this input? */ + /* Do we have an entry in the existing lock file? + And the input is not in updateInputs? */ std::shared_ptr oldLock; updatesUsed.insert(inputPath); @@ -459,7 +476,7 @@ LockedFlake lockFlake( if (oldLock && oldLock->originalRef == *input.ref - && !hasOverride) + && !hasCliOverride) { debug("keeping existing input '%s'", inputPathS); @@ -471,9 +488,8 @@ LockedFlake lockFlake( node->inputs.insert_or_assign(id, childNode); - /* If we have an --update-input flag for an input - of this input, then we must fetch the flake to - update it. */ + /* If we have this input in updateInputs, then we + must fetch the flake to update it. */ auto lb = lockFlags.inputUpdates.lower_bound(inputPath); auto mustRefetch = @@ -519,11 +535,6 @@ LockedFlake lockFlake( } } - auto localPath(parentPath); - // If this input is a path, recurse it down. - // This allows us to resolve path inputs relative to the current flake. - if ((*input.ref).input.getType() == "path") - localPath = absPath(*input.ref->input.getSourcePath(), parentPath); computeLocks( mustRefetch ? getFlake(state, oldLock->lockedRef, false, flakeCache, inputPath).inputs @@ -545,7 +556,7 @@ LockedFlake lockFlake( nuked the next time we update the lock file. That is, overrides are sticky unless you use --no-write-lock-file. */ - auto ref = input2.ref ? *input2.ref : *input.ref; + auto ref = (input2.ref && explicitCliOverrides.contains(inputPath)) ? *input2.ref : *input.ref; if (input.isFlake) { Path localPath = parentPath; @@ -578,7 +589,7 @@ LockedFlake lockFlake( oldLock ? std::dynamic_pointer_cast(oldLock) : LockFile::read( - inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root.get_ptr(), + state.store->toRealPath(inputFlake.storePath) + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root.get_ptr(), oldLock ? lockRootPath : inputPath, localPath, false); @@ -602,7 +613,7 @@ LockedFlake lockFlake( }; // Bring in the current ref for relative path resolution if we have it - auto parentPath = canonPath(flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir, true); + auto parentPath = canonPath(state.store->toRealPath(flake.storePath) + "/" + flake.lockedRef.subdir, true); computeLocks( flake.inputs, @@ -620,19 +631,14 @@ LockedFlake lockFlake( for (auto & i : lockFlags.inputUpdates) if (!updatesUsed.count(i)) - warn("the flag '--update-input %s' does not match any input", printInputPath(i)); + warn("'%s' does not match any input of this flake", printInputPath(i)); /* Check 'follows' inputs. */ newLockFile.check(); debug("new lock file: %s", newLockFile); - auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; auto sourcePath = topRef.input.getSourcePath(); - auto outputLockFilePath = sourcePath ? std::optional{*sourcePath + "/" + relPath} : std::nullopt; - if (lockFlags.outputLockFilePath) { - outputLockFilePath = lockFlags.outputLockFilePath; - } /* Check whether we need to / can write the new lock file. */ if (newLockFile != oldLockFile || lockFlags.outputLockFilePath) { @@ -640,7 +646,7 @@ LockedFlake lockFlake( auto diff = LockFile::diff(oldLockFile, newLockFile); if (lockFlags.writeLockFile) { - if (outputLockFilePath) { + if (sourcePath || lockFlags.outputLockFilePath) { if (auto unlockedInput = newLockFile.isUnlocked()) { if (fetchSettings.warnDirty) warn("will not write lock file of flake '%s' because it has an unlocked input ('%s')", topRef, *unlockedInput); @@ -648,41 +654,48 @@ LockedFlake lockFlake( if (!lockFlags.updateLockFile) throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef); - bool lockFileExists = pathExists(*outputLockFilePath); + auto newLockFileS = fmt("%s\n", newLockFile); + + if (lockFlags.outputLockFilePath) { + if (lockFlags.commitLockFile) + throw Error("'--commit-lock-file' and '--output-lock-file' are incompatible"); + writeFile(*lockFlags.outputLockFilePath, newLockFileS); + } else { + auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; + auto outputLockFilePath = *sourcePath + "/" + relPath; + + bool lockFileExists = pathExists(outputLockFilePath); - if (lockFileExists) { auto s = chomp(diff); - if (s.empty()) - warn("updating lock file '%s'", *outputLockFilePath); - else - warn("updating lock file '%s':\n%s", *outputLockFilePath, s); - } else - warn("creating lock file '%s'", *outputLockFilePath); + if (lockFileExists) { + if (s.empty()) + warn("updating lock file '%s'", outputLockFilePath); + else + warn("updating lock file '%s':\n%s", outputLockFilePath, s); + } else + warn("creating lock file '%s': \n%s", outputLockFilePath, s); - newLockFile.write(*outputLockFilePath); + std::optional commitMessage = std::nullopt; - std::optional commitMessage = std::nullopt; - if (lockFlags.commitLockFile) { - if (lockFlags.outputLockFilePath) { - throw Error("--commit-lock-file and --output-lock-file are currently incompatible"); - } - std::string cm; + if (lockFlags.commitLockFile) { + std::string cm; - cm = fetchSettings.commitLockFileSummary.get(); + cm = fetchSettings.commitLockFileSummary.get(); - if (cm == "") { - cm = fmt("%s: %s", relPath, lockFileExists ? "Update" : "Add"); + if (cm == "") { + cm = fmt("%s: %s", relPath, lockFileExists ? "Update" : "Add"); + } + + cm += "\n\nFlake lock file updates:\n\n"; + cm += filterANSIEscapes(diff, true); + commitMessage = cm; } - cm += "\n\nFlake lock file updates:\n\n"; - cm += filterANSIEscapes(diff, true); - commitMessage = cm; + topRef.input.putFile( + CanonPath((topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"), + newLockFileS, commitMessage); } - topRef.input.markChangedFile( - (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock", - commitMessage); - /* Rewriting the lockfile changed the top-level repo, so we should re-read it. FIXME: we could also just clear the 'rev' field... */ @@ -733,7 +746,7 @@ void callFlake(EvalState & state, emitTreeAttrs( state, - *lockedFlake.flake.sourceInfo, + lockedFlake.flake.storePath, lockedFlake.flake.lockedRef.input, *vRootSrc, false, @@ -741,14 +754,10 @@ void callFlake(EvalState & state, vRootSubdir->mkString(lockedFlake.flake.lockedRef.subdir); - if (!state.vCallFlake) { - state.vCallFlake = allocRootValue(state.allocValue()); - state.eval(state.parseExprFromString( - #include "call-flake.nix.gen.hh" - , CanonPath::root), **state.vCallFlake); - } + auto vCallFlake = state.allocValue(); + state.evalFile(state.callFlakeInternal, *vCallFlake); - state.callFunction(**state.vCallFlake, *vLocks, *vTmp1, noPos); + state.callFunction(*vCallFlake, *vLocks, *vTmp1, noPos); state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos); state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos); } @@ -793,6 +802,101 @@ static RegisterPrimOp r2({ .experimentalFeature = Xp::Flakes, }); +static void prim_parseFlakeRef( + EvalState & state, + const PosIdx pos, + Value * * args, + Value & v) +{ + std::string flakeRefS(state.forceStringNoCtx(*args[0], pos, + "while evaluating the argument passed to builtins.parseFlakeRef")); + auto attrs = parseFlakeRef(flakeRefS, {}, true).toAttrs(); + auto binds = state.buildBindings(attrs.size()); + for (const auto & [key, value] : attrs) { + auto s = state.symbols.create(key); + auto & vv = binds.alloc(s); + std::visit(overloaded { + [&vv](const std::string & value) { vv.mkString(value); }, + [&vv](const uint64_t & value) { vv.mkInt(value); }, + [&vv](const Explicit & value) { vv.mkBool(value.t); } + }, value); + } + v.mkAttrs(binds); +} + +static RegisterPrimOp r3({ + .name = "__parseFlakeRef", + .args = {"flake-ref"}, + .doc = R"( + Parse a flake reference, and return its exploded form. + + For example: + ```nix + builtins.parseFlakeRef "github:NixOS/nixpkgs/23.05?dir=lib" + ``` + evaluates to: + ```nix + { dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github"; } + ``` + )", + .fun = prim_parseFlakeRef, + .experimentalFeature = Xp::Flakes, +}); + + +static void prim_flakeRefToString( + EvalState & state, + const PosIdx pos, + Value * * args, + Value & v) +{ + state.forceAttrs(*args[0], noPos, + "while evaluating the argument passed to builtins.flakeRefToString"); + fetchers::Attrs attrs; + for (const auto & attr : *args[0]->attrs) { + auto t = attr.value->type(); + if (t == nInt) { + attrs.emplace(state.symbols[attr.name], + (uint64_t) attr.value->integer); + } else if (t == nBool) { + attrs.emplace(state.symbols[attr.name], + Explicit { attr.value->boolean }); + } else if (t == nString) { + attrs.emplace(state.symbols[attr.name], + std::string(attr.value->string_view())); + } else { + state.error( + "flake reference attribute sets may only contain integers, Booleans, " + "and strings, but attribute '%s' is %s", + state.symbols[attr.name], + showType(*attr.value)).debugThrow(); + } + } + auto flakeRef = FlakeRef::fromAttrs(attrs); + v.mkString(flakeRef.to_string()); +} + +static RegisterPrimOp r4({ + .name = "__flakeRefToString", + .args = {"attrs"}, + .doc = R"( + Convert a flake reference from attribute set format to URL format. + + For example: + ```nix + builtins.flakeRefToString { + dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github"; + } + ``` + evaluates to + ```nix + "github:NixOS/nixpkgs/23.05?dir=lib" + ``` + )", + .fun = prim_flakeRefToString, + .experimentalFeature = Xp::Flakes, +}); + } Fingerprint LockedFlake::getFingerprint() const @@ -802,7 +906,7 @@ Fingerprint LockedFlake::getFingerprint() const // flake.sourceInfo.storePath for the fingerprint. return hashString(htSHA256, fmt("%s;%s;%d;%d;%s", - flake.sourceInfo->storePath.to_string(), + flake.storePath.to_string(), flake.lockedRef.subdir, flake.lockedRef.input.getRevCount().value_or(0), flake.lockedRef.input.getLastModified().value_or(0), diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index c1d1b71e5096..d5ad3eaded8e 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -10,8 +10,6 @@ namespace nix { class EvalState; -namespace fetchers { struct Tree; } - namespace flake { struct FlakeInput; @@ -84,7 +82,7 @@ struct Flake */ bool forceDirty = false; std::optional description; - std::shared_ptr sourceInfo; + StorePath storePath; FlakeInputs inputs; /** * 'nixConfig' attribute @@ -193,7 +191,7 @@ void callFlake( void emitTreeAttrs( EvalState & state, - const fetchers::Tree & tree, + const StorePath & storePath, const fetchers::Input & input, Value & v, bool emptyRevFallback = false, diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 08adbe0c99d7..16f45ace77cb 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -69,32 +69,130 @@ std::optional maybeParseFlakeRef( } } -std::pair parseFlakeRefWithFragment( +std::pair parsePathFlakeRefWithFragment( const std::string & url, const std::optional & baseDir, bool allowMissing, bool isFlake) { - using namespace fetchers; + std::string path = url; + std::string fragment = ""; + std::map query; + auto pathEnd = url.find_first_of("#?"); + auto fragmentStart = pathEnd; + if (pathEnd != std::string::npos && url[pathEnd] == '?') { + fragmentStart = url.find("#"); + } + if (pathEnd != std::string::npos) { + path = url.substr(0, pathEnd); + } + if (fragmentStart != std::string::npos) { + fragment = percentDecode(url.substr(fragmentStart+1)); + } + if (pathEnd != std::string::npos && fragmentStart != std::string::npos) { + query = decodeQuery(url.substr(pathEnd+1, fragmentStart)); + } - static std::string fnRegex = "[0-9a-zA-Z-._~!$&'\"()*+,;=]+"; + if (baseDir) { + /* Check if 'url' is a path (either absolute or relative + to 'baseDir'). If so, search upward to the root of the + repo (i.e. the directory containing .git). */ + + path = absPath(path, baseDir); + + if (isFlake) { + + if (!allowMissing && !pathExists(path + "/flake.nix")){ + notice("path '%s' does not contain a 'flake.nix', searching up",path); + + // Save device to detect filesystem boundary + dev_t device = lstat(path).st_dev; + bool found = false; + while (path != "/") { + if (pathExists(path + "/flake.nix")) { + found = true; + break; + } else if (pathExists(path + "/.git")) + throw Error("path '%s' is not part of a flake (neither it nor its parent directories contain a 'flake.nix' file)", path); + else { + if (lstat(path).st_dev != device) + throw Error("unable to find a flake before encountering filesystem boundary at '%s'", path); + } + path = dirOf(path); + } + if (!found) + throw BadURL("could not find a flake.nix file"); + } - static std::regex pathUrlRegex( - "(/?" + fnRegex + "(?:/" + fnRegex + ")*/?)" - + "(?:\\?(" + queryRegex + "))?" - + "(?:#(" + queryRegex + "))?", - std::regex::ECMAScript); + if (!S_ISDIR(lstat(path).st_mode)) + throw BadURL("path '%s' is not a flake (because it's not a directory)", path); + + if (!allowMissing && !pathExists(path + "/flake.nix")) + throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path); + + auto flakeRoot = path; + std::string subdir; + + while (flakeRoot != "/") { + if (pathExists(flakeRoot + "/.git")) { + auto base = std::string("git+file://") + flakeRoot; + + auto parsedURL = ParsedURL{ + .url = base, // FIXME + .base = base, + .scheme = "git+file", + .authority = "", + .path = flakeRoot, + .query = query, + }; + + if (subdir != "") { + if (parsedURL.query.count("dir")) + throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url); + parsedURL.query.insert_or_assign("dir", subdir); + } + + if (pathExists(flakeRoot + "/.git/shallow")) + parsedURL.query.insert_or_assign("shallow", "1"); + + return std::make_pair( + FlakeRef(fetchers::Input::fromURL(parsedURL), getOr(parsedURL.query, "dir", "")), + fragment); + } + + subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir); + flakeRoot = dirOf(flakeRoot); + } + } + + } else { + if (!hasPrefix(path, "/")) + throw BadURL("flake reference '%s' is not an absolute path", url); + path = canonPath(path + "/" + getOr(query, "dir", "")); + } + + fetchers::Attrs attrs; + attrs.insert_or_assign("type", "path"); + attrs.insert_or_assign("path", path); + + return std::make_pair(FlakeRef(fetchers::Input::fromAttrs(std::move(attrs)), ""), fragment); +}; + + +/* Check if 'url' is a flake ID. This is an abbreviated syntax for + 'flake:?ref=&rev='. */ +std::optional> parseFlakeIdRef( + const std::string & url, + bool isFlake +) +{ + std::smatch match; static std::regex flakeRegex( "((" + flakeIdRegexS + ")(?:/(?:" + refAndOrRevRegex + "))?)" + "(?:#(" + queryRegex + "))?", std::regex::ECMAScript); - std::smatch match; - - /* Check if 'url' is a flake ID. This is an abbreviated syntax for - 'flake:?ref=&rev='. */ - if (std::regex_match(url, match, flakeRegex)) { auto parsedURL = ParsedURL{ .url = url, @@ -105,111 +203,53 @@ std::pair parseFlakeRefWithFragment( }; return std::make_pair( - FlakeRef(Input::fromURL(parsedURL), ""), + FlakeRef(fetchers::Input::fromURL(parsedURL, isFlake), ""), percentDecode(match.str(6))); } - else if (std::regex_match(url, match, pathUrlRegex)) { - std::string path = match[1]; - std::string fragment = percentDecode(match.str(3)); - - if (baseDir) { - /* Check if 'url' is a path (either absolute or relative - to 'baseDir'). If so, search upward to the root of the - repo (i.e. the directory containing .git). */ - - path = absPath(path, baseDir); - - if (isFlake) { - - if (!allowMissing && !pathExists(path + "/flake.nix")){ - notice("path '%s' does not contain a 'flake.nix', searching up",path); - - // Save device to detect filesystem boundary - dev_t device = lstat(path).st_dev; - bool found = false; - while (path != "/") { - if (pathExists(path + "/flake.nix")) { - found = true; - break; - } else if (pathExists(path + "/.git")) - throw Error("path '%s' is not part of a flake (neither it nor its parent directories contain a 'flake.nix' file)", path); - else { - if (lstat(path).st_dev != device) - throw Error("unable to find a flake before encountering filesystem boundary at '%s'", path); - } - path = dirOf(path); - } - if (!found) - throw BadURL("could not find a flake.nix file"); - } - - if (!S_ISDIR(lstat(path).st_mode)) - throw BadURL("path '%s' is not a flake (because it's not a directory)", path); - - if (!allowMissing && !pathExists(path + "/flake.nix")) - throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path); - - auto flakeRoot = path; - std::string subdir; - - while (flakeRoot != "/") { - if (pathExists(flakeRoot + "/.git")) { - auto base = std::string("git+file://") + flakeRoot; - - auto parsedURL = ParsedURL{ - .url = base, // FIXME - .base = base, - .scheme = "git+file", - .authority = "", - .path = flakeRoot, - .query = decodeQuery(match[2]), - }; - - if (subdir != "") { - if (parsedURL.query.count("dir")) - throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url); - parsedURL.query.insert_or_assign("dir", subdir); - } - - if (pathExists(flakeRoot + "/.git/shallow")) - parsedURL.query.insert_or_assign("shallow", "1"); - - return std::make_pair( - FlakeRef(Input::fromURL(parsedURL), getOr(parsedURL.query, "dir", "")), - fragment); - } + return {}; +} - subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir); - flakeRoot = dirOf(flakeRoot); - } - } +std::optional> parseURLFlakeRef( + const std::string & url, + const std::optional & baseDir, + bool isFlake +) +{ + ParsedURL parsedURL; + try { + parsedURL = parseURL(url); + } catch (BadURL &) { + return std::nullopt; + } - } else { - if (!hasPrefix(path, "/")) - throw BadURL("flake reference '%s' is not an absolute path", url); - auto query = decodeQuery(match[2]); - path = canonPath(path + "/" + getOr(query, "dir", "")); - } + std::string fragment; + std::swap(fragment, parsedURL.fragment); - fetchers::Attrs attrs; - attrs.insert_or_assign("type", "path"); - attrs.insert_or_assign("path", path); + auto input = fetchers::Input::fromURL(parsedURL, isFlake); + input.parent = baseDir; - return std::make_pair(FlakeRef(Input::fromAttrs(std::move(attrs)), ""), fragment); - } + return std::make_pair( + FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")), + fragment); +} - else { - auto parsedURL = parseURL(url); - std::string fragment; - std::swap(fragment, parsedURL.fragment); +std::pair parseFlakeRefWithFragment( + const std::string & url, + const std::optional & baseDir, + bool allowMissing, + bool isFlake) +{ + using namespace fetchers; - auto input = Input::fromURL(parsedURL); - input.parent = baseDir; + std::smatch match; - return std::make_pair( - FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")), - fragment); + if (auto res = parseFlakeIdRef(url, isFlake)) { + return *res; + } else if (auto res = parseURLFlakeRef(url, baseDir, isFlake)) { + return *res; + } else { + return parsePathFlakeRefWithFragment(url, baseDir, allowMissing, isFlake); } } @@ -232,10 +272,10 @@ FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs) fetchers::maybeGetStrAttr(attrs, "dir").value_or("")); } -std::pair FlakeRef::fetchTree(ref store) const +std::pair FlakeRef::fetchTree(ref store) const { - auto [tree, lockedInput] = input.fetch(store); - return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)}; + auto [storePath, lockedInput] = input.fetch(store); + return {std::move(storePath), FlakeRef(std::move(lockedInput), subdir)}; } std::tuple parseFlakeRefWithFragmentAndExtendedOutputsSpec( @@ -246,7 +286,9 @@ std::tuple parseFlakeRefWithFragment { auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(url); auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, baseDir, allowMissing, isFlake); - return {std::move(flakeRef), fragment, extendedOutputsSpec}; + return {std::move(flakeRef), fragment, std::move(extendedOutputsSpec)}; } +std::regex flakeIdRegex(flakeIdRegexS, std::regex::ECMAScript); + } diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index a7c9208c06e4..5d78f49b6834 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -6,6 +6,7 @@ #include "fetchers.hh" #include "outputs-spec.hh" +#include #include namespace nix { @@ -62,7 +63,7 @@ struct FlakeRef static FlakeRef fromAttrs(const fetchers::Attrs & attrs); - std::pair fetchTree(ref store) const; + std::pair fetchTree(ref store) const; }; std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef); @@ -91,5 +92,7 @@ std::tuple parseFlakeRefWithFragment bool allowMissing = false, bool isFlake = true); +const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*"; +extern std::regex flakeIdRegex; } diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index ba2fd46f0b9a..3e99fb2d417f 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -2,8 +2,10 @@ #include "store-api.hh" #include "url-parts.hh" +#include #include +#include #include namespace nix::flake { @@ -45,16 +47,26 @@ StorePath LockedNode::computeStorePath(Store & store) const return lockedRef.input.computeStorePath(store); } -std::shared_ptr LockFile::findInput(const InputPath & path) -{ + +static std::shared_ptr doFind(const ref& root, const InputPath & path, std::vector& visited) { auto pos = root; + auto found = std::find(visited.cbegin(), visited.cend(), path); + + if(found != visited.end()) { + std::vector cycle; + std::transform(found, visited.cend(), std::back_inserter(cycle), printInputPath); + cycle.push_back(printInputPath(path)); + throw Error("follow cycle detected: [%s]", concatStringsSep(" -> ", cycle)); + } + visited.push_back(path); + for (auto & elem : path) { if (auto i = get(pos->inputs, elem)) { if (auto node = std::get_if<0>(&*i)) pos = *node; else if (auto follows = std::get_if<1>(&*i)) { - if (auto p = findInput(*follows)) + if (auto p = doFind(root, *follows, visited)) pos = ref(p); else return {}; @@ -66,6 +78,12 @@ std::shared_ptr LockFile::findInput(const InputPath & path) return pos; } +std::shared_ptr LockFile::findInput(const InputPath & path) +{ + std::vector visited; + return doFind(root, path, visited); +} + LockFile::LockFile(const nlohmann::json & json, const Path & path) { auto version = json.value("version", 0); @@ -196,12 +214,6 @@ std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile) return stream; } -void LockFile::write(const Path & path) const -{ - createDirs(dirOf(path)); - writeFile(path, fmt("%s\n", *this)); -} - std::optional LockFile::isUnlocked() const { std::set> nodes; @@ -345,7 +357,7 @@ void LockFile::check() for (auto & [inputPath, input] : inputs) { if (auto follows = std::get_if<1>(&input)) { - if (!follows->empty() && !get(inputs, *follows)) + if (!follows->empty() && !findInput(*follows)) throw Error("input '%s' follows a non-existent input '%s'", printInputPath(inputPath), printInputPath(*follows)); diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index ba4c0c8485c0..5a1493404d6f 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -65,8 +65,6 @@ struct LockFile static LockFile read(const Path & path); - void write(const Path & path) const; - /** * Check whether this lock file has any unlocked inputs. */ diff --git a/src/libexpr/gc-small-vector.hh b/src/libexpr/gc-small-vector.hh new file mode 100644 index 000000000000..7f4f08fc7534 --- /dev/null +++ b/src/libexpr/gc-small-vector.hh @@ -0,0 +1,42 @@ +#pragma once + +#include + +#if HAVE_BOEHMGC + +#include +#include +#include + +#endif + +namespace nix { + +struct Value; + +/** + * A GC compatible vector that may used a reserved portion of `nItems` on the stack instead of allocating on the heap. + */ +#if HAVE_BOEHMGC +template +using SmallVector = boost::container::small_vector>; +#else +template +using SmallVector = boost::container::small_vector; +#endif + +/** + * A vector of value pointers. See `SmallVector`. + */ +template +using SmallValueVector = SmallVector; + +/** + * A vector of values that must not be referenced after the vector is destroyed. + * + * See also `SmallValueVector`. + */ +template +using SmallTemporaryValueVector = SmallVector; + +} \ No newline at end of file diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 506a63677638..d4e946d8104e 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -1,5 +1,4 @@ #include "get-drvs.hh" -#include "util.hh" #include "eval-inline.hh" #include "derivations.hh" #include "store-api.hh" @@ -156,7 +155,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall Outputs result; for (auto elem : outTI->listItems()) { if (elem->type() != nString) throw errMsg; - auto out = outputs.find(elem->string.s); + auto out = outputs.find(elem->c_str()); if (out == outputs.end()) throw errMsg; result.insert(*out); } @@ -230,7 +229,7 @@ std::string DrvInfo::queryMetaString(const std::string & name) { Value * v = queryMeta(name); if (!v || v->type() != nString) return ""; - return v->string.s; + return v->c_str(); } @@ -242,7 +241,7 @@ NixInt DrvInfo::queryMetaInt(const std::string & name, NixInt def) if (v->type() == nString) { /* Backwards compatibility with before we had support for integer meta fields. */ - if (auto n = string2Int(v->string.s)) + if (auto n = string2Int(v->c_str())) return *n; } return def; @@ -256,7 +255,7 @@ NixFloat DrvInfo::queryMetaFloat(const std::string & name, NixFloat def) if (v->type() == nString) { /* Backwards compatibility with before we had support for float meta fields. */ - if (auto n = string2Float(v->string.s)) + if (auto n = string2Float(v->c_str())) return *n; } return def; @@ -271,8 +270,8 @@ bool DrvInfo::queryMetaBool(const std::string & name, bool def) if (v->type() == nString) { /* Backwards compatibility with before we had support for Boolean meta fields. */ - if (strcmp(v->string.s, "true") == 0) return true; - if (strcmp(v->string.s, "false") == 0) return false; + if (v->string_view() == "true") return true; + if (v->string_view() == "false") return false; } return def; } diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index d243b9cec1d1..ed6bc761a039 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -36,15 +36,15 @@ $(d)/lexer-tab.cc $(d)/lexer-tab.hh: $(d)/lexer.l clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh -$(eval $(call install-file-in, $(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644)) +$(eval $(call install-file-in, $(buildprefix)$(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644)) $(foreach i, $(wildcard src/libexpr/value/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/value, 0644))) $(foreach i, $(wildcard src/libexpr/flake/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644))) -$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh +$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh -$(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh +$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/flake/call-flake.nix.gen.hh -src/libexpr/primops/fromTOML.o: ERROR_SWITCH_ENUM = +$(buildprefix)src/libexpr/primops/fromTOML.o: ERROR_SWITCH_ENUM = diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 4566a13887dc..22be8e68c55e 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -76,12 +76,12 @@ void Expr::show(const SymbolTable & symbols, std::ostream & str) const void ExprInt::show(const SymbolTable & symbols, std::ostream & str) const { - str << n; + str << v.integer; } void ExprFloat::show(const SymbolTable & symbols, std::ostream & str) const { - str << nf; + str << v.fpoint; } void ExprString::show(const SymbolTable & symbols, std::ostream & str) const diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 5ca3d1fa6c76..10099d49e9b5 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -20,7 +20,6 @@ MakeError(Abort, EvalError); MakeError(TypeError, EvalError); MakeError(UndefinedVarError, Error); MakeError(MissingArgumentError, EvalError); -MakeError(RestrictedPathError, Error); /** * Position objects. @@ -155,6 +154,10 @@ std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath) struct Expr { + static unsigned long nrExprs; + Expr() { + nrExprs++; + } virtual ~Expr() { }; virtual void show(const SymbolTable & symbols, std::ostream & str) const; virtual void bindVars(EvalState & es, const std::shared_ptr & env); @@ -171,18 +174,16 @@ struct Expr struct ExprInt : Expr { - NixInt n; Value v; - ExprInt(NixInt n) : n(n) { v.mkInt(n); }; + ExprInt(NixInt n) { v.mkInt(n); }; Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS }; struct ExprFloat : Expr { - NixFloat nf; Value v; - ExprFloat(NixFloat nf) : nf(nf) { v.mkFloat(nf); }; + ExprFloat(NixFloat nf) { v.mkFloat(nf); }; Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS }; @@ -198,9 +199,13 @@ struct ExprString : Expr struct ExprPath : Expr { + ref accessor; std::string s; Value v; - ExprPath(std::string s) : s(std::move(s)) { v.mkPath(this->s.c_str()); }; + ExprPath(ref accessor, std::string s) : accessor(accessor), s(std::move(s)) + { + v.mkPath(&*accessor, this->s.c_str()); + } Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS }; @@ -238,7 +243,7 @@ struct ExprSelect : Expr PosIdx pos; Expr * e, * def; AttrPath attrPath; - ExprSelect(const PosIdx & pos, Expr * e, const AttrPath && attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(std::move(attrPath)) { }; + ExprSelect(const PosIdx & pos, Expr * e, AttrPath attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(std::move(attrPath)) { }; ExprSelect(const PosIdx & pos, Expr * e, Symbol name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); }; PosIdx getPos() const override { return pos; } COMMON_METHODS @@ -248,7 +253,7 @@ struct ExprOpHasAttr : Expr { Expr * e; AttrPath attrPath; - ExprOpHasAttr(Expr * e, const AttrPath && attrPath) : e(e), attrPath(std::move(attrPath)) { }; + ExprOpHasAttr(Expr * e, AttrPath attrPath) : e(e), attrPath(std::move(attrPath)) { }; PosIdx getPos() const override { return e->getPos(); } COMMON_METHODS }; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 0d0004f9f7f8..f6cf1f6893bc 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -19,9 +19,11 @@ #include #include "util.hh" +#include "users.hh" #include "nixexpr.hh" #include "eval.hh" +#include "eval-settings.hh" #include "globals.hh" namespace nix { @@ -137,6 +139,7 @@ static void addAttr(ExprAttrs * attrs, AttrPath && attrPath, dupAttr(state, ad.first, j2->second.pos, ad.second.pos); jAttrs->attrs.emplace(ad.first, ad.second); } + jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(), ae->dynamicAttrs.begin(), ae->dynamicAttrs.end()); } else { dupAttr(state, attrPath, pos, j->second.pos); } @@ -518,7 +521,7 @@ path_start /* add back in the trailing '/' to the first segment */ if ($1.p[$1.l-1] == '/' && $1.l > 1) path += "/"; - $$ = new ExprPath(path); + $$ = new ExprPath(ref(data->state.rootFS), std::move(path)); } | HPATH { if (evalSettings.pureEval) { @@ -528,7 +531,7 @@ path_start ); } Path path(getHome() + std::string($1.p + 1, $1.l - 1)); - $$ = new ExprPath(path); + $$ = new ExprPath(ref(data->state.rootFS), std::move(path)); } ; @@ -644,13 +647,16 @@ formal #include "eval.hh" #include "filetransfer.hh" -#include "fetchers.hh" +#include "tarball.hh" #include "store-api.hh" #include "flake/flake.hh" +#include "fs-input-accessor.hh" +#include "memory-input-accessor.hh" namespace nix { +unsigned long Expr::nrExprs = 0; Expr * EvalState::parse( char * text, @@ -663,7 +669,7 @@ Expr * EvalState::parse( ParseData data { .state = *this, .symbols = symbols, - .basePath = std::move(basePath), + .basePath = basePath, .origin = {origin}, }; @@ -680,17 +686,25 @@ Expr * EvalState::parse( } -SourcePath resolveExprPath(const SourcePath & path) +SourcePath resolveExprPath(SourcePath path) { + unsigned int followCount = 0, maxFollow = 1024; + /* If `path' is a symlink, follow it. This is so that relative path references work. */ - auto path2 = path.resolveSymlinks(); + while (true) { + // Basic cycle/depth limit to avoid infinite loops. + if (++followCount >= maxFollow) + throw Error("too many symbolic links encountered while traversing the path '%s'", path); + if (path.lstat().type != InputAccessor::tSymlink) break; + path = {path.accessor, CanonPath(path.readLink(), path.path.parent().value_or(CanonPath::root))}; + } /* If `path' refers to a directory, append `/default.nix'. */ - if (path2.lstat().type == InputAccessor::tDirectory) - return path2 + "default.nix"; + if (path.lstat().type == InputAccessor::tDirectory) + return path + "default.nix"; - return path2; + return path; } @@ -734,52 +748,30 @@ Expr * EvalState::parseStdin() } -void EvalState::addToSearchPath(const std::string & s) -{ - size_t pos = s.find('='); - std::string prefix; - Path path; - if (pos == std::string::npos) { - path = s; - } else { - prefix = std::string(s, 0, pos); - path = std::string(s, pos + 1); - } - - searchPath.emplace_back(SearchPathElem { - .prefix = prefix, - .path = path, - }); -} - - SourcePath EvalState::findFile(const std::string_view path) { return findFile(searchPath, path); } -SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos) +SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos) { - for (auto & i : searchPath) { - std::string suffix; - if (i.prefix.empty()) - suffix = concatStrings("/", path); - else { - auto s = i.prefix.size(); - if (path.compare(0, s, i.prefix) != 0 || - (path.size() > s && path[s] != '/')) - continue; - suffix = path.size() == s ? "" : concatStrings("/", path.substr(s)); - } - auto r = resolveSearchPathElem(i); - if (!r.first) continue; - Path res = r.second + suffix; - if (pathExists(res)) return CanonPath(canonPath(res)); + for (auto & i : searchPath.elements) { + auto suffixOpt = i.prefix.suffixIfPotentialMatch(path); + + if (!suffixOpt) continue; + auto suffix = *suffixOpt; + + auto rOpt = resolveSearchPathPath(i.path); + if (!rOpt) continue; + auto r = *rOpt; + + Path res = suffix == "" ? r : concatStrings(r, "/", suffix); + if (pathExists(res)) return rootPath(CanonPath(canonPath(res))); } if (hasPrefix(path, "nix/")) - return CanonPath(concatStrings(corepkgsPrefix, path.substr(4))); + return {corepkgsFS, CanonPath(path.substr(3))}; debugThrow(ThrownError({ .msg = hintfmt(evalSettings.pureEval @@ -791,49 +783,53 @@ SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view p } -std::pair EvalState::resolveSearchPathElem(const SearchPathElem & elem) +std::optional EvalState::resolveSearchPathPath(const SearchPath::Path & value0) { - auto i = searchPathResolved.find(elem.path); + auto & value = value0.s; + auto i = searchPathResolved.find(value); if (i != searchPathResolved.end()) return i->second; - std::pair res; + std::optional res; - if (EvalSettings::isPseudoUrl(elem.path)) { + if (EvalSettings::isPseudoUrl(value)) { try { auto storePath = fetchers::downloadTarball( - store, EvalSettings::resolvePseudoUrl(elem.path), "source", false).tree.storePath; - res = { true, store->toRealPath(storePath) }; + store, EvalSettings::resolvePseudoUrl(value), "source", false).storePath; + res = { store->toRealPath(storePath) }; } catch (FileTransferError & e) { logWarning({ - .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.path) + .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value) }); - res = { false, "" }; + res = std::nullopt; } } - else if (hasPrefix(elem.path, "flake:")) { + else if (hasPrefix(value, "flake:")) { experimentalFeatureSettings.require(Xp::Flakes); - auto flakeRef = parseFlakeRef(elem.path.substr(6), {}, true, false); - debug("fetching flake search path element '%s''", elem.path); - auto storePath = flakeRef.resolve(store).fetchTree(store).first.storePath; - res = { true, store->toRealPath(storePath) }; + auto flakeRef = parseFlakeRef(value.substr(6), {}, true, false); + debug("fetching flake search path element '%s''", value); + auto storePath = flakeRef.resolve(store).fetchTree(store).first; + res = { store->toRealPath(storePath) }; } else { - auto path = absPath(elem.path); + auto path = absPath(value); if (pathExists(path)) - res = { true, path }; + res = { path }; else { logWarning({ - .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", elem.path) + .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", value) }); - res = { false, "" }; + res = std::nullopt; } } - debug("resolved search path element '%s' to '%s'", elem.path, res.second); + if (res) + debug("resolved search path element '%s' to '%s'", value, *res); + else + debug("failed to resolve search path element '%s'", value); - searchPathResolved[elem.path] = res; + searchPathResolved[value] = res; return res; } diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index 1d690b722101..099607638e3b 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -1,10 +1,11 @@ #include "eval.hh" +#include "fs-input-accessor.hh" namespace nix { SourcePath EvalState::rootPath(CanonPath path) { - return std::move(path); + return {rootFS, std::move(path)}; } } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index f07870715bd9..0893f8a562df 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -5,6 +5,8 @@ #include "downstream-placeholder.hh" #include "eval-inline.hh" #include "eval.hh" +#include "eval-settings.hh" +#include "gc-small-vector.hh" #include "globals.hh" #include "granular-access-store.hh" #include "json-to-value.hh" @@ -12,6 +14,7 @@ #include "path-references.hh" #include "store-api.hh" #include "util.hh" +#include "processes.hh" #include "value-to-json.hh" #include "value-to-xml.hh" #include "primops.hh" @@ -33,7 +36,6 @@ #include - namespace nix { @@ -61,7 +63,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context) .drvPath = b.drvPath, .outputs = OutputsSpec::Names { b.output }, }); - ensureValid(b.drvPath); + ensureValid(b.drvPath->getBaseStorePath()); }, [&](const NixStringContextElem::Opaque & o) { auto ctxS = store->printStorePath(o.path); @@ -74,7 +76,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context) res.insert_or_assign(ctxS, ctxS); ensureValid(d.drvPath); }, - }, c.raw()); + }, c.raw); } if (drvs.empty()) return {}; @@ -82,29 +84,32 @@ StringMap EvalState::realiseContext(const NixStringContext & context) if (!evalSettings.enableImportFromDerivation) debugThrowLastTrace(Error( "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled", - store->printStorePath(drvs.begin()->drvPath))); + drvs.begin()->to_string(*store))); /* Build/substitute the context. */ std::vector buildReqs; for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d }); store->buildPaths(buildReqs); - /* Get all the output paths corresponding to the placeholders we had */ for (auto & drv : drvs) { auto outputs = resolveDerivedPath(*store, drv); for (auto & [outputName, outputPath] : outputs) { - res.insert_or_assign( - DownstreamPlaceholder::unknownCaOutput(drv.drvPath, outputName).render(), - store->printStorePath(outputPath) - ); - } - } - - /* Add the output of this derivations to the allowed - paths. */ - if (allowedPaths) { - for (auto & [_placeholder, outputPath] : res) { - allowPath(store->toRealPath(outputPath)); + /* Add the output of this derivations to the allowed + paths. */ + if (allowedPaths) { + allowPath(outputPath); + } + /* Get all the output paths corresponding to the placeholders we had */ + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + res.insert_or_assign( + DownstreamPlaceholder::fromSingleDerivedPathBuilt( + SingleDerivedPath::Built { + .drvPath = drv.drvPath, + .output = outputName, + }).render(), + store->printStorePath(outputPath) + ); + } } } @@ -123,13 +128,15 @@ static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, co auto path = state.coerceToPath(noPos, v, context, "while realising the context of a path"); try { - StringMap rewrites = state.realiseContext(context); - - auto realPath = state.rootPath(CanonPath(state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context))); + if (!context.empty()) { + auto rewrites = state.realiseContext(context); + auto realPath = state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context); + return {path.accessor, CanonPath(realPath)}; + } return flags.checkForPureEval - ? state.checkSourcePath(realPath) - : realPath; + ? state.checkSourcePath(path) + : path; } catch (Error & e) { e.addTrace(state.positions[pos], "while realising the context of path '%s'", path); throw; @@ -202,8 +209,10 @@ static void mkOutputString( { state.mkOutputString( attrs.alloc(o.first), - drvPath, - o.first, + SingleDerivedPath::Built { + .drvPath = makeConstantStorePathRef(drvPath), + .output = o.first, + }, o.second.path(*state.store, Derivation::nameFromPath(drvPath), o.first)); } @@ -246,7 +255,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v state.vImportedDrvToDerivation = allocRootValue(state.allocValue()); state.eval(state.parseExprFromString( #include "imported-drv-to-derivation.nix.gen.hh" - , CanonPath::root), **state.vImportedDrvToDerivation); + , state.rootPath(CanonPath::root)), **state.vImportedDrvToDerivation); } state.forceFunction(**state.vImportedDrvToDerivation, pos, "while evaluating imported-drv-to-derivation.nix.gen.hh"); @@ -254,12 +263,6 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix.gen.hh"); } - else if (path2 == corepkgsPrefix + "fetchurl.nix") { - state.eval(state.parseExprFromString( - #include "fetchurl.nix.gen.hh" - , CanonPath::root), v); - } - else { if (!vScope) state.evalFile(path, v); @@ -302,64 +305,71 @@ static RegisterPrimOp primop_import({ .args = {"path"}, // TODO turn "normal path values" into link below .doc = R"( - Load, parse and return the Nix expression in the file *path*. - - The value *path* can be a path, a string, or an attribute set with an - `__toString` attribute or a `outPath` attribute (as derivations or flake - inputs typically have). - - If *path* is a directory, the file `default.nix` in that directory - is loaded. - - Evaluation aborts if the file doesn’t exist or contains - an incorrect Nix expression. `import` implements Nix’s module - system: you can put any Nix expression (such as a set or a - function) in a separate file, and use it from Nix expressions in - other files. + Load, parse, and return the Nix expression in the file *path*. > **Note** > > Unlike some languages, `import` is a regular function in Nix. - > Paths using the angle bracket syntax (e.g., `import` *\*) - > are normal [path values](@docroot@/language/values.md#type-path). - A Nix expression loaded by `import` must not contain any *free - variables* (identifiers that are not defined in the Nix expression - itself and are not built-in). Therefore, it cannot refer to - variables that are in scope at the call site. For instance, if you - have a calling expression + The *path* argument must meet the same criteria as an [interpolated expression](@docroot@/language/string-interpolation.md#interpolated-expression). - ```nix - rec { - x = 123; - y = import ./foo.nix; - } - ``` + If *path* is a directory, the file `default.nix` in that directory is used if it exists. - then the following `foo.nix` will give an error: - - ```nix - x + 456 - ``` - - since `x` is not in scope in `foo.nix`. If you want `x` to be - available in `foo.nix`, you should pass it as a function argument: - - ```nix - rec { - x = 123; - y = import ./foo.nix x; - } - ``` + > **Example** + > + > ```console + > $ echo 123 > default.nix + > ``` + > + > Import `default.nix` from the current directory. + > + > ```nix + > import ./. + > ``` + > + > 123 - and + Evaluation aborts if the file doesn’t exist or contains an invalid Nix expression. - ```nix - x: x + 456 - ``` + A Nix expression loaded by `import` must not contain any *free variables*, that is, identifiers that are not defined in the Nix expression itself and are not built-in. + Therefore, it cannot refer to variables that are in scope at the call site. - (The function argument doesn’t have to be called `x` in `foo.nix`; - any name would work.) + > **Example** + > + > If you have a calling expression + > + > ```nix + > rec { + > x = 123; + > y = import ./foo.nix; + > } + > ``` + > + > then the following `foo.nix` will give an error: + > + > ```nix + > # foo.nix + > x + 456 + > ``` + > + > since `x` is not in scope in `foo.nix`. + > If you want `x` to be available in `foo.nix`, pass it as a function argument: + > + > ```nix + > rec { + > x = 123; + > y = import ./foo.nix x; + > } + > ``` + > + > and + > + > ```nix + > # foo.nix + > x: x + 456 + > ``` + > + > The function argument doesn’t have to be called `x` in `foo.nix`; any name would work. )", .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v) { @@ -634,9 +644,12 @@ struct CompareValues case nFloat: return v1->fpoint < v2->fpoint; case nString: - return strcmp(v1->string.s, v2->string.s) < 0; + return v1->string_view().compare(v2->string_view()) < 0; case nPath: - return strcmp(v1->_path, v2->_path) < 0; + // Note: we don't take the accessor into account + // since it's not obvious how to compare them in a + // reproducible way. + return strcmp(v1->_path.path, v2->_path.path) < 0; case nList: // Lexicographic comparison for (size_t i = 0;; i++) { @@ -771,6 +784,14 @@ static RegisterPrimOp primop_genericClosure(PrimOp { ``` [ { key = 5; } { key = 16; } { key = 8; } { key = 4; } { key = 2; } { key = 1; } ] ``` + + `key` can be one of the following types: + - [Number](@docroot@/language/values.md#type-number) + - [Boolean](@docroot@/language/values.md#type-boolean) + - [String](@docroot@/language/values.md#type-string) + - [Path](@docroot@/language/values.md#type-path) + - [List](@docroot@/language/values.md#list) + )", .fun = prim_genericClosure, }); @@ -854,7 +875,7 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value * * auto message = state.coerceToString(pos, *args[0], context, "while evaluating the error message passed to builtins.addErrorContext", false, false).toOwned(); - e.addTrace(nullptr, message, true); + e.addTrace(nullptr, hintfmt(message), true); throw; } } @@ -1026,7 +1047,7 @@ static void prim_trace(EvalState & state, const PosIdx pos, Value * * args, Valu { state.forceValue(*args[0], pos); if (args[0]->type() == nString) - printError("trace: %1%", args[0]->string.s); + printError("trace: %1%", args[0]->string_view()); else printError("trace: %1%", printValue(state, *args[0])); state.forceValue(*args[1], pos); @@ -1299,17 +1320,18 @@ static void derivationStrictInternal(EvalState & state, const std::string & drvN state.store->computeFSClosure(d.drvPath, refs); for (auto & j : refs) { drv.inputSrcs.insert(j); - if (j.isDerivation()) - drv.inputDrvs[j] = state.store->readDerivation(j).outputNames(); + if (j.isDerivation()) { + drv.inputDrvs.map[j].value = state.store->readDerivation(j).outputNames(); + } } }, [&](const NixStringContextElem::Built & b) { - drv.inputDrvs[b.drvPath].insert(b.output); + drv.inputDrvs.ensureSlot(*b.drvPath).value.insert(b.output); }, [&](const NixStringContextElem::Opaque & o) { drv.inputSrcs.insert(o.path); }, - }, c.raw()); + }, c.raw); } /* Do we have all required attributes? */ @@ -1353,9 +1375,10 @@ static void derivationStrictInternal(EvalState & state, const std::string & drvN auto method = ingestionMethod.value_or(FileIngestionMethod::Flat); DerivationOutput::CAFixed dof { - .ca = ContentAddress::fromParts( - std::move(method), - std::move(h)), + .ca = ContentAddress { + .method = std::move(method), + .hash = std::move(h), + }, }; drv.env["out"] = state.store->printStorePath(dof.path(*state.store, drvName, "out")); @@ -1377,13 +1400,13 @@ static void derivationStrictInternal(EvalState & state, const std::string & drvN if (isImpure) drv.outputs.insert_or_assign(i, DerivationOutput::Impure { - .method = method.raw, + .method = method, .hashType = ht, }); else drv.outputs.insert_or_assign(i, DerivationOutput::CAFloating { - .method = method.raw, + .method = method, .hashType = ht, }); } @@ -1416,7 +1439,7 @@ static void derivationStrictInternal(EvalState & state, const std::string & drvN drv.env[i] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign( i, - DerivationOutputInputAddressed { + DerivationOutput::InputAddressed { .path = std::move(outPath), }); } @@ -1424,7 +1447,7 @@ static void derivationStrictInternal(EvalState & state, const std::string & drvN ; case DrvHash::Kind::Deferred: for (auto & i : outputs) { - drv.outputs.insert_or_assign(i, DerivationOutputDeferred {}); + drv.outputs.insert_or_assign(i, DerivationOutput::Deferred {}); } } } @@ -1574,7 +1597,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args, })); NixStringContext context; - auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.storePath")).path; + auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'")).path; /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink directly in the store. The latter condition is necessary so e.g. nix-push does the right thing. */ @@ -1606,21 +1629,33 @@ static RegisterPrimOp primop_storePath({ in a new path (e.g. `/nix/store/ld01dnzc…-source-source`). Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). + + See also [`builtins.fetchClosure`](#builtins-fetchClosure). )", .fun = prim_storePath, }); static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args, Value & v) { + auto & arg = *args[0]; + /* We don’t check the path right now, because we don’t want to throw if the path isn’t allowed, but just return false (and we can’t just catch the exception here because we still want to - throw if something in the evaluation of `*args[0]` tries to + throw if something in the evaluation of `arg` tries to access an unauthorized path). */ - auto path = realisePath(state, pos, *args[0], { .checkForPureEval = false }); + auto path = realisePath(state, pos, arg, { .checkForPureEval = false }); + + /* SourcePath doesn't know about trailing slash. */ + auto mustBeDir = arg.type() == nString + && (arg.string_view().ends_with("/") + || arg.string_view().ends_with("/.")); try { - v.mkBool(state.checkSourcePath(path).pathExists()); + auto checked = state.checkSourcePath(path); + auto st = checked.maybeLstat(); + auto exists = st && (!mustBeDir || st->type == SourceAccessor::tDirectory); + v.mkBool(exists); } catch (SysError & e) { /* Don't give away info from errors while canonicalising ‘path’ in restricted mode. */ @@ -1760,9 +1795,9 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V })); } - searchPath.emplace_back(SearchPathElem { - .prefix = prefix, - .path = path, + searchPath.elements.emplace_back(SearchPath::Elem { + .prefix = SearchPath::Prefix { .s = prefix }, + .path = SearchPath::Path { .s = path }, }); } @@ -1773,13 +1808,14 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V static RegisterPrimOp primop_findFile(PrimOp { .name = "__findFile", - .args = {"search path", "lookup path"}, + .args = {"search-path", "lookup-path"}, .doc = R"( - Look up the given path with the given search path. + Find *lookup-path* in *search-path*. - A search path is represented list of [attribute sets](./values.md#attribute-set) with two attributes, `prefix`, and `path`. - `prefix` is a relative path. - `path` denotes a file system location; the exact syntax depends on the command line interface. + A search path is represented list of [attribute sets](./values.md#attribute-set) with two attributes: + - `prefix` is a relative path. + - `path` denotes a file system location + The exact syntax depends on the command line interface. Examples of search path attribute sets: @@ -1797,15 +1833,14 @@ static RegisterPrimOp primop_findFile(PrimOp { } ``` - The lookup algorithm checks each entry until a match is found, returning a [path value](@docroot@/language/values.html#type-path) of the match. + The lookup algorithm checks each entry until a match is found, returning a [path value](@docroot@/language/values.html#type-path) of the match: - This is the process for each entry: - If the lookup path matches `prefix`, then the remainder of the lookup path (the "suffix") is searched for within the directory denoted by `patch`. - Note that the `path` may need to be downloaded at this point to look inside. - If the suffix is found inside that directory, then the entry is a match; - the combined absolute path of the directory (now downloaded if need be) and the suffix is returned. + - If *lookup-path* matches `prefix`, then the remainder of *lookup-path* (the "suffix") is searched for within the directory denoted by `path`. + Note that the `path` may need to be downloaded at this point to look inside. + - If the suffix is found inside that directory, then the entry is a match. + The combined absolute path of the directory (now downloaded if need be) and the suffix is returned. - The syntax + [Lookup path](@docroot@/language/constructs/lookup-path.md) expressions can be [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath): ```nix @@ -1833,7 +1868,7 @@ static void prim_hashFile(EvalState & state, const PosIdx pos, Value * * args, V auto path = realisePath(state, pos, *args[1]); - v.mkString(hashString(*ht, path.readFile()).to_string(Base16, false)); + v.mkString(hashString(*ht, path.readFile()).to_string(HashFormat::Base16, false)); } static RegisterPrimOp primop_hashFile({ @@ -1930,6 +1965,45 @@ static RegisterPrimOp primop_readDir({ .fun = prim_readDir, }); +/* Extend single element string context with another output. */ +static void prim_outputOf(EvalState & state, const PosIdx pos, Value * * args, Value & v) +{ + SingleDerivedPath drvPath = state.coerceToSingleDerivedPath(pos, *args[0], "while evaluating the first argument to builtins.outputOf"); + + OutputNameView outputName = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument to builtins.outputOf"); + + state.mkSingleDerivedPathString( + SingleDerivedPath::Built { + .drvPath = make_ref(drvPath), + .output = std::string { outputName }, + }, + v); +} + +static RegisterPrimOp primop_outputOf({ + .name = "__outputOf", + .args = {"derivation-reference", "output-name"}, + .doc = R"( + Return the output path of a derivation, literally or using a placeholder if needed. + + If the derivation has a statically-known output path (i.e. the derivation output is input-addressed, or fixed content-addresed), the output path will just be returned. + But if the derivation is content-addressed or if the derivation is itself not-statically produced (i.e. is the output of another derivation), a placeholder will be returned instead. + + *`derivation reference`* must be a string that may contain a regular store path to a derivation, or may be a placeholder reference. If the derivation is produced by a derivation, you must explicitly select `drv.outPath`. + This primop can be chained arbitrarily deeply. + For instance, + ```nix + builtins.outputOf + (builtins.outputOf myDrv "out) + "out" + ``` + will return a placeholder for the output of the output of `myDrv`. + + This primop corresponds to the `^` sigil for derivable paths, e.g. as part of installable syntax on the command line. + )", + .fun = prim_outputOf, + .experimentalFeature = Xp::DynamicDerivations, +}); /************************************************************* * Creating files @@ -2107,7 +2181,7 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val StorePathSet refs; for (auto c : context) { - if (auto p = std::get_if(&c)) + if (auto p = std::get_if(&c.raw)) refs.insert(p->path); else state.debugThrowLastTrace(EvalError({ @@ -2240,7 +2314,7 @@ static void addPath( path = evalSettings.pureEval && expectedHash ? path - : state.checkSourcePath(CanonPath(path)).path.abs(); + : state.checkSourcePath(state.rootPath(CanonPath(path))).path.abs(); PathFilter filter = filterFun ? ([&](const Path & path) { auto st = lstat(path); @@ -2267,28 +2341,36 @@ static void addPath( std::optional expectedStorePath; if (expectedHash) expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo { - .hash = { - .method = method, - .hash = *expectedHash, - }, + .method = method, + .hash = *expectedHash, .references = {}, }); if (accessStatus && !settings.readOnlyMode) { - StorePath dstPath = state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first; + auto source = sinkToSource([&](Sink & sink) { + if (method == FileIngestionMethod::Recursive) + dumpPath(path, sink, defaultPathFilter); + else + readFile(path, sink); + }); + StorePath dstPath = state.store->computeStorePathFromDump(*source, name, method, htSHA256).first; ensureAccess(&*accessStatus, state.store->printStorePath(dstPath)); require(*state.store).setAccessStatus(dstPath, *accessStatus); } if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { - StorePath dstPath = settings.readOnlyMode - ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first - : state.store->addToStore(name, path, method, htSHA256, filter, state.repair, refs); + auto dstPath = state.rootPath(CanonPath(path)).fetchToStore(state.store, name, method, &filter, state.repair); if (expectedHash && expectedStorePath != dstPath) state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path)); state.allowAndSetStorePathString(dstPath, v); } else if (!expectedHash && accessStatus && !settings.readOnlyMode) { - StorePath dstPath = state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first; + auto source = sinkToSource([&](Sink & sink) { + if (method == FileIngestionMethod::Recursive) + dumpPath(path, sink, defaultPathFilter); + else + readFile(path, sink); + }); + StorePath dstPath = state.store->computeStorePathFromDump(*source, name, method, htSHA256).first; state.allowAndSetStorePathString(dstPath, v); } else { state.allowAndSetStorePathString(*expectedStorePath, v); @@ -2304,7 +2386,7 @@ static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * arg { NixStringContext context; auto path = state.coerceToPath(pos, *args[1], context, - "while evaluating the second argument (the path to filter) passed to builtins.filterSource"); + "while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"); state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource"); addPath(state, pos, path.baseName(), path.path.abs(), args[0], FileIngestionMethod::Recursive, std::nullopt, std::nullopt, v, context); } @@ -2425,7 +2507,7 @@ static RegisterPrimOp primop_path({ like `@`. - filter\ - A function of the type expected by `builtins.filterSource`, + A function of the type expected by [`builtins.filterSource`](#builtins-filterSource), with the same semantics. - recursive\ @@ -2469,7 +2551,7 @@ static void prim_attrNames(EvalState & state, const PosIdx pos, Value * * args, (v.listElems()[n++] = state.allocValue())->mkString(state.symbols[i.name]); std::sort(v.listElems(), v.listElems() + n, - [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; }); + [](Value * v1, Value * v2) { return v1->string_view().compare(v2->string_view()) < 0; }); } static RegisterPrimOp primop_attrNames({ @@ -2606,11 +2688,12 @@ static void prim_removeAttrs(EvalState & state, const PosIdx pos, Value * * args /* Get the attribute names to be removed. We keep them as Attrs instead of Symbols so std::set_difference can be used to remove them from attrs[0]. */ + // 64: large enough to fit the attributes of a derivation boost::container::small_vector names; names.reserve(args[1]->listSize()); for (auto elem : args[1]->listItems()) { state.forceStringNoCtx(*elem, pos, "while evaluating the values of the second argument passed to builtins.removeAttrs"); - names.emplace_back(state.symbols.create(elem->string.s), nullptr); + names.emplace_back(state.symbols.create(elem->string_view()), nullptr); } std::sort(names.begin(), names.end()); @@ -2785,8 +2868,8 @@ static void prim_catAttrs(EvalState & state, const PosIdx pos, Value * * args, V auto attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.catAttrs")); state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.catAttrs"); - Value * res[args[1]->listSize()]; - unsigned int found = 0; + SmallValueVector res(args[1]->listSize()); + size_t found = 0; for (auto v2 : args[1]->listItems()) { state.forceAttrs(*v2, pos, "while evaluating an element in the list passed as second argument to builtins.catAttrs"); @@ -3060,7 +3143,7 @@ static RegisterPrimOp primop_tail({ .name = "__tail", .args = {"list"}, .doc = R"( - Return the second to last elements of a list; abort evaluation if + Return the list without its first item; abort evaluation if the argument isn’t a list or is an empty list. > **Warning** @@ -3120,9 +3203,8 @@ static void prim_filter(EvalState & state, const PosIdx pos, Value * * args, Val state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filter"); - // FIXME: putting this on the stack is risky. - Value * vs[args[1]->listSize()]; - unsigned int k = 0; + SmallValueVector vs(args[1]->listSize()); + size_t k = 0; bool same = true; for (unsigned int n = 0; n < args[1]->listSize(); ++n) { @@ -3235,9 +3317,16 @@ static RegisterPrimOp primop_foldlStrict({ .doc = R"( Reduce a list by applying a binary operator, from left to right, e.g. `foldl' op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2) - ...`. For example, `foldl' (x: y: x + y) 0 [1 2 3]` evaluates to 6. - The return value of each application of `op` is evaluated immediately, - even for intermediate values. + ...`. + + For example, `foldl' (acc: elem: acc + elem) 0 [1 2 3]` evaluates + to `6` and `foldl' (acc: elem: { "${elem}" = elem; } // acc) {} + ["a" "b"]` evaluates to `{ a = "a"; b = "b"; }`. + + The first argument of `op` is the accumulator whereas the second + argument is the current element being processed. The return value + of each application of `op` is evaluated immediately, even for + intermediate values. )", .fun = prim_foldlStrict, }); @@ -3247,10 +3336,14 @@ static void anyOrAll(bool any, EvalState & state, const PosIdx pos, Value * * ar state.forceFunction(*args[0], pos, std::string("while evaluating the first argument passed to builtins.") + (any ? "any" : "all")); state.forceList(*args[1], pos, std::string("while evaluating the second argument passed to builtins.") + (any ? "any" : "all")); + std::string_view errorCtx = any + ? "while evaluating the return value of the function passed to builtins.any" + : "while evaluating the return value of the function passed to builtins.all"; + Value vTmp; for (auto elem : args[1]->listItems()) { state.callFunction(*args[0], *elem, vTmp, pos); - bool res = state.forceBool(vTmp, pos, std::string("while evaluating the return value of the function passed to builtins.") + (any ? "any" : "all")); + bool res = state.forceBool(vTmp, pos, errorCtx); if (res == any) { v.mkBool(any); return; @@ -3506,13 +3599,14 @@ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args, state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.concatMap"); auto nrLists = args[1]->listSize(); - Value lists[nrLists]; + // List of returned lists before concatenation. References to these Values must NOT be persisted. + SmallTemporaryValueVector lists(nrLists); size_t len = 0; for (unsigned int n = 0; n < nrLists; ++n) { Value * vElem = args[1]->listElems()[n]; state.callFunction(*args[0], *vElem, lists[n], pos); - state.forceList(lists[n], lists[n].determinePos(args[0]->determinePos(pos)), "while evaluating the return value of the function passed to buitlins.concatMap"); + state.forceList(lists[n], lists[n].determinePos(args[0]->determinePos(pos)), "while evaluating the return value of the function passed to builtins.concatMap"); len += lists[n].listSize(); } @@ -3775,10 +3869,11 @@ static RegisterPrimOp primop_substring({ .doc = R"( Return the substring of *s* from character position *start* (zero-based) up to but not including *start + len*. If *start* is - greater than the length of the string, an empty string is returned, - and if *start + len* lies beyond the end of the string, only the - substring up to the end of the string is returned. *start* must be - non-negative. For example, + greater than the length of the string, an empty string is returned. + If *start + len* lies beyond the end of the string or *len* is `-1`, + only the substring up to the end of the string is returned. + *start* must be non-negative. + For example, ```nix builtins.substring 0 3 "nixos" @@ -3820,7 +3915,7 @@ static void prim_hashString(EvalState & state, const PosIdx pos, Value * * args, NixStringContext context; // discarded auto s = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString"); - v.mkString(hashString(*ht, s).to_string(Base16, false)); + v.mkString(hashString(*ht, s).to_string(HashFormat::Base16, false)); } static RegisterPrimOp primop_hashString({ @@ -3834,6 +3929,101 @@ static RegisterPrimOp primop_hashString({ .fun = prim_hashString, }); +static void prim_convertHash(EvalState & state, const PosIdx pos, Value * * args, Value & v) +{ + state.forceAttrs(*args[0], pos, "while evaluating the first argument passed to builtins.convertHash"); + auto &inputAttrs = args[0]->attrs; + + Bindings::iterator iteratorHash = getAttr(state, state.symbols.create("hash"), inputAttrs, "while locating the attribute 'hash'"); + auto hash = state.forceStringNoCtx(*iteratorHash->value, pos, "while evaluating the attribute 'hash'"); + + Bindings::iterator iteratorHashAlgo = inputAttrs->find(state.symbols.create("hashAlgo")); + std::optional ht = std::nullopt; + if (iteratorHashAlgo != inputAttrs->end()) { + ht = parseHashType(state.forceStringNoCtx(*iteratorHashAlgo->value, pos, "while evaluating the attribute 'hashAlgo'")); + } + + Bindings::iterator iteratorToHashFormat = getAttr(state, state.symbols.create("toHashFormat"), args[0]->attrs, "while locating the attribute 'toHashFormat'"); + HashFormat hf = parseHashFormat(state.forceStringNoCtx(*iteratorToHashFormat->value, pos, "while evaluating the attribute 'toHashFormat'")); + + v.mkString(Hash::parseAny(hash, ht).to_string(hf, hf == HashFormat::SRI)); +} + +static RegisterPrimOp primop_convertHash({ + .name = "__convertHash", + .args = {"args"}, + .doc = R"( + Return the specified representation of a hash string, based on the attributes presented in *args*: + + - `hash` + + The hash to be converted. + The hash format is detected automatically. + + - `hashAlgo` + + The algorithm used to create the hash. Must be one of + - `"md5"` + - `"sha1"` + - `"sha256"` + - `"sha512"` + + The attribute may be omitted when `hash` is an [SRI hash](https://www.w3.org/TR/SRI/#the-integrity-attribute) or when the hash is prefixed with the hash algorithm name followed by a colon. + That `:` syntax is supported for backwards compatibility with existing tooling. + + - `toHashFormat` + + The format of the resulting hash. Must be one of + - `"base16"` + - `"base32"` + - `"base64"` + - `"sri"` + + The result hash is the *toHashFormat* representation of the hash *hash*. + + > **Example** + > + > Convert a SHA256 hash in Base16 to SRI: + > + > ```nix + > builtins.convertHash { + > hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + > toHashFormat = "sri"; + > hashAlgo = "sha256"; + > } + > ``` + > + > "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" + + > **Example** + > + > Convert a SHA256 hash in SRI to Base16: + > + > ```nix + > builtins.convertHash { + > hash = "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="; + > toHashFormat = "base16"; + > } + > ``` + > + > "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + + > **Example** + > + > Convert a hash in the form `:` in Base16 to SRI: + > + > ```nix + > builtins.convertHash { + > hash = "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + > toHashFormat = "sri"; + > } + > ``` + > + > "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" + )", + .fun = prim_convertHash, +}); + struct RegexCache { // TODO use C++20 transparent comparison when available @@ -4397,7 +4587,7 @@ void EvalState::createBaseEnv() .doc = R"( Logical file system location of the [Nix store](@docroot@/glossary.md#gloss-store) currently in use. - This value is determined by the `store` parameter in [Store URLs](@docroot@/command-ref/new-cli/nix3-help-stores.md): + This value is determined by the `store` parameter in [Store URLs](@docroot@/store/types/index.md#store-url-format): ```shell-session $ nix-instantiate --store 'dummy://?store=/blah' --eval --expr builtins.storeDir @@ -4445,20 +4635,20 @@ void EvalState::createBaseEnv() }); /* Add a value containing the current Nix expression search path. */ - mkList(v, searchPath.size()); + mkList(v, searchPath.elements.size()); int n = 0; - for (auto & i : searchPath) { + for (auto & i : searchPath.elements) { auto attrs = buildBindings(2); - attrs.alloc("path").mkString(i.path); - attrs.alloc("prefix").mkString(i.prefix); + attrs.alloc("path").mkString(i.path.s); + attrs.alloc("prefix").mkString(i.prefix.s); (v.listElems()[n++] = allocValue())->mkAttrs(attrs); } addConstant("__nixPath", v, { .type = nList, .doc = R"( - The search path used to resolve angle bracket path lookups. + List of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). - Angle bracket expressions can be + Lookup path expressions can be [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.findFile`](./builtins.html#builtins-findFile): @@ -4502,12 +4692,7 @@ void EvalState::createBaseEnv() /* Note: we have to initialize the 'derivation' constant *after* building baseEnv/staticBaseEnv because it uses 'builtins'. */ - char code[] = - #include "primops/derivation.nix.gen.hh" - // the parser needs two NUL bytes as terminators; one of them - // is implied by being a C string. - "\0"; - eval(parse(code, sizeof(code), derivationInternal, {CanonPath::root}, staticBaseEnv), *vDerivation); + evalFile(derivationInternal, *vDerivation); } diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index 930e7f32a1a3..45486608f778 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -8,6 +8,22 @@ namespace nix { +/** + * For functions where we do not expect deep recursion, we can use a sizable + * part of the stack a free allocation space. + * + * Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes. + */ +constexpr size_t nonRecursiveStackReservation = 128; + +/** + * Functions that maybe applied to self-similar inputs, such as concatMap on a + * tree, should reserve a smaller part of the stack for allocation. + * + * Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes. + */ +constexpr size_t conservativeStackReservation = 16; + struct RegisterPrimOp { typedef std::vector PrimOps; diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 8b3468009bab..db940f277b29 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -30,20 +30,27 @@ static RegisterPrimOp primop_hasContext({ .name = "__hasContext", .args = {"s"}, .doc = R"( - Return `true` if string *s* has a non-empty context. The - context can be obtained with + Return `true` if string *s* has a non-empty context. + The context can be obtained with [`getContext`](#builtins-getContext). + + > **Example** + > + > Many operations require a string context to be empty because they are intended only to work with "regular" strings, and also to help users avoid unintentionally loosing track of string context elements. + > `builtins.hasContext` can help create better domain-specific errors in those case. + > + > ```nix + > name: meta: + > + > if builtins.hasContext name + > then throw "package name cannot contain string context" + > else { ${name} = meta; } + > ``` )", .fun = prim_hasContext }); -/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a - builder without causing the derivation to be built (for instance, - in the derivation that builds NARs in nix-push, when doing - source-only deployment). This primop marks the string context so - that builtins.derivation adds the path to drv.inputSrcs rather than - drv.inputDrvs. */ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx pos, Value * * args, Value & v) { NixStringContext context; @@ -51,13 +58,13 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx p NixStringContext context2; for (auto && c : context) { - if (auto * ptr = std::get_if(&c)) { + if (auto * ptr = std::get_if(&c.raw)) { context2.emplace(NixStringContextElem::Opaque { .path = ptr->drvPath }); } else { /* Can reuse original item */ - context2.emplace(std::move(c)); + context2.emplace(std::move(c).raw); } } @@ -66,11 +73,83 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx p static RegisterPrimOp primop_unsafeDiscardOutputDependency({ .name = "__unsafeDiscardOutputDependency", - .arity = 1, + .args = {"s"}, + .doc = R"( + Create a copy of the given string where every "derivation deep" string context element is turned into a constant string context element. + + This is the opposite of [`builtins.addDrvOutputDependencies`](#builtins-addDrvOutputDependencies). + + This is unsafe because it allows us to "forget" store objects we would have otherwise refered to with the string context, + whereas Nix normally tracks all dependencies consistently. + Safe operations "grow" but never "shrink" string contexts. + [`builtins.addDrvOutputDependencies`] in contrast is safe because "derivation deep" string context element always refers to the underlying derivation (among many more things). + Replacing a constant string context element with a "derivation deep" element is a safe operation that just enlargens the string context without forgetting anything. + + [`builtins.addDrvOutputDependencies`]: #builtins-addDrvOutputDependencies + )", .fun = prim_unsafeDiscardOutputDependency }); +static void prim_addDrvOutputDependencies(EvalState & state, const PosIdx pos, Value * * args, Value & v) +{ + NixStringContext context; + auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.addDrvOutputDependencies"); + + auto contextSize = context.size(); + if (contextSize != 1) { + throw EvalError({ + .msg = hintfmt("context of string '%s' must have exactly one element, but has %d", *s, contextSize), + .errPos = state.positions[pos] + }); + } + NixStringContext context2 { + (NixStringContextElem { std::visit(overloaded { + [&](const NixStringContextElem::Opaque & c) -> NixStringContextElem::DrvDeep { + if (!c.path.isDerivation()) { + throw EvalError({ + .msg = hintfmt("path '%s' is not a derivation", + state.store->printStorePath(c.path)), + .errPos = state.positions[pos], + }); + } + return NixStringContextElem::DrvDeep { + .drvPath = c.path, + }; + }, + [&](const NixStringContextElem::Built & c) -> NixStringContextElem::DrvDeep { + throw EvalError({ + .msg = hintfmt("`addDrvOutputDependencies` can only act on derivations, not on a derivation output such as '%1%'", c.output), + .errPos = state.positions[pos], + }); + }, + [&](const NixStringContextElem::DrvDeep & c) -> NixStringContextElem::DrvDeep { + /* Reuse original item because we want this to be idempotent. */ + return std::move(c); + }, + }, context.begin()->raw) }), + }; + + v.mkString(*s, context2); +} + +static RegisterPrimOp primop_addDrvOutputDependencies({ + .name = "__addDrvOutputDependencies", + .args = {"s"}, + .doc = R"( + Create a copy of the given string where a single consant string context element is turned into a "derivation deep" string context element. + + The store path that is the constant string context element should point to a valid derivation, and end in `.drv`. + + The original string context element must not be empty or have multiple elements, and it must not have any other type of element other than a constant or derivation deep element. + The latter is supported so this function is idempotent. + + This is the opposite of [`builtins.unsafeDiscardOutputDependency`](#builtins-addDrvOutputDependencies). + )", + .fun = prim_addDrvOutputDependencies +}); + + /* Extract the context of a string as a structured Nix value. The context is represented as an attribute set whose keys are the @@ -106,12 +185,15 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args, contextInfos[std::move(d.drvPath)].allOutputs = true; }, [&](NixStringContextElem::Built && b) { - contextInfos[std::move(b.drvPath)].outputs.emplace_back(std::move(b.output)); + // FIXME should eventually show string context as is, no + // resolving here. + auto drvPath = resolveDerivedPath(*state.store, *b.drvPath); + contextInfos[std::move(drvPath)].outputs.emplace_back(std::move(b.output)); }, [&](NixStringContextElem::Opaque && o) { contextInfos[std::move(o.path)].path = true; }, - }, ((NixStringContextElem &&) i).raw()); + }, ((NixStringContextElem &&) i).raw); } auto attrs = state.buildBindings(contextInfos.size()); @@ -222,7 +304,7 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar for (auto elem : iter->value->listItems()) { auto outputName = state.forceStringNoCtx(*elem, iter->pos, "while evaluating an output name within a string context"); context.emplace(NixStringContextElem::Built { - .drvPath = namePath, + .drvPath = makeConstantStorePathRef(namePath), .output = std::string { outputName }, }); } diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index bae849f617b5..b86ef6b93f24 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -5,37 +5,150 @@ namespace nix { +/** + * Handler for the content addressed case. + * + * @param state Evaluator state and store to write to. + * @param fromStore Store containing the path to rewrite. + * @param fromPath Source path to be rewritten. + * @param toPathMaybe Path to write the rewritten path to. If empty, the error shows the actual path. + * @param v Return `Value` + */ +static void runFetchClosureWithRewrite(EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, const std::optional & toPathMaybe, Value &v) { + + // establish toPath or throw + + if (!toPathMaybe || !state.store->isValidPath(*toPathMaybe)) { + auto rewrittenPath = makeContentAddressed(fromStore, *state.store, fromPath); + if (toPathMaybe && *toPathMaybe != rewrittenPath) + throw Error({ + .msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected", + state.store->printStorePath(fromPath), + state.store->printStorePath(rewrittenPath), + state.store->printStorePath(*toPathMaybe)), + .errPos = state.positions[pos] + }); + if (!toPathMaybe) + throw Error({ + .msg = hintfmt( + "rewriting '%s' to content-addressed form yielded '%s'\n" + "Use this value for the 'toPath' attribute passed to 'fetchClosure'", + state.store->printStorePath(fromPath), + state.store->printStorePath(rewrittenPath)), + .errPos = state.positions[pos] + }); + } + + auto toPath = *toPathMaybe; + + // check and return + + auto resultInfo = state.store->queryPathInfo(toPath); + + if (!resultInfo->isContentAddressed(*state.store)) { + // We don't perform the rewriting when outPath already exists, as an optimisation. + // However, we can quickly detect a mistake if the toPath is input addressed. + throw Error({ + .msg = hintfmt( + "The 'toPath' value '%s' is input-addressed, so it can't possibly be the result of rewriting to a content-addressed path.\n\n" + "Set 'toPath' to an empty string to make Nix report the correct content-addressed path.", + state.store->printStorePath(toPath)), + .errPos = state.positions[pos] + }); + } + + state.mkStorePathString(toPath, v); +} + +/** + * Fetch the closure and make sure it's content addressed. + */ +static void runFetchClosureWithContentAddressedPath(EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, Value & v) { + + if (!state.store->isValidPath(fromPath)) + copyClosure(fromStore, *state.store, RealisedPath::Set { fromPath }); + + auto info = state.store->queryPathInfo(fromPath); + + if (!info->isContentAddressed(*state.store)) { + throw Error({ + .msg = hintfmt( + "The 'fromPath' value '%s' is input-addressed, but 'inputAddressed' is set to 'false' (default).\n\n" + "If you do intend to fetch an input-addressed store path, add\n\n" + " inputAddressed = true;\n\n" + "to the 'fetchClosure' arguments.\n\n" + "Note that to ensure authenticity input-addressed store paths, users must configure a trusted binary cache public key on their systems. This is not needed for content-addressed paths.", + state.store->printStorePath(fromPath)), + .errPos = state.positions[pos] + }); + } + + state.mkStorePathString(fromPath, v); +} + +/** + * Fetch the closure and make sure it's input addressed. + */ +static void runFetchClosureWithInputAddressedPath(EvalState & state, const PosIdx pos, Store & fromStore, const StorePath & fromPath, Value & v) { + + if (!state.store->isValidPath(fromPath)) + copyClosure(fromStore, *state.store, RealisedPath::Set { fromPath }); + + auto info = state.store->queryPathInfo(fromPath); + + if (info->isContentAddressed(*state.store)) { + throw Error({ + .msg = hintfmt( + "The store object referred to by 'fromPath' at '%s' is not input-addressed, but 'inputAddressed' is set to 'true'.\n\n" + "Remove the 'inputAddressed' attribute (it defaults to 'false') to expect 'fromPath' to be content-addressed", + state.store->printStorePath(fromPath)), + .errPos = state.positions[pos] + }); + } + + state.mkStorePathString(fromPath, v); +} + +typedef std::optional StorePathOrGap; + static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * args, Value & v) { state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.fetchClosure"); std::optional fromStoreUrl; std::optional fromPath; - bool toCA = false; - std::optional toPath; + std::optional toPath; + std::optional inputAddressedMaybe; for (auto & attr : *args[0]->attrs) { const auto & attrName = state.symbols[attr.name]; + auto attrHint = [&]() -> std::string { + return "while evaluating the '" + attrName + "' attribute passed to builtins.fetchClosure"; + }; if (attrName == "fromPath") { NixStringContext context; - fromPath = state.coerceToStorePath(attr.pos, *attr.value, context, - "while evaluating the 'fromPath' attribute passed to builtins.fetchClosure"); + fromPath = state.coerceToStorePath(attr.pos, *attr.value, context, attrHint()); } else if (attrName == "toPath") { state.forceValue(*attr.value, attr.pos); - toCA = true; - if (attr.value->type() != nString || attr.value->string.s != std::string("")) { + bool isEmptyString = attr.value->type() == nString && attr.value->string_view() == ""; + if (isEmptyString) { + toPath = StorePathOrGap {}; + } + else { NixStringContext context; - toPath = state.coerceToStorePath(attr.pos, *attr.value, context, - "while evaluating the 'toPath' attribute passed to builtins.fetchClosure"); + toPath = state.coerceToStorePath(attr.pos, *attr.value, context, attrHint()); } } else if (attrName == "fromStore") fromStoreUrl = state.forceStringNoCtx(*attr.value, attr.pos, - "while evaluating the 'fromStore' attribute passed to builtins.fetchClosure"); + attrHint()); + + else if (attrName == "inputAddressed") + inputAddressedMaybe = state.forceBool(*attr.value, attr.pos, attrHint()); else throw Error({ @@ -50,6 +163,18 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg .errPos = state.positions[pos] }); + bool inputAddressed = inputAddressedMaybe.value_or(false); + + if (inputAddressed) { + if (toPath) + throw Error({ + .msg = hintfmt("attribute '%s' is set to true, but '%s' is also set. Please remove one of them", + "inputAddressed", + "toPath"), + .errPos = state.positions[pos] + }); + } + if (!fromStoreUrl) throw Error({ .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"), @@ -74,55 +199,40 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg auto fromStore = openStore(parsedURL.to_string()); - if (toCA) { - if (!toPath || !state.store->isValidPath(*toPath)) { - auto remappings = makeContentAddressed(*fromStore, *state.store, { *fromPath }); - auto i = remappings.find(*fromPath); - assert(i != remappings.end()); - if (toPath && *toPath != i->second) - throw Error({ - .msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected", - state.store->printStorePath(*fromPath), - state.store->printStorePath(i->second), - state.store->printStorePath(*toPath)), - .errPos = state.positions[pos] - }); - if (!toPath) - throw Error({ - .msg = hintfmt( - "rewriting '%s' to content-addressed form yielded '%s'; " - "please set this in the 'toPath' attribute passed to 'fetchClosure'", - state.store->printStorePath(*fromPath), - state.store->printStorePath(i->second)), - .errPos = state.positions[pos] - }); - } - } else { - if (!state.store->isValidPath(*fromPath)) - copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath }); - toPath = fromPath; - } - - /* In pure mode, require a CA path. */ - if (evalSettings.pureEval) { - auto info = state.store->queryPathInfo(*toPath); - if (!info->isContentAddressed(*state.store)) - throw Error({ - .msg = hintfmt("in pure mode, 'fetchClosure' requires a content-addressed path, which '%s' isn't", - state.store->printStorePath(*toPath)), - .errPos = state.positions[pos] - }); - } - - state.mkStorePathString(*toPath, v); + if (toPath) + runFetchClosureWithRewrite(state, pos, *fromStore, *fromPath, *toPath, v); + else if (inputAddressed) + runFetchClosureWithInputAddressedPath(state, pos, *fromStore, *fromPath, v); + else + runFetchClosureWithContentAddressedPath(state, pos, *fromStore, *fromPath, v); } static RegisterPrimOp primop_fetchClosure({ .name = "__fetchClosure", .args = {"args"}, .doc = R"( - Fetch a Nix store closure from a binary cache, rewriting it into - content-addressed form. For example, + Fetch a store path [closure](@docroot@/glossary.md#gloss-closure) from a binary cache, and return the store path as a string with context. + + This function can be invoked in three ways, that we will discuss in order of preference. + + **Fetch a content-addressed store path** + + Example: + + ```nix + builtins.fetchClosure { + fromStore = "https://cache.nixos.org"; + fromPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1; + } + ``` + + This is the simplest invocation, and it does not require the user of the expression to configure [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) to ensure their authenticity. + + If your store path is [input addressed](@docroot@/glossary.md#gloss-input-addressed-store-object) instead of content addressed, consider the other two invocations. + + **Fetch any store path and rewrite it to a fully content-addressed store path** + + Example: ```nix builtins.fetchClosure { @@ -132,28 +242,42 @@ static RegisterPrimOp primop_fetchClosure({ } ``` - fetches `/nix/store/r2jd...` from the specified binary cache, + This example fetches `/nix/store/r2jd...` from the specified binary cache, and rewrites it into the content-addressed store path `/nix/store/ldbh...`. - If `fromPath` is already content-addressed, or if you are - allowing impure evaluation (`--impure`), then `toPath` may be - omitted. + Like the previous example, no extra configuration or privileges are required. To find out the correct value for `toPath` given a `fromPath`, - you can use `nix store make-content-addressed`: + use [`nix store make-content-addressed`](@docroot@/command-ref/new-cli/nix3-store-make-content-addressed.md): ```console # nix store make-content-addressed --from https://cache.nixos.org /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1 rewrote '/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1' to '/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1' ``` - This function is similar to `builtins.storePath` in that it - allows you to use a previously built store path in a Nix - expression. However, it is more reproducible because it requires - specifying a binary cache from which the path can be fetched. - Also, requiring a content-addressed final store path avoids the - need for users to configure binary cache public keys. + Alternatively, set `toPath = ""` and find the correct `toPath` in the error message. + + **Fetch an input-addressed store path as is** + + Example: + + ```nix + builtins.fetchClosure { + fromStore = "https://cache.nixos.org"; + fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1; + inputAddressed = true; + } + ``` + + It is possible to fetch an [input-addressed store path](@docroot@/glossary.md#gloss-input-addressed-store-object) and return it as is. + However, this is the least preferred way of invoking `fetchClosure`, because it requires that the input-addressed paths are trusted by the Nix configuration. + + **`builtins.storePath`** + + `fetchClosure` is similar to [`builtins.storePath`](#builtins-storePath) in that it allows you to use a previously built store path in a Nix expression. + However, `fetchClosure` is more reproducible because it specifies a binary cache from which the path can be fetched. + Also, using content-addressed store paths does not require users to configure [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) to ensure their authenticity. )", .fun = prim_fetchClosure, .experimentalFeature = Xp::FetchClosure, diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 322692b52b2b..e76ce455d384 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -1,5 +1,6 @@ #include "primops.hh" #include "eval-inline.hh" +#include "eval-settings.hh" #include "store-api.hh" #include "fetchers.hh" #include "url.hh" @@ -70,10 +71,10 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a auto input = fetchers::Input::fromAttrs(std::move(attrs)); // FIXME: use name - auto [tree, input2] = input.fetch(state.store); + auto [storePath, input2] = input.fetch(state.store); auto attrs2 = state.buildBindings(8); - state.mkStorePathString(tree.storePath, attrs2.alloc(state.sOutPath)); + state.mkStorePathString(storePath, attrs2.alloc(state.sOutPath)); if (input2.getRef()) attrs2.alloc("branch").mkString(*input2.getRef()); // Backward compatibility: set 'rev' to @@ -85,7 +86,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a attrs2.alloc("revCount").mkInt(*revCount); v.mkAttrs(attrs2); - state.allowPath(tree.storePath); + state.allowPath(storePath); } static RegisterPrimOp r_fetchMercurial({ diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 579a45f9243c..383ec7c5824d 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -1,10 +1,13 @@ #include "primops.hh" #include "eval-inline.hh" +#include "eval-settings.hh" #include "store-api.hh" #include "fetchers.hh" #include "filetransfer.hh" #include "registry.hh" +#include "tarball.hh" #include "url.hh" +#include "value-to-json.hh" #include #include @@ -14,7 +17,7 @@ namespace nix { void emitTreeAttrs( EvalState & state, - const fetchers::Tree & tree, + const StorePath & storePath, const fetchers::Input & input, Value & v, bool emptyRevFallback, @@ -24,14 +27,13 @@ void emitTreeAttrs( auto attrs = state.buildBindings(10); - - state.mkStorePathString(tree.storePath, attrs.alloc(state.sOutPath)); + state.mkStorePathString(storePath, attrs.alloc(state.sOutPath)); // FIXME: support arbitrary input attributes. auto narHash = input.getNarHash(); assert(narHash); - attrs.alloc("narHash").mkString(narHash->to_string(SRI, true)); + attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true)); if (input.getType() == "git") attrs.alloc("submodules").mkBool( @@ -70,36 +72,10 @@ void emitTreeAttrs( v.mkAttrs(attrs); } -std::string fixURI(std::string uri, EvalState & state, const std::string & defaultScheme = "file") -{ - state.checkURI(uri); - if (uri.find("://") == std::string::npos) { - const auto p = ParsedURL { - .scheme = defaultScheme, - .authority = "", - .path = uri - }; - return p.to_string(); - } else { - return uri; - } -} - -std::string fixURIForGit(std::string uri, EvalState & state) -{ - /* Detects scp-style uris (e.g. git@github.com:NixOS/nix) and fixes - * them by removing the `:` and assuming a scheme of `ssh://` - * */ - static std::regex scp_uri("([^/]*)@(.*):(.*)"); - if (uri[0] != '/' && std::regex_match(uri, scp_uri)) - return fixURI(std::regex_replace(uri, scp_uri, "$1@$2/$3"), state, "ssh"); - else - return fixURI(uri, state); -} - struct FetchTreeParams { bool emptyRevFallback = false; bool allowNameArgument = false; + bool isFetchGit = false; }; static void fetchTree( @@ -107,11 +83,12 @@ static void fetchTree( const PosIdx pos, Value * * args, Value & v, - std::optional type, const FetchTreeParams & params = FetchTreeParams{} ) { fetchers::Input input; NixStringContext context; + std::optional type; + if (params.isFetchGit) type = "git"; state.forceValue(*args[0], pos); @@ -141,16 +118,18 @@ static void fetchTree( if (attr.value->type() == nPath || attr.value->type() == nString) { auto s = state.coerceToString(attr.pos, *attr.value, context, "", false, false).toOwned(); attrs.emplace(state.symbols[attr.name], - state.symbols[attr.name] == "url" - ? type == "git" - ? fixURIForGit(s, state) - : fixURI(s, state) + params.isFetchGit && state.symbols[attr.name] == "url" + ? fixGitURL(s) : s); } else if (attr.value->type() == nBool) attrs.emplace(state.symbols[attr.name], Explicit{attr.value->boolean}); else if (attr.value->type() == nInt) attrs.emplace(state.symbols[attr.name], uint64_t(attr.value->integer)); + else if (state.symbols[attr.name] == "publicKeys") { + experimentalFeatureSettings.require(Xp::VerifiedFetches); + attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, context).dump()); + } else state.debugThrowLastTrace(TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected", state.symbols[attr.name], showType(*attr.value))); @@ -169,40 +148,87 @@ static void fetchTree( "while evaluating the first argument passed to the fetcher", false, false).toOwned(); - if (type == "git") { + if (params.isFetchGit) { fetchers::Attrs attrs; attrs.emplace("type", "git"); - attrs.emplace("url", fixURIForGit(url, state)); + attrs.emplace("url", fixGitURL(url)); input = fetchers::Input::fromAttrs(std::move(attrs)); } else { - input = fetchers::Input::fromURL(fixURI(url, state)); + if (!experimentalFeatureSettings.isEnabled(Xp::Flakes)) + state.debugThrowLastTrace(EvalError({ + .msg = hintfmt("passing a string argument to 'fetchTree' requires the 'flakes' experimental feature"), + .errPos = state.positions[pos] + })); + input = fetchers::Input::fromURL(url); } } - if (!evalSettings.pureEval && !input.isDirect()) + if (!evalSettings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes)) input = lookupInRegistries(state.store, input).first; if (evalSettings.pureEval && !input.isLocked()) state.debugThrowLastTrace(EvalError("in pure evaluation mode, 'fetchTree' requires a locked input, at %s", state.positions[pos])); - auto [tree, input2] = input.fetch(state.store); + state.checkURI(input.toURLString()); - state.allowPath(tree.storePath); + auto [storePath, input2] = input.fetch(state.store); - emitTreeAttrs(state, tree, input2, v, params.emptyRevFallback, false); + state.allowPath(storePath); + + emitTreeAttrs(state, storePath, input2, v, params.emptyRevFallback, false); } static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - experimentalFeatureSettings.require(Xp::Flakes); - fetchTree(state, pos, args, v, std::nullopt, FetchTreeParams { .allowNameArgument = false }); + fetchTree(state, pos, args, v, { }); } -// FIXME: document static RegisterPrimOp primop_fetchTree({ .name = "fetchTree", - .arity = 1, - .fun = prim_fetchTree + .args = {"input"}, + .doc = R"( + Fetch a source tree or a plain file using one of the supported backends. + *input* must be a [flake reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references), either in attribute set representation or in the URL-like syntax. + The input should be "locked", that is, it should contain a commit hash or content hash unless impure evaluation (`--impure`) is enabled. + + > **Note** + > + > The URL-like syntax requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled. + + Here are some examples of how to use `fetchTree`: + + - Fetch a GitHub repository using the attribute set representation: + + ```nix + builtins.fetchTree { + type = "github"; + owner = "NixOS"; + repo = "nixpkgs"; + rev = "ae2e6b3958682513d28f7d633734571fb18285dd"; + } + ``` + + This evaluates to the following attribute set: + + ``` + { + lastModified = 1686503798; + lastModifiedDate = "20230611171638"; + narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc="; + outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source"; + rev = "ae2e6b3958682513d28f7d633734571fb18285dd"; + shortRev = "ae2e6b3"; + } + ``` + + - Fetch the same GitHub repository using the URL-like syntax: + + ``` + builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd" + ``` + )", + .fun = prim_fetchTree, + .experimentalFeature = Xp::FetchTree, }); static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v, @@ -254,10 +280,8 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v auto expectedPath = state.store->makeFixedOutputPath( name, FixedOutputInfo { - .hash = { - .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, - .hash = *expectedHash, - }, + .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, + .hash = *expectedHash, .references = {} }); @@ -271,7 +295,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v // https://github.com/NixOS/nix/issues/4313 auto storePath = unpack - ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).tree.storePath + ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; if (expectedHash) { @@ -280,7 +304,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v : hashFile(htSHA256, state.store->toRealPath(storePath)); if (hash != *expectedHash) state.debugThrowLastTrace(EvalError((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", - *url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true))); + *url, expectedHash->to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true))); } state.allowAndSetStorePathString(storePath, v); @@ -354,7 +378,12 @@ static RegisterPrimOp primop_fetchTarball({ static void prim_fetchGit(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - fetchTree(state, pos, args, v, "git", FetchTreeParams { .emptyRevFallback = true, .allowNameArgument = true }); + fetchTree(state, pos, args, v, + FetchTreeParams { + .emptyRevFallback = true, + .allowNameArgument = true, + .isFetchGit = true + }); } static RegisterPrimOp primop_fetchGit({ @@ -369,7 +398,7 @@ static RegisterPrimOp primop_fetchGit({ The URL of the repo. - - `name` (default: *basename of the URL*) + - `name` (default: `source`) The name of the directory the repo should be exported to in the store. @@ -396,7 +425,8 @@ static RegisterPrimOp primop_fetchGit({ - `shallow` (default: `false`) - A Boolean parameter that specifies whether fetching a shallow clone is allowed. + A Boolean parameter that specifies whether fetching from a shallow remote repository is allowed. + This still performs a full clone of what is available on the remote. - `allRefs` @@ -404,6 +434,42 @@ static RegisterPrimOp primop_fetchGit({ With this argument being true, it's possible to load a `rev` from *any* `ref` (by default only `rev`s from the specified `ref` are supported). + - `verifyCommit` (default: `true` if `publicKey` or `publicKeys` are provided, otherwise `false`) + + Whether to check `rev` for a signature matching `publicKey` or `publicKeys`. + If `verifyCommit` is enabled, then `fetchGit` cannot use a local repository with uncommitted changes. + Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + + - `publicKey` + + The public key against which `rev` is verified if `verifyCommit` is enabled. + Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + + - `keytype` (default: `"ssh-ed25519"`) + + The key type of `publicKey`. + Possible values: + - `"ssh-dsa"` + - `"ssh-ecdsa"` + - `"ssh-ecdsa-sk"` + - `"ssh-ed25519"` + - `"ssh-ed25519-sk"` + - `"ssh-rsa"` + Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + + - `publicKeys` + + The public keys against which `rev` is verified if `verifyCommit` is enabled. + Must be given as a list of attribute sets with the following form: + ```nix + { + key = ""; + type = ""; # optional, default: "ssh-ed25519" + } + ``` + Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + + Here are some examples of how to use `fetchGit`. - To fetch a private repository over SSH: @@ -478,6 +544,21 @@ static RegisterPrimOp primop_fetchGit({ } ``` + - To verify the commit signature: + + ```nix + builtins.fetchGit { + url = "ssh://git@github.com/nixos/nix.git"; + verifyCommit = true; + publicKeys = [ + { + type = "ssh-ed25519"; + key = "AAAAC3NzaC1lZDI1NTE5AAAAIArPKULJOid8eS6XETwUjO48/HKBWl7FTCK0Z//fplDi"; + } + ]; + } + ``` + Nix will refetch the branch according to the [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl) setting. This behavior is disabled in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). diff --git a/src/libexpr/search-path.cc b/src/libexpr/search-path.cc new file mode 100644 index 000000000000..a25767496368 --- /dev/null +++ b/src/libexpr/search-path.cc @@ -0,0 +1,55 @@ +#include "search-path.hh" + +namespace nix { + +std::optional SearchPath::Prefix::suffixIfPotentialMatch( + std::string_view path) const +{ + auto n = s.size(); + + /* Non-empty prefix and suffix must be separated by a /, or the + prefix is not a valid path prefix. */ + bool needSeparator = n > 0 && n < path.size(); + + if (needSeparator && path[n] != '/') { + return std::nullopt; + } + + /* Prefix must be prefix of this path. */ + if (path.compare(0, n, s) != 0) { + return std::nullopt; + } + + /* Skip next path separator. */ + return { + path.substr(needSeparator ? n + 1 : n) + }; +} + + +SearchPath::Elem SearchPath::Elem::parse(std::string_view rawElem) +{ + size_t pos = rawElem.find('='); + + return SearchPath::Elem { + .prefix = Prefix { + .s = pos == std::string::npos + ? std::string { "" } + : std::string { rawElem.substr(0, pos) }, + }, + .path = Path { + .s = std::string { rawElem.substr(pos + 1) }, + }, + }; +} + + +SearchPath parseSearchPath(const Strings & rawElems) +{ + SearchPath res; + for (auto & rawElem : rawElems) + res.elements.emplace_back(SearchPath::Elem::parse(rawElem)); + return res; +} + +} diff --git a/src/libexpr/search-path.hh b/src/libexpr/search-path.hh new file mode 100644 index 000000000000..ce78135b51f1 --- /dev/null +++ b/src/libexpr/search-path.hh @@ -0,0 +1,108 @@ +#pragma once +///@file + +#include + +#include "types.hh" +#include "comparator.hh" + +namespace nix { + +/** + * A "search path" is a list of ways look for something, used with + * `builtins.findFile` and `< >` lookup expressions. + */ +struct SearchPath +{ + /** + * A single element of a `SearchPath`. + * + * Each element is tried in succession when looking up a path. The first + * element to completely match wins. + */ + struct Elem; + + /** + * The first part of a `SearchPath::Elem` pair. + * + * Called a "prefix" because it takes the form of a prefix of a file + * path (first `n` path components). When looking up a path, to use + * a `SearchPath::Elem`, its `Prefix` must match the path. + */ + struct Prefix; + + /** + * The second part of a `SearchPath::Elem` pair. + * + * It is either a path or a URL (with certain restrictions / extra + * structure). + * + * If the prefix of the path we are looking up matches, we then + * check if the rest of the path points to something that exists + * within the directory denoted by this. If so, the + * `SearchPath::Elem` as a whole matches, and that *something* being + * pointed to by the rest of the path we are looking up is the + * result. + */ + struct Path; + + /** + * The list of search path elements. Each one is checked for a path + * when looking up. (The actual lookup entry point is in `EvalState` + * not in this class.) + */ + std::list elements; + + /** + * Parse a string into a `SearchPath` + */ + static SearchPath parse(const Strings & rawElems); +}; + +struct SearchPath::Prefix +{ + /** + * Underlying string + * + * @todo Should we normalize this when constructing a `SearchPath::Prefix`? + */ + std::string s; + + GENERATE_CMP(SearchPath::Prefix, me->s); + + /** + * If the path possibly matches this search path element, return the + * suffix that we should look for inside the resolved value of the + * element + * Note the double optionality in the name. While we might have a matching prefix, the suffix may not exist. + */ + std::optional suffixIfPotentialMatch(std::string_view path) const; +}; + +struct SearchPath::Path +{ + /** + * The location of a search path item, as a path or URL. + * + * @todo Maybe change this to `std::variant`. + */ + std::string s; + + GENERATE_CMP(SearchPath::Path, me->s); +}; + +struct SearchPath::Elem +{ + + Prefix prefix; + Path path; + + GENERATE_CMP(SearchPath::Elem, me->prefix, me->path); + + /** + * Parse a string into a `SearchPath::Elem` + */ + static SearchPath::Elem parse(std::string_view rawElem); +}; + +} diff --git a/src/libexpr/tests/derived-path.cc b/src/libexpr/tests/derived-path.cc index 8210efef22d0..d5fc6f201557 100644 --- a/src/libexpr/tests/derived-path.cc +++ b/src/libexpr/tests/derived-path.cc @@ -18,15 +18,17 @@ TEST_F(DerivedPathExpressionTest, force_init) { } +#ifndef COVERAGE + RC_GTEST_FIXTURE_PROP( DerivedPathExpressionTest, prop_opaque_path_round_trip, - (const DerivedPath::Opaque & o)) + (const SingleDerivedPath::Opaque & o)) { auto * v = state.allocValue(); state.mkStorePathString(o.path, *v); - auto d = state.coerceToDerivedPath(noPos, *v, ""); - RC_ASSERT(DerivedPath { o } == d); + auto d = state.coerceToSingleDerivedPath(noPos, *v, ""); + RC_ASSERT(SingleDerivedPath { o } == d); } // TODO use DerivedPath::Built for parameter once it supports a single output @@ -34,32 +36,33 @@ RC_GTEST_FIXTURE_PROP( RC_GTEST_FIXTURE_PROP( DerivedPathExpressionTest, - prop_built_path_placeholder_round_trip, - (const StorePath & drvPath, const StorePathName & outputName)) + prop_derived_path_built_placeholder_round_trip, + (const SingleDerivedPath::Built & b)) { + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "ca-derivations"); + auto * v = state.allocValue(); - state.mkOutputString(*v, drvPath, outputName.name, std::nullopt); - auto [d, _] = state.coerceToDerivedPathUnchecked(noPos, *v, ""); - DerivedPath::Built b { - .drvPath = drvPath, - .outputs = OutputsSpec::Names { outputName.name }, - }; - RC_ASSERT(DerivedPath { b } == d); + state.mkOutputString(*v, b, std::nullopt, mockXpSettings); + auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, ""); + RC_ASSERT(SingleDerivedPath { b } == d); } RC_GTEST_FIXTURE_PROP( DerivedPathExpressionTest, - prop_built_path_out_path_round_trip, - (const StorePath & drvPath, const StorePathName & outputName, const StorePath & outPath)) + prop_derived_path_built_out_path_round_trip, + (const SingleDerivedPath::Built & b, const StorePath & outPath)) { auto * v = state.allocValue(); - state.mkOutputString(*v, drvPath, outputName.name, outPath); - auto [d, _] = state.coerceToDerivedPathUnchecked(noPos, *v, ""); - DerivedPath::Built b { - .drvPath = drvPath, - .outputs = OutputsSpec::Names { outputName.name }, - }; - RC_ASSERT(DerivedPath { b } == d); + state.mkOutputString(*v, b, outPath); + auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, ""); + RC_ASSERT(SingleDerivedPath { b } == d); } +#endif + } /* namespace nix */ diff --git a/src/libexpr/tests/error_traces.cc b/src/libexpr/tests/error_traces.cc index 285651256ca8..81498f65a3f8 100644 --- a/src/libexpr/tests/error_traces.cc +++ b/src/libexpr/tests/error_traces.cc @@ -310,7 +310,7 @@ namespace nix { ASSERT_TRACE2("storePath true", TypeError, hintfmt("cannot coerce %s to a string", "a Boolean"), - hintfmt("while evaluating the first argument passed to builtins.storePath")); + hintfmt("while evaluating the first argument passed to 'builtins.storePath'")); } @@ -378,12 +378,12 @@ namespace nix { ASSERT_TRACE2("filterSource [] []", TypeError, hintfmt("cannot coerce %s to a string", "a list"), - hintfmt("while evaluating the second argument (the path to filter) passed to builtins.filterSource")); + hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); ASSERT_TRACE2("filterSource [] \"foo\"", EvalError, hintfmt("string '%s' doesn't represent an absolute path", "foo"), - hintfmt("while evaluating the second argument (the path to filter) passed to builtins.filterSource")); + hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); ASSERT_TRACE2("filterSource [] ./.", TypeError, @@ -906,12 +906,12 @@ namespace nix { ASSERT_TRACE2("concatMap (x: 1) [ \"foo\" ] # TODO", TypeError, hintfmt("value is %s while a list was expected", "an integer"), - hintfmt("while evaluating the return value of the function passed to buitlins.concatMap")); + hintfmt("while evaluating the return value of the function passed to builtins.concatMap")); ASSERT_TRACE2("concatMap (x: \"foo\") [ 1 2 ] # TODO", TypeError, hintfmt("value is %s while a list was expected", "a string"), - hintfmt("while evaluating the return value of the function passed to buitlins.concatMap")); + hintfmt("while evaluating the return value of the function passed to builtins.concatMap")); } @@ -1084,7 +1084,7 @@ namespace nix { ASSERT_TRACE1("hashString \"foo\" \"content\"", UsageError, - hintfmt("unknown hash algorithm '%s'", "foo")); + hintfmt("unknown hash algorithm '%s', expect 'md5', 'sha1', 'sha256', or 'sha512'", "foo")); ASSERT_TRACE2("hashString \"sha256\" {}", TypeError, diff --git a/src/libexpr/tests/flakeref.cc b/src/libexpr/tests/flakeref.cc new file mode 100644 index 000000000000..2b7809b938d3 --- /dev/null +++ b/src/libexpr/tests/flakeref.cc @@ -0,0 +1,22 @@ +#include + +#include "flake/flakeref.hh" + +namespace nix { + +/* ----------- tests for flake/flakeref.hh --------------------------------------------------*/ + + /* ---------------------------------------------------------------------------- + * to_string + * --------------------------------------------------------------------------*/ + + TEST(to_string, doesntReencodeUrl) { + auto s = "http://localhost:8181/test/+3d.tar.gz"; + auto flakeref = parseFlakeRef(s); + auto parsed = flakeref.to_string(); + auto expected = "http://localhost:8181/test/%2B3d.tar.gz"; + + ASSERT_EQ(parsed, expected); + } + +} diff --git a/src/libexpr/tests/json.cc b/src/libexpr/tests/json.cc index 7586bdd9b34e..f4cc118d6645 100644 --- a/src/libexpr/tests/json.cc +++ b/src/libexpr/tests/json.cc @@ -62,7 +62,7 @@ namespace nix { // not supported by store 'dummy'" thrown in the test body. TEST_F(JSONValueTest, DISABLED_Path) { Value v; - v.mkPath("test"); + v.mkPath(state.rootPath(CanonPath("/test"))); ASSERT_EQ(getJSONValue(v), "\"/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x\""); } } /* namespace nix */ diff --git a/src/libexpr/tests/libexpr.hh b/src/libexpr/tests/libexpr.hh index b8e65aafe43b..9684314469e8 100644 --- a/src/libexpr/tests/libexpr.hh +++ b/src/libexpr/tests/libexpr.hh @@ -71,7 +71,7 @@ namespace nix { if (arg.type() != nString) { return false; } - return std::string_view(arg.string.s) == s; + return std::string_view(arg.c_str()) == s; } MATCHER_P(IsIntEq, v, fmt("The string is equal to \"%1%\"", v)) { @@ -103,14 +103,17 @@ namespace nix { } MATCHER_P(IsPathEq, p, fmt("Is a path equal to \"%1%\"", p)) { - if (arg.type() != nPath) { - *result_listener << "Expected a path got " << arg.type(); - return false; - } else if (std::string_view(arg.string.s) != p) { - *result_listener << "Expected a path that equals \"" << p << "\" but got: " << arg.string.s; + if (arg.type() != nPath) { + *result_listener << "Expected a path got " << arg.type(); + return false; + } else { + auto path = arg.path(); + if (path.path != CanonPath(p)) { + *result_listener << "Expected a path that equals \"" << p << "\" but got: " << path.path; return false; } - return true; + } + return true; } diff --git a/src/libexpr/tests/local.mk b/src/libexpr/tests/local.mk index 331a5ead60e4..7689a03e0a60 100644 --- a/src/libexpr/tests/local.mk +++ b/src/libexpr/tests/local.mk @@ -6,7 +6,11 @@ libexpr-tests_NAME := libnixexpr-tests libexpr-tests_DIR := $(d) -libexpr-tests_INSTALL_DIR := +ifeq ($(INSTALL_UNIT_TESTS), yes) + libexpr-tests_INSTALL_DIR := $(checkbindir) +else + libexpr-tests_INSTALL_DIR := +endif libexpr-tests_SOURCES := \ $(wildcard $(d)/*.cc) \ @@ -16,4 +20,4 @@ libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/l libexpr-tests_LIBS = libstore-tests libutils-tests libexpr libutil libstore libfetchers -libexpr-tests_LDFLAGS := $(GTEST_LIBS) -lgmock +libexpr-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock diff --git a/src/libexpr/tests/primops.cc b/src/libexpr/tests/primops.cc index ce3b5d11fae8..7485fa0d0eab 100644 --- a/src/libexpr/tests/primops.cc +++ b/src/libexpr/tests/primops.cc @@ -711,14 +711,14 @@ namespace nix { // FIXME: add a test that verifies the string context is as expected auto v = eval("builtins.replaceStrings [\"oo\" \"a\"] [\"a\" \"i\"] \"foobar\""); ASSERT_EQ(v.type(), nString); - ASSERT_EQ(v.string.s, std::string_view("fabir")); + ASSERT_EQ(v.string_view(), "fabir"); } TEST_F(PrimOpTest, concatStringsSep) { // FIXME: add a test that verifies the string context is as expected auto v = eval("builtins.concatStringsSep \"%\" [\"foo\" \"bar\" \"baz\"]"); ASSERT_EQ(v.type(), nString); - ASSERT_EQ(std::string_view(v.string.s), "foo%bar%baz"); + ASSERT_EQ(v.string_view(), "foo%bar%baz"); } TEST_F(PrimOpTest, split1) { @@ -814,6 +814,14 @@ namespace nix { ASSERT_THAT(*v.listElems()[0], IsStringEq("FOO")); } + TEST_F(PrimOpTest, match5) { + // The regex "\\{}" is valid and matches the string "{}". + // Caused a regression before when trying to switch from std::regex to boost::regex. + // See https://github.com/NixOS/nix/pull/7762#issuecomment-1834303659 + auto v = eval("builtins.match \"\\\\{}\" \"{}\""); + ASSERT_THAT(v, IsListOfSize(0)); + } + TEST_F(PrimOpTest, attrNames) { auto v = eval("builtins.attrNames { x = 1; y = 2; z = 3; a = 2; }"); ASSERT_THAT(v, IsListOfSize(4)); diff --git a/src/libexpr/tests/search-path.cc b/src/libexpr/tests/search-path.cc new file mode 100644 index 000000000000..dbe7ab95fb57 --- /dev/null +++ b/src/libexpr/tests/search-path.cc @@ -0,0 +1,90 @@ +#include +#include + +#include "search-path.hh" + +namespace nix { + +TEST(SearchPathElem, parse_justPath) { + ASSERT_EQ( + SearchPath::Elem::parse("foo"), + (SearchPath::Elem { + .prefix = SearchPath::Prefix { .s = "" }, + .path = SearchPath::Path { .s = "foo" }, + })); +} + +TEST(SearchPathElem, parse_emptyPrefix) { + ASSERT_EQ( + SearchPath::Elem::parse("=foo"), + (SearchPath::Elem { + .prefix = SearchPath::Prefix { .s = "" }, + .path = SearchPath::Path { .s = "foo" }, + })); +} + +TEST(SearchPathElem, parse_oneEq) { + ASSERT_EQ( + SearchPath::Elem::parse("foo=bar"), + (SearchPath::Elem { + .prefix = SearchPath::Prefix { .s = "foo" }, + .path = SearchPath::Path { .s = "bar" }, + })); +} + +TEST(SearchPathElem, parse_twoEqs) { + ASSERT_EQ( + SearchPath::Elem::parse("foo=bar=baz"), + (SearchPath::Elem { + .prefix = SearchPath::Prefix { .s = "foo" }, + .path = SearchPath::Path { .s = "bar=baz" }, + })); +} + + +TEST(SearchPathElem, suffixIfPotentialMatch_justPath) { + SearchPath::Prefix prefix { .s = "" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("any/thing"), std::optional { "any/thing" }); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_misleadingPrefix1) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("fooX"), std::nullopt); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_misleadingPrefix2) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("fooX/bar"), std::nullopt); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_partialPrefix) { + SearchPath::Prefix prefix { .s = "fooX" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo"), std::nullopt); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_exactPrefix) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo"), std::optional { "" }); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_multiKey) { + SearchPath::Prefix prefix { .s = "foo/bar" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/bar/baz"), std::optional { "baz" }); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_trailingSlash) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/"), std::optional { "" }); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_trailingDoubleSlash) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo//"), std::optional { "/" }); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_trailingPath) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/bar/baz"), std::optional { "bar/baz" }); +} + +} diff --git a/src/libexpr/tests/value/context.cc b/src/libexpr/tests/value/context.cc index 0d93815776fe..92d4889abde6 100644 --- a/src/libexpr/tests/value/context.cc +++ b/src/libexpr/tests/value/context.cc @@ -8,6 +8,8 @@ namespace nix { +// Test a few cases of invalid string context elements. + TEST(NixStringContextElemTest, empty_invalid) { EXPECT_THROW( NixStringContextElem::parse(""), @@ -38,46 +40,88 @@ TEST(NixStringContextElemTest, slash_invalid) { BadStorePath); } +/** + * Round trip (string <-> data structure) test for + * `NixStringContextElem::Opaque`. + */ TEST(NixStringContextElemTest, opaque) { std::string_view opaque = "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x"; auto elem = NixStringContextElem::parse(opaque); - auto * p = std::get_if(&elem); + auto * p = std::get_if(&elem.raw); ASSERT_TRUE(p); ASSERT_EQ(p->path, StorePath { opaque }); ASSERT_EQ(elem.to_string(), opaque); } +/** + * Round trip (string <-> data structure) test for + * `NixStringContextElem::DrvDeep`. + */ TEST(NixStringContextElemTest, drvDeep) { std::string_view drvDeep = "=g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; auto elem = NixStringContextElem::parse(drvDeep); - auto * p = std::get_if(&elem); + auto * p = std::get_if(&elem.raw); ASSERT_TRUE(p); ASSERT_EQ(p->drvPath, StorePath { drvDeep.substr(1) }); ASSERT_EQ(elem.to_string(), drvDeep); } -TEST(NixStringContextElemTest, built) { +/** + * Round trip (string <-> data structure) test for a simpler + * `NixStringContextElem::Built`. + */ +TEST(NixStringContextElemTest, built_opaque) { std::string_view built = "!foo!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; auto elem = NixStringContextElem::parse(built); - auto * p = std::get_if(&elem); + auto * p = std::get_if(&elem.raw); ASSERT_TRUE(p); ASSERT_EQ(p->output, "foo"); - ASSERT_EQ(p->drvPath, StorePath { built.substr(5) }); + ASSERT_EQ(*p->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { + .path = StorePath { built.substr(5) }, + })); ASSERT_EQ(elem.to_string(), built); } +/** + * Round trip (string <-> data structure) test for a more complex, + * inductive `NixStringContextElem::Built`. + */ +TEST(NixStringContextElemTest, built_built) { + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations"); + + std::string_view built = "!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; + auto elem = NixStringContextElem::parse(built, mockXpSettings); + auto * p = std::get_if(&elem.raw); + ASSERT_TRUE(p); + ASSERT_EQ(p->output, "foo"); + auto * drvPath = std::get_if(&*p->drvPath); + ASSERT_TRUE(drvPath); + ASSERT_EQ(drvPath->output, "bar"); + ASSERT_EQ(*drvPath->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { + .path = StorePath { built.substr(9) }, + })); + ASSERT_EQ(elem.to_string(), built); } -namespace rc { -using namespace nix; +/** + * Without the right experimental features enabled, we cannot parse a + * complex inductive string context element. + */ +TEST(NixStringContextElemTest, built_built_xp) { + ASSERT_THROW( + NixStringContextElem::parse("!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"), MissingExperimentalFeature); +} -Gen Arbitrary::arbitrary() -{ - return gen::just(NixStringContextElem::Opaque { - .path = *gen::arbitrary(), - }); } +namespace rc { +using namespace nix; + Gen Arbitrary::arbitrary() { return gen::just(NixStringContextElem::DrvDeep { @@ -85,14 +129,6 @@ Gen Arbitrary::arb }); } -Gen Arbitrary::arbitrary() -{ - return gen::just(NixStringContextElem::Built { - .drvPath = *gen::arbitrary(), - .output = (*gen::arbitrary()).name, - }); -} - Gen Arbitrary::arbitrary() { switch (*gen::inRange(0, std::variant_size_v)) { @@ -111,6 +147,8 @@ Gen Arbitrary::arbitrary() namespace nix { +#ifndef COVERAGE + RC_GTEST_PROP( NixStringContextElemTest, prop_round_rip, @@ -119,4 +157,6 @@ RC_GTEST_PROP( RC_ASSERT(o == NixStringContextElem::parse(o.to_string())); } +#endif + } diff --git a/src/libexpr/tests/value/print.cc b/src/libexpr/tests/value/print.cc index 5e96e12ec7d3..a4f6fc014f35 100644 --- a/src/libexpr/tests/value/print.cc +++ b/src/libexpr/tests/value/print.cc @@ -114,7 +114,8 @@ TEST_F(ValuePrintingTests, vLambda) TEST_F(ValuePrintingTests, vPrimOp) { Value vPrimOp; - vPrimOp.mkPrimOp(nullptr); + PrimOp primOp{}; + vPrimOp.mkPrimOp(&primOp); test(vPrimOp, ""); } diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 4996a5bde87b..74b3ebf136ba 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -1,7 +1,7 @@ #include "value-to-json.hh" #include "eval-inline.hh" -#include "util.hh" #include "store-api.hh" +#include "signals.hh" #include #include @@ -31,7 +31,7 @@ json printValueAsJSON(EvalState & state, bool strict, case nString: copyContext(v, context); - out = v.string.s; + out = v.c_str(); break; case nPath: @@ -43,6 +43,7 @@ json printValueAsJSON(EvalState & state, bool strict, break; case nNull: + // already initialized as null break; case nAttrs: { @@ -59,7 +60,13 @@ json printValueAsJSON(EvalState & state, bool strict, names.emplace(state.symbols[j.name]); for (auto & j : names) { Attr & a(*v.attrs->find(state.symbols.create(j))); - out[j] = printValueAsJSON(state, strict, *a.value, a.pos, context, copyToStore); + try { + out[j] = printValueAsJSON(state, strict, *a.value, a.pos, context, copyToStore); + } catch (Error & e) { + e.addTrace(state.positions[a.pos], + hintfmt("while evaluating attribute '%1%'", j)); + throw; + } } } else return printValueAsJSON(state, strict, *i->value, i->pos, context, copyToStore); @@ -68,8 +75,17 @@ json printValueAsJSON(EvalState & state, bool strict, case nList: { out = json::array(); - for (auto elem : v.listItems()) - out.push_back(printValueAsJSON(state, strict, *elem, pos, context, copyToStore)); + int i = 0; + for (auto elem : v.listItems()) { + try { + out.push_back(printValueAsJSON(state, strict, *elem, pos, context, copyToStore)); + } catch (Error & e) { + e.addTrace({}, + hintfmt("while evaluating list element at index %1%", i)); + throw; + } + i++; + } break; } diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 2539ad1c1284..5032115bbb51 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -1,7 +1,7 @@ #include "value-to-xml.hh" #include "xml-writer.hh" #include "eval-inline.hh" -#include "util.hh" +#include "signals.hh" #include @@ -74,7 +74,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, case nString: /* !!! show the context? */ copyContext(v, context); - doc.writeEmptyElement("string", singletonAttrs("value", v.string.s)); + doc.writeEmptyElement("string", singletonAttrs("value", v.c_str())); break; case nPath: @@ -96,14 +96,14 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, if (a != v.attrs->end()) { if (strict) state.forceValue(*a->value, a->pos); if (a->value->type() == nString) - xmlAttrs["drvPath"] = drvPath = a->value->string.s; + xmlAttrs["drvPath"] = drvPath = a->value->c_str(); } a = v.attrs->find(state.sOutPath); if (a != v.attrs->end()) { if (strict) state.forceValue(*a->value, a->pos); if (a->value->type() == nString) - xmlAttrs["outPath"] = a->value->string.s; + xmlAttrs["outPath"] = a->value->c_str(); } XMLOpenElement _(doc, "derivation", xmlAttrs); diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index c44683e501ec..bcff8ae55b7d 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -3,6 +3,7 @@ #include #include +#include #include "symbol-table.hh" #include "value/context.hh" @@ -158,60 +159,72 @@ public: inline bool isPrimOp() const { return internalType == tPrimOp; }; inline bool isPrimOpApp() const { return internalType == tPrimOpApp; }; + /** + * Strings in the evaluator carry a so-called `context` which + * is a list of strings representing store paths. This is to + * allow users to write things like + * + * "--with-freetype2-library=" + freetype + "/lib" + * + * where `freetype` is a derivation (or a source to be copied + * to the store). If we just concatenated the strings without + * keeping track of the referenced store paths, then if the + * string is used as a derivation attribute, the derivation + * will not have the correct dependencies in its inputDrvs and + * inputSrcs. + + * The semantics of the context is as follows: when a string + * with context C is used as a derivation attribute, then the + * derivations in C will be added to the inputDrvs of the + * derivation, and the other store paths in C will be added to + * the inputSrcs of the derivations. + + * For canonicity, the store paths should be in sorted order. + */ + struct StringWithContext { + const char * c_str; + const char * * context; // must be in sorted order + }; + + struct Path { + InputAccessor * accessor; + const char * path; + }; + + struct ClosureThunk { + Env * env; + Expr * expr; + }; + + struct FunctionApplicationThunk { + Value * left, * right; + }; + + struct Lambda { + Env * env; + ExprLambda * fun; + }; + union { NixInt integer; bool boolean; - /** - * Strings in the evaluator carry a so-called `context` which - * is a list of strings representing store paths. This is to - * allow users to write things like - - * "--with-freetype2-library=" + freetype + "/lib" + StringWithContext string; - * where `freetype` is a derivation (or a source to be copied - * to the store). If we just concatenated the strings without - * keeping track of the referenced store paths, then if the - * string is used as a derivation attribute, the derivation - * will not have the correct dependencies in its inputDrvs and - * inputSrcs. + Path _path; - * The semantics of the context is as follows: when a string - * with context C is used as a derivation attribute, then the - * derivations in C will be added to the inputDrvs of the - * derivation, and the other store paths in C will be added to - * the inputSrcs of the derivations. - - * For canonicity, the store paths should be in sorted order. - */ - struct { - const char * s; - const char * * context; // must be in sorted order - } string; - - const char * _path; Bindings * attrs; struct { size_t size; Value * * elems; } bigList; Value * smallList[2]; - struct { - Env * env; - Expr * expr; - } thunk; - struct { - Value * left, * right; - } app; - struct { - Env * env; - ExprLambda * fun; - } lambda; + ClosureThunk thunk; + FunctionApplicationThunk app; + Lambda lambda; PrimOp * primOp; - struct { - Value * left, * right; - } primOpApp; + FunctionApplicationThunk primOpApp; ExternalValueBase * external; NixFloat fpoint; }; @@ -270,7 +283,7 @@ public: inline void mkString(const char * s, const char * * context = 0) { internalType = tString; - string.s = s; + string.c_str = s; string.context = context; } @@ -287,11 +300,12 @@ public: void mkPath(const SourcePath & path); - inline void mkPath(const char * path) + inline void mkPath(InputAccessor * accessor, const char * path) { clearValue(); internalType = tPath; - _path = path; + _path.accessor = accessor; + _path.path = path; } inline void mkNull() @@ -349,13 +363,7 @@ public: // Value will be overridden anyways } - inline void mkPrimOp(PrimOp * p) - { - clearValue(); - internalType = tPrimOp; - primOp = p; - } - + void mkPrimOp(PrimOp * p); inline void mkPrimOpApp(Value * l, Value * r) { @@ -388,7 +396,13 @@ public: return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems; } - const Value * const * listElems() const + std::span listItems() const + { + assert(isList()); + return std::span(listElems(), listSize()); + } + + Value * const * listElems() const { return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems; } @@ -407,44 +421,30 @@ public: */ bool isTrivial() const; - auto listItems() + SourcePath path() const { - struct ListIterable - { - typedef Value * const * iterator; - iterator _begin, _end; - iterator begin() const { return _begin; } - iterator end() const { return _end; } + assert(internalType == tPath); + return SourcePath { + .accessor = ref(_path.accessor->shared_from_this()), + .path = CanonPath(CanonPath::unchecked_t(), _path.path) }; - assert(isList()); - auto begin = listElems(); - return ListIterable { begin, begin + listSize() }; } - auto listItems() const + std::string_view string_view() const { - struct ConstListIterable - { - typedef const Value * const * iterator; - iterator _begin, _end; - iterator begin() const { return _begin; } - iterator end() const { return _end; } - }; - assert(isList()); - auto begin = listElems(); - return ConstListIterable { begin, begin + listSize() }; + assert(internalType == tString); + return std::string_view(string.c_str); } - SourcePath path() const + const char * const c_str() const { - assert(internalType == tPath); - return SourcePath{CanonPath(_path)}; + assert(internalType == tString); + return string.c_str; } - std::string_view str() const + const char * * context() const { - assert(internalType == tString); - return std::string_view(string.s); + return string.context; } }; diff --git a/src/libexpr/value/context.cc b/src/libexpr/value/context.cc index f76fc76e48d2..6d9633268dfe 100644 --- a/src/libexpr/value/context.cc +++ b/src/libexpr/value/context.cc @@ -1,32 +1,56 @@ +#include "util.hh" #include "value/context.hh" #include namespace nix { -NixStringContextElem NixStringContextElem::parse(std::string_view s0) +NixStringContextElem NixStringContextElem::parse( + std::string_view s0, + const ExperimentalFeatureSettings & xpSettings) { std::string_view s = s0; + std::function parseRest; + parseRest = [&]() -> SingleDerivedPath { + // Case on whether there is a '!' + size_t index = s.find("!"); + if (index == std::string_view::npos) { + return SingleDerivedPath::Opaque { + .path = StorePath { s }, + }; + } else { + std::string output { s.substr(0, index) }; + // Advance string to parse after the '!' + s = s.substr(index + 1); + auto drv = make_ref(parseRest()); + drvRequireExperiment(*drv, xpSettings); + return SingleDerivedPath::Built { + .drvPath = std::move(drv), + .output = std::move(output), + }; + } + }; + if (s.size() == 0) { throw BadNixStringContextElem(s0, "String context element should never be an empty string"); } + switch (s.at(0)) { case '!': { - s = s.substr(1); // advance string to parse after first ! - size_t index = s.find("!"); - // This makes index + 1 safe. Index can be the length (one after index - // of last character), so given any valid character index --- a - // successful find --- we can add one. - if (index == std::string_view::npos) { + // Advance string to parse after the '!' + s = s.substr(1); + + // Find *second* '!' + if (s.find("!") == std::string_view::npos) { throw BadNixStringContextElem(s0, "String content element beginning with '!' should have a second '!'"); } - return NixStringContextElem::Built { - .drvPath = StorePath { s.substr(index + 1) }, - .output = std::string(s.substr(0, index)), - }; + + return std::visit( + [&](auto x) -> NixStringContextElem { return std::move(x); }, + parseRest()); } case '=': { return NixStringContextElem::DrvDeep { @@ -34,33 +58,51 @@ NixStringContextElem NixStringContextElem::parse(std::string_view s0) }; } default: { - return NixStringContextElem::Opaque { - .path = StorePath { s }, - }; + // Ensure no '!' + if (s.find("!") != std::string_view::npos) { + throw BadNixStringContextElem(s0, + "String content element not beginning with '!' should not have a second '!'"); + } + return std::visit( + [&](auto x) -> NixStringContextElem { return std::move(x); }, + parseRest()); } } } -std::string NixStringContextElem::to_string() const { - return std::visit(overloaded { +std::string NixStringContextElem::to_string() const +{ + std::string res; + + std::function toStringRest; + toStringRest = [&](auto & p) { + std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) { + res += o.path.to_string(); + }, + [&](const SingleDerivedPath::Built & o) { + res += o.output; + res += '!'; + toStringRest(*o.drvPath); + }, + }, p.raw()); + }; + + std::visit(overloaded { [&](const NixStringContextElem::Built & b) { - std::string res; - res += '!'; - res += b.output; res += '!'; - res += b.drvPath.to_string(); - return res; + toStringRest(b); + }, + [&](const NixStringContextElem::Opaque & o) { + toStringRest(o); }, [&](const NixStringContextElem::DrvDeep & d) { - std::string res; res += '='; res += d.drvPath.to_string(); - return res; }, - [&](const NixStringContextElem::Opaque & o) { - return std::string { o.path.to_string() }; - }, - }, raw()); + }, raw); + + return res; } } diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh index 287ae08a942d..51fd30a44cb2 100644 --- a/src/libexpr/value/context.hh +++ b/src/libexpr/value/context.hh @@ -1,11 +1,9 @@ #pragma once ///@file -#include "util.hh" #include "comparator.hh" -#include "path.hh" - -#include +#include "derived-path.hh" +#include "variant-wrapper.hh" #include @@ -26,75 +24,59 @@ public: } }; -/** - * Plain opaque path to some store object. - * - * Encoded as just the path: ‘’. - */ -struct NixStringContextElem_Opaque { - StorePath path; - - GENERATE_CMP(NixStringContextElem_Opaque, me->path); -}; +struct NixStringContextElem { + /** + * Plain opaque path to some store object. + * + * Encoded as just the path: ‘’. + */ + using Opaque = SingleDerivedPath::Opaque; -/** - * Path to a derivation and its entire build closure. - * - * The path doesn't just refer to derivation itself and its closure, but - * also all outputs of all derivations in that closure (including the - * root derivation). - * - * Encoded in the form ‘=’. - */ -struct NixStringContextElem_DrvDeep { - StorePath drvPath; - - GENERATE_CMP(NixStringContextElem_DrvDeep, me->drvPath); -}; + /** + * Path to a derivation and its entire build closure. + * + * The path doesn't just refer to derivation itself and its closure, but + * also all outputs of all derivations in that closure (including the + * root derivation). + * + * Encoded in the form ‘=’. + */ + struct DrvDeep { + StorePath drvPath; -/** - * Derivation output. - * - * Encoded in the form ‘!!’. - */ -struct NixStringContextElem_Built { - StorePath drvPath; - std::string output; + GENERATE_CMP(DrvDeep, me->drvPath); + }; - GENERATE_CMP(NixStringContextElem_Built, me->drvPath, me->output); -}; + /** + * Derivation output. + * + * Encoded in the form ‘!!’. + */ + using Built = SingleDerivedPath::Built; -using _NixStringContextElem_Raw = std::variant< - NixStringContextElem_Opaque, - NixStringContextElem_DrvDeep, - NixStringContextElem_Built ->; + using Raw = std::variant< + Opaque, + DrvDeep, + Built + >; -struct NixStringContextElem : _NixStringContextElem_Raw { - using Raw = _NixStringContextElem_Raw; - using Raw::Raw; + Raw raw; - using Opaque = NixStringContextElem_Opaque; - using DrvDeep = NixStringContextElem_DrvDeep; - using Built = NixStringContextElem_Built; + GENERATE_CMP(NixStringContextElem, me->raw); - inline const Raw & raw() const & { - return static_cast(*this); - } - inline Raw & raw() & { - return static_cast(*this); - } - inline Raw && raw() && { - return static_cast(*this); - } + MAKE_WRAPPER_CONSTRUCTOR(NixStringContextElem); /** * Decode a context string, one of: * - ‘’ * - ‘=’ * - ‘!!’ + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ - static NixStringContextElem parse(std::string_view s); + static NixStringContextElem parse( + std::string_view s, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); std::string to_string() const; }; diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh index 9f885a7935e4..b9a2c824ea7b 100644 --- a/src/libfetchers/attrs.hh +++ b/src/libfetchers/attrs.hh @@ -13,6 +13,12 @@ namespace nix::fetchers { typedef std::variant> Attr; + +/** + * An `Attrs` can be thought of a JSON object restricted or simplified + * to be "flat", not containing any subcontainers (arrays or objects) + * and also not containing any `null`s. + */ typedef std::map Attrs; Attrs jsonToAttrs(const nlohmann::json & json); diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index 0c8ecac9d48a..63b05bdab19a 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -1,4 +1,5 @@ #include "cache.hh" +#include "users.hh" #include "sqlite.hh" #include "sync.hh" #include "store-api.hh" @@ -19,6 +20,9 @@ create table if not exists Cache ( ); )sql"; +// FIXME: we should periodically purge/nuke this cache to prevent it +// from growing too big. + struct CacheImpl : Cache { struct State @@ -47,6 +51,60 @@ struct CacheImpl : Cache "select info, path, immutable, timestamp from Cache where input = ?"); } + void upsert( + const Attrs & inAttrs, + const Attrs & infoAttrs) override + { + _state.lock()->add.use() + (attrsToJSON(inAttrs).dump()) + (attrsToJSON(infoAttrs).dump()) + ("") // no path + (false) + (time(0)).exec(); + } + + std::optional lookup(const Attrs & inAttrs) override + { + if (auto res = lookupExpired(inAttrs)) + return std::move(res->infoAttrs); + return {}; + } + + std::optional lookupWithTTL(const Attrs & inAttrs) override + { + if (auto res = lookupExpired(inAttrs)) { + if (!res->expired) + return std::move(res->infoAttrs); + debug("ignoring expired cache entry '%s'", + attrsToJSON(inAttrs).dump()); + } + return {}; + } + + std::optional lookupExpired(const Attrs & inAttrs) override + { + auto state(_state.lock()); + + auto inAttrsJSON = attrsToJSON(inAttrs).dump(); + + auto stmt(state->lookup.use()(inAttrsJSON)); + if (!stmt.next()) { + debug("did not find cache entry for '%s'", inAttrsJSON); + return {}; + } + + auto infoJSON = stmt.getStr(0); + auto locked = stmt.getInt(2) != 0; + auto timestamp = stmt.getInt(3); + + debug("using cache entry '%s' -> '%s'", inAttrsJSON, infoJSON); + + return Result2 { + .expired = !locked && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)), + .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJSON)), + }; + } + void add( ref store, const Attrs & inAttrs, diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh index ae398d0404ba..f70589267f76 100644 --- a/src/libfetchers/cache.hh +++ b/src/libfetchers/cache.hh @@ -2,13 +2,53 @@ ///@file #include "fetchers.hh" +#include "path.hh" namespace nix::fetchers { +/** + * A cache for arbitrary `Attrs` -> `Attrs` mappings with a timestamp + * for expiration. + */ struct Cache { virtual ~Cache() { } + /** + * Add a value to the cache. The cache is an arbitrary mapping of + * Attrs to Attrs. + */ + virtual void upsert( + const Attrs & inAttrs, + const Attrs & infoAttrs) = 0; + + /** + * Look up a key with infinite TTL. + */ + virtual std::optional lookup( + const Attrs & inAttrs) = 0; + + /** + * Look up a key. Return nothing if its TTL has exceeded + * `settings.tarballTTL`. + */ + virtual std::optional lookupWithTTL( + const Attrs & inAttrs) = 0; + + struct Result2 + { + bool expired = false; + Attrs infoAttrs; + }; + + /** + * Look up a key. Return a bool denoting whether its TTL has + * exceeded `settings.tarballTTL`. + */ + virtual std::optional lookupExpired( + const Attrs & inAttrs) = 0; + + /* Old cache for things that have a store path. */ virtual void add( ref store, const Attrs & inAttrs, diff --git a/src/libfetchers/fetch-settings.hh b/src/libfetchers/fetch-settings.hh index 6108a179cda4..f095963a8340 100644 --- a/src/libfetchers/fetch-settings.hh +++ b/src/libfetchers/fetch-settings.hh @@ -3,7 +3,6 @@ #include "types.hh" #include "config.hh" -#include "util.hh" #include #include diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 2860c1ceb286..60208619e87b 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -1,21 +1,41 @@ #include "fetchers.hh" #include "store-api.hh" +#include "input-accessor.hh" #include namespace nix::fetchers { -std::unique_ptr>> inputSchemes = nullptr; +using InputSchemeMap = std::map>; + +std::unique_ptr inputSchemes = nullptr; void registerInputScheme(std::shared_ptr && inputScheme) { - if (!inputSchemes) inputSchemes = std::make_unique>>(); - inputSchemes->push_back(std::move(inputScheme)); + if (!inputSchemes) + inputSchemes = std::make_unique(); + auto schemeName = inputScheme->schemeName(); + if (inputSchemes->count(schemeName) > 0) + throw Error("Input scheme with name %s already registered", schemeName); + inputSchemes->insert_or_assign(schemeName, std::move(inputScheme)); +} + +nlohmann::json dumpRegisterInputSchemeInfo() { + using nlohmann::json; + + auto res = json::object(); + + for (auto & [name, scheme] : *inputSchemes) { + auto & r = res[name] = json::object(); + r["allowedAttrs"] = scheme->allowedAttrs(); + } + + return res; } -Input Input::fromURL(const std::string & url) +Input Input::fromURL(const std::string & url, bool requireTree) { - return fromURL(parseURL(url)); + return fromURL(parseURL(url), requireTree); } static void fixupInput(Input & input) @@ -31,11 +51,12 @@ static void fixupInput(Input & input) input.locked = true; } -Input Input::fromURL(const ParsedURL & url) +Input Input::fromURL(const ParsedURL & url, bool requireTree) { - for (auto & inputScheme : *inputSchemes) { - auto res = inputScheme->inputFromURL(url); + for (auto & [_, inputScheme] : *inputSchemes) { + auto res = inputScheme->inputFromURL(url, requireTree); if (res) { + experimentalFeatureSettings.require(inputScheme->experimentalFeature()); res->scheme = inputScheme; fixupInput(*res); return std::move(*res); @@ -47,19 +68,49 @@ Input Input::fromURL(const ParsedURL & url) Input Input::fromAttrs(Attrs && attrs) { - for (auto & inputScheme : *inputSchemes) { - auto res = inputScheme->inputFromAttrs(attrs); - if (res) { - res->scheme = inputScheme; - fixupInput(*res); - return std::move(*res); - } - } + auto schemeName = ({ + auto schemeNameOpt = maybeGetStrAttr(attrs, "type"); + if (!schemeNameOpt) + throw Error("'type' attribute to specify input scheme is required but not provided"); + *std::move(schemeNameOpt); + }); - Input input; - input.attrs = attrs; - fixupInput(input); - return input; + auto raw = [&]() { + // Return an input without a scheme; most operations will fail, + // but not all of them. Doing this is to support those other + // operations which are supposed to be robust on + // unknown/uninterpretable inputs. + Input input; + input.attrs = attrs; + fixupInput(input); + return input; + }; + + std::shared_ptr inputScheme = ({ + auto i = inputSchemes->find(schemeName); + i == inputSchemes->end() ? nullptr : i->second; + }); + + if (!inputScheme) return raw(); + + experimentalFeatureSettings.require(inputScheme->experimentalFeature()); + + auto allowedAttrs = inputScheme->allowedAttrs(); + + for (auto & [name, _] : attrs) + if (name != "type" && allowedAttrs.count(name) == 0) + throw Error("input attribute '%s' not supported by scheme '%s'", name, schemeName); + + auto res = inputScheme->inputFromAttrs(attrs); + if (!res) return raw(); + res->scheme = inputScheme; + fixupInput(*res); + return std::move(*res); +} + +std::optional Input::getFingerprint(ref store) const +{ + return scheme ? scheme->getFingerprint(store, *this) : std::nullopt; } ParsedURL Input::toURL() const @@ -82,14 +133,14 @@ std::string Input::to_string() const return toURL().to_string(); } -Attrs Input::toAttrs() const +bool Input::isDirect() const { - return attrs; + return !scheme || scheme->isDirect(*this); } -bool Input::hasAllInfo() const +Attrs Input::toAttrs() const { - return getNarHash() && scheme && scheme->hasAllInfo(*this); + return attrs; } bool Input::operator ==(const Input & other) const @@ -107,7 +158,7 @@ bool Input::contains(const Input & other) const return false; } -std::pair Input::fetch(ref store) const +std::pair Input::fetch(ref store) const { if (!scheme) throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs())); @@ -115,7 +166,7 @@ std::pair Input::fetch(ref store) const /* The tree may already be in the Nix store, or it could be substituted (which is often faster than fetching from the original source). So check that. */ - if (hasAllInfo()) { + if (getNarHash()) { try { auto storePath = computeStorePath(*store); @@ -124,7 +175,7 @@ std::pair Input::fetch(ref store) const debug("using substituted/cached input '%s' in '%s'", to_string(), store->printStorePath(storePath)); - return {Tree { .actualPath = store->toRealPath(storePath), .storePath = std::move(storePath) }, *this}; + return {std::move(storePath), *this}; } catch (Error & e) { debug("substitution of input '%s' failed: %s", to_string(), e.what()); } @@ -139,18 +190,16 @@ std::pair Input::fetch(ref store) const } }(); - Tree tree { - .actualPath = store->toRealPath(storePath), - .storePath = storePath, - }; - - auto narHash = store->queryPathInfo(tree.storePath)->narHash; - input.attrs.insert_or_assign("narHash", narHash.to_string(SRI, true)); + auto narHash = store->queryPathInfo(storePath)->narHash; + input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); if (auto prevNarHash = getNarHash()) { if (narHash != *prevNarHash) throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", - to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash.to_string(SRI, true)); + to_string(), + store->printStorePath(storePath), + prevNarHash->to_string(HashFormat::SRI, true), + narHash.to_string(HashFormat::SRI, true)); } if (auto prevLastModified = getLastModified()) { @@ -173,9 +222,17 @@ std::pair Input::fetch(ref store) const input.locked = true; - assert(input.hasAllInfo()); + return {std::move(storePath), input}; +} - return {std::move(tree), input}; +std::pair, Input> Input::getAccessor(ref store) const +{ + try { + return scheme->getAccessor(store, *this); + } catch (Error & e) { + e.addTrace({}, "while fetching the input '%s'", to_string()); + throw; + } } Input Input::applyOverrides( @@ -198,12 +255,13 @@ std::optional Input::getSourcePath() const return scheme->getSourcePath(*this); } -void Input::markChangedFile( - std::string_view file, +void Input::putFile( + const CanonPath & path, + std::string_view contents, std::optional commitMsg) const { assert(scheme); - return scheme->markChangedFile(*this, file, commitMsg); + return scheme->putFile(*this, path, contents, commitMsg); } std::string Input::getName() const @@ -217,10 +275,8 @@ StorePath Input::computeStorePath(Store & store) const if (!narHash) throw Error("cannot compute store path for unlocked input '%s'", to_string()); return store.makeFixedOutputPath(getName(), FixedOutputInfo { - .hash = { - .method = FileIngestionMethod::Recursive, - .hash = *narHash, - }, + .method = FileIngestionMethod::Recursive, + .hash = *narHash, .references = {}, }); } @@ -256,7 +312,8 @@ std::optional Input::getRev() const try { hash = Hash::parseAnyPrefixed(*s); } catch (BadHash &e) { - // Default to sha1 for backwards compatibility with existing flakes + // Default to sha1 for backwards compatibility with existing + // usages (e.g. `builtins.fetchTree` calls or flake inputs). hash = Hash::parseAny(*s, htSHA1); } } @@ -295,14 +352,18 @@ Input InputScheme::applyOverrides( return input; } -std::optional InputScheme::getSourcePath(const Input & input) +std::optional InputScheme::getSourcePath(const Input & input) const { return {}; } -void InputScheme::markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) +void InputScheme::putFile( + const Input & input, + const CanonPath & path, + std::string_view contents, + std::optional commitMsg) const { - assert(false); + throw Error("input '%s' does not support modifying file '%s'", input.to_string(), path); } void InputScheme::clone(const Input & input, const Path & destDir) const @@ -310,4 +371,26 @@ void InputScheme::clone(const Input & input, const Path & destDir) const throw Error("do not know how to clone input '%s'", input.to_string()); } +std::pair InputScheme::fetch(ref store, const Input & input) +{ + auto [accessor, input2] = getAccessor(store, input); + auto storePath = accessor->root().fetchToStore(store, input2.getName()); + return {storePath, input2}; +} + +std::pair, Input> InputScheme::getAccessor(ref store, const Input & input) const +{ + throw UnimplementedError("InputScheme must implement fetch() or getAccessor()"); +} + +std::optional InputScheme::experimentalFeature() const +{ + return {}; +} + +std::string publicKeys_to_string(const std::vector& publicKeys) +{ + return ((nlohmann::json) publicKeys).dump(); +} + } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index d0738f61976c..5f3254b6d888 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -3,31 +3,25 @@ #include "types.hh" #include "hash.hh" -#include "path.hh" +#include "canon-path.hh" #include "attrs.hh" #include "url.hh" #include +#include -namespace nix { class Store; } +namespace nix { class Store; class StorePath; struct InputAccessor; } namespace nix::fetchers { -struct Tree -{ - Path actualPath; - StorePath storePath; -}; - struct InputScheme; /** - * The Input object is generated by a specific fetcher, based on the - * user-supplied input attribute in the flake.nix file, and contains + * The `Input` object is generated by a specific fetcher, based on + * user-supplied information, and contains * the information that the specific fetcher needs to perform the * actual fetch. The Input object is most commonly created via the - * "fromURL()" or "fromAttrs()" static functions which are provided - * the url or attrset specified in the flake file. + * `fromURL()` or `fromAttrs()` static functions. */ struct Input { @@ -36,7 +30,6 @@ struct Input std::shared_ptr scheme; // note: can be null Attrs attrs; bool locked = false; - bool direct = true; /** * path of the parent of this input, used for relative path resolution @@ -44,10 +37,20 @@ struct Input std::optional parent; public: - static Input fromURL(const std::string & url); + /** + * Create an `Input` from a URL. + * + * The URL indicate which sort of fetcher, and provides information to that fetcher. + */ + static Input fromURL(const std::string & url, bool requireTree = true); - static Input fromURL(const ParsedURL & url); + static Input fromURL(const ParsedURL & url, bool requireTree = true); + /** + * Create an `Input` from a an `Attrs`. + * + * The URL indicate which sort of fetcher, and provides information to that fetcher. + */ static Input fromAttrs(Attrs && attrs); ParsedURL toURL() const; @@ -62,7 +65,7 @@ public: * Check whether this is a "direct" input, that is, not * one that goes through a registry. */ - bool isDirect() const { return direct; } + bool isDirect() const; /** * Check whether this is a "locked" input, that is, @@ -70,24 +73,17 @@ public: */ bool isLocked() const { return locked; } - /** - * Check whether the input carries all necessary info required - * for cache insertion and substitution. - * These fields are used to uniquely identify cached trees - * within the "tarball TTL" window without necessarily - * indicating that the input's origin is unchanged. - */ - bool hasAllInfo() const; - bool operator ==(const Input & other) const; bool contains(const Input & other) const; /** - * Fetch the input into the Nix store, returning the location in - * the Nix store and the locked input. + * Fetch the entire input into the Nix store, returning the + * location in the Nix store and the locked input. */ - std::pair fetch(ref store) const; + std::pair fetch(ref store) const; + + std::pair, Input> getAccessor(ref store) const; Input applyOverrides( std::optional ref, @@ -97,8 +93,13 @@ public: std::optional getSourcePath() const; - void markChangedFile( - std::string_view file, + /** + * Write a file to this input, for input types that support + * writing. Optionally commit the change (for e.g. Git inputs). + */ + void putFile( + const CanonPath & path, + std::string_view contents, std::optional commitMsg) const; std::string getName() const; @@ -112,30 +113,52 @@ public: std::optional getRev() const; std::optional getRevCount() const; std::optional getLastModified() const; + + /** + * For locked inputs, return a string that uniquely specifies the + * content of the input (typically a commit hash or content hash). + */ + std::optional getFingerprint(ref store) const; }; /** - * The InputScheme represents a type of fetcher. Each fetcher - * registers with nix at startup time. When processing an input for a - * flake, each scheme is given an opportunity to "recognize" that - * input from the url or attributes in the flake file's specification - * and return an Input object to represent the input if it is - * recognized. The Input object contains the information the fetcher - * needs to actually perform the "fetch()" when called. + * The `InputScheme` represents a type of fetcher. Each fetcher + * registers with nix at startup time. When processing an `Input`, + * each scheme is given an opportunity to "recognize" that + * input from the user-provided url or attributes + * and return an `Input` object to represent the input if it is + * recognized. The `Input` object contains the information the fetcher + * needs to actually perform the `fetch()` when called. */ struct InputScheme { virtual ~InputScheme() { } - virtual std::optional inputFromURL(const ParsedURL & url) const = 0; + virtual std::optional inputFromURL(const ParsedURL & url, bool requireTree) const = 0; virtual std::optional inputFromAttrs(const Attrs & attrs) const = 0; - virtual ParsedURL toURL(const Input & input) const; + /** + * What is the name of the scheme? + * + * The `type` attribute is used to select which input scheme is + * used, and then the other fields are forwarded to that input + * scheme. + */ + virtual std::string_view schemeName() const = 0; - virtual bool hasAllInfo(const Input & input) const = 0; + /** + * Allowed attributes in an attribute set that is converted to an + * input. + * + * `type` is not included from this set, because the `type` field is + parsed first to choose which scheme; `type` is always required. + */ + virtual StringSet allowedAttrs() const = 0; + + virtual ParsedURL toURL(const Input & input) const; virtual Input applyOverrides( const Input & input, @@ -144,42 +167,41 @@ struct InputScheme virtual void clone(const Input & input, const Path & destDir) const; - virtual std::optional getSourcePath(const Input & input); + virtual std::optional getSourcePath(const Input & input) const; - virtual void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg); + virtual void putFile( + const Input & input, + const CanonPath & path, + std::string_view contents, + std::optional commitMsg) const; - virtual std::pair fetch(ref store, const Input & input) = 0; -}; + virtual std::pair fetch(ref store, const Input & input); -void registerInputScheme(std::shared_ptr && fetcher); + virtual std::pair, Input> getAccessor(ref store, const Input & input) const; -struct DownloadFileResult -{ - StorePath storePath; - std::string etag; - std::string effectiveUrl; - std::optional immutableUrl; + /** + * Is this `InputScheme` part of an experimental feature? + */ + virtual std::optional experimentalFeature() const; + + virtual bool isDirect(const Input & input) const + { return true; } + + virtual std::optional getFingerprint(ref store, const Input & input) const + { return std::nullopt; } }; -DownloadFileResult downloadFile( - ref store, - const std::string & url, - const std::string & name, - bool locked, - const Headers & headers = {}); +void registerInputScheme(std::shared_ptr && fetcher); + +nlohmann::json dumpRegisterInputSchemeInfo(); -struct DownloadTarballResult +struct PublicKey { - Tree tree; - time_t lastModified; - std::optional immutableUrl; + std::string type = "ssh-ed25519"; + std::string key; }; +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(PublicKey, type, key) -DownloadTarballResult downloadTarball( - ref store, - const std::string & url, - const std::string & name, - bool locked, - const Headers & headers = {}); +std::string publicKeys_to_string(const std::vector&); } diff --git a/src/libfetchers/fs-input-accessor.cc b/src/libfetchers/fs-input-accessor.cc new file mode 100644 index 000000000000..2efee932ddc5 --- /dev/null +++ b/src/libfetchers/fs-input-accessor.cc @@ -0,0 +1,131 @@ +#include "fs-input-accessor.hh" +#include "posix-source-accessor.hh" +#include "store-api.hh" + +namespace nix { + +struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor +{ + CanonPath root; + std::optional> allowedPaths; + MakeNotAllowedError makeNotAllowedError; + + FSInputAccessorImpl( + const CanonPath & root, + std::optional> && allowedPaths, + MakeNotAllowedError && makeNotAllowedError) + : root(root) + , allowedPaths(std::move(allowedPaths)) + , makeNotAllowedError(std::move(makeNotAllowedError)) + { + displayPrefix = root.isRoot() ? "" : root.abs(); + } + + void readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) override + { + auto absPath = makeAbsPath(path); + checkAllowed(absPath); + PosixSourceAccessor::readFile(absPath, sink, sizeCallback); + } + + bool pathExists(const CanonPath & path) override + { + auto absPath = makeAbsPath(path); + return isAllowed(absPath) && PosixSourceAccessor::pathExists(absPath); + } + + std::optional maybeLstat(const CanonPath & path) override + { + auto absPath = makeAbsPath(path); + checkAllowed(absPath); + return PosixSourceAccessor::maybeLstat(absPath); + } + + DirEntries readDirectory(const CanonPath & path) override + { + auto absPath = makeAbsPath(path); + checkAllowed(absPath); + DirEntries res; + for (auto & entry : PosixSourceAccessor::readDirectory(absPath)) + if (isAllowed(absPath + entry.first)) + res.emplace(entry); + return res; + } + + std::string readLink(const CanonPath & path) override + { + auto absPath = makeAbsPath(path); + checkAllowed(absPath); + return PosixSourceAccessor::readLink(absPath); + } + + CanonPath makeAbsPath(const CanonPath & path) + { + return root + path; + } + + void checkAllowed(const CanonPath & absPath) override + { + if (!isAllowed(absPath)) + throw makeNotAllowedError + ? makeNotAllowedError(absPath) + : RestrictedPathError("access to path '%s' is forbidden", absPath); + } + + bool isAllowed(const CanonPath & absPath) + { + if (!absPath.isWithin(root)) + return false; + + if (allowedPaths) { + auto p = absPath.removePrefix(root); + if (!p.isAllowed(*allowedPaths)) + return false; + } + + return true; + } + + void allowPath(CanonPath path) override + { + if (allowedPaths) + allowedPaths->insert(std::move(path)); + } + + bool hasAccessControl() override + { + return (bool) allowedPaths; + } + + std::optional getPhysicalPath(const CanonPath & path) override + { + return makeAbsPath(path); + } +}; + +ref makeFSInputAccessor( + const CanonPath & root, + std::optional> && allowedPaths, + MakeNotAllowedError && makeNotAllowedError) +{ + return make_ref(root, std::move(allowedPaths), std::move(makeNotAllowedError)); +} + +ref makeStorePathAccessor( + ref store, + const StorePath & storePath, + MakeNotAllowedError && makeNotAllowedError) +{ + return makeFSInputAccessor(CanonPath(store->toRealPath(storePath)), {}, std::move(makeNotAllowedError)); +} + +SourcePath getUnfilteredRootPath(CanonPath path) +{ + static auto rootFS = makeFSInputAccessor(CanonPath::root); + return {rootFS, path}; +} + +} diff --git a/src/libfetchers/fs-input-accessor.hh b/src/libfetchers/fs-input-accessor.hh new file mode 100644 index 000000000000..19a5211c8b47 --- /dev/null +++ b/src/libfetchers/fs-input-accessor.hh @@ -0,0 +1,33 @@ +#pragma once + +#include "input-accessor.hh" + +namespace nix { + +class StorePath; +class Store; + +struct FSInputAccessor : InputAccessor +{ + virtual void checkAllowed(const CanonPath & absPath) = 0; + + virtual void allowPath(CanonPath path) = 0; + + virtual bool hasAccessControl() = 0; +}; + +typedef std::function MakeNotAllowedError; + +ref makeFSInputAccessor( + const CanonPath & root, + std::optional> && allowedPaths = {}, + MakeNotAllowedError && makeNotAllowedError = {}); + +ref makeStorePathAccessor( + ref store, + const StorePath & storePath, + MakeNotAllowedError && makeNotAllowedError = {}); + +SourcePath getUnfilteredRootPath(CanonPath path); + +} diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc new file mode 100644 index 000000000000..19eae0e1da54 --- /dev/null +++ b/src/libfetchers/git-utils.cc @@ -0,0 +1,677 @@ +#include "git-utils.hh" +#include "input-accessor.hh" +#include "cache.hh" +#include "finally.hh" +#include "processes.hh" +#include "signals.hh" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace std { + +template<> struct hash +{ + size_t operator()(const git_oid & oid) const + { + return * (size_t *) oid.id; + } +}; + +} + +std::ostream & operator << (std::ostream & str, const git_oid & oid) +{ + str << git_oid_tostr_s(&oid); + return str; +} + +bool operator == (const git_oid & oid1, const git_oid & oid2) +{ + return git_oid_equal(&oid1, &oid2); +} + +namespace nix { + +// Some wrapper types that ensure that the git_*_free functions get called. +template +struct Deleter +{ + template + void operator()(T * p) const { del(p); }; +}; + +typedef std::unique_ptr> Repository; +typedef std::unique_ptr> TreeEntry; +typedef std::unique_ptr> Tree; +typedef std::unique_ptr> TreeBuilder; +typedef std::unique_ptr> Blob; +typedef std::unique_ptr> Object; +typedef std::unique_ptr> Commit; +typedef std::unique_ptr> Reference; +typedef std::unique_ptr> DescribeResult; +typedef std::unique_ptr> StatusList; +typedef std::unique_ptr> Remote; +typedef std::unique_ptr> GitConfig; +typedef std::unique_ptr> ConfigIterator; + +// A helper to ensure that we don't leak objects returned by libgit2. +template +struct Setter +{ + T & t; + typename T::pointer p = nullptr; + + Setter(T & t) : t(t) { } + + ~Setter() { if (p) t = T(p); } + + operator typename T::pointer * () { return &p; } +}; + +Hash toHash(const git_oid & oid) +{ + #ifdef GIT_EXPERIMENTAL_SHA256 + assert(oid.type == GIT_OID_SHA1); + #endif + Hash hash(htSHA1); + memcpy(hash.hash, oid.id, hash.hashSize); + return hash; +} + +static void initLibGit2() +{ + if (git_libgit2_init() < 0) + throw Error("initialising libgit2: %s", git_error_last()->message); +} + +git_oid hashToOID(const Hash & hash) +{ + git_oid oid; + if (git_oid_fromstr(&oid, hash.gitRev().c_str())) + throw Error("cannot convert '%s' to a Git OID", hash.gitRev()); + return oid; +} + +Object lookupObject(git_repository * repo, const git_oid & oid) +{ + Object obj; + if (git_object_lookup(Setter(obj), repo, &oid, GIT_OBJECT_ANY)) { + auto err = git_error_last(); + throw Error("getting Git object '%s': %s", oid, err->message); + } + return obj; +} + +template +T peelObject(git_repository * repo, git_object * obj, git_object_t type) +{ + T obj2; + if (git_object_peel((git_object * *) (typename T::pointer *) Setter(obj2), obj, type)) { + auto err = git_error_last(); + throw Error("peeling Git object '%s': %s", git_object_id(obj), err->message); + } + return obj2; +} + +struct GitRepoImpl : GitRepo, std::enable_shared_from_this +{ + CanonPath path; + Repository repo; + + GitRepoImpl(CanonPath _path, bool create, bool bare) + : path(std::move(_path)) + { + initLibGit2(); + + if (pathExists(path.abs())) { + if (git_repository_open(Setter(repo), path.c_str())) + throw Error("opening Git repository '%s': %s", path, git_error_last()->message); + } else { + if (git_repository_init(Setter(repo), path.c_str(), bare)) + throw Error("creating Git repository '%s': %s", path, git_error_last()->message); + } + } + + operator git_repository * () + { + return repo.get(); + } + + uint64_t getRevCount(const Hash & rev) override + { + std::unordered_set done; + std::queue todo; + + todo.push(peelObject(*this, lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT)); + + while (auto commit = pop(todo)) { + if (!done.insert(*git_commit_id(commit->get())).second) continue; + + for (size_t n = 0; n < git_commit_parentcount(commit->get()); ++n) { + git_commit * parent; + if (git_commit_parent(&parent, commit->get(), n)) + throw Error("getting parent of Git commit '%s': %s", *git_commit_id(commit->get()), git_error_last()->message); + todo.push(Commit(parent)); + } + } + + return done.size(); + } + + uint64_t getLastModified(const Hash & rev) override + { + auto commit = peelObject(*this, lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT); + + return git_commit_time(commit.get()); + } + + bool isShallow() override + { + return git_repository_is_shallow(*this); + } + + Hash resolveRef(std::string ref) override + { + // Handle revisions used as refs. + { + git_oid oid; + if (git_oid_fromstr(&oid, ref.c_str()) == 0) + return toHash(oid); + } + + // Resolve short names like 'master'. + Reference ref2; + if (!git_reference_dwim(Setter(ref2), *this, ref.c_str())) + ref = git_reference_name(ref2.get()); + + // Resolve full references like 'refs/heads/master'. + Reference ref3; + if (git_reference_lookup(Setter(ref3), *this, ref.c_str())) + throw Error("resolving Git reference '%s': %s", ref, git_error_last()->message); + + auto oid = git_reference_target(ref3.get()); + if (!oid) + throw Error("cannot get OID for Git reference '%s'", git_reference_name(ref3.get())); + + return toHash(*oid); + } + + std::vector parseSubmodules(const CanonPath & configFile) + { + GitConfig config; + if (git_config_open_ondisk(Setter(config), configFile.abs().c_str())) + throw Error("parsing .gitmodules file: %s", git_error_last()->message); + + ConfigIterator it; + if (git_config_iterator_glob_new(Setter(it), config.get(), "^submodule\\..*\\.(path|url|branch)$")) + throw Error("iterating over .gitmodules: %s", git_error_last()->message); + + std::map entries; + + while (true) { + git_config_entry * entry = nullptr; + if (auto err = git_config_next(&entry, it.get())) { + if (err == GIT_ITEROVER) break; + throw Error("iterating over .gitmodules: %s", git_error_last()->message); + } + entries.emplace(entry->name + 10, entry->value); + } + + std::vector result; + + for (auto & [key, value] : entries) { + if (!hasSuffix(key, ".path")) continue; + std::string key2(key, 0, key.size() - 5); + auto path = CanonPath(value); + result.push_back(Submodule { + .path = path, + .url = entries[key2 + ".url"], + .branch = entries[key2 + ".branch"], + }); + } + + return result; + } + + // Helper for statusCallback below. + static int statusCallbackTrampoline(const char * path, unsigned int statusFlags, void * payload) + { + return (*((std::function *) payload))(path, statusFlags); + } + + WorkdirInfo getWorkdirInfo() override + { + WorkdirInfo info; + + /* Get the head revision, if any. */ + git_oid headRev; + if (auto err = git_reference_name_to_id(&headRev, *this, "HEAD")) { + if (err != GIT_ENOTFOUND) + throw Error("resolving HEAD: %s", git_error_last()->message); + } else + info.headRev = toHash(headRev); + + /* Get all tracked files and determine whether the working + directory is dirty. */ + std::function statusCallback = [&](const char * path, unsigned int statusFlags) + { + if (!(statusFlags & GIT_STATUS_INDEX_DELETED) && + !(statusFlags & GIT_STATUS_WT_DELETED)) + info.files.insert(CanonPath(path)); + if (statusFlags != GIT_STATUS_CURRENT) + info.isDirty = true; + return 0; + }; + + git_status_options options = GIT_STATUS_OPTIONS_INIT; + options.flags |= GIT_STATUS_OPT_INCLUDE_UNMODIFIED; + options.flags |= GIT_STATUS_OPT_EXCLUDE_SUBMODULES; + if (git_status_foreach_ext(*this, &options, &statusCallbackTrampoline, &statusCallback)) + throw Error("getting working directory status: %s", git_error_last()->message); + + /* Get submodule info. */ + auto modulesFile = path + ".gitmodules"; + if (pathExists(modulesFile.abs())) + info.submodules = parseSubmodules(modulesFile); + + return info; + } + + std::optional getWorkdirRef() override + { + Reference ref; + if (git_reference_lookup(Setter(ref), *this, "HEAD")) + throw Error("looking up HEAD: %s", git_error_last()->message); + + if (auto target = git_reference_symbolic_target(ref.get())) + return target; + + return std::nullopt; + } + + std::vector> getSubmodules(const Hash & rev) override; + + std::string resolveSubmoduleUrl( + const std::string & url, + const std::string & base) override + { + git_buf buf = GIT_BUF_INIT; + if (git_submodule_resolve_url(&buf, *this, url.c_str())) + throw Error("resolving Git submodule URL '%s'", url); + Finally cleanup = [&]() { git_buf_dispose(&buf); }; + + std::string res(buf.ptr); + + if (!hasPrefix(res, "/") && res.find("://") == res.npos) + res = parseURL(base + "/" + res).canonicalise().to_string(); + + return res; + } + + bool hasObject(const Hash & oid_) override + { + auto oid = hashToOID(oid_); + + Object obj; + if (auto errCode = git_object_lookup(Setter(obj), *this, &oid, GIT_OBJECT_ANY)) { + if (errCode == GIT_ENOTFOUND) return false; + auto err = git_error_last(); + throw Error("getting Git object '%s': %s", oid, err->message); + } + + return true; + } + + ref getAccessor(const Hash & rev) override; + + static int sidebandProgressCallback(const char * str, int len, void * payload) + { + auto act = (Activity *) payload; + act->result(resFetchStatus, trim(std::string_view(str, len))); + return _isInterrupted ? -1 : 0; + } + + static int transferProgressCallback(const git_indexer_progress * stats, void * payload) + { + auto act = (Activity *) payload; + act->result(resFetchStatus, + fmt("%d/%d objects received, %d/%d deltas indexed, %.1f MiB", + stats->received_objects, + stats->total_objects, + stats->indexed_deltas, + stats->total_deltas, + stats->received_bytes / (1024.0 * 1024.0))); + return _isInterrupted ? -1 : 0; + } + + void fetch( + const std::string & url, + const std::string & refspec, + bool shallow) override + { + Activity act(*logger, lvlTalkative, actFetchTree, fmt("fetching Git repository '%s'", url)); + + Remote remote; + + if (git_remote_create_anonymous(Setter(remote), *this, url.c_str())) + throw Error("cannot create Git remote '%s': %s", url, git_error_last()->message); + + char * refspecs[] = {(char *) refspec.c_str()}; + git_strarray refspecs2 { + .strings = refspecs, + .count = 1 + }; + + git_fetch_options opts = GIT_FETCH_OPTIONS_INIT; + // FIXME: for some reason, shallow fetching over ssh barfs + // with "could not read from remote repository". + opts.depth = shallow && parseURL(url).scheme != "ssh" ? 1 : GIT_FETCH_DEPTH_FULL; + opts.callbacks.payload = &act; + opts.callbacks.sideband_progress = sidebandProgressCallback; + opts.callbacks.transfer_progress = transferProgressCallback; + + if (git_remote_fetch(remote.get(), &refspecs2, &opts, nullptr)) + throw Error("fetching '%s' from '%s': %s", refspec, url, git_error_last()->message); + } + + void verifyCommit( + const Hash & rev, + const std::vector & publicKeys) override + { + // Create ad-hoc allowedSignersFile and populate it with publicKeys + auto allowedSignersFile = createTempFile().second; + std::string allowedSigners; + for (const fetchers::PublicKey & k : publicKeys) { + if (k.type != "ssh-dsa" + && k.type != "ssh-ecdsa" + && k.type != "ssh-ecdsa-sk" + && k.type != "ssh-ed25519" + && k.type != "ssh-ed25519-sk" + && k.type != "ssh-rsa") + throw Error("Unknown key type '%s'.\n" + "Please use one of\n" + "- ssh-dsa\n" + " ssh-ecdsa\n" + " ssh-ecdsa-sk\n" + " ssh-ed25519\n" + " ssh-ed25519-sk\n" + " ssh-rsa", k.type); + allowedSigners += "* " + k.type + " " + k.key + "\n"; + } + writeFile(allowedSignersFile, allowedSigners); + + // Run verification command + auto [status, output] = runProgram(RunOptions { + .program = "git", + .args = { + "-c", + "gpg.ssh.allowedSignersFile=" + allowedSignersFile, + "-C", path.abs(), + "verify-commit", + rev.gitRev() + }, + .mergeStderrToStdout = true, + }); + + /* Evaluate result through status code and checking if public + key fingerprints appear on stderr. This is neccessary + because the git command might also succeed due to the + commit being signed by gpg keys that are present in the + users key agent. */ + std::string re = R"(Good "git" signature for \* with .* key SHA256:[)"; + for (const fetchers::PublicKey & k : publicKeys){ + // Calculate sha256 fingerprint from public key and escape the regex symbol '+' to match the key literally + auto fingerprint = trim(hashString(htSHA256, base64Decode(k.key)).to_string(nix::HashFormat::Base64, false), "="); + auto escaped_fingerprint = std::regex_replace(fingerprint, std::regex("\\+"), "\\+" ); + re += "(" + escaped_fingerprint + ")"; + } + re += "]"; + if (status == 0 && std::regex_search(output, std::regex(re))) + printTalkative("Signature verification on commit %s succeeded.", rev.gitRev()); + else + throw Error("Commit signature verification on commit %s failed: %s", rev.gitRev(), output); + } +}; + +ref GitRepo::openRepo(const CanonPath & path, bool create, bool bare) +{ + return make_ref(path, create, bare); +} + +struct GitInputAccessor : InputAccessor +{ + ref repo; + Tree root; + + GitInputAccessor(ref repo_, const Hash & rev) + : repo(repo_) + , root(peelObject(*repo, lookupObject(*repo, hashToOID(rev)).get(), GIT_OBJECT_TREE)) + { + } + + std::string readBlob(const CanonPath & path, bool symlink) + { + auto blob = getBlob(path, symlink); + + auto data = std::string_view((const char *) git_blob_rawcontent(blob.get()), git_blob_rawsize(blob.get())); + + return std::string(data); + } + + std::string readFile(const CanonPath & path) override + { + return readBlob(path, false); + } + + bool pathExists(const CanonPath & path) override + { + return path.isRoot() ? true : (bool) lookup(path); + } + + std::optional maybeLstat(const CanonPath & path) override + { + if (path.isRoot()) + return Stat { .type = tDirectory }; + + auto entry = lookup(path); + if (!entry) + return std::nullopt; + + auto mode = git_tree_entry_filemode(entry); + + if (mode == GIT_FILEMODE_TREE) + return Stat { .type = tDirectory }; + + else if (mode == GIT_FILEMODE_BLOB) + return Stat { .type = tRegular }; + + else if (mode == GIT_FILEMODE_BLOB_EXECUTABLE) + return Stat { .type = tRegular, .isExecutable = true }; + + else if (mode == GIT_FILEMODE_LINK) + return Stat { .type = tSymlink }; + + else if (mode == GIT_FILEMODE_COMMIT) + // Treat submodules as an empty directory. + return Stat { .type = tDirectory }; + + else + throw Error("file '%s' has an unsupported Git file type"); + } + + DirEntries readDirectory(const CanonPath & path) override + { + return std::visit(overloaded { + [&](Tree tree) { + DirEntries res; + + auto count = git_tree_entrycount(tree.get()); + + for (size_t n = 0; n < count; ++n) { + auto entry = git_tree_entry_byindex(tree.get(), n); + // FIXME: add to cache + res.emplace(std::string(git_tree_entry_name(entry)), DirEntry{}); + } + + return res; + }, + [&](Submodule) { + return DirEntries(); + } + }, getTree(path)); + } + + std::string readLink(const CanonPath & path) override + { + return readBlob(path, true); + } + + Hash getSubmoduleRev(const CanonPath & path) + { + auto entry = need(path); + + if (git_tree_entry_type(entry) != GIT_OBJECT_COMMIT) + throw Error("'%s' is not a submodule", showPath(path)); + + return toHash(*git_tree_entry_id(entry)); + } + + std::map lookupCache; + + /* Recursively look up 'path' relative to the root. */ + git_tree_entry * lookup(const CanonPath & path) + { + if (path.isRoot()) return nullptr; + + auto i = lookupCache.find(path); + if (i == lookupCache.end()) { + TreeEntry entry; + if (auto err = git_tree_entry_bypath(Setter(entry), root.get(), std::string(path.rel()).c_str())) { + if (err != GIT_ENOTFOUND) + throw Error("looking up '%s': %s", showPath(path), git_error_last()->message); + } + + i = lookupCache.emplace(path, std::move(entry)).first; + } + + return &*i->second; + } + + git_tree_entry * need(const CanonPath & path) + { + auto entry = lookup(path); + if (!entry) + throw Error("'%s' does not exist", showPath(path)); + return entry; + } + + struct Submodule { }; + + std::variant getTree(const CanonPath & path) + { + if (path.isRoot()) { + Tree tree; + if (git_tree_dup(Setter(tree), root.get())) + throw Error("duplicating directory '%s': %s", showPath(path), git_error_last()->message); + return tree; + } + + auto entry = need(path); + + if (git_tree_entry_type(entry) == GIT_OBJECT_COMMIT) + return Submodule(); + + if (git_tree_entry_type(entry) != GIT_OBJECT_TREE) + throw Error("'%s' is not a directory", showPath(path)); + + Tree tree; + if (git_tree_entry_to_object((git_object * *) (git_tree * *) Setter(tree), *repo, entry)) + throw Error("looking up directory '%s': %s", showPath(path), git_error_last()->message); + + return tree; + } + + Blob getBlob(const CanonPath & path, bool expectSymlink) + { + auto notExpected = [&]() + { + throw Error( + expectSymlink + ? "'%s' is not a symlink" + : "'%s' is not a regular file", + showPath(path)); + }; + + if (path.isRoot()) notExpected(); + + auto entry = need(path); + + if (git_tree_entry_type(entry) != GIT_OBJECT_BLOB) + notExpected(); + + auto mode = git_tree_entry_filemode(entry); + if (expectSymlink) { + if (mode != GIT_FILEMODE_LINK) + notExpected(); + } else { + if (mode != GIT_FILEMODE_BLOB && mode != GIT_FILEMODE_BLOB_EXECUTABLE) + notExpected(); + } + + Blob blob; + if (git_tree_entry_to_object((git_object * *) (git_blob * *) Setter(blob), *repo, entry)) + throw Error("looking up file '%s': %s", showPath(path), git_error_last()->message); + + return blob; + } +}; + +ref GitRepoImpl::getAccessor(const Hash & rev) +{ + return make_ref(ref(shared_from_this()), rev); +} + +std::vector> GitRepoImpl::getSubmodules(const Hash & rev) +{ + /* Read the .gitmodules files from this revision. */ + CanonPath modulesFile(".gitmodules"); + + auto accessor = getAccessor(rev); + if (!accessor->pathExists(modulesFile)) return {}; + + /* Parse it and get the revision of each submodule. */ + auto configS = accessor->readFile(modulesFile); + + auto [fdTemp, pathTemp] = createTempFile("nix-git-submodules"); + writeFull(fdTemp.get(), configS); + + std::vector> result; + + for (auto & submodule : parseSubmodules(CanonPath(pathTemp))) { + auto rev = accessor.dynamic_pointer_cast()->getSubmoduleRev(submodule.path); + result.push_back({std::move(submodule), rev}); + } + + return result; +} + + +} diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh new file mode 100644 index 000000000000..1def82071efe --- /dev/null +++ b/src/libfetchers/git-utils.hh @@ -0,0 +1,90 @@ +#pragma once + +#include "input-accessor.hh" + +namespace nix { + +namespace fetchers { struct PublicKey; } + +struct GitRepo +{ + virtual ~GitRepo() + { } + + static ref openRepo(const CanonPath & path, bool create = false, bool bare = false); + + virtual uint64_t getRevCount(const Hash & rev) = 0; + + virtual uint64_t getLastModified(const Hash & rev) = 0; + + virtual bool isShallow() = 0; + + /* Return the commit hash to which a ref points. */ + virtual Hash resolveRef(std::string ref) = 0; + + /** + * Info about a submodule. + */ + struct Submodule + { + CanonPath path; + std::string url; + std::string branch; + }; + + struct WorkdirInfo + { + bool isDirty = false; + + /* The checked out commit, or nullopt if there are no commits + in the repo yet. */ + std::optional headRev; + + /* All files in the working directory that are unchanged, + modified or added, but excluding deleted files. */ + std::set files; + + /* The submodules listed in .gitmodules of this workdir. */ + std::vector submodules; + }; + + virtual WorkdirInfo getWorkdirInfo() = 0; + + /* Get the ref that HEAD points to. */ + virtual std::optional getWorkdirRef() = 0; + + /** + * Return the submodules of this repo at the indicated revision, + * along with the revision of each submodule. + */ + virtual std::vector> getSubmodules(const Hash & rev) = 0; + + virtual std::string resolveSubmoduleUrl( + const std::string & url, + const std::string & base) = 0; + + struct TarballInfo + { + Hash treeHash; + time_t lastModified; + }; + + virtual bool hasObject(const Hash & oid) = 0; + + virtual ref getAccessor(const Hash & rev) = 0; + + virtual void fetch( + const std::string & url, + const std::string & refspec, + bool shallow) = 0; + + /** + * Verify that commit `rev` is signed by one of the keys in + * `publicKeys`. Throw an error if it isn't. + */ + virtual void verifyCommit( + const Hash & rev, + const std::vector & publicKeys) = 0; +}; + +} diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index be5842d531b9..8cd74057ce0e 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -1,12 +1,18 @@ #include "fetchers.hh" +#include "users.hh" #include "cache.hh" #include "globals.hh" #include "tarfile.hh" #include "store-api.hh" #include "url-parts.hh" #include "pathlocks.hh" -#include "util.hh" +#include "processes.hh" #include "git.hh" +#include "fs-input-accessor.hh" +#include "mounted-input-accessor.hh" +#include "git-utils.hh" +#include "logging.hh" +#include "finally.hh" #include "fetch-settings.hh" @@ -46,7 +52,7 @@ bool touchCacheFile(const Path & path, time_t touch_time) Path getCachePath(std::string_view key) { return getCacheDir() + "/nix/gitv3/" + - hashString(htSHA256, key).to_string(Base32, false); + hashString(htSHA256, key).to_string(HashFormat::Base32, false); } // Returns the name of the HEAD branch. @@ -132,131 +138,24 @@ std::optional readHeadCached(const std::string & actualUrl) return std::nullopt; } -bool isNotDotGitDirectory(const Path & path) +std::vector getPublicKeys(const Attrs & attrs) { - return baseNameOf(path) != ".git"; -} - -struct WorkdirInfo -{ - bool clean = false; - bool hasHead = false; -}; - -// Returns whether a git workdir is clean and has commits. -WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir) -{ - const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); - std::string gitDir(".git"); - - auto env = getEnv(); - // Set LC_ALL to C: because we rely on the error messages from git rev-parse to determine what went wrong - // that way unknown errors can lead to a failure instead of continuing through the wrong code path - env["LC_ALL"] = "C"; - - /* Check whether HEAD points to something that looks like a commit, - since that is the refrence we want to use later on. */ - auto result = runProgram(RunOptions { - .program = "git", - .args = { "-C", workdir, "--git-dir", gitDir, "rev-parse", "--verify", "--no-revs", "HEAD^{commit}" }, - .environment = env, - .mergeStderrToStdout = true - }); - auto exitCode = WEXITSTATUS(result.first); - auto errorMessage = result.second; - - if (errorMessage.find("fatal: not a git repository") != std::string::npos) { - throw Error("'%s' is not a Git repository", workdir); - } else if (errorMessage.find("fatal: Needed a single revision") != std::string::npos) { - // indicates that the repo does not have any commits - // we want to proceed and will consider it dirty later - } else if (exitCode != 0) { - // any other errors should lead to a failure - throw Error("getting the HEAD of the Git tree '%s' failed with exit code %d:\n%s", workdir, exitCode, errorMessage); + std::vector publicKeys; + if (attrs.contains("publicKeys")) { + nlohmann::json publicKeysJson = nlohmann::json::parse(getStrAttr(attrs, "publicKeys")); + ensureType(publicKeysJson, nlohmann::json::value_t::array); + publicKeys = publicKeysJson.get>(); } - - bool clean = false; - bool hasHead = exitCode == 0; - - try { - if (hasHead) { - // Using git diff is preferrable over lower-level operations here, - // because its conceptually simpler and we only need the exit code anyways. - auto gitDiffOpts = Strings({ "-C", workdir, "--git-dir", gitDir, "diff", "HEAD", "--quiet"}); - if (!submodules) { - // Changes in submodules should only make the tree dirty - // when those submodules will be copied as well. - gitDiffOpts.emplace_back("--ignore-submodules"); - } - gitDiffOpts.emplace_back("--"); - runProgram("git", true, gitDiffOpts); - - clean = true; - } - } catch (ExecError & e) { - if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw; - } - - return WorkdirInfo { .clean = clean, .hasHead = hasHead }; + if (attrs.contains("publicKey")) + publicKeys.push_back(PublicKey{maybeGetStrAttr(attrs, "keytype").value_or("ssh-ed25519"),getStrAttr(attrs, "publicKey")}); + return publicKeys; } -std::pair fetchFromWorkdir(ref store, Input & input, const Path & workdir, const WorkdirInfo & workdirInfo) -{ - const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); - auto gitDir = ".git"; - - if (!fetchSettings.allowDirty) - throw Error("Git tree '%s' is dirty", workdir); - - if (fetchSettings.warnDirty) - warn("Git tree '%s' is dirty", workdir); - - auto gitOpts = Strings({ "-C", workdir, "--git-dir", gitDir, "ls-files", "-z" }); - if (submodules) - gitOpts.emplace_back("--recurse-submodules"); - - auto files = tokenizeString>( - runProgram("git", true, gitOpts), "\0"s); - - Path actualPath(absPath(workdir)); - - PathFilter filter = [&](const Path & p) -> bool { - assert(hasPrefix(p, actualPath)); - std::string file(p, actualPath.size() + 1); - - auto st = lstat(p); - - if (S_ISDIR(st.st_mode)) { - auto prefix = file + "/"; - auto i = files.lower_bound(prefix); - return i != files.end() && hasPrefix(*i, prefix); - } - - return files.count(file); - }; - - auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter); - - // FIXME: maybe we should use the timestamp of the last - // modified dirty file? - input.attrs.insert_or_assign( - "lastModified", - workdirInfo.hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0); - - if (workdirInfo.hasHead) { - input.attrs.insert_or_assign("dirtyRev", chomp( - runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "rev-parse", "--verify", "HEAD" })) + "-dirty"); - input.attrs.insert_or_assign("dirtyShortRev", chomp( - runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "rev-parse", "--verify", "--short", "HEAD" })) + "-dirty"); - } - - return {std::move(storePath), input}; -} } // end namespace struct GitInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) const override + std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override { if (url.scheme != "git" && url.scheme != "git+http" && @@ -272,9 +171,9 @@ struct GitInputScheme : InputScheme attrs.emplace("type", "git"); for (auto & [name, value] : url.query) { - if (name == "rev" || name == "ref") + if (name == "rev" || name == "ref" || name == "keytype" || name == "publicKey" || name == "publicKeys") attrs.emplace(name, value); - else if (name == "shallow" || name == "submodules" || name == "allRefs") + else if (name == "shallow" || name == "submodules" || name == "allRefs" || name == "verifyCommit") attrs.emplace(name, Explicit { value == "1" }); else url2.query.emplace(name, value); @@ -285,18 +184,44 @@ struct GitInputScheme : InputScheme return inputFromAttrs(attrs); } - std::optional inputFromAttrs(const Attrs & attrs) const override + + std::string_view schemeName() const override + { + return "git"; + } + + StringSet allowedAttrs() const override { - if (maybeGetStrAttr(attrs, "type") != "git") return {}; + return { + "url", + "ref", + "rev", + "shallow", + "submodules", + "lastModified", + "revCount", + "narHash", + "allRefs", + "name", + "dirtyRev", + "dirtyShortRev", + "verifyCommit", + "keytype", + "publicKey", + "publicKeys", + }; + } - for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name" && name != "dirtyRev" && name != "dirtyShortRev") - throw Error("unsupported Git input attribute '%s'", name); + std::optional inputFromAttrs(const Attrs & attrs) const override + { + for (auto & [name, _] : attrs) + if (name == "verifyCommit" + || name == "keytype" + || name == "publicKey" + || name == "publicKeys") + experimentalFeatureSettings.require(Xp::VerifiedFetches); - parseURL(getStrAttr(attrs, "url")); - maybeGetBoolAttr(attrs, "shallow"); - maybeGetBoolAttr(attrs, "submodules"); - maybeGetBoolAttr(attrs, "allRefs"); + maybeGetBoolAttr(attrs, "verifyCommit"); if (auto ref = maybeGetStrAttr(attrs, "ref")) { if (std::regex_search(*ref, badGitRefRegex)) @@ -305,6 +230,12 @@ struct GitInputScheme : InputScheme Input input; input.attrs = attrs; + auto url = fixGitURL(getStrAttr(attrs, "url")); + parseURL(url); + input.attrs["url"] = url; + getShallowAttr(input); + getSubmodulesAttr(input); + getAllRefsAttr(input); return input; } @@ -314,20 +245,22 @@ struct GitInputScheme : InputScheme if (url.scheme != "git") url.scheme = "git+" + url.scheme; if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev()); if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref); - if (maybeGetBoolAttr(input.attrs, "shallow").value_or(false)) + if (getShallowAttr(input)) url.query.insert_or_assign("shallow", "1"); + if (getSubmodulesAttr(input)) + url.query.insert_or_assign("submodules", "1"); + if (maybeGetBoolAttr(input.attrs, "verifyCommit").value_or(false)) + url.query.insert_or_assign("verifyCommit", "1"); + auto publicKeys = getPublicKeys(input.attrs); + if (publicKeys.size() == 1) { + url.query.insert_or_assign("keytype", publicKeys.at(0).type); + url.query.insert_or_assign("publicKey", publicKeys.at(0).key); + } + else if (publicKeys.size() > 1) + url.query.insert_or_assign("publicKeys", publicKeys_to_string(publicKeys)); return url; } - bool hasAllInfo(const Input & input) const override - { - bool maybeDirty = !input.getRef(); - bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); - return - maybeGetIntAttr(input.attrs, "lastModified") - && (shallow || maybeDirty || maybeGetIntAttr(input.attrs, "revCount")); - } - Input applyOverrides( const Input & input, std::optional ref, @@ -343,11 +276,11 @@ struct GitInputScheme : InputScheme void clone(const Input & input, const Path & destDir) const override { - auto [isLocal, actualUrl] = getActualUrl(input); + auto repoInfo = getRepoInfo(input); Strings args = {"clone"}; - args.push_back(actualUrl); + args.push_back(repoInfo.url); if (auto ref = input.getRef()) { args.push_back("--branch"); @@ -361,30 +294,90 @@ struct GitInputScheme : InputScheme runProgram("git", true, args, {}, true); } - std::optional getSourcePath(const Input & input) override + std::optional getSourcePath(const Input & input) const override { - auto url = parseURL(getStrAttr(input.attrs, "url")); - if (url.scheme == "file" && !input.getRef() && !input.getRev()) - return url.path; - return {}; + auto repoInfo = getRepoInfo(input); + if (repoInfo.isLocal) return repoInfo.url; + return std::nullopt; } - void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) override + void putFile( + const Input & input, + const CanonPath & path, + std::string_view contents, + std::optional commitMsg) const override { - auto sourcePath = getSourcePath(input); - assert(sourcePath); - auto gitDir = ".git"; + auto repoInfo = getRepoInfo(input); + if (!repoInfo.isLocal) + throw Error("cannot commit '%s' to Git repository '%s' because it's not a working tree", path, input.to_string()); + + writeFile((CanonPath(repoInfo.url) + path).abs(), contents); runProgram("git", true, - { "-C", *sourcePath, "--git-dir", gitDir, "add", "--intent-to-add", "--", std::string(file) }); + { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) }); + // Pause the logger to allow for user input (such as a gpg passphrase) in `git commit` + logger->pause(); + Finally restoreLogger([]() { logger->resume(); }); if (commitMsg) runProgram("git", true, - { "-C", *sourcePath, "--git-dir", gitDir, "commit", std::string(file), "-m", *commitMsg }); + { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "commit", std::string(path.rel()), "-m", *commitMsg }); } - std::pair getActualUrl(const Input & input) const + struct RepoInfo { + /* Whether this is a local, non-bare repository. */ + bool isLocal = false; + + /* Working directory info: the complete list of files, and + whether the working directory is dirty compared to HEAD. */ + GitRepo::WorkdirInfo workdirInfo; + + /* URL of the repo, or its path if isLocal. Never a `file` URL. */ + std::string url; + + void warnDirty() const + { + if (workdirInfo.isDirty) { + if (!fetchSettings.allowDirty) + throw Error("Git tree '%s' is dirty", url); + + if (fetchSettings.warnDirty) + warn("Git tree '%s' is dirty", url); + } + } + + std::string gitDir = ".git"; + }; + + bool getShallowAttr(const Input & input) const + { + return maybeGetBoolAttr(input.attrs, "shallow").value_or(false); + } + + bool getSubmodulesAttr(const Input & input) const + { + return maybeGetBoolAttr(input.attrs, "submodules").value_or(false); + } + + bool getAllRefsAttr(const Input & input) const + { + return maybeGetBoolAttr(input.attrs, "allRefs").value_or(false); + } + + RepoInfo getRepoInfo(const Input & input) const + { + auto checkHashType = [&](const std::optional & hash) + { + if (hash.has_value() && !(hash->type == htSHA1 || hash->type == htSHA256)) + throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(HashFormat::Base16, true)); + }; + + if (auto rev = input.getRev()) + checkHashType(rev); + + RepoInfo repoInfo; + // file:// URIs are normally not cloned (but otherwise treated the // same as remote URIs, i.e. we don't use the working tree or // HEAD). Exception: If _NIX_FORCE_HTTP is set, or the repo is a bare git @@ -392,153 +385,132 @@ struct GitInputScheme : InputScheme static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing auto url = parseURL(getStrAttr(input.attrs, "url")); bool isBareRepository = url.scheme == "file" && !pathExists(url.path + "/.git"); - bool isLocal = url.scheme == "file" && !forceHttp && !isBareRepository; - return {isLocal, isLocal ? url.path : url.base}; + repoInfo.isLocal = url.scheme == "file" && !forceHttp && !isBareRepository; + repoInfo.url = repoInfo.isLocal ? url.path : url.base; + + // If this is a local directory and no ref or revision is + // given, then allow the use of an unclean working tree. + if (!input.getRef() && !input.getRev() && repoInfo.isLocal) + repoInfo.workdirInfo = GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirInfo(); + + return repoInfo; } - std::pair fetch(ref store, const Input & _input) override + uint64_t getLastModified(const RepoInfo & repoInfo, const std::string & repoDir, const Hash & rev) const { - Input input(_input); - auto gitDir = ".git"; + Attrs key{{"_what", "gitLastModified"}, {"rev", rev.gitRev()}}; - std::string name = input.getName(); + auto cache = getCache(); - bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); - bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); - bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false); + if (auto res = cache->lookup(key)) + return getIntAttr(*res, "lastModified"); - std::string cacheType = "git"; - if (shallow) cacheType += "-shallow"; - if (submodules) cacheType += "-submodules"; - if (allRefs) cacheType += "-all-refs"; + auto lastModified = GitRepo::openRepo(CanonPath(repoDir))->getLastModified(rev); - auto checkHashType = [&](const std::optional & hash) - { - if (hash.has_value() && !(hash->type == htSHA1 || hash->type == htSHA256)) - throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(Base16, true)); - }; + cache->upsert(key, Attrs{{"lastModified", lastModified}}); - auto getLockedAttrs = [&]() - { - checkHashType(input.getRev()); + return lastModified; + } - return Attrs({ - {"type", cacheType}, - {"name", name}, - {"rev", input.getRev()->gitRev()}, - }); - }; + uint64_t getRevCount(const RepoInfo & repoInfo, const std::string & repoDir, const Hash & rev) const + { + Attrs key{{"_what", "gitRevCount"}, {"rev", rev.gitRev()}}; + + auto cache = getCache(); + + if (auto revCountAttrs = cache->lookup(key)) + return getIntAttr(*revCountAttrs, "revCount"); - auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) - -> std::pair + Activity act(*logger, lvlChatty, actUnknown, fmt("getting Git revision count of '%s'", repoInfo.url)); + + auto revCount = GitRepo::openRepo(CanonPath(repoDir))->getRevCount(rev); + + cache->upsert(key, Attrs{{"revCount", revCount}}); + + return revCount; + } + + std::string getDefaultRef(const RepoInfo & repoInfo) const + { + auto head = repoInfo.isLocal + ? GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirRef() + : readHeadCached(repoInfo.url); + if (!head) { + warn("could not read HEAD ref from repo at '%s', using 'master'", repoInfo.url); + return "master"; + } + return *head; + } + + static MakeNotAllowedError makeNotAllowedError(std::string url) + { + return [url{std::move(url)}](const CanonPath & path) -> RestrictedPathError { - assert(input.getRev()); - assert(!_input.getRev() || _input.getRev() == input.getRev()); - if (!shallow) - input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); - input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified")); - return {std::move(storePath), input}; + if (nix::pathExists(path.abs())) + return RestrictedPathError("access to path '%s' is forbidden because it is not under Git control; maybe you should 'git add' it to the repository '%s'?", path, url); + else + return RestrictedPathError("path '%s' does not exist in Git repository '%s'", path, url); }; + } - if (input.getRev()) { - if (auto res = getCache()->lookup(store, getLockedAttrs())) - return makeResult(res->first, std::move(res->second)); + void verifyCommit(const Input & input, std::shared_ptr repo) const + { + auto publicKeys = getPublicKeys(input.attrs); + auto verifyCommit = maybeGetBoolAttr(input.attrs, "verifyCommit").value_or(!publicKeys.empty()); + + if (verifyCommit) { + if (input.getRev() && repo) + repo->verifyCommit(*input.getRev(), publicKeys); + else + throw Error("commit verification is required for Git repository '%s', but it's dirty", input.to_string()); } + } - auto [isLocal, actualUrl_] = getActualUrl(input); - auto actualUrl = actualUrl_; // work around clang bug + std::pair, Input> getAccessorFromCommit( + ref store, + RepoInfo & repoInfo, + Input && input) const + { + assert(!repoInfo.workdirInfo.isDirty); - /* If this is a local directory and no ref or revision is given, - allow fetching directly from a dirty workdir. */ - if (!input.getRef() && !input.getRev() && isLocal) { - auto workdirInfo = getWorkdirInfo(input, actualUrl); - if (!workdirInfo.clean) { - return fetchFromWorkdir(store, input, actualUrl, workdirInfo); - } - } + auto origRev = input.getRev(); - Attrs unlockedAttrs({ - {"type", cacheType}, - {"name", name}, - {"url", actualUrl}, - }); + std::string name = input.getName(); - Path repoDir; + auto originalRef = input.getRef(); + auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo); + input.attrs.insert_or_assign("ref", ref); - if (isLocal) { - if (!input.getRef()) { - auto head = readHead(actualUrl); - if (!head) { - warn("could not read HEAD ref from repo at '%s', using 'master'", actualUrl); - head = "master"; - } - input.attrs.insert_or_assign("ref", *head); - unlockedAttrs.insert_or_assign("ref", *head); - } + Path repoDir; + if (repoInfo.isLocal) { + repoDir = repoInfo.url; if (!input.getRev()) - input.attrs.insert_or_assign("rev", - Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "--git-dir", gitDir, "rev-parse", *input.getRef() })), htSHA1).gitRev()); - - repoDir = actualUrl; + input.attrs.insert_or_assign("rev", GitRepo::openRepo(CanonPath(repoDir))->resolveRef(ref).gitRev()); } else { - const bool useHeadRef = !input.getRef(); - if (useHeadRef) { - auto head = readHeadCached(actualUrl); - if (!head) { - warn("could not read HEAD ref from repo at '%s', using 'master'", actualUrl); - head = "master"; - } - input.attrs.insert_or_assign("ref", *head); - unlockedAttrs.insert_or_assign("ref", *head); - } else { - if (!input.getRev()) { - unlockedAttrs.insert_or_assign("ref", input.getRef().value()); - } - } - - if (auto res = getCache()->lookup(store, unlockedAttrs)) { - auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1); - if (!input.getRev() || input.getRev() == rev2) { - input.attrs.insert_or_assign("rev", rev2.gitRev()); - return makeResult(res->first, std::move(res->second)); - } - } - - Path cacheDir = getCachePath(actualUrl); + Path cacheDir = getCachePath(repoInfo.url); repoDir = cacheDir; - gitDir = "."; + repoInfo.gitDir = "."; createDirs(dirOf(cacheDir)); - PathLocks cacheDirLock({cacheDir + ".lock"}); + PathLocks cacheDirLock({cacheDir}); - if (!pathExists(cacheDir)) { - runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", "--bare", repoDir }); - } + auto repo = GitRepo::openRepo(CanonPath(cacheDir), true, true); Path localRefFile = - input.getRef()->compare(0, 5, "refs/") == 0 - ? cacheDir + "/" + *input.getRef() - : cacheDir + "/refs/heads/" + *input.getRef(); + ref.compare(0, 5, "refs/") == 0 + ? cacheDir + "/" + ref + : cacheDir + "/refs/heads/" + ref; bool doFetch; time_t now = time(0); /* If a rev was specified, we need to fetch if it's not in the repo. */ - if (input.getRev()) { - try { - runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "cat-file", "-e", input.getRev()->gitRev() }); - doFetch = false; - } catch (ExecError & e) { - if (WIFEXITED(e.status)) { - doFetch = true; - } else { - throw; - } - } + if (auto rev = input.getRev()) { + doFetch = !repo->hasObject(*rev); } else { - if (allRefs) { + if (getAllRefsAttr(input)) { doFetch = true; } else { /* If the local ref is older than ‘tarball-ttl’ seconds, do a @@ -550,160 +522,210 @@ struct GitInputScheme : InputScheme } if (doFetch) { - Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", actualUrl)); - - // FIXME: git stderr messes up our progress indicator, so - // we're using --quiet for now. Should process its stderr. try { - auto ref = input.getRef(); - auto fetchRef = allRefs + auto fetchRef = + getAllRefsAttr(input) ? "refs/*" - : ref->compare(0, 5, "refs/") == 0 - ? *ref - : ref == "HEAD" - ? *ref - : "refs/heads/" + *ref; - runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }, {}, true); + : input.getRev() + ? input.getRev()->gitRev() + : ref.compare(0, 5, "refs/") == 0 + ? ref + : ref == "HEAD" + ? ref + : "refs/heads/" + ref; + + repo->fetch(repoInfo.url, fmt("%s:%s", fetchRef, fetchRef), getShallowAttr(input)); } catch (Error & e) { if (!pathExists(localRefFile)) throw; - warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl); + logError(e.info()); + warn("could not update local clone of Git repository '%s'; continuing with the most recent version", repoInfo.url); } if (!touchCacheFile(localRefFile, now)) warn("could not update mtime for file '%s': %s", localRefFile, strerror(errno)); - if (useHeadRef && !storeCachedHead(actualUrl, *input.getRef())) - warn("could not update cached head '%s' for '%s'", *input.getRef(), actualUrl); + if (!originalRef && !storeCachedHead(repoInfo.url, ref)) + warn("could not update cached head '%s' for '%s'", ref, repoInfo.url); } - if (!input.getRev()) + if (auto rev = input.getRev()) { + if (!repo->hasObject(*rev)) + throw Error( + "Cannot find Git revision '%s' in ref '%s' of repository '%s'! " + "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the " + ANSI_BOLD "ref" ANSI_NORMAL " you've specified or add " ANSI_BOLD + "allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".", + rev->gitRev(), + ref, + repoInfo.url + ); + } else input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev()); // cache dir lock is removed at scope end; we will only use read-only operations on specific revisions in the remainder } - bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "rev-parse", "--is-shallow-repository" })) == "true"; - - if (isShallow && !shallow) - throw Error("'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified.", actualUrl); + auto repo = GitRepo::openRepo(CanonPath(repoDir)); - // FIXME: check whether rev is an ancestor of ref. + auto isShallow = repo->isShallow(); - printTalkative("using revision %s of repo '%s'", input.getRev()->gitRev(), actualUrl); + if (isShallow && !getShallowAttr(input)) + throw Error("'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified", repoInfo.url); - /* Now that we know the ref, check again whether we have it in - the store. */ - if (auto res = getCache()->lookup(store, getLockedAttrs())) - return makeResult(res->first, std::move(res->second)); + // FIXME: check whether rev is an ancestor of ref? - Path tmpDir = createTempDir(); - AutoDelete delTmpDir(tmpDir, true); - PathFilter filter = defaultPathFilter; + auto rev = *input.getRev(); - auto result = runProgram(RunOptions { - .program = "git", - .args = { "-C", repoDir, "--git-dir", gitDir, "cat-file", "commit", input.getRev()->gitRev() }, - .mergeStderrToStdout = true + Attrs infoAttrs({ + {"rev", rev.gitRev()}, + {"lastModified", getLastModified(repoInfo, repoDir, rev)}, }); - if (WEXITSTATUS(result.first) == 128 - && result.second.find("bad file") != std::string::npos) - { - throw Error( - "Cannot find Git revision '%s' in ref '%s' of repository '%s'! " - "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the " - ANSI_BOLD "ref" ANSI_NORMAL " you've specified or add " ANSI_BOLD - "allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".", - input.getRev()->gitRev(), - *input.getRef(), - actualUrl - ); + + if (!getShallowAttr(input)) + infoAttrs.insert_or_assign("revCount", + getRevCount(repoInfo, repoDir, rev)); + + printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.url); + + verifyCommit(input, repo); + + auto accessor = repo->getAccessor(rev); + + accessor->setPathDisplay("«" + input.to_string() + "»"); + + /* If the repo has submodules, fetch them and return a mounted + input accessor consisting of the accessor for the top-level + repo and the accessors for the submodules. */ + if (getSubmodulesAttr(input)) { + std::map> mounts; + + for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev)) { + auto resolved = repo->resolveSubmoduleUrl(submodule.url, repoInfo.url); + debug("Git submodule %s: %s %s %s -> %s", + submodule.path, submodule.url, submodule.branch, submoduleRev.gitRev(), resolved); + fetchers::Attrs attrs; + attrs.insert_or_assign("type", "git"); + attrs.insert_or_assign("url", resolved); + if (submodule.branch != "") + attrs.insert_or_assign("ref", submodule.branch); + attrs.insert_or_assign("rev", submoduleRev.gitRev()); + auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs)); + auto [submoduleAccessor, submoduleInput2] = + submoduleInput.getAccessor(store); + mounts.insert_or_assign(submodule.path, submoduleAccessor); + } + + if (!mounts.empty()) { + mounts.insert_or_assign(CanonPath::root, accessor); + accessor = makeMountedInputAccessor(std::move(mounts)); + } } - if (submodules) { - Path tmpGitDir = createTempDir(); - AutoDelete delTmpGitDir(tmpGitDir, true); + assert(!origRev || origRev == rev); + if (!getShallowAttr(input)) + input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); + input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified")); - runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", tmpDir, "--separate-git-dir", tmpGitDir }); + return {accessor, std::move(input)}; + } - { - // TODO: repoDir might lack the ref (it only checks if rev - // exists, see FIXME above) so use a big hammer and fetch - // everything to ensure we get the rev. - Activity act(*logger, lvlTalkative, actUnknown, fmt("making temporary clone of '%s'", repoDir)); - runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", - "--update-head-ok", "--", repoDir, "refs/*:refs/*" }, {}, true); + std::pair, Input> getAccessorFromWorkdir( + ref store, + RepoInfo & repoInfo, + Input && input) const + { + if (getSubmodulesAttr(input)) + /* Create mountpoints for the submodules. */ + for (auto & submodule : repoInfo.workdirInfo.submodules) + repoInfo.workdirInfo.files.insert(submodule.path); + + ref accessor = + makeFSInputAccessor(CanonPath(repoInfo.url), repoInfo.workdirInfo.files, makeNotAllowedError(repoInfo.url)); + + /* If the repo has submodules, return a mounted input accessor + consisting of the accessor for the top-level repo and the + accessors for the submodule workdirs. */ + if (getSubmodulesAttr(input) && !repoInfo.workdirInfo.submodules.empty()) { + std::map> mounts; + + for (auto & submodule : repoInfo.workdirInfo.submodules) { + auto submodulePath = CanonPath(repoInfo.url) + submodule.path; + fetchers::Attrs attrs; + attrs.insert_or_assign("type", "git"); + attrs.insert_or_assign("url", submodulePath.abs()); + auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs)); + auto [submoduleAccessor, submoduleInput2] = + submoduleInput.getAccessor(store); + + /* If the submodule is dirty, mark this repo dirty as + well. */ + if (!submoduleInput2.getRev()) + repoInfo.workdirInfo.isDirty = true; + + mounts.insert_or_assign(submodule.path, submoduleAccessor); } - runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() }); + mounts.insert_or_assign(CanonPath::root, accessor); + accessor = makeMountedInputAccessor(std::move(mounts)); + } - /* Ensure that we use the correct origin for fetching - submodules. This matters for submodules with relative - URLs. */ - if (isLocal) { - writeFile(tmpGitDir + "/config", readFile(repoDir + "/" + gitDir + "/config")); + if (!repoInfo.workdirInfo.isDirty) { + auto repo = GitRepo::openRepo(CanonPath(repoInfo.url)); - /* Restore the config.bare setting we may have just - copied erroneously from the user's repo. */ - runProgram("git", true, { "-C", tmpDir, "config", "core.bare", "false" }); - } else - runProgram("git", true, { "-C", tmpDir, "config", "remote.origin.url", actualUrl }); - - /* As an optimisation, copy the modules directory of the - source repo if it exists. */ - auto modulesPath = repoDir + "/" + gitDir + "/modules"; - if (pathExists(modulesPath)) { - Activity act(*logger, lvlTalkative, actUnknown, fmt("copying submodules of '%s'", actualUrl)); - runProgram("cp", true, { "-R", "--", modulesPath, tmpGitDir + "/modules" }); - } + if (auto ref = repo->getWorkdirRef()) + input.attrs.insert_or_assign("ref", *ref); - { - Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", actualUrl)); - runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }, {}, true); - } + auto rev = repoInfo.workdirInfo.headRev.value(); - filter = isNotDotGitDirectory; + input.attrs.insert_or_assign("rev", rev.gitRev()); + input.attrs.insert_or_assign("revCount", getRevCount(repoInfo, repoInfo.url, rev)); + + verifyCommit(input, repo); } else { - // FIXME: should pipe this, or find some better way to extract a - // revision. - auto source = sinkToSource([&](Sink & sink) { - runProgram2({ - .program = "git", - .args = { "-C", repoDir, "--git-dir", gitDir, "archive", input.getRev()->gitRev() }, - .standardOut = &sink - }); - }); - - unpackTarfile(*source, tmpDir); + repoInfo.warnDirty(); + + if (repoInfo.workdirInfo.headRev) { + input.attrs.insert_or_assign("dirtyRev", + repoInfo.workdirInfo.headRev->gitRev() + "-dirty"); + input.attrs.insert_or_assign("dirtyShortRev", + repoInfo.workdirInfo.headRev->gitShortRev() + "-dirty"); + } + + verifyCommit(input, nullptr); } - auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter); + input.attrs.insert_or_assign( + "lastModified", + repoInfo.workdirInfo.headRev + ? getLastModified(repoInfo, repoInfo.url, *repoInfo.workdirInfo.headRev) + : 0); - auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() })); + input.locked = true; // FIXME - Attrs infoAttrs({ - {"rev", input.getRev()->gitRev()}, - {"lastModified", lastModified}, - }); + return {accessor, std::move(input)}; + } - if (!shallow) - infoAttrs.insert_or_assign("revCount", - std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "rev-list", "--count", input.getRev()->gitRev() }))); - - if (!_input.getRev()) - getCache()->add( - store, - unlockedAttrs, - infoAttrs, - storePath, - false); - - getCache()->add( - store, - getLockedAttrs(), - infoAttrs, - storePath, - true); - - return makeResult(infoAttrs, std::move(storePath)); + std::pair, Input> getAccessor(ref store, const Input & _input) const override + { + Input input(_input); + + auto repoInfo = getRepoInfo(input); + + auto [accessor, final] = + input.getRef() || input.getRev() || !repoInfo.isLocal + ? getAccessorFromCommit(store, repoInfo, std::move(input)) + : getAccessorFromWorkdir(store, repoInfo, std::move(input)); + + accessor->fingerprint = final.getFingerprint(store); + + return {accessor, std::move(final)}; + } + + std::optional getFingerprint(ref store, const Input & input) const override + { + if (auto rev = input.getRev()) + return rev->gitRev() + (getSubmodulesAttr(input) ? ";s" : ""); + else + return std::nullopt; } }; diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 80598e7f836e..661ad48844aa 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -7,6 +7,7 @@ #include "git.hh" #include "fetchers.hh" #include "fetch-settings.hh" +#include "tarball.hh" #include #include @@ -26,13 +27,11 @@ std::regex hostRegex(hostRegexS, std::regex::ECMAScript); struct GitArchiveInputScheme : InputScheme { - virtual std::string type() const = 0; - virtual std::optional> accessHeaderFromToken(const std::string & token) const = 0; - std::optional inputFromURL(const ParsedURL & url) const override + std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override { - if (url.scheme != type()) return {}; + if (url.scheme != schemeName()) return {}; auto path = tokenizeString>(url.path, "/"); @@ -90,7 +89,7 @@ struct GitArchiveInputScheme : InputScheme throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url.url, *ref, rev->gitRev()); Input input; - input.attrs.insert_or_assign("type", type()); + input.attrs.insert_or_assign("type", std::string { schemeName() }); input.attrs.insert_or_assign("owner", path[0]); input.attrs.insert_or_assign("repo", path[1]); if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); @@ -100,14 +99,21 @@ struct GitArchiveInputScheme : InputScheme return input; } - std::optional inputFromAttrs(const Attrs & attrs) const override + StringSet allowedAttrs() const override { - if (maybeGetStrAttr(attrs, "type") != type()) return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified" && name != "host") - throw Error("unsupported input attribute '%s'", name); + return { + "owner", + "repo", + "ref", + "rev", + "narHash", + "lastModified", + "host", + }; + } + std::optional inputFromAttrs(const Attrs & attrs) const override + { getStrAttr(attrs, "owner"); getStrAttr(attrs, "repo"); @@ -125,18 +131,13 @@ struct GitArchiveInputScheme : InputScheme auto path = owner + "/" + repo; assert(!(ref && rev)); if (ref) path += "/" + *ref; - if (rev) path += "/" + rev->to_string(Base16, false); + if (rev) path += "/" + rev->to_string(HashFormat::Base16, false); return ParsedURL { - .scheme = type(), + .scheme = std::string { schemeName() }, .path = path, }; } - bool hasAllInfo(const Input & input) const override - { - return input.getRev() && maybeGetIntAttr(input.attrs, "lastModified"); - } - Input applyOverrides( const Input & _input, std::optional ref, @@ -218,16 +219,29 @@ struct GitArchiveInputScheme : InputScheme {"rev", rev->gitRev()}, {"lastModified", uint64_t(result.lastModified)} }, - result.tree.storePath, + result.storePath, true); - return {result.tree.storePath, input}; + return {result.storePath, input}; + } + + std::optional experimentalFeature() const override + { + return Xp::Flakes; + } + + std::optional getFingerprint(ref store, const Input & input) const override + { + if (auto rev = input.getRev()) + return rev->gitRev(); + else + return std::nullopt; } }; struct GitHubInputScheme : GitArchiveInputScheme { - std::string type() const override { return "github"; } + std::string_view schemeName() const override { return "github"; } std::optional> accessHeaderFromToken(const std::string & token) const override { @@ -291,7 +305,7 @@ struct GitHubInputScheme : GitArchiveInputScheme : "https://api.%s/repos/%s/%s/tarball/%s"; const auto url = fmt(urlFmt, host, getOwner(input), getRepo(input), - input.getRev()->to_string(Base16, false)); + input.getRev()->to_string(HashFormat::Base16, false)); return DownloadUrl { url, headers }; } @@ -308,7 +322,7 @@ struct GitHubInputScheme : GitArchiveInputScheme struct GitLabInputScheme : GitArchiveInputScheme { - std::string type() const override { return "gitlab"; } + std::string_view schemeName() const override { return "gitlab"; } std::optional> accessHeaderFromToken(const std::string & token) const override { @@ -357,7 +371,7 @@ struct GitLabInputScheme : GitArchiveInputScheme auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), - input.getRev()->to_string(Base16, false)); + input.getRev()->to_string(HashFormat::Base16, false)); Headers headers = makeHeadersWithAuthTokens(host); return DownloadUrl { url, headers }; @@ -376,7 +390,7 @@ struct GitLabInputScheme : GitArchiveInputScheme struct SourceHutInputScheme : GitArchiveInputScheme { - std::string type() const override { return "sourcehut"; } + std::string_view schemeName() const override { return "sourcehut"; } std::optional> accessHeaderFromToken(const std::string & token) const override { @@ -444,7 +458,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht"); auto url = fmt("https://%s/%s/%s/archive/%s.tar.gz", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), - input.getRev()->to_string(Base16, false)); + input.getRev()->to_string(HashFormat::Base16, false)); Headers headers = makeHeadersWithAuthTokens(host); return DownloadUrl { url, headers }; diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index b99504a16009..8e30284c6acd 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -1,5 +1,6 @@ #include "fetchers.hh" #include "url-parts.hh" +#include "path.hh" namespace nix::fetchers { @@ -7,7 +8,7 @@ std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); struct IndirectInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) const override + std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override { if (url.scheme != "flake") return {}; @@ -41,7 +42,6 @@ struct IndirectInputScheme : InputScheme // FIXME: forbid query params? Input input; - input.direct = false; input.attrs.insert_or_assign("type", "indirect"); input.attrs.insert_or_assign("id", id); if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); @@ -50,20 +50,28 @@ struct IndirectInputScheme : InputScheme return input; } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::string_view schemeName() const override { - if (maybeGetStrAttr(attrs, "type") != "indirect") return {}; + return "indirect"; + } - for (auto & [name, value] : attrs) - if (name != "type" && name != "id" && name != "ref" && name != "rev" && name != "narHash") - throw Error("unsupported indirect input attribute '%s'", name); + StringSet allowedAttrs() const override + { + return { + "id", + "ref", + "rev", + "narHash", + }; + } + std::optional inputFromAttrs(const Attrs & attrs) const override + { auto id = getStrAttr(attrs, "id"); if (!std::regex_match(id, flakeRegex)) throw BadURL("'%s' is not a valid flake ID", id); Input input; - input.direct = false; input.attrs = attrs; return input; } @@ -78,11 +86,6 @@ struct IndirectInputScheme : InputScheme return url; } - bool hasAllInfo(const Input & input) const override - { - return false; - } - Input applyOverrides( const Input & _input, std::optional ref, @@ -98,6 +101,14 @@ struct IndirectInputScheme : InputScheme { throw Error("indirect input '%s' cannot be fetched directly", input.to_string()); } + + std::optional experimentalFeature() const override + { + return Xp::Flakes; + } + + bool isDirect(const Input & input) const override + { return false; } }; static auto rIndirectInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc index f37a8058b68c..85dc4609fe5e 100644 --- a/src/libfetchers/input-accessor.cc +++ b/src/libfetchers/input-accessor.cc @@ -1,77 +1,94 @@ #include "input-accessor.hh" #include "store-api.hh" +#include "cache.hh" namespace nix { -std::ostream & operator << (std::ostream & str, const SourcePath & path) +StorePath InputAccessor::fetchToStore( + ref store, + const CanonPath & path, + std::string_view name, + FileIngestionMethod method, + PathFilter * filter, + RepairFlag repair) { - str << path.to_string(); - return str; -} + // FIXME: add an optimisation for the case where the accessor is + // an FSInputAccessor pointing to a store path. -std::string_view SourcePath::baseName() const -{ - return path.baseName().value_or("source"); -} + std::optional cacheKey; -SourcePath SourcePath::parent() const -{ - auto p = path.parent(); - assert(p); - return std::move(*p); -} + if (!filter && fingerprint) { + cacheKey = fetchers::Attrs{ + {"_what", "fetchToStore"}, + {"store", store->storeDir}, + {"name", std::string(name)}, + {"fingerprint", *fingerprint}, + {"method", (uint8_t) method}, + {"path", path.abs()} + }; + if (auto res = fetchers::getCache()->lookup(store, *cacheKey)) { + debug("store path cache hit for '%s'", showPath(path)); + return res->second; + } + } else + debug("source path '%s' is uncacheable", showPath(path)); -InputAccessor::Stat SourcePath::lstat() const -{ - auto st = nix::lstat(path.abs()); - return InputAccessor::Stat { - .type = - S_ISREG(st.st_mode) ? InputAccessor::tRegular : - S_ISDIR(st.st_mode) ? InputAccessor::tDirectory : - S_ISLNK(st.st_mode) ? InputAccessor::tSymlink : - InputAccessor::tMisc, - .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR - }; + Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path))); + + auto source = sinkToSource([&](Sink & sink) { + if (method == FileIngestionMethod::Recursive) + dumpPath(path, sink, filter ? *filter : defaultPathFilter); + else + readFile(path, sink); + }); + + auto storePath = + settings.readOnlyMode + ? store->computeStorePathFromDump(*source, name, method, htSHA256).first + : store->addToStoreFromDump(*source, name, method, htSHA256, repair); + + if (cacheKey) + fetchers::getCache()->add(store, *cacheKey, {}, storePath, true); + + return storePath; } -std::optional SourcePath::maybeLstat() const +SourcePath InputAccessor::root() { - // FIXME: merge these into one operation. - if (!pathExists()) - return {}; - return lstat(); + return {ref(shared_from_this()), CanonPath::root}; } -InputAccessor::DirEntries SourcePath::readDirectory() const +std::ostream & operator << (std::ostream & str, const SourcePath & path) { - InputAccessor::DirEntries res; - for (auto & entry : nix::readDirectory(path.abs())) { - std::optional type; - switch (entry.type) { - case DT_REG: type = InputAccessor::Type::tRegular; break; - case DT_LNK: type = InputAccessor::Type::tSymlink; break; - case DT_DIR: type = InputAccessor::Type::tDirectory; break; - } - res.emplace(entry.name, type); - } - return res; + str << path.to_string(); + return str; } StorePath SourcePath::fetchToStore( ref store, std::string_view name, + FileIngestionMethod method, PathFilter * filter, RepairFlag repair) const { - return - settings.readOnlyMode - ? store->computeStorePathForPath(name, path.abs(), FileIngestionMethod::Recursive, htSHA256, filter ? *filter : defaultPathFilter).first - : store->addToStore(name, path.abs(), FileIngestionMethod::Recursive, htSHA256, filter ? *filter : defaultPathFilter, repair); + return accessor->fetchToStore(store, path, name, method, filter, repair); +} + +std::string_view SourcePath::baseName() const +{ + return path.baseName().value_or("source"); +} + +SourcePath SourcePath::parent() const +{ + auto p = path.parent(); + assert(p); + return {accessor, std::move(*p)}; } SourcePath SourcePath::resolveSymlinks() const { - SourcePath res(CanonPath::root); + auto res = accessor->root(); int linksAllowed = 1024; diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh index 5a2f17f62888..26d17f064285 100644 --- a/src/libfetchers/input-accessor.hh +++ b/src/libfetchers/input-accessor.hh @@ -1,40 +1,43 @@ #pragma once +///@file +#include "source-accessor.hh" #include "ref.hh" #include "types.hh" -#include "archive.hh" -#include "canon-path.hh" +#include "file-system.hh" #include "repair-flag.hh" +#include "content-address.hh" namespace nix { +MakeError(RestrictedPathError, Error); + +struct SourcePath; class StorePath; class Store; -struct InputAccessor +struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this { - enum Type { - tRegular, tSymlink, tDirectory, - /** - Any other node types that may be encountered on the file system, such as device nodes, sockets, named pipe, and possibly even more exotic things. - - Responsible for `"unknown"` from `builtins.readFileType "/dev/null"`. - - Unlike `DT_UNKNOWN`, this must not be used for deferring the lookup of types. - */ - tMisc - }; + std::optional fingerprint; - struct Stat + /** + * Return the maximum last-modified time of the files in this + * tree, if available. + */ + virtual std::optional getLastModified() { - Type type = tMisc; - //uint64_t fileSize = 0; // regular files only - bool isExecutable = false; // regular files only - }; + return std::nullopt; + } - typedef std::optional DirEntry; + StorePath fetchToStore( + ref store, + const CanonPath & path, + std::string_view name = "source", + FileIngestionMethod method = FileIngestionMethod::Recursive, + PathFilter * filter = nullptr, + RepairFlag repair = NoRepair); - typedef std::map DirEntries; + SourcePath root(); }; /** @@ -45,12 +48,9 @@ struct InputAccessor */ struct SourcePath { + ref accessor; CanonPath path; - SourcePath(CanonPath path) - : path(std::move(path)) - { } - std::string_view baseName() const; /** @@ -64,39 +64,42 @@ struct SourcePath * return its contents; otherwise throw an error. */ std::string readFile() const - { return nix::readFile(path.abs()); } + { return accessor->readFile(path); } /** * Return whether this `SourcePath` denotes a file (of any type) * that exists */ bool pathExists() const - { return nix::pathExists(path.abs()); } + { return accessor->pathExists(path); } /** * Return stats about this `SourcePath`, or throw an exception if * it doesn't exist. */ - InputAccessor::Stat lstat() const; + InputAccessor::Stat lstat() const + { return accessor->lstat(path); } /** * Return stats about this `SourcePath`, or std::nullopt if it * doesn't exist. */ - std::optional maybeLstat() const; + std::optional maybeLstat() const + { return accessor->maybeLstat(path); } /** * If this `SourcePath` denotes a directory (not a symlink), * return its directory entries; otherwise throw an error. */ - InputAccessor::DirEntries readDirectory() const; + InputAccessor::DirEntries readDirectory() const + { return accessor->readDirectory(path); } /** * If this `SourcePath` denotes a symlink, return its target; * otherwise throw an error. */ std::string readLink() const - { return nix::readLink(path.abs()); } + { return accessor->readLink(path); } /** * Dump this `SourcePath` to `sink` as a NAR archive. @@ -104,7 +107,7 @@ struct SourcePath void dumpPath( Sink & sink, PathFilter & filter = defaultPathFilter) const - { return nix::dumpPath(path.abs(), sink, filter); } + { return accessor->dumpPath(path, sink, filter); } /** * Copy this `SourcePath` to the Nix store. @@ -112,6 +115,7 @@ struct SourcePath StorePath fetchToStore( ref store, std::string_view name = "source", + FileIngestionMethod method = FileIngestionMethod::Recursive, PathFilter * filter = nullptr, RepairFlag repair = NoRepair) const; @@ -120,7 +124,7 @@ struct SourcePath * it has a physical location. */ std::optional getPhysicalPath() const - { return path; } + { return accessor->getPhysicalPath(path); } std::string to_string() const { return path.abs(); } @@ -129,7 +133,7 @@ struct SourcePath * Append a `CanonPath` to this path. */ SourcePath operator + (const CanonPath & x) const - { return {path + x}; } + { return {accessor, path + x}; } /** * Append a single component `c` to this path. `c` must not @@ -137,21 +141,21 @@ struct SourcePath * and `c`. */ SourcePath operator + (std::string_view c) const - { return {path + c}; } + { return {accessor, path + c}; } bool operator == (const SourcePath & x) const { - return path == x.path; + return std::tie(accessor, path) == std::tie(x.accessor, x.path); } bool operator != (const SourcePath & x) const { - return path != x.path; + return std::tie(accessor, path) != std::tie(x.accessor, x.path); } bool operator < (const SourcePath & x) const { - return path < x.path; + return std::tie(accessor, path) < std::tie(x.accessor, x.path); } /** diff --git a/src/libfetchers/local.mk b/src/libfetchers/local.mk index 2e8869d83fa8..266e7a21149b 100644 --- a/src/libfetchers/local.mk +++ b/src/libfetchers/local.mk @@ -8,6 +8,6 @@ libfetchers_SOURCES := $(wildcard $(d)/*.cc) libfetchers_CXXFLAGS += -I src/libutil -I src/libstore -libfetchers_LDFLAGS += -pthread +libfetchers_LDFLAGS += -pthread $(LIBGIT2_LIBS) -larchive libfetchers_LIBS = libutil libstore diff --git a/src/libfetchers/memory-input-accessor.cc b/src/libfetchers/memory-input-accessor.cc new file mode 100644 index 000000000000..057f3e37f730 --- /dev/null +++ b/src/libfetchers/memory-input-accessor.cc @@ -0,0 +1,22 @@ +#include "memory-input-accessor.hh" +#include "memory-source-accessor.hh" + +namespace nix { + +struct MemoryInputAccessorImpl : MemoryInputAccessor, MemorySourceAccessor +{ + SourcePath addFile(CanonPath path, std::string && contents) override + { + return { + ref(shared_from_this()), + MemorySourceAccessor::addFile(path, std::move(contents)) + }; + } +}; + +ref makeMemoryInputAccessor() +{ + return make_ref(); +} + +} diff --git a/src/libfetchers/memory-input-accessor.hh b/src/libfetchers/memory-input-accessor.hh new file mode 100644 index 000000000000..b75b02bfd617 --- /dev/null +++ b/src/libfetchers/memory-input-accessor.hh @@ -0,0 +1,15 @@ +#include "input-accessor.hh" + +namespace nix { + +/** + * An input accessor for an in-memory file system. + */ +struct MemoryInputAccessor : InputAccessor +{ + virtual SourcePath addFile(CanonPath path, std::string && contents) = 0; +}; + +ref makeMemoryInputAccessor(); + +} diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 86e8f81f44fe..aa991a75d6f4 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -1,4 +1,6 @@ #include "fetchers.hh" +#include "processes.hh" +#include "users.hh" #include "cache.hh" #include "globals.hh" #include "tarfile.hh" @@ -43,7 +45,7 @@ static std::string runHg(const Strings & args, const std::optional struct MercurialInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) const override + std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override { if (url.scheme != "hg+http" && url.scheme != "hg+https" && @@ -69,14 +71,25 @@ struct MercurialInputScheme : InputScheme return inputFromAttrs(attrs); } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::string_view schemeName() const override { - if (maybeGetStrAttr(attrs, "type") != "hg") return {}; + return "hg"; + } - for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash" && name != "name") - throw Error("unsupported Mercurial input attribute '%s'", name); + StringSet allowedAttrs() const override + { + return { + "url", + "ref", + "rev", + "revCount", + "narHash", + "name", + }; + } + std::optional inputFromAttrs(const Attrs & attrs) const override + { parseURL(getStrAttr(attrs, "url")); if (auto ref = maybeGetStrAttr(attrs, "ref")) { @@ -98,13 +111,6 @@ struct MercurialInputScheme : InputScheme return url; } - bool hasAllInfo(const Input & input) const override - { - // FIXME: ugly, need to distinguish between dirty and clean - // default trees. - return input.getRef() == "default" || maybeGetIntAttr(input.attrs, "revCount"); - } - Input applyOverrides( const Input & input, std::optional ref, @@ -116,7 +122,7 @@ struct MercurialInputScheme : InputScheme return res; } - std::optional getSourcePath(const Input & input) override + std::optional getSourcePath(const Input & input) const override { auto url = parseURL(getStrAttr(input.attrs, "url")); if (url.scheme == "file" && !input.getRef() && !input.getRev()) @@ -124,18 +130,27 @@ struct MercurialInputScheme : InputScheme return {}; } - void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) override + void putFile( + const Input & input, + const CanonPath & path, + std::string_view contents, + std::optional commitMsg) const override { - auto sourcePath = getSourcePath(input); - assert(sourcePath); + auto [isLocal, repoPath] = getActualUrl(input); + if (!isLocal) + throw Error("cannot commit '%s' to Mercurial repository '%s' because it's not a working tree", path, input.to_string()); + + auto absPath = CanonPath(repoPath) + path; + + writeFile(absPath.abs(), contents); // FIXME: shut up if file is already tracked. runHg( - { "add", *sourcePath + "/" + std::string(file) }); + { "add", absPath.abs() }); if (commitMsg) runHg( - { "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg }); + { "commit", absPath.abs(), "-m", *commitMsg }); } std::pair getActualUrl(const Input & input) const @@ -206,7 +221,7 @@ struct MercurialInputScheme : InputScheme auto checkHashType = [&](const std::optional & hash) { if (hash.has_value() && hash->type != htSHA1) - throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(Base16, true)); + throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(HashFormat::Base16, true)); }; @@ -252,7 +267,7 @@ struct MercurialInputScheme : InputScheme } } - Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false)); + Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(HashFormat::Base32, false)); /* If this is a commit hash that we already have, we don't have to pull again. */ @@ -324,6 +339,14 @@ struct MercurialInputScheme : InputScheme return makeResult(infoAttrs, std::move(storePath)); } + + std::optional getFingerprint(ref store, const Input & input) const override + { + if (auto rev = input.getRev()) + return rev->gitRev(); + else + return std::nullopt; + } }; static auto rMercurialInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); diff --git a/src/libfetchers/mounted-input-accessor.cc b/src/libfetchers/mounted-input-accessor.cc new file mode 100644 index 000000000000..6f397eb17401 --- /dev/null +++ b/src/libfetchers/mounted-input-accessor.cc @@ -0,0 +1,77 @@ +#include "mounted-input-accessor.hh" + +namespace nix { + +struct MountedInputAccessor : InputAccessor +{ + std::map> mounts; + + MountedInputAccessor(std::map> _mounts) + : mounts(std::move(_mounts)) + { + // Currently we require a root filesystem. This could be relaxed. + assert(mounts.contains(CanonPath::root)); + + // FIXME: return dummy parent directories automatically? + } + + std::string readFile(const CanonPath & path) override + { + auto [accessor, subpath] = resolve(path); + return accessor->readFile(subpath); + } + + bool pathExists(const CanonPath & path) override + { + auto [accessor, subpath] = resolve(path); + return accessor->pathExists(subpath); + } + + std::optional maybeLstat(const CanonPath & path) override + { + auto [accessor, subpath] = resolve(path); + return accessor->maybeLstat(subpath); + } + + DirEntries readDirectory(const CanonPath & path) override + { + auto [accessor, subpath] = resolve(path); + return accessor->readDirectory(subpath); + } + + std::string readLink(const CanonPath & path) override + { + auto [accessor, subpath] = resolve(path); + return accessor->readLink(subpath); + } + + std::string showPath(const CanonPath & path) override + { + auto [accessor, subpath] = resolve(path); + return accessor->showPath(subpath); + } + + std::pair, CanonPath> resolve(CanonPath path) + { + // Find the nearest parent of `path` that is a mount point. + std::vector subpath; + while (true) { + auto i = mounts.find(path); + if (i != mounts.end()) { + std::reverse(subpath.begin(), subpath.end()); + return {i->second, CanonPath(subpath)}; + } + + assert(!path.isRoot()); + subpath.push_back(std::string(*path.baseName())); + path.pop(); + } + } +}; + +ref makeMountedInputAccessor(std::map> mounts) +{ + return make_ref(std::move(mounts)); +} + +} diff --git a/src/libfetchers/mounted-input-accessor.hh b/src/libfetchers/mounted-input-accessor.hh new file mode 100644 index 000000000000..b557c5dad7fb --- /dev/null +++ b/src/libfetchers/mounted-input-accessor.hh @@ -0,0 +1,9 @@ +#pragma once + +#include "input-accessor.hh" + +namespace nix { + +ref makeMountedInputAccessor(std::map> mounts); + +} diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 61541e69dfb0..f9b973320b08 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -6,7 +6,7 @@ namespace nix::fetchers { struct PathInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) const override + std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override { if (url.scheme != "path") return {}; @@ -32,22 +32,29 @@ struct PathInputScheme : InputScheme return input; } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::string_view schemeName() const override { - if (maybeGetStrAttr(attrs, "type") != "path") return {}; - - getStrAttr(attrs, "path"); + return "path"; + } - for (auto & [name, value] : attrs) + StringSet allowedAttrs() const override + { + return { + "path", /* Allow the user to pass in "fake" tree info - attributes. This is useful for making a pinned tree - work the same as the repository from which is exported - (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */ - if (name == "type" || name == "rev" || name == "revCount" || name == "lastModified" || name == "narHash" || name == "path") - // checked in Input::fromAttrs - ; - else - throw Error("unsupported path input attribute '%s'", name); + attributes. This is useful for making a pinned tree work + the same as the repository from which is exported (e.g. + path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). + */ + "rev", + "revCount", + "lastModified", + "narHash", + }; + } + std::optional inputFromAttrs(const Attrs & attrs) const override + { + getStrAttr(attrs, "path"); Input input; input.attrs = attrs; @@ -66,19 +73,28 @@ struct PathInputScheme : InputScheme }; } - bool hasAllInfo(const Input & input) const override + std::optional getSourcePath(const Input & input) const override { - return true; + return getStrAttr(input.attrs, "path"); } - std::optional getSourcePath(const Input & input) override + void putFile( + const Input & input, + const CanonPath & path, + std::string_view contents, + std::optional commitMsg) const override { - return getStrAttr(input.attrs, "path"); + writeFile((CanonPath(getAbsPath(input)) + path).abs(), contents); } - void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) override + CanonPath getAbsPath(const Input & input) const { - // nothing to do + auto path = getStrAttr(input.attrs, "path"); + + if (path[0] == '/') + return CanonPath(path); + + throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string()); } std::pair fetch(ref store, const Input & _input) override @@ -125,6 +141,11 @@ struct PathInputScheme : InputScheme return {std::move(*storePath), input}; } + + std::optional experimentalFeature() const override + { + return Xp::Flakes; + } }; static auto rPathInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 43c03beec17f..9c7bc0cfe194 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -1,6 +1,6 @@ #include "registry.hh" -#include "fetchers.hh" -#include "util.hh" +#include "tarball.hh" +#include "users.hh" #include "globals.hh" #include "store-api.hh" #include "local-fs-store.hh" diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index e42aca6db3af..0062878a9d48 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -1,3 +1,4 @@ +#include "tarball.hh" #include "fetchers.hh" #include "cache.hh" #include "filetransfer.hh" @@ -77,10 +78,8 @@ DownloadFileResult downloadFile( *store, name, FixedOutputInfo { - .hash = { - .method = FileIngestionMethod::Flat, - .hash = hash, - }, + .method = FileIngestionMethod::Flat, + .hash = hash, .references = {}, }, hashString(htSHA256, sink.s), @@ -135,7 +134,7 @@ DownloadTarballResult downloadTarball( if (cached && !cached->expired) return { - .tree = Tree { .actualPath = store->toRealPath(cached->storePath), .storePath = std::move(cached->storePath) }, + .storePath = std::move(cached->storePath), .lastModified = (time_t) getIntAttr(cached->infoAttrs, "lastModified"), .immutableUrl = maybeGetStrAttr(cached->infoAttrs, "immutableUrl"), }; @@ -176,7 +175,7 @@ DownloadTarballResult downloadTarball( locked); return { - .tree = Tree { .actualPath = store->toRealPath(*unpackedStorePath), .storePath = std::move(*unpackedStorePath) }, + .storePath = std::move(*unpackedStorePath), .lastModified = lastModified, .immutableUrl = res.immutableUrl, }; @@ -185,7 +184,6 @@ DownloadTarballResult downloadTarball( // An input scheme corresponding to a curl-downloadable resource. struct CurlInputScheme : InputScheme { - virtual const std::string inputType() const = 0; const std::set transportUrlSchemes = {"file", "http", "https"}; const bool hasTarballExtension(std::string_view path) const @@ -196,11 +194,11 @@ struct CurlInputScheme : InputScheme || hasSuffix(path, ".tar.zst"); } - virtual bool isValidURL(const ParsedURL & url) const = 0; + virtual bool isValidURL(const ParsedURL & url, bool requireTree) const = 0; - std::optional inputFromURL(const ParsedURL & _url) const override + std::optional inputFromURL(const ParsedURL & _url, bool requireTree) const override { - if (!isValidURL(_url)) + if (!isValidURL(_url, requireTree)) return std::nullopt; Input input; @@ -223,22 +221,27 @@ struct CurlInputScheme : InputScheme url.query.erase("rev"); url.query.erase("revCount"); - input.attrs.insert_or_assign("type", inputType()); + input.attrs.insert_or_assign("type", std::string { schemeName() }); input.attrs.insert_or_assign("url", url.to_string()); return input; } - std::optional inputFromAttrs(const Attrs & attrs) const override + StringSet allowedAttrs() const override { - auto type = maybeGetStrAttr(attrs, "type"); - if (type != inputType()) return {}; - - // FIXME: some of these only apply to TarballInputScheme. - std::set allowedNames = {"type", "url", "narHash", "name", "unpack", "rev", "revCount"}; - for (auto & [name, value] : attrs) - if (!allowedNames.count(name)) - throw Error("unsupported %s input attribute '%s'", *type, name); + return { + "type", + "url", + "narHash", + "name", + "unpack", + "rev", + "revCount", + "lastModified", + }; + } + std::optional inputFromAttrs(const Attrs & attrs) const override + { Input input; input.attrs = attrs; @@ -252,28 +255,22 @@ struct CurlInputScheme : InputScheme // NAR hashes are preferred over file hashes since tar/zip // files don't have a canonical representation. if (auto narHash = input.getNarHash()) - url.query.insert_or_assign("narHash", narHash->to_string(SRI, true)); + url.query.insert_or_assign("narHash", narHash->to_string(HashFormat::SRI, true)); return url; } - - bool hasAllInfo(const Input & input) const override - { - return true; - } - }; struct FileInputScheme : CurlInputScheme { - const std::string inputType() const override { return "file"; } + std::string_view schemeName() const override { return "file"; } - bool isValidURL(const ParsedURL & url) const override + bool isValidURL(const ParsedURL & url, bool requireTree) const override { auto parsedUrlScheme = parseUrlScheme(url.scheme); return transportUrlSchemes.count(std::string(parsedUrlScheme.transport)) && (parsedUrlScheme.application - ? parsedUrlScheme.application.value() == inputType() - : !hasTarballExtension(url.path)); + ? parsedUrlScheme.application.value() == schemeName() + : (!requireTree && !hasTarballExtension(url.path))); } std::pair fetch(ref store, const Input & input) override @@ -285,16 +282,16 @@ struct FileInputScheme : CurlInputScheme struct TarballInputScheme : CurlInputScheme { - const std::string inputType() const override { return "tarball"; } + std::string_view schemeName() const override { return "tarball"; } - bool isValidURL(const ParsedURL & url) const override + bool isValidURL(const ParsedURL & url, bool requireTree) const override { auto parsedUrlScheme = parseUrlScheme(url.scheme); return transportUrlSchemes.count(std::string(parsedUrlScheme.transport)) && (parsedUrlScheme.application - ? parsedUrlScheme.application.value() == inputType() - : hasTarballExtension(url.path)); + ? parsedUrlScheme.application.value() == schemeName() + : (requireTree || hasTarballExtension(url.path))); } std::pair fetch(ref store, const Input & _input) override @@ -312,7 +309,10 @@ struct TarballInputScheme : CurlInputScheme input = immutableInput; } - return {result.tree.storePath, std::move(input)}; + if (result.lastModified && !input.attrs.contains("lastModified")) + input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified)); + + return {result.storePath, std::move(input)}; } }; diff --git a/src/libfetchers/tarball.hh b/src/libfetchers/tarball.hh new file mode 100644 index 000000000000..9e6b50b31ded --- /dev/null +++ b/src/libfetchers/tarball.hh @@ -0,0 +1,43 @@ +#pragma once + +#include "types.hh" +#include "path.hh" + +#include + +namespace nix { +class Store; +} + +namespace nix::fetchers { + +struct DownloadFileResult +{ + StorePath storePath; + std::string etag; + std::string effectiveUrl; + std::optional immutableUrl; +}; + +DownloadFileResult downloadFile( + ref store, + const std::string & url, + const std::string & name, + bool locked, + const Headers & headers = {}); + +struct DownloadTarballResult +{ + StorePath storePath; + time_t lastModified; + std::optional immutableUrl; +}; + +DownloadTarballResult downloadTarball( + ref store, + const std::string & url, + const std::string & name, + bool locked, + const Headers & headers = {}); + +} diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index f92920d18e5d..5b49aaabcef9 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -1,6 +1,9 @@ #include "common-args.hh" +#include "args/root.hh" #include "globals.hh" +#include "logging.hh" #include "loggers.hh" +#include "util.hh" namespace nix { @@ -34,21 +37,21 @@ MixCommonArgs::MixCommonArgs(const std::string & programName) .description = "Set the Nix configuration setting *name* to *value* (overriding `nix.conf`).", .category = miscCategory, .labels = {"name", "value"}, - .handler = {[](std::string name, std::string value) { + .handler = {[this](std::string name, std::string value) { try { globalConfig.set(name, value); } catch (UsageError & e) { - if (!completions) + if (!getRoot().completions) warn(e.what()); } }}, - .completer = [](size_t index, std::string_view prefix) { + .completer = [](AddCompletions & completions, size_t index, std::string_view prefix) { if (index == 0) { std::map settings; globalConfig.getSettings(settings); for (auto & s : settings) if (hasPrefix(s.first, prefix)) - completions->add(s.first, fmt("Set the `%s` setting.", s.first)); + completions.add(s.first, fmt("Set the `%s` setting.", s.first)); } } }); diff --git a/src/libmain/local.mk b/src/libmain/local.mk index 99da95e27975..5c7061863021 100644 --- a/src/libmain/local.mk +++ b/src/libmain/local.mk @@ -14,4 +14,4 @@ libmain_LIBS = libstore libutil libmain_ALLOW_UNDEFINED = 1 -$(eval $(call install-file-in, $(d)/nix-main.pc, $(libdir)/pkgconfig, 0644)) +$(eval $(call install-file-in, $(buildprefix)$(d)/nix-main.pc, $(libdir)/pkgconfig, 0644)) diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc index cda5cb939ac2..9829859de326 100644 --- a/src/libmain/loggers.cc +++ b/src/libmain/loggers.cc @@ -1,6 +1,6 @@ #include "loggers.hh" +#include "environment-variables.hh" #include "progress-bar.hh" -#include "util.hh" namespace nix { diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 6600ec177289..3aa012ee1cc5 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -1,5 +1,5 @@ #include "progress-bar.hh" -#include "util.hh" +#include "terminal.hh" #include "sync.hh" #include "store-api.hh" #include "names.hh" @@ -108,7 +108,8 @@ class ProgressBar : public Logger stop(); } - void stop() override + /* Called by destructor, can't be overridden */ + void stop() override final { { auto state(state_.lock()); @@ -339,6 +340,14 @@ class ProgressBar : public Logger state->activitiesByType[type].expected += j; update(*state); } + + else if (type == resFetchStatus) { + auto i = state->its.find(act); + assert(i != state->its.end()); + ActInfo & actInfo = *i->second; + actInfo.lastLine = getS(fields, 0); + update(*state); + } } void update(State & state) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 56f47a4ac818..862ef355b958 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -1,10 +1,11 @@ #include "globals.hh" +#include "current-process.hh" #include "shared.hh" #include "store-api.hh" #include "gc-store.hh" -#include "util.hh" #include "loggers.hh" #include "progress-bar.hh" +#include "signals.hh" #include #include @@ -379,9 +380,9 @@ RunPager::RunPager() }); pid.setKillSignal(SIGINT); - stdout = fcntl(STDOUT_FILENO, F_DUPFD_CLOEXEC, 0); + std_out = fcntl(STDOUT_FILENO, F_DUPFD_CLOEXEC, 0); if (dup2(toPager.writeSide.get(), STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); + throw SysError("dupping standard output"); } @@ -390,7 +391,7 @@ RunPager::~RunPager() try { if (pid != -1) { std::cout.flush(); - dup2(stdout, STDOUT_FILENO); + dup2(std_out, STDOUT_FILENO); pid.wait(); } } catch (...) { diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 7a9e83c6c298..c68f6cd83e9c 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -1,8 +1,9 @@ #pragma once ///@file -#include "util.hh" +#include "processes.hh" #include "args.hh" +#include "args/root.hh" #include "common-args.hh" #include "path.hh" #include "derived-path.hh" @@ -66,7 +67,7 @@ template N getIntArg(const std::string & opt, } -struct LegacyArgs : public MixCommonArgs +struct LegacyArgs : public MixCommonArgs, public RootArgs { std::function parseArg; @@ -85,8 +86,9 @@ struct LegacyArgs : public MixCommonArgs void showManPage(const std::string & name); /** - * The constructor of this class starts a pager if stdout is a - * terminal and $PAGER is set. Stdout is redirected to the pager. + * The constructor of this class starts a pager if standard output is a + * terminal and $PAGER is set. Standard output is redirected to the + * pager. */ class RunPager { @@ -96,7 +98,7 @@ public: private: Pid pid; - int stdout; + int std_out; }; extern volatile ::sig_atomic_t blockInt; diff --git a/src/libstore/access-status.hh b/src/libstore/access-status.hh index c1045681c654..0a77e1732cd0 100644 --- a/src/libstore/access-status.hh +++ b/src/libstore/access-status.hh @@ -7,6 +7,7 @@ #include "comparator.hh" #include "globals.hh" #include "acl.hh" +#include "util.hh" namespace nix { template diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index fcd763a9d3ea..ae483c95efc5 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -2,7 +2,7 @@ #include "binary-cache-store.hh" #include "compression.hh" #include "derivations.hh" -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "globals.hh" #include "nar-info.hh" #include "sync.hh" @@ -11,6 +11,7 @@ #include "nar-accessor.hh" #include "thread-pool.hh" #include "callback.hh" +#include "signals.hh" #include #include @@ -143,7 +144,7 @@ ref BinaryCacheStore::addToStoreCommon( write the compressed NAR to disk), into a HashSink (to get the NAR hash), and into a NarAccessor (to get the NAR listing). */ HashSink fileHashSink { htSHA256 }; - std::shared_ptr narAccessor; + std::shared_ptr narAccessor; HashSink narHashSink { htSHA256 }; { FdSink fileSink(fdTemp.get()); @@ -164,7 +165,7 @@ ref BinaryCacheStore::addToStoreCommon( auto [fileHash, fileSize] = fileHashSink.finish(); narInfo->fileHash = fileHash; narInfo->fileSize = fileSize; - narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar" + narInfo->url = "nar/" + narInfo->fileHash->to_string(HashFormat::Base32, false) + ".nar" + (compression == "xz" ? ".xz" : compression == "bzip2" ? ".bz2" : compression == "zstd" ? ".zst" : @@ -195,7 +196,7 @@ ref BinaryCacheStore::addToStoreCommon( if (writeNARListing) { nlohmann::json j = { {"version", 1}, - {"root", listNar(ref(narAccessor), "", true)}, + {"root", listNar(ref(narAccessor), CanonPath::root, true)}, }; upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json"); @@ -206,9 +207,9 @@ ref BinaryCacheStore::addToStoreCommon( specify the NAR file and member containing the debug info. */ if (writeDebugInfo) { - std::string buildIdDir = "/lib/debug/.build-id"; + CanonPath buildIdDir("lib/debug/.build-id"); - if (narAccessor->stat(buildIdDir).type == FSAccessor::tDirectory) { + if (auto st = narAccessor->maybeLstat(buildIdDir); st && st->type == SourceAccessor::tDirectory) { ThreadPool threadPool(25); @@ -231,17 +232,17 @@ ref BinaryCacheStore::addToStoreCommon( std::regex regex1("^[0-9a-f]{2}$"); std::regex regex2("^[0-9a-f]{38}\\.debug$"); - for (auto & s1 : narAccessor->readDirectory(buildIdDir)) { - auto dir = buildIdDir + "/" + s1; + for (auto & [s1, _type] : narAccessor->readDirectory(buildIdDir)) { + auto dir = buildIdDir + s1; - if (narAccessor->stat(dir).type != FSAccessor::tDirectory + if (narAccessor->lstat(dir).type != SourceAccessor::tDirectory || !std::regex_match(s1, regex1)) continue; - for (auto & s2 : narAccessor->readDirectory(dir)) { - auto debugPath = dir + "/" + s2; + for (auto & [s2, _type] : narAccessor->readDirectory(dir)) { + auto debugPath = dir + s2; - if (narAccessor->stat(debugPath).type != FSAccessor::tRegular + if (narAccessor->lstat(debugPath).type != SourceAccessor::tRegular || !std::regex_match(s2, regex2)) continue; @@ -250,7 +251,7 @@ ref BinaryCacheStore::addToStoreCommon( std::string key = "debuginfo/" + buildId; std::string target = "../" + narInfo->url; - threadPool.enqueue(std::bind(doFile, std::string(debugPath, 1), key, target)); + threadPool.enqueue(std::bind(doFile, std::string(debugPath.rel()), key, target)); } } @@ -309,10 +310,8 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n *this, name, FixedOutputInfo { - .hash = { - .method = method, - .hash = nar.first, - }, + .method = method, + .hash = nar.first, .references = { .others = references, // caller is not capable of creating a self-reference, because this is content-addressed without modulus @@ -428,10 +427,8 @@ StorePath BinaryCacheStore::addToStore( *this, name, FixedOutputInfo { - .hash = { - .method = method, - .hash = h, - }, + .method = method, + .hash = h, .references = { .others = references, // caller is not capable of creating a self-reference, because this is content-addressed without modulus @@ -465,8 +462,8 @@ StorePath BinaryCacheStore::addTextToStore( *this, std::string { name }, TextInfo { - { .hash = textHash }, - references, + .hash = textHash, + .references = references, }, nar.first, }; @@ -507,9 +504,9 @@ void BinaryCacheStore::registerDrvOutput(const Realisation& info) { upsertFile(filePath, info.toJSON().dump(), "application/json"); } -ref BinaryCacheStore::getFSAccessor() +ref BinaryCacheStore::getFSAccessor(bool requireValidPath) { - return make_ref(ref(shared_from_this()), localNarCache); + return make_ref(ref(shared_from_this()), requireValidPath, localNarCache); } void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSet & sigs) diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 49f271d248c2..cea2a571f133 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -17,28 +17,28 @@ struct BinaryCacheStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - const Setting compression{(StoreConfig*) this, "xz", "compression", + const Setting compression{this, "xz", "compression", "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)."}; - const Setting writeNARListing{(StoreConfig*) this, false, "write-nar-listing", + const Setting writeNARListing{this, false, "write-nar-listing", "Whether to write a JSON file that lists the files in each NAR."}; - const Setting writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", + const Setting writeDebugInfo{this, false, "index-debug-info", R"( Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to fetch debug info on demand )"}; - const Setting secretKeyFile{(StoreConfig*) this, "", "secret-key", + const Setting secretKeyFile{this, "", "secret-key", "Path to the secret key used to sign the binary cache."}; - const Setting localNarCache{(StoreConfig*) this, "", "local-nar-cache", + const Setting localNarCache{this, "", "local-nar-cache", "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`."}; - const Setting parallelCompression{(StoreConfig*) this, false, "parallel-compression", + const Setting parallelCompression{this, false, "parallel-compression", "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."}; - const Setting compressionLevel{(StoreConfig*) this, -1, "compression-level", + const Setting compressionLevel{this, -1, "compression-level", R"( The *preset level* to be used when compressing NARs. The meaning and accepted values depend on the compression method selected. @@ -148,7 +148,7 @@ public: void narFromPath(const StorePath & path, Sink & sink) override; - ref getFSAccessor() override; + ref getFSAccessor(bool requireValidPath) override; void addSignatures(const StorePath & storePath, const StringSet & sigs) override; diff --git a/src/libstore/build-result.cc b/src/libstore/build-result.cc new file mode 100644 index 000000000000..18f519c5c61d --- /dev/null +++ b/src/libstore/build-result.cc @@ -0,0 +1,18 @@ +#include "build-result.hh" + +namespace nix { + +GENERATE_CMP_EXT( + , + BuildResult, + me->status, + me->errorMsg, + me->timesBuilt, + me->isNonDeterministic, + me->builtOutputs, + me->startTime, + me->stopTime, + me->cpuUser, + me->cpuSystem); + +} diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index b7a56e7917d7..8840fa7e3404 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -3,6 +3,7 @@ #include "realisation.hh" #include "derived-path.hh" +#include "comparator.hh" #include #include @@ -100,6 +101,8 @@ struct BuildResult */ std::optional cpuUser, cpuSystem; + DECLARE_CMP(BuildResult); + bool success() { return status == Built || status == Substituted || status == AlreadyValid || status == ResolvesToAlreadyValid; diff --git a/src/libstore/build/child.cc b/src/libstore/build/child.cc new file mode 100644 index 000000000000..aa31c3caf241 --- /dev/null +++ b/src/libstore/build/child.cc @@ -0,0 +1,37 @@ +#include "child.hh" +#include "current-process.hh" +#include "logging.hh" + +#include +#include + +namespace nix { + +void commonChildInit() +{ + logger = makeSimpleLogger(); + + const static std::string pathNullDevice = "/dev/null"; + restoreProcessContext(false); + + /* Put the child in a separate session (and thus a separate + process group) so that it has no controlling terminal (meaning + that e.g. ssh cannot open /dev/tty) and it doesn't receive + terminal signals. */ + if (setsid() == -1) + throw SysError("creating a new session"); + + /* Dup stderr to stdout. */ + if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) + throw SysError("cannot dup stderr into stdout"); + + /* Reroute stdin to /dev/null. */ + int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); + if (fdDevNull == -1) + throw SysError("cannot open '%1%'", pathNullDevice); + if (dup2(fdDevNull, STDIN_FILENO) == -1) + throw SysError("cannot dup null device into stdin"); + close(fdDevNull); +} + +} diff --git a/src/libstore/build/child.hh b/src/libstore/build/child.hh new file mode 100644 index 000000000000..3dfc552b93db --- /dev/null +++ b/src/libstore/build/child.hh @@ -0,0 +1,11 @@ +#pragma once +///@file + +namespace nix { + +/** + * Common initialisation performed in child processes. + */ +void commonChildInit(); + +} diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index cab890634a0c..ddf206cc8306 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -10,8 +10,8 @@ #include "util.hh" #include "archive.hh" #include "compression.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" #include "topo-sort.hh" #include "callback.hh" #include "local-store.hh" // TODO remove, along with remaining downcasts @@ -67,7 +67,7 @@ namespace nix { DerivationGoal::DerivationGoal(const StorePath & drvPath, const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs }) + : Goal(worker, DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = wantedOutputs }) , useDerivation(true) , drvPath(drvPath) , wantedOutputs(wantedOutputs) @@ -76,7 +76,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, state = &DerivationGoal::getDerivation; name = fmt( "building of '%s' from .drv file", - DerivedPath::Built { drvPath, wantedOutputs }.to_string(worker.store)); + DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); @@ -86,7 +86,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs }) + : Goal(worker, DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = wantedOutputs }) , useDerivation(false) , drvPath(drvPath) , wantedOutputs(wantedOutputs) @@ -97,7 +97,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation state = &DerivationGoal::haveDerivation; name = fmt( "building of '%s' from in-memory derivation", - DerivedPath::Built { drvPath, drv.outputNames() }.to_string(worker.store)); + DerivedPath::Built { makeConstantStorePathRef(drvPath), drv.outputNames() }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); @@ -370,20 +370,37 @@ void DerivationGoal::gaveUpOnSubstitution() /* The inputs must be built before we can build this goal. */ inputDrvOutputs.clear(); - if (useDerivation) - for (auto & i : dynamic_cast(drv.get())->inputDrvs) { + if (useDerivation) { + std::function, const DerivedPathMap::ChildNode &)> addWaiteeDerivedPath; + + addWaiteeDerivedPath = [&](ref inputDrv, const DerivedPathMap::ChildNode & inputNode) { + if (!inputNode.value.empty()) + addWaitee(worker.makeGoal( + DerivedPath::Built { + .drvPath = inputDrv, + .outputs = inputNode.value, + }, + buildMode == bmRepair ? bmRepair : bmNormal)); + for (const auto & [outputName, childNode] : inputNode.childMap) + addWaiteeDerivedPath( + make_ref(SingleDerivedPath::Built { inputDrv, outputName }), + childNode); + }; + + for (const auto & [inputDrvPath, inputNode] : dynamic_cast(drv.get())->inputDrvs.map) { /* Ensure that pure, non-fixed-output derivations don't depend on impure derivations. */ if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && drv->type().isPure() && !drv->type().isFixed()) { - auto inputDrv = worker.evalStore.readDerivation(i.first); + auto inputDrv = worker.evalStore.readDerivation(inputDrvPath); if (!inputDrv.type().isPure()) throw Error("pure derivation '%s' depends on impure derivation '%s'", worker.store.printStorePath(drvPath), - worker.store.printStorePath(i.first)); + worker.store.printStorePath(inputDrvPath)); } - addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal)); + addWaiteeDerivedPath(makeConstantStorePathRef(inputDrvPath), inputNode); } + } /* Copy the input sources from the eval store to the build store. */ @@ -454,7 +471,12 @@ void DerivationGoal::repairClosure() if (drvPath2 == outputsToDrv.end()) addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i, Repair))); else - addWaitee(worker.makeDerivationGoal(drvPath2->second, OutputsSpec::All(), bmRepair)); + addWaitee(worker.makeGoal( + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(drvPath2->second), + .outputs = OutputsSpec::All { }, + }, + bmRepair)); } if (waitees.empty()) { @@ -511,7 +533,7 @@ void DerivationGoal::inputsRealised() return ia.deferred; }, [&](const DerivationType::ContentAddressed & ca) { - return !fullDrv.inputDrvs.empty() && ( + return !fullDrv.inputDrvs.map.empty() && ( ca.fixed /* Can optionally resolve if fixed, which is good for avoiding unnecessary rebuilds. */ @@ -523,9 +545,9 @@ void DerivationGoal::inputsRealised() [&](const DerivationType::Impure &) { return true; } - }, drvType.raw()); + }, drvType.raw); - if (resolveDrv && !fullDrv.inputDrvs.empty()) { + if (resolveDrv && !fullDrv.inputDrvs.map.empty()) { experimentalFeatureSettings.require(Xp::CaDerivations); /* We are be able to resolve this derivation based on the @@ -541,7 +563,7 @@ void DerivationGoal::inputsRealised() attempt = fullDrv.tryResolve(worker.store); } assert(attempt); - Derivation drvResolved { *std::move(attempt) }; + Derivation drvResolved { std::move(*attempt) }; auto pathResolved = writeDerivation(worker.store, drvResolved); @@ -562,11 +584,13 @@ void DerivationGoal::inputsRealised() return; } - for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) { + std::function::ChildNode &)> accumInputPaths; + + accumInputPaths = [&](const StorePath & depDrvPath, const DerivedPathMap::ChildNode & inputNode) { /* Add the relevant output closures of the input derivation `i' as input paths. Only add the closures of output paths that are specified as inputs. */ - for (auto & j : wantedDepOutputs) { + auto getOutput = [&](const std::string & outputName) { /* TODO (impure derivations-induced tech debt): Tracking input derivation outputs statefully through the goals is error prone and has led to bugs. @@ -578,21 +602,30 @@ void DerivationGoal::inputsRealised() a representation in the store, which is a usability problem in itself. When implementing this logic entirely with lookups make sure that they're cached. */ - if (auto outPath = get(inputDrvOutputs, { depDrvPath, j })) { - worker.store.computeFSClosure(*outPath, inputPaths); + if (auto outPath = get(inputDrvOutputs, { depDrvPath, outputName })) { + return *outPath; } else { auto outMap = worker.evalStore.queryDerivationOutputMap(depDrvPath); - auto outMapPath = outMap.find(j); + auto outMapPath = outMap.find(outputName); if (outMapPath == outMap.end()) { throw Error( "derivation '%s' requires non-existent output '%s' from input derivation '%s'", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); + worker.store.printStorePath(drvPath), outputName, worker.store.printStorePath(depDrvPath)); } - worker.store.computeFSClosure(outMapPath->second, inputPaths); + return outMapPath->second; } - } - } + }; + + for (auto & outputName : inputNode.value) + worker.store.computeFSClosure(getOutput(outputName), inputPaths); + + for (auto & [outputName, childNode] : inputNode.childMap) + accumInputPaths(getOutput(outputName), childNode); + }; + + for (auto & [depDrvPath, depNode] : fullDrv.inputDrvs.map) + accumInputPaths(depDrvPath, depNode); } /* Second, the input sources. */ @@ -1005,10 +1038,11 @@ void DerivationGoal::buildDone() } else { + assert(derivationType); st = dynamic_cast(&e) ? BuildResult::NotDeterministic : statusOk(status) ? BuildResult::OutputRejected : - !derivationType.isSandboxed() || diskFull ? BuildResult::TransientFailure : + !derivationType->isSandboxed() || diskFull ? BuildResult::TransientFailure : BuildResult::PermanentFailure; } @@ -1160,11 +1194,11 @@ HookReply DerivationGoal::tryBuildHook() throw; } - WorkerProto::WriteConn conn { hook->sink }; + CommonProto::WriteConn conn { hook->sink }; /* Tell the hook all the inputs that have to be copied to the remote system. */ - WorkerProto::write(worker.store, conn, inputPaths); + CommonProto::write(worker.store, conn, inputPaths); /* Tell the hooks the missing outputs that have to be copied back from the remote system. */ @@ -1175,7 +1209,7 @@ HookReply DerivationGoal::tryBuildHook() if (buildMode != bmCheck && status.known && status.known->isValid()) continue; missingOutputs.insert(outputName); } - WorkerProto::write(worker.store, conn, missingOutputs); + CommonProto::write(worker.store, conn, missingOutputs); } hook->sink = FdSink(); @@ -1306,9 +1340,26 @@ void DerivationGoal::handleChildOutput(int fd, std::string_view data) auto s = handleJSONLogMessage(*json, worker.act, hook->activities, true); // ensure that logs from a builder using `ssh-ng://` as protocol // are also available to `nix log`. - if (s && !isWrittenToLog && logSink && (*json)["type"] == resBuildLogLine) { - auto f = (*json)["fields"]; - (*logSink)((f.size() > 0 ? f.at(0).get() : "") + "\n"); + if (s && !isWrittenToLog && logSink) { + const auto type = (*json)["type"]; + const auto fields = (*json)["fields"]; + if (type == resBuildLogLine) { + (*logSink)((fields.size() > 0 ? fields[0].get() : "") + "\n"); + } else if (type == resSetPhase && ! fields.is_null()) { + const auto phase = fields[0]; + if (! phase.is_null()) { + // nixpkgs' stdenv produces lines in the log to signal + // phase changes. + // We want to get the same lines in case of remote builds. + // The format is: + // @nix { "action": "setPhase", "phase": "$curPhase" } + const auto logLine = nlohmann::json::object({ + {"action", "setPhase"}, + {"phase", phase} + }); + (*logSink)("@nix " + logLine.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace) + "\n"); + } + } } } currentHookLine.clear(); @@ -1381,7 +1432,7 @@ std::pair DerivationGoal::checkPathValidity() [&](const OutputsSpec::Names & names) { return static_cast(names); }, - }, wantedOutputs.raw()); + }, wantedOutputs.raw); SingleDrvOutputs validOutputs; for (auto & i : queryPartialDerivationOutputMap()) { @@ -1473,6 +1524,7 @@ void DerivationGoal::done( SingleDrvOutputs builtOutputs, std::optional ex) { + outputLocks.unlock(); buildResult.status = status; if (ex) buildResult.errorMsg = fmt("%s", normaltxt(ex->info().msg)); @@ -1518,12 +1570,13 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) auto * dg = dynamic_cast(&*waitee); if (!dg) return; - auto outputs = fullDrv.inputDrvs.find(dg->drvPath); - if (outputs == fullDrv.inputDrvs.end()) return; + auto * nodeP = fullDrv.inputDrvs.findSlot(DerivedPath::Opaque { .path = dg->drvPath }); + if (!nodeP) return; + auto & outputs = nodeP->value; - for (auto & outputName : outputs->second) { + for (auto & outputName : outputs) { auto buildResult = dg->getBuildResult(DerivedPath::Built { - .drvPath = dg->drvPath, + .drvPath = makeConstantStorePathRef(dg->drvPath), .outputs = OutputsSpec::Names { outputName }, }); if (buildResult.success()) { diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index f396eef92bc1..a80c33db34d7 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -52,6 +52,9 @@ struct InitialOutput { std::optional known; }; +/** + * A goal for building some or all of the outputs of a derivation. + */ struct DerivationGoal : public Goal { /** @@ -68,8 +71,7 @@ struct DerivationGoal : public Goal std::shared_ptr resolvedDrvGoal; /** - * The specific outputs that we need to build. Empty means all of - * them. + * The specific outputs that we need to build. */ OutputsSpec wantedOutputs; @@ -186,7 +188,7 @@ struct DerivationGoal : public Goal /** * The sort of derivation we are building. */ - DerivationType derivationType; + std::optional derivationType; typedef void (DerivationGoal::*GoalState)(); GoalState state; @@ -336,7 +338,9 @@ struct DerivationGoal : public Goal StorePathSet exportReferences(const StorePathSet & storePaths); - JobCategory jobCategory() override { return JobCategory::Build; }; + JobCategory jobCategory() const override { + return JobCategory::Build; + }; }; MakeError(NotDeterministic, BuildError); diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index 5d1253a71efc..da2426e5e86c 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -73,7 +73,9 @@ public: void work() override; void handleEOF(int fd) override; - JobCategory jobCategory() override { return JobCategory::Substitution; }; + JobCategory jobCategory() const override { + return JobCategory::Substitution; + }; }; } diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 4aa4d6dcaf1e..74eca63f3166 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -15,7 +15,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod worker.run(goals); - StorePathSet failed; + StringSet failed; std::optional ex; for (auto & i : goals) { if (i->ex) { @@ -25,8 +25,10 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod ex = std::move(i->ex); } if (i->exitCode != Goal::ecSuccess) { - if (auto i2 = dynamic_cast(i.get())) failed.insert(i2->drvPath); - else if (auto i2 = dynamic_cast(i.get())) failed.insert(i2->storePath); + if (auto i2 = dynamic_cast(i.get())) + failed.insert(std::string { i2->drvPath.to_string() }); + else if (auto i2 = dynamic_cast(i.get())) + failed.insert(std::string { i2->storePath.to_string()}); } } @@ -35,7 +37,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod throw std::move(*ex); } else if (!failed.empty()) { if (ex) logError(ex->info()); - throw Error(worker.failingExitStatus(), "build of %s failed", showPaths(failed)); + throw Error(worker.failingExitStatus(), "build of %s failed", concatStringsSep(", ", quoteStrings(failed))); } } @@ -77,7 +79,7 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat try { worker.run(Goals{goal}); return goal->getBuildResult(DerivedPath::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All {}, }); } catch (Error & e) { @@ -124,8 +126,11 @@ void Store::repairPath(const StorePath & path) auto info = queryPathInfo(path); if (info->deriver && isValidPath(*info->deriver)) { goals.clear(); - // FIXME: Should just build the specific output we need. - goals.insert(worker.makeDerivationGoal(*info->deriver, OutputsSpec::All { }, bmRepair)); + goals.insert(worker.makeGoal(DerivedPath::Built { + .drvPath = makeConstantStorePathRef(*info->deriver), + // FIXME: Should just build the specific output we need. + .outputs = OutputsSpec::All { }, + }, bmRepair)); worker.run(goals); } else throw Error(worker.failingExitStatus(), "cannot repair path '%s'", printStorePath(path)); diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index ca7097a68df8..f8db98280766 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -11,7 +11,7 @@ bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { } -BuildResult Goal::getBuildResult(const DerivedPath & req) { +BuildResult Goal::getBuildResult(const DerivedPath & req) const { BuildResult res { buildResult }; if (auto pbp = std::get_if(&req)) { diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index a313bf22c29e..9af083230eed 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -41,7 +41,13 @@ typedef std::map WeakGoalMap; * of each category in parallel. */ enum struct JobCategory { + /** + * A build of a derivation; it will use CPU and disk resources. + */ Build, + /** + * A substitution an arbitrary store object; it will use network resources. + */ Substitution, }; @@ -110,7 +116,7 @@ public: * sake of both privacy and determinism, and this "safe accessor" * ensures we don't. */ - BuildResult getBuildResult(const DerivedPath &); + BuildResult getBuildResult(const DerivedPath &) const; /** * Exception containing an error message, if any. @@ -144,7 +150,7 @@ public: void trace(std::string_view s); - std::string getName() + std::string getName() const { return name; } @@ -162,7 +168,11 @@ public: virtual void cleanup() { } - virtual JobCategory jobCategory() = 0; + /** + * @brief Hint for the scheduler, which concurrency limit applies. + * @see JobCategory + */ + virtual JobCategory jobCategory() const = 0; }; void addToWeakGoals(WeakGoals & goals, GoalPtr p); diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/build/hook-instance.cc index 337c60bd460f..5d045ec3d07a 100644 --- a/src/libstore/build/hook-instance.cc +++ b/src/libstore/build/hook-instance.cc @@ -1,5 +1,7 @@ #include "globals.hh" #include "hook-instance.hh" +#include "file-system.hh" +#include "child.hh" namespace nix { diff --git a/src/libstore/build/hook-instance.hh b/src/libstore/build/hook-instance.hh index d84f62877756..61cf534f4e9c 100644 --- a/src/libstore/build/hook-instance.hh +++ b/src/libstore/build/hook-instance.hh @@ -3,6 +3,7 @@ #include "logging.hh" #include "serialise.hh" +#include "processes.hh" namespace nix { diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index a9aa1db0592d..74357c0210e1 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -3,6 +3,7 @@ #include "config.hh" #include "gc-store.hh" #include "granular-access-store.hh" +#include "indirect-root-store.hh" #include "hook-instance.hh" #include "local-fs-store.hh" #include "worker.hh" @@ -19,7 +20,11 @@ #include "json-utils.hh" #include "cgroup.hh" #include "personality.hh" +#include "current-process.hh" #include "namespaces.hh" +#include "child.hh" +#include "unix-domain-socket.hh" +#include "posix-fs-canonicalise.hh" #include #include @@ -184,6 +189,8 @@ void LocalDerivationGoal::tryLocalBuild() return; } + assert(derivationType); + /* Are we doing a chroot build? */ { auto noChroot = parsedDrv->getBoolAttr("__noChroot"); @@ -201,7 +208,7 @@ void LocalDerivationGoal::tryLocalBuild() else if (settings.sandboxMode == smDisabled) useChroot = false; else if (settings.sandboxMode == smRelaxed) - useChroot = derivationType.isSandboxed() && !noChroot; + useChroot = derivationType->isSandboxed() && !noChroot; } auto & localStore = getLocalStore(); @@ -231,7 +238,7 @@ void LocalDerivationGoal::tryLocalBuild() if (!buildUser) { if (!actLock) actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); + fmt("waiting for a free build user ID for '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); worker.waitForAWhile(shared_from_this()); return; } @@ -419,26 +426,54 @@ void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() cleanupPostOutputsRegisteredModeCheck(); } +static void doBind(const Path & source, const Path & target, Store & store, bool optional = false) { + auto doMount = [&](const Path & source, const Path & target) { + debug("bind mounting '%1%' to '%2%'", source, target); + struct stat st; + if (stat(source.c_str(), &st) == -1) { + if (optional && errno == ENOENT) + return; + else + throw SysError("getting attributes of path '%1%'", source); + } -#if __linux__ -static void linkOrCopy(const Path & from, const Path & to) -{ - if (link(from.c_str(), to.c_str()) == -1) { - /* Hard-linking fails if we exceed the maximum link count on a - file (e.g. 32000 of ext3), which is quite possible after a - 'nix-store --optimise'. FIXME: actually, why don't we just - bind-mount in this case? - - It can also fail with EPERM in BeegFS v7 and earlier versions - or fail with EXDEV in OpenAFS - which don't allow hard-links to other directories */ - if (errno != EMLINK && errno != EPERM && errno != EXDEV) - throw SysError("linking '%s' to '%s'", to, from); - copyPath(from, to); - } -} -#endif + if (S_ISDIR(st.st_mode)) + createDirs(target); + else { + createDirs(dirOf(target)); + writeFile(target, ""); + } + + if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) + throw SysError("bind mount from '%1%' to '%2%' failed", source, target); + }; + + + if (experimentalFeatureSettings.isEnabled(Xp::ACLs) && store.isInStore(source)) { + auto [storePath, subPath] = store.toStorePath(source); + // TODO(ACL) Add tests to check that ACL information is never leaked + // FIXME probably should use a FUSE fs or something? + ssize_t eaSize = llistxattr(source.c_str(), nullptr, 0); + if (subPath == "" && eaSize > 0) { + // The source store path contains extended attributes + // mounting it as-is would preserve them, which is undesireable. + if (std::filesystem::is_directory(source)) { + createDirs(target); // In case the directory is empty + for (auto dirent : std::filesystem::directory_iterator(std::filesystem::directory_entry(source))) + doMount(dirent.path().c_str(), (target + "/" + baseNameOf(dirent.path().c_str())).c_str()); + } + else { + std::filesystem::copy(source, target); + } + using namespace std::filesystem; + auto p = status(target).permissions(); + permissions(target, (p | ((p & perms::owner_read) != perms::none ? perms::others_read : perms::none) | ((p & perms::owner_exec) != perms::none ? perms::others_exec : perms::none)), perm_options::add); + return; + } + } + doMount(source, target); +}; void LocalDerivationGoal::startBuilder() { @@ -614,7 +649,7 @@ void LocalDerivationGoal::startBuilder() /* Allow a user-configurable set of directories from the host file system. */ - dirsInChroot.clear(); + pathsInChroot.clear(); for (auto i : settings.sandboxPaths.get()) { if (i.empty()) continue; @@ -625,15 +660,19 @@ void LocalDerivationGoal::startBuilder() } size_t p = i.find('='); if (p == std::string::npos) - dirsInChroot[i] = {i, optional}; + pathsInChroot[i] = {i, optional}; else - dirsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional}; + pathsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional}; + } + if (hasPrefix(worker.store.storeDir, tmpDirInSandbox)) + { + throw Error("`sandbox-build-dir` must not contain the storeDir"); } - dirsInChroot[tmpDirInSandbox] = tmpDir; + pathsInChroot[tmpDirInSandbox] = tmpDir; /* Add the closure of store paths to the chroot. */ StorePathSet closure; - for (auto & i : dirsInChroot) + for (auto & i : pathsInChroot) try { if (worker.store.isInStore(i.second.source)) worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure); @@ -644,7 +683,7 @@ void LocalDerivationGoal::startBuilder() } for (auto & i : closure) { auto p = worker.store.printStorePath(i); - dirsInChroot.insert_or_assign(p, p); + pathsInChroot.insert_or_assign(p, p); } PathSet allowedPaths = settings.allowedImpureHostPrefixes; @@ -672,14 +711,14 @@ void LocalDerivationGoal::startBuilder() /* Allow files in __impureHostDeps to be missing; e.g. macOS 11+ has no /usr/lib/libSystem*.dylib */ - dirsInChroot[i] = {i, true}; + pathsInChroot[i] = {i, true}; } #if __linux__ /* Create a temporary directory in which we set up the chroot environment using bind-mounts. We put it in the Nix store - to ensure that we can create hard-links to non-directory - inputs in the fake Nix store in the chroot (see below). */ + so that the build outputs can be moved efficiently from the + chroot to their final location. */ chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot"; deletePath(chrootRootDir); @@ -720,7 +759,7 @@ void LocalDerivationGoal::startBuilder() "nogroup:x:65534:\n", sandboxGid())); /* Create /etc/hosts with localhost entry. */ - if (derivationType.isSandboxed()) + if (derivationType->isSandboxed()) writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); /* Make the closure of the inputs available in the chroot, @@ -741,12 +780,12 @@ void LocalDerivationGoal::startBuilder() for (auto & i : inputPaths) { auto p = worker.store.printStorePath(i); Path r = worker.store.toRealPath(p); - dirsInChroot.insert_or_assign(p, r); + pathsInChroot.insert_or_assign(p, r); } /* If we're repairing, checking or rebuilding part of a multiple-outputs derivation, it's possible that we're - rebuilding a path that is in settings.dirsInChroot + rebuilding a path that is in settings.sandbox-paths (typically the dependencies of /bin/sh). Throw them out. */ for (auto & i : drv->outputsAndOptPaths(worker.store)) { @@ -756,7 +795,7 @@ void LocalDerivationGoal::startBuilder() is already in the sandbox, so we don't need to worry about removing it. */ if (i.second.second) - dirsInChroot.erase(worker.store.printStorePath(*i.second.second)); + pathsInChroot.erase(worker.store.printStorePath(*i.second.second)); } if (cgroup) { @@ -814,9 +853,9 @@ void LocalDerivationGoal::startBuilder() } else { auto p = line.find('='); if (p == std::string::npos) - dirsInChroot[line] = line; + pathsInChroot[line] = line; else - dirsInChroot[line.substr(0, p)] = line.substr(p + 1); + pathsInChroot[line.substr(0, p)] = line.substr(p + 1); } } } @@ -923,7 +962,7 @@ void LocalDerivationGoal::startBuilder() us. */ - if (derivationType.isSandboxed()) + if (derivationType->isSandboxed()) privateNetwork = true; userNamespaceSync.create(); @@ -942,15 +981,13 @@ void LocalDerivationGoal::startBuilder() openSlave(); /* Drop additional groups here because we can't do it - after we've created the new user namespace. FIXME: - this means that if we're not root in the parent - namespace, we can't drop additional groups; they will - be mapped to nogroup in the child namespace. There does - not seem to be a workaround for this. (But who can tell - from reading user_namespaces(7)?) - See also https://lwn.net/Articles/621612/. */ - if (getuid() == 0 && setgroups(0, 0) == -1) - throw SysError("setgroups failed"); + after we've created the new user namespace. */ + if (setgroups(0, 0) == -1) { + if (errno != EPERM) + throw SysError("setgroups failed"); + if (settings.requireDropSupplementaryGroups) + throw Error("setgroups failed. Set the require-drop-supplementary-groups option to false to skip this step."); + } ProcessOptions options; options.cloneFlags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; @@ -1096,7 +1133,7 @@ void LocalDerivationGoal::initTmpDir() { env[i.first] = i.second; } else { auto hash = hashString(htSHA256, i.first); - std::string fn = ".attr-" + hash.to_string(Base32, false); + std::string fn = ".attr-" + hash.to_string(HashFormat::Base32, false); Path p = tmpDir + "/" + fn; writeFile(p, rewriteStrings(i.second, inputRewrites)); chownToBuilder(p); @@ -1153,7 +1190,7 @@ void LocalDerivationGoal::initEnv() derivation, tell the builder, so that for instance `fetchurl' can skip checking the output. On older Nixes, this environment variable won't be set, so `fetchurl' will do the check. */ - if (derivationType.isFixed()) env["NIX_OUTPUT_CHECKED"] = "1"; + if (derivationType->isFixed()) env["NIX_OUTPUT_CHECKED"] = "1"; /* *Only* if this is a fixed-output derivation, propagate the values of the environment variables specified in the @@ -1164,9 +1201,19 @@ void LocalDerivationGoal::initEnv() to the builder is generally impure, but the output of fixed-output derivations is by definition pure (since we already know the cryptographic hash of the output). */ - if (!derivationType.isSandboxed()) { - for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) - env[i] = getEnv(i).value_or(""); + if (!derivationType->isSandboxed()) { + auto & impureEnv = settings.impureEnv.get(); + if (!impureEnv.empty()) + experimentalFeatureSettings.require(Xp::ConfigurableImpureEnv); + + for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) { + auto envVar = impureEnv.find(i); + if (envVar != impureEnv.end()) { + env[i] = envVar->second; + } else { + env[i] = getEnv(i).value_or(""); + } + } } /* Currently structured log messages piggyback on stderr, but we @@ -1204,6 +1251,19 @@ void LocalDerivationGoal::writeStructuredAttrs() } +static StorePath pathPartOfReq(const SingleDerivedPath & req) +{ + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) { + return bo.path; + }, + [&](const SingleDerivedPath::Built & bfd) { + return pathPartOfReq(*bfd.drvPath); + }, + }, req.raw()); +} + + static StorePath pathPartOfReq(const DerivedPath & req) { return std::visit(overloaded { @@ -1211,7 +1271,7 @@ static StorePath pathPartOfReq(const DerivedPath & req) return bo.path; }, [&](const DerivedPath::Built & bfd) { - return bfd.drvPath; + return pathPartOfReq(*bfd.drvPath); }, }, req.raw()); } @@ -1232,7 +1292,7 @@ struct RestrictedStoreConfig : virtual LocalFSStoreConfig /* A wrapper around LocalStore that only allows building/querying of paths that are in the input closures of the build or were added via recursive Nix calls. */ -struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore, public virtual GcStore +struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual IndirectRootStore, public virtual GcStore { ref next; @@ -1283,11 +1343,13 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo void queryReferrers(const StorePath & path, StorePathSet & referrers) override { } - std::map> queryPartialDerivationOutputMap(const StorePath & path) override + std::map> queryPartialDerivationOutputMap( + const StorePath & path, + Store * evalStore = nullptr) override { if (!goal.isAllowed(path)) throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path)); - return next->queryPartialDerivationOutputMap(path); + return next->queryPartialDerivationOutputMap(path, evalStore); } std::optional queryPathFromHashPart(const std::string & hashPart) override @@ -1576,41 +1638,33 @@ void LocalDerivationGoal::addDependency(const StorePath & path) Path source = worker.store.Store::toRealPath(path); Path target = chrootRootDir + worker.store.printStorePath(path); - debug("bind-mounting %s -> %s", target, source); - if (pathExists(target)) + if (pathExists(target)) { + // There is a similar debug message in doBind, so only run it in this block to not have double messages. + debug("bind-mounting %s -> %s", target, source); throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); + } - auto st = lstat(source); - - if (S_ISDIR(st.st_mode)) { - - /* Bind-mount the path into the sandbox. This requires - entering its mount namespace, which is not possible - in multithreaded programs. So we do this in a - child process.*/ - Pid child(startProcess([&]() { - - if (usingUserNamespace && (setns(sandboxUserNamespace.get(), 0) == -1)) - throw SysError("entering sandbox user namespace"); + /* Bind-mount the path into the sandbox. This requires + entering its mount namespace, which is not possible + in multithreaded programs. So we do this in a + child process.*/ + Pid child(startProcess([&]() { - if (setns(sandboxMountNamespace.get(), 0) == -1) - throw SysError("entering sandbox mount namespace"); + if (usingUserNamespace && (setns(sandboxUserNamespace.get(), 0) == -1)) + throw SysError("entering sandbox user namespace"); - createDirs(target); + if (setns(sandboxMountNamespace.get(), 0) == -1) + throw SysError("entering sandbox mount namespace"); - if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1) - throw SysError("bind mount from '%s' to '%s' failed", source, target); + doBind(source, target, worker.store); - _exit(0); - })); + _exit(0); + })); - int status = child.wait(); - if (status != 0) - throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); - - } else - linkOrCopy(source, target); + int status = child.wait(); + if (status != 0) + throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); #else throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", @@ -1641,6 +1695,8 @@ void setupSeccomp() seccomp_release(ctx); }); + constexpr std::string_view nativeSystem = SYSTEM; + if (nativeSystem == "x86_64-linux" && seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) throw SysError("unable to add 32-bit seccomp architecture"); @@ -1799,7 +1855,7 @@ void LocalDerivationGoal::runChild() /* Set up a nearly empty /dev, unless the user asked to bind-mount the host /dev. */ Strings ss; - if (dirsInChroot.find("/dev") == dirsInChroot.end()) { + if (pathsInChroot.find("/dev") == pathsInChroot.end()) { createDirs(chrootRootDir + "/dev/shm"); createDirs(chrootRootDir + "/dev/pts"); ss.push_back("/dev/full"); @@ -1819,7 +1875,7 @@ void LocalDerivationGoal::runChild() /* Fixed-output derivations typically need to access the network, so give them access to /etc/resolv.conf and so on. */ - if (!derivationType.isSandboxed()) { + if (!derivationType->isSandboxed()) { // Only use nss functions to resolve hosts and // services. Don’t use it for anything else that may // be configured for this system. This limits the @@ -1834,64 +1890,15 @@ void LocalDerivationGoal::runChild() ss.push_back(path); if (settings.caFile != "") - dirsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", settings.caFile, true); + pathsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", settings.caFile, true); } - for (auto & i : ss) dirsInChroot.emplace(i, i); + for (auto & i : ss) pathsInChroot.emplace(i, i); /* Bind-mount all the directories from the "host" filesystem that we want in the chroot environment. */ - auto doBind = [&](const Path & source, const Path & target, bool optional = false) { - auto doMount = [&](const Path & source, const Path & target) { - debug("bind mounting '%1%' to '%2%'", source, target); - struct stat st; - if (stat(source.c_str(), &st) == -1) { - if (optional && errno == ENOENT) - return; - else - throw SysError("getting attributes of path '%1%'", source); - } - - if (S_ISDIR(st.st_mode)) - createDirs(target); - else { - createDirs(dirOf(target)); - writeFile(target, ""); - } - - if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError("bind mount from '%1%' to '%2%' failed", source, target); - }; - - - if (experimentalFeatureSettings.isEnabled(Xp::ACLs) && worker.store.isInStore(source)) { - auto [storePath, subPath] = worker.store.toStorePath(source); - - // TODO(ACL) Add tests to check that ACL information is never leaked - // FIXME probably should use a FUSE fs or something? - ssize_t eaSize = llistxattr(source.c_str(), nullptr, 0); - if (subPath == "" && eaSize > 0) { - // The source store path contains extended attributes - // mounting it as-is would preserve them, which is undesireable. - if (std::filesystem::is_directory(source)) { - createDirs(target); // In case the directory is empty - for (auto dirent : std::filesystem::directory_iterator(std::filesystem::directory_entry(source))) - doMount(dirent.path().c_str(), (target + "/" + baseNameOf(dirent.path().c_str())).c_str()); - } - else { - std::filesystem::copy(source, target); - } - using namespace std::filesystem; - auto p = status(target).permissions(); - permissions(target, (p | ((p & perms::owner_read) != perms::none ? perms::others_read : perms::none) | ((p & perms::owner_exec) != perms::none ? perms::others_exec : perms::none)), perm_options::add); - return; - } - } - doMount(source, target); - }; - - for (auto & i : dirsInChroot) { + for (auto & i : pathsInChroot) { if (i.second.source == "/proc") continue; // backwards compatibility #if HAVE_EMBEDDED_SANDBOX_SHELL @@ -1905,7 +1912,7 @@ void LocalDerivationGoal::runChild() chmod_(dst, 0555); } else #endif - doBind(i.second.source, chrootRootDir + i.first, i.second.optional); + doBind(i.second.source, chrootRootDir + i.first, worker.store, i.second.optional); } /* Bind a new instance of procfs on /proc. */ @@ -1932,7 +1939,7 @@ void LocalDerivationGoal::runChild() if /dev/ptx/ptmx exists). */ if (pathExists("/dev/pts/ptmx") && !pathExists(chrootRootDir + "/dev/ptmx") - && !dirsInChroot.count("/dev/pts")) + && !pathsInChroot.count("/dev/pts")) { if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) { @@ -1944,8 +1951,8 @@ void LocalDerivationGoal::runChild() } else { if (errno != EINVAL) throw SysError("mounting /dev/pts"); - doBind("/dev/pts", chrootRootDir + "/dev/pts"); - doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx"); + doBind("/dev/pts", chrootRootDir + "/dev/pts", worker.store); + doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx", worker.store); } } @@ -2067,7 +2074,7 @@ void LocalDerivationGoal::runChild() /* We build the ancestry before adding all inputPaths to the store because we know they'll all have the same parents (the store), and there might be lots of inputs. This isn't particularly efficient... I doubt it'll be a bottleneck in practice */ - for (auto & i : dirsInChroot) { + for (auto & i : pathsInChroot) { Path cur = i.first; while (cur.compare("/") != 0) { cur = dirOf(cur); @@ -2075,7 +2082,7 @@ void LocalDerivationGoal::runChild() } } - /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost + /* And we want the store in there regardless of how empty pathsInChroot. We include the innermost path component this time, since it's typically /nix/store and we care about that. */ Path cur = worker.store.storeDir; while (cur.compare("/") != 0) { @@ -2086,8 +2093,7 @@ void LocalDerivationGoal::runChild() /* Add all our input paths to the chroot */ for (auto & i : inputPaths) { auto p = worker.store.printStorePath(i); - - dirsInChroot[p] = p; + pathsInChroot[p] = p; } /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ @@ -2101,7 +2107,7 @@ void LocalDerivationGoal::runChild() #include "sandbox-defaults.sb" ; - if (!derivationType.isSandboxed()) + if (!derivationType->isSandboxed()) sandboxProfile += #include "sandbox-network.sb" ; @@ -2118,7 +2124,7 @@ void LocalDerivationGoal::runChild() without file-write* allowed, access() incorrectly returns EPERM */ sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & i : dirsInChroot) { + for (auto & i : pathsInChroot) { if (i.first != i.second.source) throw Error( "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", @@ -2373,7 +2379,6 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() bool discardReferences = false; if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { if (auto udr = get(*structuredAttrs, "unsafeDiscardReferences")) { - experimentalFeatureSettings.require(Xp::DiscardReferences); if (auto output = get(*udr, outputName)) { if (!output->is_boolean()) throw Error("attribute 'unsafeDiscardReferences.\"%s\"' of derivation '%s' must be a Boolean", outputName, drvPath.to_string()); @@ -2569,7 +2574,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() ValidPathInfo newInfo0 { worker.store, outputPathName(drv->name, outputName), - *std::move(optCA), + std::move(*optCA), Hash::dummy, }; if (*scratchPath != newInfo0.path) { @@ -2614,16 +2619,16 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() }, [&](const DerivationOutput::CAFixed & dof) { - auto wanted = dof.ca.getHash(); + auto & wanted = dof.ca.hash; auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating { - .method = dof.ca.getMethod(), + .method = dof.ca.method, .hashType = wanted.type, }); /* Check wanted hash */ assert(newInfo0.ca); - auto got = newInfo0.ca->getHash(); + auto & got = newInfo0.ca->hash; if (wanted != got) { /* Throw an error after registering the path as valid. */ @@ -2631,8 +2636,8 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() delayedException = std::make_exception_ptr( BuildError("hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s", worker.store.printStorePath(drvPath), - wanted.to_string(SRI, true), - got.to_string(SRI, true))); + wanted.to_string(HashFormat::SRI, true), + got.to_string(HashFormat::SRI, true))); } if (!newInfo0.references.empty()) delayedException = std::make_exception_ptr( @@ -2659,7 +2664,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() }); }, - }, output->raw()); + }, output->raw); /* FIXME: set proper permissions in restorePath() so we don't have to do another traversal. */ @@ -3022,7 +3027,7 @@ bool LocalDerivationGoal::isReadDesc(int fd) } -StorePath LocalDerivationGoal::makeFallbackPath(std::string_view outputName) +StorePath LocalDerivationGoal::makeFallbackPath(OutputNameView outputName) { return worker.store.makeStorePath( "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName), diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 9acd7593d6c3..88152a645cf3 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -3,6 +3,7 @@ #include "derivation-goal.hh" #include "local-store.hh" +#include "processes.hh" namespace nix { @@ -86,8 +87,8 @@ struct LocalDerivationGoal : public DerivationGoal : source(source), optional(optional) { } }; - typedef map DirsInChroot; // maps target path to source path - DirsInChroot dirsInChroot; + typedef map PathsInChroot; // maps target path to source path + PathsInChroot pathsInChroot; typedef map Environment; Environment env; @@ -120,14 +121,6 @@ struct LocalDerivationGoal : public DerivationGoal */ OutputPathMap scratchOutputs; - /** - * Path registration info from the previous round, if we're - * building multiple times. Since this contains the hash, it - * allows us to compare whether two rounds produced the same - * result. - */ - std::map prevInfos; - uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); } gid_t sandboxGid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 100 : 0) : buildUser->getGID(); } @@ -272,8 +265,10 @@ struct LocalDerivationGoal : public DerivationGoal /** * Forcibly kill the child process, if any. + * + * Called by destructor, can't be overridden */ - void killChild() override; + void killChild() override final; /** * Kill any processes running under the build user UID or in the @@ -295,7 +290,7 @@ struct LocalDerivationGoal : public DerivationGoal * @todo Add option to randomize, so we can audit whether our * rewrites caught everything */ - StorePath makeFallbackPath(std::string_view outputName); + StorePath makeFallbackPath(OutputNameView outputName); }; } diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 9fc041920280..1d389d328ffc 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -114,9 +114,12 @@ public: void handleChildOutput(int fd, std::string_view data) override; void handleEOF(int fd) override; - void cleanup() override; + /* Called by destructor, can't be overridden */ + void cleanup() override final; - JobCategory jobCategory() override { return JobCategory::Substitution; }; + JobCategory jobCategory() const override { + return JobCategory::Substitution; + }; }; } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index a9ca9cbbc8c0..01f52e7aba7a 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -4,6 +4,7 @@ #include "drv-output-substitution-goal.hh" #include "local-derivation-goal.hh" #include "hook-instance.hh" +#include "signals.hh" #include @@ -111,7 +112,10 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) { return std::visit(overloaded { [&](const DerivedPath::Built & bfd) -> GoalPtr { - return makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode); + if (auto bop = std::get_if(&*bfd.drvPath)) + return makeDerivationGoal(bop->path, bfd.outputs, buildMode); + else + throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented."); }, [&](const DerivedPath::Opaque & bo) -> GoalPtr { return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair); @@ -195,8 +199,16 @@ void Worker::childStarted(GoalPtr goal, const std::set & fds, child.respectTimeouts = respectTimeouts; children.emplace_back(child); if (inBuildSlot) { - if (goal->jobCategory() == JobCategory::Substitution) nrSubstitutions++; - else nrLocalBuilds++; + switch (goal->jobCategory()) { + case JobCategory::Substitution: + nrSubstitutions++; + break; + case JobCategory::Build: + nrLocalBuilds++; + break; + default: + abort(); + } } } @@ -208,12 +220,17 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers) if (i == children.end()) return; if (i->inBuildSlot) { - if (goal->jobCategory() == JobCategory::Substitution) { + switch (goal->jobCategory()) { + case JobCategory::Substitution: assert(nrSubstitutions > 0); nrSubstitutions--; - } else { + break; + case JobCategory::Build: assert(nrLocalBuilds > 0); nrLocalBuilds--; + break; + default: + abort(); } } @@ -265,7 +282,10 @@ void Worker::run(const Goals & _topGoals) for (auto & i : _topGoals) { topGoals.insert(i); if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back(DerivedPath::Built{goal->drvPath, goal->wantedOutputs}); + topPaths.push_back(DerivedPath::Built { + .drvPath = makeConstantStorePathRef(goal->drvPath), + .outputs = goal->wantedOutputs, + }); } else if (auto goal = dynamic_cast(i.get())) { topPaths.push_back(DerivedPath::Opaque{goal->storePath}); } @@ -516,10 +536,13 @@ void Worker::markContentsGood(const StorePath & path) } -GoalPtr upcast_goal(std::shared_ptr subGoal) { +GoalPtr upcast_goal(std::shared_ptr subGoal) +{ return subGoal; } -GoalPtr upcast_goal(std::shared_ptr subGoal) { + +GoalPtr upcast_goal(std::shared_ptr subGoal) +{ return subGoal; } diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 5abceca0d71a..23ad879148db 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -34,7 +34,6 @@ GoalPtr upcast_goal(std::shared_ptr subGoal); typedef std::chrono::time_point steady_time_point; - /** * A mapping used to remember for each child process to what goal it * belongs, and file descriptors for receiving log data and output diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 7bba33fb9a60..c8911d153fe2 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -174,15 +174,19 @@ void builtinBuildenv(const BasicDerivation & drv) /* Convert the stuff we get from the environment back into a * coherent data type. */ Packages pkgs; - auto derivations = tokenizeString(getAttr("derivations")); - while (!derivations.empty()) { - /* !!! We're trusting the caller to structure derivations env var correctly */ - auto active = derivations.front(); derivations.pop_front(); - auto priority = stoi(derivations.front()); derivations.pop_front(); - auto outputs = stoi(derivations.front()); derivations.pop_front(); - for (auto n = 0; n < outputs; n++) { - auto path = derivations.front(); derivations.pop_front(); - pkgs.emplace_back(path, active != "false", priority); + { + auto derivations = tokenizeString(getAttr("derivations")); + + auto itemIt = derivations.begin(); + while (itemIt != derivations.end()) { + /* !!! We're trusting the caller to structure derivations env var correctly */ + const bool active = "false" != *itemIt++; + const int priority = stoi(*itemIt++); + const size_t outputs = stoul(*itemIt++); + + for (size_t n {0}; n < outputs; n++) { + pkgs.emplace_back(std::move(*itemIt++), active, priority); + } } } diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 7d7924d77339..357800333709 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -65,7 +65,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; std::optional ht = parseHashTypeOpt(getAttr("outputHashAlgo")); Hash h = newHashAllowEmpty(getAttr("outputHash"), ht); - fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false)); + fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(HashFormat::Base16, false)); return; } catch (Error & e) { debug(e.what()); diff --git a/src/libstore/common-protocol-impl.hh b/src/libstore/common-protocol-impl.hh new file mode 100644 index 000000000000..079c182b86a1 --- /dev/null +++ b/src/libstore/common-protocol-impl.hh @@ -0,0 +1,41 @@ +#pragma once +/** + * @file + * + * Template implementations (as opposed to mere declarations). + * + * This file is an exmample of the "impl.hh" pattern. See the + * contributing guide. + */ + +#include "common-protocol.hh" +#include "length-prefixed-protocol-helper.hh" + +namespace nix { + +/* protocol-agnostic templates */ + +#define COMMON_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \ + TEMPLATE T CommonProto::Serialise< T >::read(const Store & store, CommonProto::ReadConn conn) \ + { \ + return LengthPrefixedProtoHelper::read(store, conn); \ + } \ + TEMPLATE void CommonProto::Serialise< T >::write(const Store & store, CommonProto::WriteConn conn, const T & t) \ + { \ + LengthPrefixedProtoHelper::write(store, conn, t); \ + } + +COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) +COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::set) +COMMON_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) + +#define COMMA_ , +COMMON_USE_LENGTH_PREFIX_SERIALISER( + template, + std::map) +#undef COMMA_ + + +/* protocol-specific templates */ + +} diff --git a/src/libstore/common-protocol.cc b/src/libstore/common-protocol.cc new file mode 100644 index 000000000000..68445258ff17 --- /dev/null +++ b/src/libstore/common-protocol.cc @@ -0,0 +1,97 @@ +#include "serialise.hh" +#include "path-with-outputs.hh" +#include "store-api.hh" +#include "build-result.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" +#include "archive.hh" +#include "derivations.hh" + +#include + +namespace nix { + +/* protocol-agnostic definitions */ + +std::string CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +{ + return readString(conn.from); +} + +void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const std::string & str) +{ + conn.to << str; +} + + +StorePath CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +{ + return store.parseStorePath(readString(conn.from)); +} + +void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const StorePath & storePath) +{ + conn.to << store.printStorePath(storePath); +} + + +ContentAddress CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +{ + return ContentAddress::parse(readString(conn.from)); +} + +void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const ContentAddress & ca) +{ + conn.to << renderContentAddress(ca); +} + + +Realisation CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +{ + std::string rawInput = readString(conn.from); + return Realisation::fromJSON( + nlohmann::json::parse(rawInput), + "remote-protocol" + ); +} + +void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const Realisation & realisation) +{ + conn.to << realisation.toJSON().dump(); +} + + +DrvOutput CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +{ + return DrvOutput::parse(readString(conn.from)); +} + +void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput) +{ + conn.to << drvOutput.to_string(); +} + + +std::optional CommonProto::Serialise>::read(const Store & store, CommonProto::ReadConn conn) +{ + auto s = readString(conn.from); + return s == "" ? std::optional {} : store.parseStorePath(s); +} + +void CommonProto::Serialise>::write(const Store & store, CommonProto::WriteConn conn, const std::optional & storePathOpt) +{ + conn.to << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); +} + + +std::optional CommonProto::Serialise>::read(const Store & store, CommonProto::ReadConn conn) +{ + return ContentAddress::parseOpt(readString(conn.from)); +} + +void CommonProto::Serialise>::write(const Store & store, CommonProto::WriteConn conn, const std::optional & caOpt) +{ + conn.to << (caOpt ? renderContentAddress(*caOpt) : ""); +} + +} diff --git a/src/libstore/common-protocol.hh b/src/libstore/common-protocol.hh new file mode 100644 index 000000000000..f3f28972afd4 --- /dev/null +++ b/src/libstore/common-protocol.hh @@ -0,0 +1,106 @@ +#pragma once +///@file + +#include "serialise.hh" + +namespace nix { + +class Store; +struct Source; + +// items being serialized +class StorePath; +struct ContentAddress; +struct DrvOutput; +struct Realisation; + + +/** + * Shared serializers between the worker protocol, serve protocol, and a + * few others. + * + * This `struct` is basically just a `namespace`; We use a type rather + * than a namespace just so we can use it as a template argument. + */ +struct CommonProto +{ + /** + * A unidirectional read connection, to be used by the read half of the + * canonical serializers below. + */ + struct ReadConn { + Source & from; + }; + + /** + * A unidirectional write connection, to be used by the write half of the + * canonical serializers below. + */ + struct WriteConn { + Sink & to; + }; + + template + struct Serialise; + + /** + * Wrapper function around `CommonProto::Serialise::write` that allows us to + * infer the type instead of having to write it down explicitly. + */ + template + static void write(const Store & store, WriteConn conn, const T & t) + { + CommonProto::Serialise::write(store, conn, t); + } +}; + +#define DECLARE_COMMON_SERIALISER(T) \ + struct CommonProto::Serialise< T > \ + { \ + static T read(const Store & store, CommonProto::ReadConn conn); \ + static void write(const Store & store, CommonProto::WriteConn conn, const T & str); \ + } + +template<> +DECLARE_COMMON_SERIALISER(std::string); +template<> +DECLARE_COMMON_SERIALISER(StorePath); +template<> +DECLARE_COMMON_SERIALISER(ContentAddress); +template<> +DECLARE_COMMON_SERIALISER(DrvOutput); +template<> +DECLARE_COMMON_SERIALISER(Realisation); + +template +DECLARE_COMMON_SERIALISER(std::vector); +template +DECLARE_COMMON_SERIALISER(std::set); +template +DECLARE_COMMON_SERIALISER(std::tuple); + +#define COMMA_ , +template +DECLARE_COMMON_SERIALISER(std::map); +#undef COMMA_ + +/** + * These use the empty string for the null case, relying on the fact + * that the underlying types never serialize to the empty string. + * + * We do this instead of a generic std::optional instance because + * ordinal tags (0 or 1, here) are a bit of a compatability hazard. For + * the same reason, we don't have a std::variant instances (ordinal + * tags 0...n). + * + * We could the generic instances and then these as specializations for + * compatability, but that's proven a bit finnicky, and also makes the + * worker protocol harder to implement in other languages where such + * specializations may not be allowed. + */ +template<> +DECLARE_COMMON_SERIALISER(std::optional); +template<> +DECLARE_COMMON_SERIALISER(std::optional); + +} diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 04f7ac214f44..a5f7cdf8124f 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -4,11 +4,6 @@ namespace nix { -std::string FixedOutputHash::printMethodAlgo() const -{ - return makeFileIngestionPrefix(method) + printHashType(hash.type); -} - std::string makeFileIngestionPrefix(FileIngestionMethod m) { switch (m) { @@ -34,27 +29,13 @@ std::string ContentAddressMethod::renderPrefix() const ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m) { - ContentAddressMethod method = FileIngestionMethod::Flat; - if (splitPrefix(m, "r:")) - method = FileIngestionMethod::Recursive; - else if (splitPrefix(m, "text:")) - method = TextIngestionMethod {}; - return method; -} - -std::string ContentAddress::render() const -{ - return std::visit(overloaded { - [](const TextHash & th) { - return "text:" - + th.hash.to_string(Base32, true); - }, - [](const FixedOutputHash & fsh) { - return "fixed:" - + makeFileIngestionPrefix(fsh.method) - + fsh.hash.to_string(Base32, true); - } - }, raw); + if (splitPrefix(m, "r:")) { + return FileIngestionMethod::Recursive; + } + else if (splitPrefix(m, "text:")) { + return TextIngestionMethod {}; + } + return FileIngestionMethod::Flat; } std::string ContentAddressMethod::render(HashType ht) const @@ -69,6 +50,20 @@ std::string ContentAddressMethod::render(HashType ht) const }, raw); } +std::string ContentAddress::render() const +{ + return std::visit(overloaded { + [](const TextIngestionMethod &) -> std::string { + return "text:"; + }, + [](const FileIngestionMethod & method) { + return "fixed:" + + makeFileIngestionPrefix(method); + }, + }, method.raw) + + this->hash.to_string(HashFormat::Base32, true); +} + /** * Parses content address strings up to the hash. */ @@ -89,7 +84,7 @@ static std::pair parseContentAddressMethodPrefix if (!hashTypeRaw) throw UsageError("content address hash must be in form ':', but found: %s", wholeInput); HashType hashType = parseHashType(*hashTypeRaw); - return std::move(hashType); + return hashType; }; // Switch on prefix @@ -118,22 +113,12 @@ ContentAddress ContentAddress::parse(std::string_view rawCa) { auto rest = rawCa; - auto [caMethod, hashType_] = parseContentAddressMethodPrefix(rest); - auto hashType = hashType_; // work around clang bug + auto [caMethod, hashType] = parseContentAddressMethodPrefix(rest); - return std::visit(overloaded { - [&](TextIngestionMethod &) { - return ContentAddress(TextHash { - .hash = Hash::parseNonSRIUnprefixed(rest, hashType) - }); - }, - [&](FileIngestionMethod & fim) { - return ContentAddress(FixedOutputHash { - .method = fim, - .hash = Hash::parseNonSRIUnprefixed(rest, hashType), - }); - }, - }, caMethod.raw); + return ContentAddress { + .method = std::move(caMethod), + .hash = Hash::parseNonSRIUnprefixed(rest, hashType), + }; } std::pair ContentAddressMethod::parse(std::string_view caMethod) @@ -156,52 +141,10 @@ std::string renderContentAddress(std::optional ca) return ca ? ca->render() : ""; } -ContentAddress ContentAddress::fromParts( - ContentAddressMethod method, Hash hash) noexcept -{ - return std::visit(overloaded { - [&](TextIngestionMethod _) -> ContentAddress { - return TextHash { - .hash = std::move(hash), - }; - }, - [&](FileIngestionMethod m2) -> ContentAddress { - return FixedOutputHash { - .method = std::move(m2), - .hash = std::move(hash), - }; - }, - }, method.raw); -} - -ContentAddressMethod ContentAddress::getMethod() const -{ - return std::visit(overloaded { - [](const TextHash & th) -> ContentAddressMethod { - return TextIngestionMethod {}; - }, - [](const FixedOutputHash & fsh) -> ContentAddressMethod { - return fsh.method; - }, - }, raw); -} - -const Hash & ContentAddress::getHash() const -{ - return std::visit(overloaded { - [](const TextHash & th) -> auto & { - return th.hash; - }, - [](const FixedOutputHash & fsh) -> auto & { - return fsh.hash; - }, - }, raw); -} - std::string ContentAddress::printMethodAlgo() const { - return getMethod().renderPrefix() - + printHashType(getHash().type); + return method.renderPrefix() + + printHashType(hash.type); } bool StoreReferences::empty() const @@ -217,19 +160,20 @@ size_t StoreReferences::size() const ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) noexcept { return std::visit(overloaded { - [&](const TextHash & h) -> ContentAddressWithReferences { + [&](const TextIngestionMethod &) -> ContentAddressWithReferences { return TextInfo { - .hash = h, + .hash = ca.hash, .references = {}, }; }, - [&](const FixedOutputHash & h) -> ContentAddressWithReferences { + [&](const FileIngestionMethod & method) -> ContentAddressWithReferences { return FixedOutputInfo { - .hash = h, + .method = method, + .hash = ca.hash, .references = {}, }; }, - }, ca.raw); + }, ca.method.raw); } std::optional ContentAddressWithReferences::fromPartsOpt( @@ -241,7 +185,7 @@ std::optional ContentAddressWithReferences::fromPa return std::nullopt; return ContentAddressWithReferences { TextInfo { - .hash = { .hash = std::move(hash) }, + .hash = std::move(hash), .references = std::move(refs.others), } }; @@ -249,10 +193,8 @@ std::optional ContentAddressWithReferences::fromPa [&](FileIngestionMethod m2) -> std::optional { return ContentAddressWithReferences { FixedOutputInfo { - .hash = { - .method = m2, - .hash = std::move(hash), - }, + .method = m2, + .hash = std::move(hash), .references = std::move(refs), } }; @@ -267,7 +209,7 @@ ContentAddressMethod ContentAddressWithReferences::getMethod() const return TextIngestionMethod {}; }, [](const FixedOutputInfo & fsh) -> ContentAddressMethod { - return fsh.hash.method; + return fsh.method; }, }, raw); } @@ -276,10 +218,10 @@ Hash ContentAddressWithReferences::getHash() const { return std::visit(overloaded { [](const TextInfo & th) { - return th.hash.hash; + return th.hash; }, [](const FixedOutputInfo & fsh) { - return fsh.hash.hash; + return fsh.hash; }, }, raw); } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index e1e80448ba0b..bdb558907b6a 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -5,6 +5,7 @@ #include "hash.hh" #include "path.hh" #include "comparator.hh" +#include "variant-wrapper.hh" namespace nix { @@ -38,12 +39,12 @@ enum struct FileIngestionMethod : uint8_t { /** * Flat-file hashing. Directly ingest the contents of a single file */ - Flat = false, + Flat = 0, /** * Recursive (or NAR) hashing. Serializes the file-system object in Nix * Archive format and ingest that */ - Recursive = true + Recursive = 1 }; /** @@ -71,11 +72,7 @@ struct ContentAddressMethod GENERATE_CMP(ContentAddressMethod, me->raw); - /* The moral equivalent of `using Raw::Raw;` */ - ContentAddressMethod(auto &&... arg) - : raw(std::forward(arg)...) - { } - + MAKE_WRAPPER_CONSTRUCTOR(ContentAddressMethod); /** * Parse the prefix tag which indicates how the files @@ -113,37 +110,6 @@ struct ContentAddressMethod * Mini content address */ -/** - * Somewhat obscure, used by \ref Derivation derivations and - * `builtins.toFile` currently. - */ -struct TextHash { - /** - * Hash of the contents of the text/file. - */ - Hash hash; - - GENERATE_CMP(TextHash, me->hash); -}; - -/** - * Used by most store objects that are content-addressed. - */ -struct FixedOutputHash { - /** - * How the file system objects are serialized - */ - FileIngestionMethod method; - /** - * Hash of that serialization - */ - Hash hash; - - std::string printMethodAlgo() const; - - GENERATE_CMP(FixedOutputHash, me->method, me->hash); -}; - /** * We've accumulated several types of content-addressed paths over the * years; fixed-output derivations support multiple hash algorithms and @@ -158,19 +124,17 @@ struct FixedOutputHash { */ struct ContentAddress { - typedef std::variant< - TextHash, - FixedOutputHash - > Raw; - - Raw raw; + /** + * How the file system objects are serialized + */ + ContentAddressMethod method; - GENERATE_CMP(ContentAddress, me->raw); + /** + * Hash of that serialization + */ + Hash hash; - /* The moral equivalent of `using Raw::Raw;` */ - ContentAddress(auto &&... arg) - : raw(std::forward(arg)...) - { } + GENERATE_CMP(ContentAddress, me->method, me->hash); /** * Compute the content-addressability assertion @@ -183,20 +147,6 @@ struct ContentAddress static std::optional parseOpt(std::string_view rawCaOpt); - /** - * Create a `ContentAddress` from 2 parts: - * - * @param method Way ingesting the file system data. - * - * @param hash Hash of ingested file system data. - */ - static ContentAddress fromParts( - ContentAddressMethod method, Hash hash) noexcept; - - ContentAddressMethod getMethod() const; - - const Hash & getHash() const; - std::string printMethodAlgo() const; }; @@ -219,7 +169,8 @@ std::string renderContentAddress(std::optional ca); * References to other store objects are tracked with store paths, self * references however are tracked with a boolean. */ -struct StoreReferences { +struct StoreReferences +{ /** * References to other store objects */ @@ -246,8 +197,13 @@ struct StoreReferences { }; // This matches the additional info that we need for makeTextPath -struct TextInfo { - TextHash hash; +struct TextInfo +{ + /** + * Hash of the contents of the text/file. + */ + Hash hash; + /** * References to other store objects only; self references * disallowed @@ -257,8 +213,18 @@ struct TextInfo { GENERATE_CMP(TextInfo, me->hash, me->references); }; -struct FixedOutputInfo { - FixedOutputHash hash; +struct FixedOutputInfo +{ + /** + * How the file system objects are serialized + */ + FileIngestionMethod method; + + /** + * Hash of that serialization + */ + Hash hash; + /** * References to other store objects or this one. */ @@ -283,10 +249,7 @@ struct ContentAddressWithReferences GENERATE_CMP(ContentAddressWithReferences, me->raw); - /* The moral equivalent of `using Raw::Raw;` */ - ContentAddressWithReferences(auto &&... arg) - : raw(std::forward(arg)...) - { } + MAKE_WRAPPER_CONSTRUCTOR(ContentAddressWithReferences); /** * Create a `ContentAddressWithReferences` from a mere diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc index 1027469c9eee..1b705733c30b 100644 --- a/src/libstore/crypto.cc +++ b/src/libstore/crypto.cc @@ -1,4 +1,5 @@ #include "crypto.hh" +#include "file-system.hh" #include "util.hh" #include "globals.hh" diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 903dc9684322..20d25211dea7 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -10,6 +10,7 @@ #include "store-cast.hh" #include "gc-store.hh" #include "log-store.hh" +#include "indirect-root-store.hh" #include "path-with-outputs.hh" #include "finally.hh" #include "archive.hh" @@ -47,9 +48,9 @@ struct TunnelLogger : public Logger Sync state_; - unsigned int clientVersion; + WorkerProto::Version clientVersion; - TunnelLogger(FdSink & to, unsigned int clientVersion) + TunnelLogger(FdSink & to, WorkerProto::Version clientVersion) : to(to), clientVersion(clientVersion) { } void enqueueMsg(const std::string & s) @@ -263,24 +264,18 @@ struct ClientSettings } }; -static std::vector readDerivedPaths(Store & store, unsigned int clientVersion, WorkerProto::ReadConn conn) -{ - std::vector reqs; - if (GET_PROTOCOL_MINOR(clientVersion) >= 30) { - reqs = WorkerProto::Serialise>::read(store, conn); - } else { - for (auto & s : readStrings(conn.from)) - reqs.push_back(parsePathWithOutputs(store, s).toDerivedPath()); - } - return reqs; -} - static void performOp(TunnelLogger * logger, ref store, - AuthenticatedUser user, RecursiveFlag recursive, unsigned int clientVersion, + AuthenticatedUser user, RecursiveFlag recursive, WorkerProto::Version clientVersion, Source & from, BufferedSink & to, WorkerProto::Op op) { - WorkerProto::ReadConn rconn { .from = from }; - WorkerProto::WriteConn wconn { .to = to }; + WorkerProto::ReadConn rconn { + .from = from, + .version = clientVersion, + }; + WorkerProto::WriteConn wconn { + .to = to, + .version = clientVersion, + }; switch (op) { @@ -336,7 +331,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto hash = store->queryPathInfo(path)->narHash; logger->stopWork(); - to << hash.to_string(Base16, false); + to << hash.to_string(HashFormat::Base16, false); break; } @@ -430,7 +425,7 @@ static void performOp(TunnelLogger * logger, ref store, }(); logger->stopWork(); - pathInfo->write(to, *store, GET_PROTOCOL_MINOR(clientVersion)); + WorkerProto::Serialise::write(*store, wconn, *pathInfo); } else { HashType hashAlgo; std::string baseName; @@ -462,13 +457,13 @@ static void performOp(TunnelLogger * logger, ref store, eagerly consume the entire stream it's given, past the length of the Nar. */ TeeSource savedNARSource(from, saved); - ParseSink sink; /* null sink; just parse the NAR */ + NullParseSink sink; /* just parse the NAR */ parseDump(sink, savedNARSource); } else { /* Incrementally parse the NAR file, stripping the metadata, and streaming the sole file we expect into `saved`. */ - RetrieveRegularNARSink savedRegular { saved }; + RegularFileSink savedRegular { saved }; parseDump(savedRegular, from); if (!savedRegular.regular) throw Error("regular file expected"); } @@ -535,7 +530,7 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::BuildPaths: { - auto drvs = readDerivedPaths(*store, clientVersion, rconn); + auto drvs = WorkerProto::Serialise::read(*store, rconn); BuildMode mode = bmNormal; if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { mode = (BuildMode) readInt(from); @@ -560,7 +555,7 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::BuildPathsWithResults: { - auto drvs = readDerivedPaths(*store, clientVersion, rconn); + auto drvs = WorkerProto::Serialise::read(*store, rconn); BuildMode mode = bmNormal; mode = (BuildMode) readInt(from); @@ -644,16 +639,7 @@ static void performOp(TunnelLogger * logger, ref store, auto res = store->buildDerivation(drvPath, drv, buildMode); logger->stopWork(); - to << res.status << res.errorMsg; - if (GET_PROTOCOL_MINOR(clientVersion) >= 29) { - to << res.timesBuilt << res.isNonDeterministic << res.startTime << res.stopTime; - } - if (GET_PROTOCOL_MINOR(clientVersion) >= 28) { - DrvOutputs builtOutputs; - for (auto & [output, realisation] : res.builtOutputs) - builtOutputs.insert_or_assign(realisation.id, realisation); - WorkerProto::write(*store, wconn, builtOutputs); - } + WorkerProto::write(*store, wconn, res); break; } @@ -675,12 +661,27 @@ static void performOp(TunnelLogger * logger, ref store, break; } + case WorkerProto::Op::AddPermRoot: { + if (!user.trusted) + throw Error( + "you are not privileged to create perm roots\n\n" + "hint: you can just do this client-side without special privileges, and probably want to do that instead."); + auto storePath = WorkerProto::Serialise::read(*store, rconn); + Path gcRoot = absPath(readString(from)); + logger->startWork(); + auto & localFSStore = require(*store); + localFSStore.addPermRoot(storePath, gcRoot); + logger->stopWork(); + to << gcRoot; + break; + } + case WorkerProto::Op::AddIndirectRoot: { Path path = absPath(readString(from)); logger->startWork(); - auto & gcStore = require(*store); - gcStore.addIndirectRoot(path); + auto & indirectRootStore = require(*store); + indirectRootStore.addIndirectRoot(path); logger->stopWork(); to << 1; @@ -837,7 +838,7 @@ static void performOp(TunnelLogger * logger, ref store, if (info) { if (GET_PROTOCOL_MINOR(clientVersion) >= 17) to << 1; - info->write(to, *store, GET_PROTOCOL_MINOR(clientVersion), false); + WorkerProto::write(*store, wconn, static_cast(*info)); } else { assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); to << 0; @@ -917,7 +918,7 @@ static void performOp(TunnelLogger * logger, ref store, source = std::make_unique(from, to); else { TeeSource tee { from, saved }; - ParseSink ether; + NullParseSink ether; parseDump(ether, tee); source = std::make_unique(saved.s); } @@ -935,7 +936,7 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::QueryMissing: { - auto targets = readDerivedPaths(*store, clientVersion, rconn); + auto targets = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); StorePathSet willBuild, willSubstitute, unknown; uint64_t downloadSize, narSize; @@ -1090,7 +1091,7 @@ void processConnection( if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch"); to << WORKER_MAGIC_2 << PROTOCOL_VERSION; to.flush(); - unsigned int clientVersion = readInt(from); + WorkerProto::Version clientVersion = readInt(from); if (clientVersion < 0x10a) throw Error("the Nix client version is too old"); @@ -1125,7 +1126,10 @@ void processConnection( auto temp = user.trusted ? store->isTrustedClient() : std::optional { NotTrusted }; - WorkerProto::WriteConn wconn { .to = to }; + WorkerProto::WriteConn wconn { + .to = to, + .version = clientVersion, + }; WorkerProto::write(*store, wconn, temp); } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 6f63685d462c..6d9c8b9d6fea 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -4,15 +4,14 @@ #include "globals.hh" #include "util.hh" #include "split.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" -#include "fs-accessor.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" #include #include namespace nix { -std::optional DerivationOutput::path(const Store & store, std::string_view drvName, std::string_view outputName) const +std::optional DerivationOutput::path(const Store & store, std::string_view drvName, OutputNameView outputName) const { return std::visit(overloaded { [](const DerivationOutput::InputAddressed & doi) -> std::optional { @@ -32,11 +31,11 @@ std::optional DerivationOutput::path(const Store & store, std::string [](const DerivationOutput::Impure &) -> std::optional { return std::nullopt; }, - }, raw()); + }, raw); } -StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const +StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, OutputNameView outputName) const { return store.makeFixedOutputPathFromCA( outputPathName(drvName, outputName), @@ -60,7 +59,7 @@ bool DerivationType::isCA() const [](const Impure &) { return true; }, - }, raw()); + }, raw); } bool DerivationType::isFixed() const @@ -75,7 +74,7 @@ bool DerivationType::isFixed() const [](const Impure &) { return false; }, - }, raw()); + }, raw); } bool DerivationType::hasKnownOutputPaths() const @@ -90,7 +89,7 @@ bool DerivationType::hasKnownOutputPaths() const [](const Impure &) { return false; }, - }, raw()); + }, raw); } @@ -106,7 +105,7 @@ bool DerivationType::isSandboxed() const [](const Impure &) { return false; }, - }, raw()); + }, raw); } @@ -122,7 +121,7 @@ bool DerivationType::isPure() const [](const Impure &) { return false; }, - }, raw()); + }, raw); } @@ -136,7 +135,7 @@ StorePath writeDerivation(Store & store, const Derivation & drv, RepairFlag repair, bool readOnly) { auto references = drv.inputSrcs; - for (auto & i : drv.inputDrvs) + for (auto & i : drv.inputDrvs.map) references.insert(i.first); /* Note that the outputs of a derivation are *not* references (that can be missing (of course) and should not necessarily be @@ -152,10 +151,10 @@ StorePath writeDerivation(Store & store, /* Read string `s' from stream `str'. */ static void expect(std::istream & str, std::string_view s) { - char s2[s.size()]; - str.read(s2, s.size()); - if (std::string(s2, s.size()) != s) - throw FormatError("expected string '%1%'", s); + for (auto & c : s) { + if (str.get() != c) + throw FormatError("expected string '%1%'", s); + } } @@ -207,23 +206,27 @@ static bool endOfList(std::istream & str) static StringSet parseStrings(std::istream & str, bool arePaths) { StringSet res; + expect(str, "["); while (!endOfList(str)) res.insert(arePaths ? parsePath(str) : parseString(str)); return res; } -static DerivationOutput parseDerivationOutput(const Store & store, - std::string_view pathS, std::string_view hashAlgo, std::string_view hashS) +static DerivationOutput parseDerivationOutput( + const Store & store, + std::string_view pathS, std::string_view hashAlgo, std::string_view hashS, + const ExperimentalFeatureSettings & xpSettings) { if (hashAlgo != "") { ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgo); if (method == TextIngestionMethod {}) - experimentalFeatureSettings.require(Xp::DynamicDerivations); + xpSettings.require(Xp::DynamicDerivations); const auto hashType = parseHashType(hashAlgo); if (hashS == "impure") { - experimentalFeatureSettings.require(Xp::ImpureDerivations); - assert(pathS == ""); + xpSettings.require(Xp::ImpureDerivations); + if (pathS != "") + throw FormatError("impure derivation output should not specify output path"); return DerivationOutput::Impure { .method = std::move(method), .hashType = std::move(hashType), @@ -232,13 +235,15 @@ static DerivationOutput parseDerivationOutput(const Store & store, validatePath(pathS); auto hash = Hash::parseNonSRIUnprefixed(hashS, hashType); return DerivationOutput::CAFixed { - .ca = ContentAddress::fromParts( - std::move(method), - std::move(hash)), + .ca = ContentAddress { + .method = std::move(method), + .hash = std::move(hash), + }, }; } else { - experimentalFeatureSettings.require(Xp::CaDerivations); - assert(pathS == ""); + xpSettings.require(Xp::CaDerivations); + if (pathS != "") + throw FormatError("content-addressed derivation output should not specify output path"); return DerivationOutput::CAFloating { .method = std::move(method), .hashType = std::move(hashType), @@ -255,29 +260,119 @@ static DerivationOutput parseDerivationOutput(const Store & store, } } -static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str) +static DerivationOutput parseDerivationOutput( + const Store & store, std::istringstream & str, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings) { expect(str, ","); const auto pathS = parseString(str); expect(str, ","); const auto hashAlgo = parseString(str); expect(str, ","); const auto hash = parseString(str); expect(str, ")"); - return parseDerivationOutput(store, pathS, hashAlgo, hash); + return parseDerivationOutput(store, pathS, hashAlgo, hash, xpSettings); +} + +/** + * All ATerm Derivation format versions currently known. + * + * Unknown versions are rejected at the parsing stage. + */ +enum struct DerivationATermVersion { + /** + * Older unversioned form + */ + Traditional, + + /** + * Newer versioned form; only this version so far. + */ + DynamicDerivations, +}; + +static DerivedPathMap::ChildNode parseDerivedPathMapNode( + const Store & store, + std::istringstream & str, + DerivationATermVersion version) +{ + DerivedPathMap::ChildNode node; + + auto parseNonDynamic = [&]() { + node.value = parseStrings(str, false); + }; + + // Older derivation should never use new form, but newer + // derivaiton can use old form. + switch (version) { + case DerivationATermVersion::Traditional: + parseNonDynamic(); + break; + case DerivationATermVersion::DynamicDerivations: + switch (str.peek()) { + case '[': + parseNonDynamic(); + break; + case '(': + expect(str, "("); + node.value = parseStrings(str, false); + expect(str, ",["); + while (!endOfList(str)) { + expect(str, "("); + auto outputName = parseString(str); + expect(str, ","); + node.childMap.insert_or_assign(outputName, parseDerivedPathMapNode(store, str, version)); + expect(str, ")"); + } + expect(str, ")"); + break; + default: + throw FormatError("invalid inputDrvs entry in derivation"); + } + break; + default: + // invalid format, not a parse error but internal error + assert(false); + } + return node; } -Derivation parseDerivation(const Store & store, std::string && s, std::string_view name) +Derivation parseDerivation( + const Store & store, std::string && s, std::string_view name, + const ExperimentalFeatureSettings & xpSettings) { Derivation drv; drv.name = name; std::istringstream str(std::move(s)); - expect(str, "Derive(["); + expect(str, "D"); + DerivationATermVersion version; + switch (str.peek()) { + case 'e': + expect(str, "erive("); + version = DerivationATermVersion::Traditional; + break; + case 'r': { + expect(str, "rvWithVersion("); + auto versionS = parseString(str); + if (versionS == "xp-dyn-drv") { + // Only verison we have so far + version = DerivationATermVersion::DynamicDerivations; + xpSettings.require(Xp::DynamicDerivations); + } else { + throw FormatError("Unknown derivation ATerm format version '%s'", versionS); + } + expect(str, ","); + break; + } + default: + throw Error("derivation does not start with 'Derive' or 'DrvWithVersion'"); + } /* Parse the list of outputs. */ + expect(str, "["); while (!endOfList(str)) { expect(str, "("); std::string id = parseString(str); - auto output = parseDerivationOutput(store, str); + auto output = parseDerivationOutput(store, str, xpSettings); drv.outputs.emplace(std::move(id), std::move(output)); } @@ -286,12 +381,12 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi while (!endOfList(str)) { expect(str, "("); Path drvPath = parsePath(str); - expect(str, ",["); - drv.inputDrvs.insert_or_assign(store.parseStorePath(drvPath), parseStrings(str, false)); + expect(str, ","); + drv.inputDrvs.map.insert_or_assign(store.parseStorePath(drvPath), parseDerivedPathMapNode(store, str, version)); expect(str, ")"); } - expect(str, ",["); drv.inputSrcs = store.parseStorePathSet(parseStrings(str, true)); + expect(str, ","); drv.inputSrcs = store.parseStorePathSet(parseStrings(str, true)); expect(str, ","); drv.platform = parseString(str); expect(str, ","); drv.builder = parseString(str); @@ -375,14 +470,67 @@ static void printUnquotedStrings(std::string & res, ForwardIterator i, ForwardIt } +static void unparseDerivedPathMapNode(const Store & store, std::string & s, const DerivedPathMap::ChildNode & node) +{ + s += ','; + if (node.childMap.empty()) { + printUnquotedStrings(s, node.value.begin(), node.value.end()); + } else { + s += "("; + printUnquotedStrings(s, node.value.begin(), node.value.end()); + s += ",["; + bool first = true; + for (auto & [outputName, childNode] : node.childMap) { + if (first) first = false; else s += ','; + s += '('; printUnquotedString(s, outputName); + unparseDerivedPathMapNode(store, s, childNode); + s += ')'; + } + s += "])"; + } +} + + +/** + * Does the derivation have a dependency on the output of a dynamic + * derivation? + * + * In other words, does it on the output of derivation that is itself an + * ouput of a derivation? This corresponds to a dependency that is an + * inductive derived path with more than one layer of + * `DerivedPath::Built`. + */ +static bool hasDynamicDrvDep(const Derivation & drv) +{ + return + std::find_if( + drv.inputDrvs.map.begin(), + drv.inputDrvs.map.end(), + [](auto & kv) { return !kv.second.childMap.empty(); }) + != drv.inputDrvs.map.end(); +} + + std::string Derivation::unparse(const Store & store, bool maskOutputs, - std::map * actualInputs) const + DerivedPathMap::ChildNode::Map * actualInputs) const { std::string s; s.reserve(65536); - s += "Derive(["; + + /* Use older unversioned form if possible, for wider compat. Use + newer form only if we need it, which we do for + `Xp::DynamicDerivations`. */ + if (hasDynamicDrvDep(*this)) { + s += "DrvWithVersion("; + // Only version we have so far + printUnquotedString(s, "xp-dyn-drv"); + s += ","; + } else { + s += "Derive("; + } bool first = true; + s += "["; for (auto & i : outputs) { if (first) first = false; else s += ','; s += '('; printUnquotedString(s, i.first); @@ -395,7 +543,7 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs, [&](const DerivationOutput::CAFixed & dof) { s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first))); s += ','; printUnquotedString(s, dof.ca.printMethodAlgo()); - s += ','; printUnquotedString(s, dof.ca.getHash().to_string(Base16, false)); + s += ','; printUnquotedString(s, dof.ca.hash.to_string(HashFormat::Base16, false)); }, [&](const DerivationOutput::CAFloating & dof) { s += ','; printUnquotedString(s, ""); @@ -407,30 +555,30 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs, s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, ""); }, - [&](const DerivationOutputImpure & doi) { + [&](const DerivationOutput::Impure & doi) { // FIXME s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, doi.method.renderPrefix() + printHashType(doi.hashType)); s += ','; printUnquotedString(s, "impure"); } - }, i.second.raw()); + }, i.second.raw); s += ')'; } s += "],["; first = true; if (actualInputs) { - for (auto & i : *actualInputs) { + for (auto & [drvHashModulo, childMap] : *actualInputs) { if (first) first = false; else s += ','; - s += '('; printUnquotedString(s, i.first); - s += ','; printUnquotedStrings(s, i.second.begin(), i.second.end()); + s += '('; printUnquotedString(s, drvHashModulo); + unparseDerivedPathMapNode(store, s, childMap); s += ')'; } } else { - for (auto & i : inputDrvs) { + for (auto & [drvPath, childMap] : inputDrvs.map) { if (first) first = false; else s += ','; - s += '('; printUnquotedString(s, store.printStorePath(i.first)); - s += ','; printUnquotedStrings(s, i.second.begin(), i.second.end()); + s += '('; printUnquotedString(s, store.printStorePath(drvPath)); + unparseDerivedPathMapNode(store, s, childMap); s += ')'; } } @@ -465,7 +613,7 @@ bool isDerivation(std::string_view fileName) } -std::string outputPathName(std::string_view drvName, std::string_view outputName) { +std::string outputPathName(std::string_view drvName, OutputNameView outputName) { std::string res { drvName }; if (outputName != "out") { res += "-"; @@ -508,7 +656,7 @@ DerivationType BasicDerivation::type() const [&](const DerivationOutput::Impure &) { impureOutputs.insert(i.first); }, - }, i.second.raw()); + }, i.second.raw); } if (inputAddressedOutputs.empty() @@ -625,10 +773,10 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut if (type.isFixed()) { std::map outputHashes; for (const auto & i : drv.outputs) { - auto & dof = std::get(i.second.raw()); + auto & dof = std::get(i.second.raw); auto hash = hashString(htSHA256, "fixed:out:" + dof.ca.printMethodAlgo() + ":" - + dof.ca.getHash().to_string(Base16, false) + ":" + + dof.ca.hash.to_string(HashFormat::Base16, false) + ":" + store.printStorePath(dof.path(store, drv.name, i.first))); outputHashes.insert_or_assign(i.first, std::move(hash)); } @@ -662,20 +810,18 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut [](const DerivationType::Impure &) -> DrvHash::Kind { assert(false); } - }, drv.type().raw()); + }, drv.type().raw); - std::map inputs2; - for (auto & [drvPath, inputOutputs0] : drv.inputDrvs) { - // Avoid lambda capture restriction with standard / Clang - auto & inputOutputs = inputOutputs0; + DerivedPathMap::ChildNode::Map inputs2; + for (auto & [drvPath, node] : drv.inputDrvs.map) { const auto & res = pathDerivationModulo(store, drvPath); if (res.kind == DrvHash::Kind::Deferred) kind = DrvHash::Kind::Deferred; - for (auto & outputName : inputOutputs) { + for (auto & outputName : node.value) { const auto h = get(res.hashes, outputName); if (!h) throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); - inputs2[h->to_string(Base16, false)].insert(outputName); + inputs2[h->to_string(HashFormat::Base16, false)].value.insert(outputName); } } @@ -705,7 +851,7 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store) const auto hashAlgo = readString(in); const auto hash = readString(in); - return parseDerivationOutput(store, pathS, hashAlgo, hash); + return parseDerivationOutput(store, pathS, hashAlgo, hash, experimentalFeatureSettings); } StringSet BasicDerivation::outputNames() const @@ -719,10 +865,10 @@ StringSet BasicDerivation::outputNames() const DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const { DerivationOutputsAndOptPaths outsAndOptPaths; - for (auto output : outputs) + for (auto & [outputName, output] : outputs) outsAndOptPaths.insert(std::make_pair( - output.first, - std::make_pair(output.second, output.second.path(store, name, output.first)) + outputName, + std::make_pair(output, output.path(store, name, outputName)) ) ); return outsAndOptPaths; @@ -750,8 +896,8 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, drv.outputs.emplace(std::move(name), std::move(output)); } - drv.inputSrcs = WorkerProto::Serialise::read(store, - WorkerProto::ReadConn { .from = in }); + drv.inputSrcs = CommonProto::Serialise::read(store, + CommonProto::ReadConn { .from = in }); in >> drv.platform >> drv.builder; drv.args = readStrings(in); @@ -780,7 +926,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr [&](const DerivationOutput::CAFixed & dof) { out << store.printStorePath(dof.path(store, drv.name, i.first)) << dof.ca.printMethodAlgo() - << dof.ca.getHash().to_string(Base16, false); + << dof.ca.hash.to_string(HashFormat::Base16, false); }, [&](const DerivationOutput::CAFloating & dof) { out << "" @@ -797,10 +943,10 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr << (doi.method.renderPrefix() + printHashType(doi.hashType)) << "impure"; }, - }, i.second.raw()); + }, i.second.raw); } - WorkerProto::write(store, - WorkerProto::WriteConn { .to = out }, + CommonProto::write(store, + CommonProto::WriteConn { .to = out }, drv.inputSrcs); out << drv.platform << drv.builder << drv.args; out << drv.env.size(); @@ -809,10 +955,10 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr } -std::string hashPlaceholder(const std::string_view outputName) +std::string hashPlaceholder(const OutputNameView outputName) { // FIXME: memoize? - return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(Base32, false); + return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(HashFormat::Base32, false); } @@ -820,6 +966,8 @@ std::string hashPlaceholder(const std::string_view outputName) static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) { + debug("Rewriting the derivation"); + for (auto & rewrite : rewrites) { debug("rewriting %s as %s", rewrite.first, rewrite.second); } @@ -839,7 +987,7 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String auto hashModulo = hashDerivationModulo(store, Derivation(drv), true); for (auto & [outputName, output] : drv.outputs) { - if (std::holds_alternative(output.raw())) { + if (std::holds_alternative(output.raw)) { auto h = get(hashModulo.hashes, outputName); if (!h) throw Error("derivation '%s' output '%s' has no hash (derivations.cc/rewriteDerivation)", @@ -858,14 +1006,70 @@ std::optional Derivation::tryResolve(Store & store) const { std::map, StorePath> inputDrvOutputs; - for (auto & input : inputDrvs) - for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(input.first)) - if (outputPath) - inputDrvOutputs.insert_or_assign({input.first, outputName}, *outputPath); + std::function::ChildNode &)> accum; + accum = [&](auto & inputDrv, auto & node) { + for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(inputDrv)) { + if (outputPath) { + inputDrvOutputs.insert_or_assign({inputDrv, outputName}, *outputPath); + if (auto p = get(node.childMap, outputName)) + accum(*outputPath, *p); + } + } + }; + + for (auto & [inputDrv, node] : inputDrvs.map) + accum(inputDrv, node); return tryResolve(store, inputDrvOutputs); } +static bool tryResolveInput( + Store & store, StorePathSet & inputSrcs, StringMap & inputRewrites, + const DownstreamPlaceholder * placeholderOpt, + const StorePath & inputDrv, const DerivedPathMap::ChildNode & inputNode, + const std::map, StorePath> & inputDrvOutputs) +{ + auto getOutput = [&](const std::string & outputName) { + auto * actualPathOpt = get(inputDrvOutputs, { inputDrv, outputName }); + if (!actualPathOpt) + warn("output %s of input %s missing, aborting the resolving", + outputName, + store.printStorePath(inputDrv) + ); + return actualPathOpt; + }; + + auto getPlaceholder = [&](const std::string & outputName) { + return placeholderOpt + ? DownstreamPlaceholder::unknownDerivation(*placeholderOpt, outputName) + : DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName); + }; + + for (auto & outputName : inputNode.value) { + auto actualPathOpt = getOutput(outputName); + if (!actualPathOpt) return false; + auto actualPath = *actualPathOpt; + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + inputRewrites.emplace( + getPlaceholder(outputName).render(), + store.printStorePath(actualPath)); + } + inputSrcs.insert(std::move(actualPath)); + } + + for (auto & [outputName, childNode] : inputNode.childMap) { + auto actualPathOpt = getOutput(outputName); + if (!actualPathOpt) return false; + auto actualPath = *actualPathOpt; + auto nextPlaceholder = getPlaceholder(outputName); + if (!tryResolveInput(store, inputSrcs, inputRewrites, + &nextPlaceholder, actualPath, childNode, + inputDrvOutputs)) + return false; + } + return true; +} + std::optional Derivation::tryResolve( Store & store, const std::map, StorePath> & inputDrvOutputs) const @@ -875,21 +1079,10 @@ std::optional Derivation::tryResolve( // Input paths that we'll want to rewrite in the derivation StringMap inputRewrites; - for (auto & [inputDrv, inputOutputs] : inputDrvs) { - for (auto & outputName : inputOutputs) { - if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) { - inputRewrites.emplace( - DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName).render(), - store.printStorePath(*actualPath)); - resolved.inputSrcs.insert(*actualPath); - } else { - warn("output '%s' of input '%s' missing, aborting the resolving", - outputName, - store.printStorePath(inputDrv)); - return {}; - } - } - } + for (auto & [inputDrv, inputNode] : inputDrvs.map) + if (!tryResolveInput(store, resolved.inputSrcs, inputRewrites, + nullptr, inputDrv, inputNode, inputDrvOutputs)) + return std::nullopt; rewriteDerivation(store, resolved, inputRewrites); @@ -952,7 +1145,7 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const [&](const DerivationOutput::Impure &) { /* Nothing to check */ }, - }, i.second.raw()); + }, i.second.raw); } } @@ -960,7 +1153,7 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const const Hash impureOutputHash = hashString(htSHA256, "impure"); nlohmann::json DerivationOutput::toJSON( - const Store & store, std::string_view drvName, std::string_view outputName) const + const Store & store, std::string_view drvName, OutputNameView outputName) const { nlohmann::json res = nlohmann::json::object(); std::visit(overloaded { @@ -970,7 +1163,7 @@ nlohmann::json DerivationOutput::toJSON( [&](const DerivationOutput::CAFixed & dof) { res["path"] = store.printStorePath(dof.path(store, drvName, outputName)); res["hashAlgo"] = dof.ca.printMethodAlgo(); - res["hash"] = dof.ca.getHash().to_string(Base16, false); + res["hash"] = dof.ca.hash.to_string(HashFormat::Base16, false); // FIXME print refs? }, [&](const DerivationOutput::CAFloating & dof) { @@ -981,17 +1174,18 @@ nlohmann::json DerivationOutput::toJSON( res["hashAlgo"] = doi.method.renderPrefix() + printHashType(doi.hashType); res["impure"] = true; }, - }, raw()); + }, raw); return res; } DerivationOutput DerivationOutput::fromJSON( - const Store & store, std::string_view drvName, std::string_view outputName, + const Store & store, std::string_view drvName, OutputNameView outputName, const nlohmann::json & _json, const ExperimentalFeatureSettings & xpSettings) { std::set keys; + ensureType(_json, nlohmann::detail::value_t::object); auto json = (std::map) _json; for (const auto & [key, _] : json) @@ -1017,9 +1211,10 @@ DerivationOutput DerivationOutput::fromJSON( else if (keys == (std::set { "path", "hashAlgo", "hash" })) { auto [method, hashType] = methodAlgo(); auto dof = DerivationOutput::CAFixed { - .ca = ContentAddress::fromParts( - std::move(method), - Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType)), + .ca = ContentAddress { + .method = std::move(method), + .hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType), + }, }; if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"])) throw Error("Path doesn't match derivation output"); @@ -1076,10 +1271,25 @@ nlohmann::json Derivation::toJSON(const Store & store) const } { - auto& inputDrvsObj = res["inputDrvs"]; - inputDrvsObj = nlohmann::json ::object(); - for (auto & input : inputDrvs) - inputDrvsObj[store.printStorePath(input.first)] = input.second; + std::function::ChildNode &)> doInput; + doInput = [&](const auto & inputNode) { + auto value = nlohmann::json::object(); + value["outputs"] = inputNode.value; + { + auto next = nlohmann::json::object(); + for (auto & [outputId, childNode] : inputNode.childMap) + next[outputId] = doInput(childNode); + value["dynamicOutputs"] = std::move(next); + } + return value; + }; + { + auto& inputDrvsObj = res["inputDrvs"]; + inputDrvsObj = nlohmann::json::object(); + for (auto & [inputDrv, inputNode] : inputDrvs.map) { + inputDrvsObj[store.printStorePath(inputDrv)] = doInput(inputNode); + } + } } res["system"] = platform; @@ -1093,38 +1303,63 @@ nlohmann::json Derivation::toJSON(const Store & store) const Derivation Derivation::fromJSON( const Store & store, - const nlohmann::json & json) + const nlohmann::json & json, + const ExperimentalFeatureSettings & xpSettings) { + using nlohmann::detail::value_t; + Derivation res; - res.name = json["name"]; + ensureType(json, value_t::object); - { - auto & outputsObj = json["outputs"]; + res.name = ensureType(valueAt(json, "name"), value_t::string); + + try { + auto & outputsObj = ensureType(valueAt(json, "outputs"), value_t::object); for (auto & [outputName, output] : outputsObj.items()) { res.outputs.insert_or_assign( outputName, DerivationOutput::fromJSON(store, res.name, outputName, output)); } + } catch (Error & e) { + e.addTrace({}, "while reading key 'outputs'"); + throw; } - { - auto & inputsList = json["inputSrcs"]; + try { + auto & inputsList = ensureType(valueAt(json, "inputSrcs"), value_t::array); for (auto & input : inputsList) res.inputSrcs.insert(store.parseStorePath(static_cast(input))); + } catch (Error & e) { + e.addTrace({}, "while reading key 'inputSrcs'"); + throw; } - { - auto & inputDrvsObj = json["inputDrvs"]; + try { + std::function::ChildNode(const nlohmann::json &)> doInput; + doInput = [&](const auto & json) { + DerivedPathMap::ChildNode node; + node.value = static_cast( + ensureType(valueAt(json, "outputs"), value_t::array)); + for (auto & [outputId, childNode] : ensureType(valueAt(json, "dynamicOutputs"), value_t::object).items()) { + xpSettings.require(Xp::DynamicDerivations); + node.childMap[outputId] = doInput(childNode); + } + return node; + }; + auto & inputDrvsObj = ensureType(valueAt(json, "inputDrvs"), value_t::object); for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items()) - res.inputDrvs[store.parseStorePath(inputDrvPath)] = - static_cast(inputOutputs); + res.inputDrvs.map[store.parseStorePath(inputDrvPath)] = + doInput(inputOutputs); + } catch (Error & e) { + e.addTrace({}, "while reading key 'inputDrvs'"); + throw; } - res.platform = json["system"]; - res.builder = json["builder"]; - res.args = json["args"]; - res.env = json["env"]; + res.platform = ensureType(valueAt(json, "system"), value_t::string); + res.builder = ensureType(valueAt(json, "builder"), value_t::string); + res.args = ensureType(valueAt(json, "args"), value_t::array); + res.env = ensureType(valueAt(json, "env"), value_t::object); return res; } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index fa79f77fd0be..fa14e75362d9 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -6,9 +6,10 @@ #include "hash.hh" #include "content-address.hh" #include "repair-flag.hh" -#include "derived-path.hh" +#include "derived-path-map.hh" #include "sync.hh" #include "comparator.hh" +#include "variant-wrapper.hh" #include #include @@ -21,107 +22,109 @@ class Store; /* Abstract syntax of derivations. */ /** - * The traditional non-fixed-output derivation type. - */ -struct DerivationOutputInputAddressed -{ - StorePath path; - - GENERATE_CMP(DerivationOutputInputAddressed, me->path); -}; - -/** - * Fixed-output derivations, whose output paths are content - * addressed according to that fixed output. + * A single output of a BasicDerivation (and Derivation). */ -struct DerivationOutputCAFixed +struct DerivationOutput { /** - * Method and hash used for expected hash computation. - * - * References are not allowed by fiat. + * The traditional non-fixed-output derivation type. */ - ContentAddress ca; + struct InputAddressed + { + StorePath path; + + GENERATE_CMP(InputAddressed, me->path); + }; /** - * Return the \ref StorePath "store path" corresponding to this output - * - * @param drvName The name of the derivation this is an output of, without the `.drv`. - * @param outputName The name of this output. + * Fixed-output derivations, whose output paths are content + * addressed according to that fixed output. */ - StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const; + struct CAFixed + { + /** + * Method and hash used for expected hash computation. + * + * References are not allowed by fiat. + */ + ContentAddress ca; - GENERATE_CMP(DerivationOutputCAFixed, me->ca); -}; + /** + * Return the \ref StorePath "store path" corresponding to this output + * + * @param drvName The name of the derivation this is an output of, without the `.drv`. + * @param outputName The name of this output. + */ + StorePath path(const Store & store, std::string_view drvName, OutputNameView outputName) const; -/** - * Floating-output derivations, whose output paths are content - * addressed, but not fixed, and so are dynamically calculated from - * whatever the output ends up being. - * */ -struct DerivationOutputCAFloating -{ - /** - * How the file system objects will be serialized for hashing - */ - ContentAddressMethod method; + GENERATE_CMP(CAFixed, me->ca); + }; /** - * How the serialization will be hashed - */ - HashType hashType; + * Floating-output derivations, whose output paths are content + * addressed, but not fixed, and so are dynamically calculated from + * whatever the output ends up being. + * */ + struct CAFloating + { + /** + * How the file system objects will be serialized for hashing + */ + ContentAddressMethod method; - GENERATE_CMP(DerivationOutputCAFloating, me->method, me->hashType); -}; + /** + * How the serialization will be hashed + */ + HashType hashType; -/** - * Input-addressed output which depends on a (CA) derivation whose hash - * isn't known yet. - */ -struct DerivationOutputDeferred { - GENERATE_CMP(DerivationOutputDeferred); -}; + GENERATE_CMP(CAFloating, me->method, me->hashType); + }; -/** - * Impure output which is moved to a content-addressed location (like - * CAFloating) but isn't registered as a realization. - */ -struct DerivationOutputImpure -{ /** - * How the file system objects will be serialized for hashing + * Input-addressed output which depends on a (CA) derivation whose hash + * isn't known yet. */ - ContentAddressMethod method; + struct Deferred { + GENERATE_CMP(Deferred); + }; /** - * How the serialization will be hashed + * Impure output which is moved to a content-addressed location (like + * CAFloating) but isn't registered as a realization. */ - HashType hashType; + struct Impure + { + /** + * How the file system objects will be serialized for hashing + */ + ContentAddressMethod method; - GENERATE_CMP(DerivationOutputImpure, me->method, me->hashType); -}; + /** + * How the serialization will be hashed + */ + HashType hashType; + + GENERATE_CMP(Impure, me->method, me->hashType); + }; -typedef std::variant< - DerivationOutputInputAddressed, - DerivationOutputCAFixed, - DerivationOutputCAFloating, - DerivationOutputDeferred, - DerivationOutputImpure -> _DerivationOutputRaw; + typedef std::variant< + InputAddressed, + CAFixed, + CAFloating, + Deferred, + Impure + > Raw; -/** - * A single output of a BasicDerivation (and Derivation). - */ -struct DerivationOutput : _DerivationOutputRaw -{ - using Raw = _DerivationOutputRaw; - using Raw::Raw; + Raw raw; - using InputAddressed = DerivationOutputInputAddressed; - using CAFixed = DerivationOutputCAFixed; - using CAFloating = DerivationOutputCAFloating; - using Deferred = DerivationOutputDeferred; - using Impure = DerivationOutputImpure; + GENERATE_CMP(DerivationOutput, me->raw); + + MAKE_WRAPPER_CONSTRUCTOR(DerivationOutput); + + /** + * Force choosing a variant + */ + DerivationOutput() = delete; /** * \note when you use this function you should make sure that you're @@ -129,23 +132,19 @@ struct DerivationOutput : _DerivationOutputRaw * the safer interface provided by * BasicDerivation::outputsAndOptPaths */ - std::optional path(const Store & store, std::string_view drvName, std::string_view outputName) const; - - inline const Raw & raw() const { - return static_cast(*this); - } + std::optional path(const Store & store, std::string_view drvName, OutputNameView outputName) const; nlohmann::json toJSON( const Store & store, std::string_view drvName, - std::string_view outputName) const; + OutputNameView outputName) const; /** * @param xpSettings Stop-gap to avoid globals during unit tests. */ static DerivationOutput fromJSON( const Store & store, std::string_view drvName, - std::string_view outputName, + OutputNameView outputName, const nlohmann::json & json, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); }; @@ -167,61 +166,71 @@ typedef std::map DerivationInputs; -/** - * Input-addressed derivation types - */ -struct DerivationType_InputAddressed { +struct DerivationType { /** - * True iff the derivation type can't be determined statically, - * for instance because it (transitively) depends on a content-addressed - * derivation. - */ - bool deferred; -}; + * Input-addressed derivation types + */ + struct InputAddressed { + /** + * True iff the derivation type can't be determined statically, + * for instance because it (transitively) depends on a content-addressed + * derivation. + */ + bool deferred; + + GENERATE_CMP(InputAddressed, me->deferred); + }; -/** - * Content-addressed derivation types - */ -struct DerivationType_ContentAddressed { /** - * Whether the derivation should be built safely inside a sandbox. + * Content-addressed derivation types */ - bool sandboxed; + struct ContentAddressed { + /** + * Whether the derivation should be built safely inside a sandbox. + */ + bool sandboxed; + /** + * Whether the derivation's outputs' content-addresses are "fixed" + * or "floating". + * + * - Fixed: content-addresses are written down as part of the + * derivation itself. If the outputs don't end up matching the + * build fails. + * + * - Floating: content-addresses are not written down, we do not + * know them until we perform the build. + */ + bool fixed; + + GENERATE_CMP(ContentAddressed, me->sandboxed, me->fixed); + }; + /** - * Whether the derivation's outputs' content-addresses are "fixed" - * or "floating. - * - * - Fixed: content-addresses are written down as part of the - * derivation itself. If the outputs don't end up matching the - * build fails. + * Impure derivation type * - * - Floating: content-addresses are not written down, we do not - * know them until we perform the build. + * This is similar at buil-time to the content addressed, not standboxed, not fixed + * type, but has some restrictions on its usage. */ - bool fixed; -}; + struct Impure { + GENERATE_CMP(Impure); + }; -/** - * Impure derivation type - * - * This is similar at buil-time to the content addressed, not standboxed, not fixed - * type, but has some restrictions on its usage. - */ -struct DerivationType_Impure { -}; + typedef std::variant< + InputAddressed, + ContentAddressed, + Impure + > Raw; + + Raw raw; + + GENERATE_CMP(DerivationType, me->raw); -typedef std::variant< - DerivationType_InputAddressed, - DerivationType_ContentAddressed, - DerivationType_Impure -> _DerivationTypeRaw; + MAKE_WRAPPER_CONSTRUCTOR(DerivationType); -struct DerivationType : _DerivationTypeRaw { - using Raw = _DerivationTypeRaw; - using Raw::Raw; - using InputAddressed = DerivationType_InputAddressed; - using ContentAddressed = DerivationType_ContentAddressed; - using Impure = DerivationType_Impure; + /** + * Force choosing a variant + */ + DerivationType() = delete; /** * Do the outputs of the derivation have paths calculated from their @@ -257,10 +266,6 @@ struct DerivationType : _DerivationTypeRaw { * closure, or if fixed output. */ bool hasKnownOutputPaths() const; - - inline const Raw & raw() const { - return static_cast(*this); - } }; struct BasicDerivation @@ -318,13 +323,13 @@ struct Derivation : BasicDerivation /** * inputs that are sub-derivations */ - DerivationInputs inputDrvs; + DerivedPathMap> inputDrvs; /** * Print a derivation. */ std::string unparse(const Store & store, bool maskOutputs, - std::map * actualInputs = nullptr) const; + DerivedPathMap::ChildNode::Map * actualInputs = nullptr) const; /** * Return the underlying basic derivation but with these changes: @@ -363,7 +368,8 @@ struct Derivation : BasicDerivation nlohmann::json toJSON(const Store & store) const; static Derivation fromJSON( const Store & store, - const nlohmann::json & json); + const nlohmann::json & json, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); GENERATE_CMP(Derivation, static_cast(*me), @@ -384,7 +390,11 @@ StorePath writeDerivation(Store & store, /** * Read a derivation from a file. */ -Derivation parseDerivation(const Store & store, std::string && s, std::string_view name); +Derivation parseDerivation( + const Store & store, + std::string && s, + std::string_view name, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); /** * \todo Remove. @@ -400,7 +410,7 @@ bool isDerivation(std::string_view fileName); * This is usually -, but is just when * the output name is "out". */ -std::string outputPathName(std::string_view drvName, std::string_view outputName); +std::string outputPathName(std::string_view drvName, OutputNameView outputName); /** @@ -494,7 +504,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr * own outputs without needing to use the hash of a derivation in * itself, making the hash near-impossible to calculate. */ -std::string hashPlaceholder(const std::string_view outputName); +std::string hashPlaceholder(const OutputNameView outputName); extern const Hash impureOutputHash; diff --git a/src/libstore/derived-path-map.cc b/src/libstore/derived-path-map.cc new file mode 100644 index 000000000000..4c1ea417a36d --- /dev/null +++ b/src/libstore/derived-path-map.cc @@ -0,0 +1,70 @@ +#include "derived-path-map.hh" +#include "util.hh" + +namespace nix { + +template +typename DerivedPathMap::ChildNode & DerivedPathMap::ensureSlot(const SingleDerivedPath & k) +{ + std::function initIter; + initIter = [&](const auto & k) -> auto & { + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) -> auto & { + // will not overwrite if already there + return map[bo.path]; + }, + [&](const SingleDerivedPath::Built & bfd) -> auto & { + auto & n = initIter(*bfd.drvPath); + return n.childMap[bfd.output]; + }, + }, k.raw()); + }; + return initIter(k); +} + +template +typename DerivedPathMap::ChildNode * DerivedPathMap::findSlot(const SingleDerivedPath & k) +{ + std::function initIter; + initIter = [&](const auto & k) { + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) { + auto it = map.find(bo.path); + return it != map.end() + ? &it->second + : nullptr; + }, + [&](const SingleDerivedPath::Built & bfd) { + auto * n = initIter(*bfd.drvPath); + if (!n) return (ChildNode *)nullptr; + + auto it = n->childMap.find(bfd.output); + return it != n->childMap.end() + ? &it->second + : nullptr; + }, + }, k.raw()); + }; + return initIter(k); +} + +} + +// instantiations + +namespace nix { + +GENERATE_CMP_EXT( + template<>, + DerivedPathMap>::ChildNode, + me->value, + me->childMap); + +GENERATE_CMP_EXT( + template<>, + DerivedPathMap>, + me->map); + +template struct DerivedPathMap>; + +}; diff --git a/src/libstore/derived-path-map.hh b/src/libstore/derived-path-map.hh new file mode 100644 index 000000000000..393cdedf747f --- /dev/null +++ b/src/libstore/derived-path-map.hh @@ -0,0 +1,96 @@ +#pragma once +///@file + +#include "types.hh" +#include "derived-path.hh" + +namespace nix { + +/** + * A simple Trie, of sorts. Conceptually a map of `SingleDerivedPath` to + * values. + * + * Concretely, an n-ary tree, as described below. A + * `SingleDerivedPath::Opaque` maps to the value of an immediate child + * of the root node. A `SingleDerivedPath::Built` maps to a deeper child + * node: the `SingleDerivedPath::Built::drvPath` is first mapped to a a + * child node (inductively), and then the + * `SingleDerivedPath::Built::output` is used to look up that child's + * child via its map. In this manner, every `SingleDerivedPath` is + * mapped to a child node. + * + * @param V A type to instantiate for each output. It should probably + * should be an "optional" type so not every interior node has to have a + * value. `* const Something` or `std::optional` would be + * good choices for "optional" types. + */ +template +struct DerivedPathMap { + /** + * A child node (non-root node). + */ + struct ChildNode { + /** + * Value of this child node. + * + * @see DerivedPathMap for what `V` should be. + */ + V value; + + /** + * The map type for the root node. + */ + using Map = std::map; + + /** + * The map of the root node. + */ + Map childMap; + + DECLARE_CMP(ChildNode); + }; + + /** + * The map type for the root node. + */ + using Map = std::map; + + /** + * The map of root node. + */ + Map map; + + DECLARE_CMP(DerivedPathMap); + + /** + * Find the node for `k`, creating it if needed. + * + * The node is referred to as a "slot" on the assumption that `V` is + * some sort of optional type, so the given key can be set or unset + * by changing this node. + */ + ChildNode & ensureSlot(const SingleDerivedPath & k); + + /** + * Like `ensureSlot` but does not create the slot if it doesn't exist. + * + * Read the entire description of `ensureSlot` to understand an + * important caveat here that "have slot" does *not* imply "key is + * set in map". To ensure a key is set one would need to get the + * child node (with `findSlot` or `ensureSlot`) *and* check the + * `ChildNode::value`. + */ + ChildNode * findSlot(const SingleDerivedPath & k); +}; + + +DECLARE_CMP_EXT( + template<>, + DerivedPathMap>::, + DerivedPathMap>); +DECLARE_CMP_EXT( + template<>, + DerivedPathMap>::ChildNode::, + DerivedPathMap>::ChildNode); + +} diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 9a2ffda39139..47d784deb373 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -1,5 +1,4 @@ #include "derived-path.hh" -#include "derivations.hh" #include "store-api.hh" #include @@ -8,50 +7,83 @@ namespace nix { -nlohmann::json DerivedPath::Opaque::toJSON(ref store) const { +#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \ + bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ + { \ + const MY_TYPE* me = this; \ + auto fields1 = std::make_tuple(*me->drvPath, me->FIELD); \ + me = &other; \ + auto fields2 = std::make_tuple(*me->drvPath, me->FIELD); \ + return fields1 COMPARATOR fields2; \ + } +#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <) + +#define FIELD_TYPE std::string +CMP(SingleDerivedPath, SingleDerivedPathBuilt, output) +#undef FIELD_TYPE + +#define FIELD_TYPE OutputsSpec +CMP(SingleDerivedPath, DerivedPathBuilt, outputs) +#undef FIELD_TYPE + +#undef CMP +#undef CMP_ONE + +nlohmann::json DerivedPath::Opaque::toJSON(const Store & store) const +{ + return store.printStorePath(path); +} + +nlohmann::json SingleDerivedPath::Built::toJSON(Store & store) const { nlohmann::json res; - res["path"] = store->printStorePath(path); + res["drvPath"] = drvPath->toJSON(store); + // Fallback for the input-addressed derivation case: We expect to always be + // able to print the output paths, so let’s do it + // FIXME try-resolve on drvPath + const auto outputMap = store.queryPartialDerivationOutputMap(resolveDerivedPath(store, *drvPath)); + res["output"] = output; + auto outputPathIter = outputMap.find(output); + if (outputPathIter == outputMap.end()) + res["outputPath"] = nullptr; + else if (std::optional p = outputPathIter->second) + res["outputPath"] = store.printStorePath(*p); + else + res["outputPath"] = nullptr; return res; } -nlohmann::json DerivedPath::Built::toJSON(ref store) const { +nlohmann::json DerivedPath::Built::toJSON(Store & store) const { nlohmann::json res; - res["drvPath"] = store->printStorePath(drvPath); + res["drvPath"] = drvPath->toJSON(store); // Fallback for the input-addressed derivation case: We expect to always be // able to print the output paths, so let’s do it - const auto outputMap = store->queryPartialDerivationOutputMap(drvPath); + // FIXME try-resolve on drvPath + const auto outputMap = store.queryPartialDerivationOutputMap(resolveDerivedPath(store, *drvPath)); for (const auto & [output, outputPathOpt] : outputMap) { if (!outputs.contains(output)) continue; if (outputPathOpt) - res["outputs"][output] = store->printStorePath(*outputPathOpt); + res["outputs"][output] = store.printStorePath(*outputPathOpt); else res["outputs"][output] = nullptr; } return res; } -nlohmann::json BuiltPath::Built::toJSON(ref store) const { - nlohmann::json res; - res["drvPath"] = store->printStorePath(drvPath); - for (const auto& [output, path] : outputs) { - res["outputs"][output] = store->printStorePath(path); - } - return res; +nlohmann::json SingleDerivedPath::toJSON(Store & store) const +{ + return std::visit([&](const auto & buildable) { + return buildable.toJSON(store); + }, raw()); } -StorePathSet BuiltPath::outPaths() const +nlohmann::json DerivedPath::toJSON(Store & store) const { - return std::visit( - overloaded{ - [](const BuiltPath::Opaque & p) { return StorePathSet{p.path}; }, - [](const BuiltPath::Built & b) { - StorePathSet res; - for (auto & [_, path] : b.outputs) - res.insert(path); - return res; - }, - }, raw() - ); + return std::visit([&](const auto & buildable) { + return buildable.toJSON(store); + }, raw()); } std::string DerivedPath::Opaque::to_string(const Store & store) const @@ -59,25 +91,49 @@ std::string DerivedPath::Opaque::to_string(const Store & store) const return store.printStorePath(path); } +std::string SingleDerivedPath::Built::to_string(const Store & store) const +{ + return drvPath->to_string(store) + "^" + output; +} + +std::string SingleDerivedPath::Built::to_string_legacy(const Store & store) const +{ + return drvPath->to_string(store) + "!" + output; +} + std::string DerivedPath::Built::to_string(const Store & store) const { - return store.printStorePath(drvPath) + return drvPath->to_string(store) + '^' + outputs.to_string(); } std::string DerivedPath::Built::to_string_legacy(const Store & store) const { - return store.printStorePath(drvPath) - + '!' + return drvPath->to_string_legacy(store) + + "!" + outputs.to_string(); } +std::string SingleDerivedPath::to_string(const Store & store) const +{ + return std::visit( + [&](const auto & req) { return req.to_string(store); }, + raw()); +} + std::string DerivedPath::to_string(const Store & store) const +{ + return std::visit( + [&](const auto & req) { return req.to_string(store); }, + raw()); +} + +std::string SingleDerivedPath::to_string_legacy(const Store & store) const { return std::visit(overloaded { - [&](const DerivedPath::Built & req) { return req.to_string(store); }, - [&](const DerivedPath::Opaque & req) { return req.to_string(store); }, + [&](const SingleDerivedPath::Built & req) { return req.to_string_legacy(store); }, + [&](const SingleDerivedPath::Opaque & req) { return req.to_string(store); }, }, this->raw()); } @@ -95,61 +151,156 @@ DerivedPath::Opaque DerivedPath::Opaque::parse(const Store & store, std::string_ return {store.parseStorePath(s)}; } -DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_view drvS, std::string_view outputsS) +void drvRequireExperiment( + const SingleDerivedPath & drv, + const ExperimentalFeatureSettings & xpSettings) +{ + std::visit(overloaded { + [&](const SingleDerivedPath::Opaque &) { + // plain drv path; no experimental features required. + }, + [&](const SingleDerivedPath::Built &) { + xpSettings.require(Xp::DynamicDerivations); + }, + }, drv.raw()); +} + +SingleDerivedPath::Built SingleDerivedPath::Built::parse( + const Store & store, ref drv, + OutputNameView output, + const ExperimentalFeatureSettings & xpSettings) +{ + drvRequireExperiment(*drv, xpSettings); + return { + .drvPath = drv, + .output = std::string { output }, + }; +} + +DerivedPath::Built DerivedPath::Built::parse( + const Store & store, ref drv, + OutputNameView outputsS, + const ExperimentalFeatureSettings & xpSettings) { + drvRequireExperiment(*drv, xpSettings); return { - .drvPath = store.parseStorePath(drvS), + .drvPath = drv, .outputs = OutputsSpec::parse(outputsS), }; } -static inline DerivedPath parseWith(const Store & store, std::string_view s, std::string_view separator) +static SingleDerivedPath parseWithSingle( + const Store & store, std::string_view s, std::string_view separator, + const ExperimentalFeatureSettings & xpSettings) { - size_t n = s.find(separator); + size_t n = s.rfind(separator); + return n == s.npos + ? (SingleDerivedPath) SingleDerivedPath::Opaque::parse(store, s) + : (SingleDerivedPath) SingleDerivedPath::Built::parse(store, + make_ref(parseWithSingle( + store, + s.substr(0, n), + separator, + xpSettings)), + s.substr(n + 1), + xpSettings); +} + +SingleDerivedPath SingleDerivedPath::parse( + const Store & store, + std::string_view s, + const ExperimentalFeatureSettings & xpSettings) +{ + return parseWithSingle(store, s, "^", xpSettings); +} + +SingleDerivedPath SingleDerivedPath::parseLegacy( + const Store & store, + std::string_view s, + const ExperimentalFeatureSettings & xpSettings) +{ + return parseWithSingle(store, s, "!", xpSettings); +} + +static DerivedPath parseWith( + const Store & store, std::string_view s, std::string_view separator, + const ExperimentalFeatureSettings & xpSettings) +{ + size_t n = s.rfind(separator); return n == s.npos ? (DerivedPath) DerivedPath::Opaque::parse(store, s) - : (DerivedPath) DerivedPath::Built::parse(store, s.substr(0, n), s.substr(n + 1)); -} - -DerivedPath DerivedPath::parse(const Store & store, std::string_view s) -{ - return parseWith(store, s, "^"); -} - -DerivedPath DerivedPath::parseLegacy(const Store & store, std::string_view s) -{ - return parseWith(store, s, "!"); -} - -RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const -{ - RealisedPath::Set res; - std::visit( - overloaded{ - [&](const BuiltPath::Opaque & p) { res.insert(p.path); }, - [&](const BuiltPath::Built & p) { - auto drvHashes = - staticOutputHashes(store, store.readDerivation(p.drvPath)); - for (auto& [outputName, outputPath] : p.outputs) { - if (experimentalFeatureSettings.isEnabled( - Xp::CaDerivations)) { - auto drvOutput = get(drvHashes, outputName); - if (!drvOutput) - throw Error( - "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)", - store.printStorePath(p.drvPath), outputName); - auto thisRealisation = store.queryRealisation( - DrvOutput{*drvOutput, outputName}); - assert(thisRealisation); // We’ve built it, so we must - // have the realisation - res.insert(*thisRealisation); - } else { - res.insert(outputPath); - } - } - }, + : (DerivedPath) DerivedPath::Built::parse(store, + make_ref(parseWithSingle( + store, + s.substr(0, n), + separator, + xpSettings)), + s.substr(n + 1), + xpSettings); +} + +DerivedPath DerivedPath::parse( + const Store & store, + std::string_view s, + const ExperimentalFeatureSettings & xpSettings) +{ + return parseWith(store, s, "^", xpSettings); +} + +DerivedPath DerivedPath::parseLegacy( + const Store & store, + std::string_view s, + const ExperimentalFeatureSettings & xpSettings) +{ + return parseWith(store, s, "!", xpSettings); +} + +DerivedPath DerivedPath::fromSingle(const SingleDerivedPath & req) +{ + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) -> DerivedPath { + return o; }, - raw()); - return res; + [&](const SingleDerivedPath::Built & b) -> DerivedPath { + return DerivedPath::Built { + .drvPath = b.drvPath, + .outputs = OutputsSpec::Names { b.output }, + }; + }, + }, req.raw()); +} + +const StorePath & SingleDerivedPath::Built::getBaseStorePath() const +{ + return drvPath->getBaseStorePath(); } + +const StorePath & DerivedPath::Built::getBaseStorePath() const +{ + return drvPath->getBaseStorePath(); +} + +template +static inline const StorePath & getBaseStorePath_(const DP & derivedPath) +{ + return std::visit(overloaded { + [&](const typename DP::Built & bfd) -> auto & { + return bfd.drvPath->getBaseStorePath(); + }, + [&](const typename DP::Opaque & bo) -> auto & { + return bo.path; + }, + }, derivedPath.raw()); +} + +const StorePath & SingleDerivedPath::getBaseStorePath() const +{ + return getBaseStorePath_(*this); +} + +const StorePath & DerivedPath::getBaseStorePath() const +{ + return getBaseStorePath_(*this); +} + } diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 5f7acbebc25b..6c5dfeed9cc2 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -1,11 +1,10 @@ #pragma once ///@file -#include "util.hh" #include "path.hh" -#include "realisation.hh" #include "outputs-spec.hh" #include "comparator.hh" +#include "config.hh" #include @@ -25,28 +24,37 @@ class Store; struct DerivedPathOpaque { StorePath path; - nlohmann::json toJSON(ref store) const; std::string to_string(const Store & store) const; static DerivedPathOpaque parse(const Store & store, std::string_view); + nlohmann::json toJSON(const Store & store) const; GENERATE_CMP(DerivedPathOpaque, me->path); }; +struct SingleDerivedPath; + /** - * A derived path that is built from a derivation - * - * Built derived paths are pair of a derivation and some output names. - * They are evaluated by building the derivation, and then replacing the - * output names with the resulting outputs. + * A single derived path that is built from a derivation * - * Note that does mean a derived store paths evaluates to multiple - * opaque paths, which is sort of icky as expressions are supposed to - * evaluate to single values. Perhaps this should have just a single - * output name. + * Built derived paths are pair of a derivation and an output name. They are + * evaluated by building the derivation, and then taking the resulting output + * path of the given output name. */ -struct DerivedPathBuilt { - StorePath drvPath; - OutputsSpec outputs; +struct SingleDerivedPathBuilt { + ref drvPath; + OutputName output; + + /** + * Get the store path this is ultimately derived from (by realising + * and projecting outputs). + * + * Note that this is *not* a property of the store object being + * referred to, but just of this path --- how we happened to be + * referring to that store object. In other words, this means this + * function breaks "referential transparency". It should therefore + * be used only with great care. + */ + const StorePath & getBaseStorePath() const; /** * Uses `^` as the separator @@ -58,21 +66,27 @@ struct DerivedPathBuilt { std::string to_string_legacy(const Store & store) const; /** * The caller splits on the separator, so it works for both variants. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ - static DerivedPathBuilt parse(const Store & store, std::string_view drvPath, std::string_view outputs); - nlohmann::json toJSON(ref store) const; + static SingleDerivedPathBuilt parse( + const Store & store, ref drvPath, + OutputNameView outputs, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + nlohmann::json toJSON(Store & store) const; - GENERATE_CMP(DerivedPathBuilt, me->drvPath, me->outputs); + DECLARE_CMP(SingleDerivedPathBuilt); }; -using _DerivedPathRaw = std::variant< +using _SingleDerivedPathRaw = std::variant< DerivedPathOpaque, - DerivedPathBuilt + SingleDerivedPathBuilt >; /** - * A "derived path" is a very simple sort of expression that evaluates - * to (concrete) store path. It is either: + * A "derived path" is a very simple sort of expression (not a Nix + * language expression! But an expression in a the general sense) that + * evaluates to (concrete) store path. It is either: * * - opaque, in which case it is just a concrete store path with * possibly no known derivation @@ -80,17 +94,29 @@ using _DerivedPathRaw = std::variant< * - built, in which case it is a pair of a derivation path and an * output name. */ -struct DerivedPath : _DerivedPathRaw { - using Raw = _DerivedPathRaw; +struct SingleDerivedPath : _SingleDerivedPathRaw { + using Raw = _SingleDerivedPathRaw; using Raw::Raw; using Opaque = DerivedPathOpaque; - using Built = DerivedPathBuilt; + using Built = SingleDerivedPathBuilt; inline const Raw & raw() const { return static_cast(*this); } + /** + * Get the store path this is ultimately derived from (by realising + * and projecting outputs). + * + * Note that this is *not* a property of the store object being + * referred to, but just of this path --- how we happened to be + * referring to that store object. In other words, this means this + * function breaks "referential transparency". It should therefore + * be used only with great care. + */ + const StorePath & getBaseStorePath() const; + /** * Uses `^` as the separator */ @@ -101,55 +127,165 @@ struct DerivedPath : _DerivedPathRaw { std::string to_string_legacy(const Store & store) const; /** * Uses `^` as the separator + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ - static DerivedPath parse(const Store & store, std::string_view); + static SingleDerivedPath parse( + const Store & store, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); /** * Uses `!` as the separator + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ - static DerivedPath parseLegacy(const Store & store, std::string_view); + static SingleDerivedPath parseLegacy( + const Store & store, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + nlohmann::json toJSON(Store & store) const; }; +static inline ref makeConstantStorePathRef(StorePath drvPath) +{ + return make_ref(SingleDerivedPath::Opaque { drvPath }); +} + /** - * A built derived path with hints in the form of optional concrete output paths. + * A set of derived paths that are built from a derivation * - * See 'BuiltPath' for more an explanation. + * Built derived paths are pair of a derivation and some output names. + * They are evaluated by building the derivation, and then replacing the + * output names with the resulting outputs. + * + * Note that does mean a derived store paths evaluates to multiple + * opaque paths, which is sort of icky as expressions are supposed to + * evaluate to single values. Perhaps this should have just a single + * output name. */ -struct BuiltPathBuilt { - StorePath drvPath; - std::map outputs; +struct DerivedPathBuilt { + ref drvPath; + OutputsSpec outputs; - nlohmann::json toJSON(ref store) const; - static BuiltPathBuilt parse(const Store & store, std::string_view); + /** + * Get the store path this is ultimately derived from (by realising + * and projecting outputs). + * + * Note that this is *not* a property of the store object being + * referred to, but just of this path --- how we happened to be + * referring to that store object. In other words, this means this + * function breaks "referential transparency". It should therefore + * be used only with great care. + */ + const StorePath & getBaseStorePath() const; - GENERATE_CMP(BuiltPathBuilt, me->drvPath, me->outputs); + /** + * Uses `^` as the separator + */ + std::string to_string(const Store & store) const; + /** + * Uses `!` as the separator + */ + std::string to_string_legacy(const Store & store) const; + /** + * The caller splits on the separator, so it works for both variants. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static DerivedPathBuilt parse( + const Store & store, ref, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + nlohmann::json toJSON(Store & store) const; + + DECLARE_CMP(DerivedPathBuilt); }; -using _BuiltPathRaw = std::variant< - DerivedPath::Opaque, - BuiltPathBuilt +using _DerivedPathRaw = std::variant< + DerivedPathOpaque, + DerivedPathBuilt >; /** - * A built path. Similar to a DerivedPath, but enriched with the corresponding - * output path(s). + * A "derived path" is a very simple sort of expression that evaluates + * to one or more (concrete) store paths. It is either: + * + * - opaque, in which case it is just a single concrete store path with + * possibly no known derivation + * + * - built, in which case it is a pair of a derivation path and some + * output names. */ -struct BuiltPath : _BuiltPathRaw { - using Raw = _BuiltPathRaw; +struct DerivedPath : _DerivedPathRaw { + using Raw = _DerivedPathRaw; using Raw::Raw; using Opaque = DerivedPathOpaque; - using Built = BuiltPathBuilt; + using Built = DerivedPathBuilt; inline const Raw & raw() const { return static_cast(*this); } - StorePathSet outPaths() const; - RealisedPath::Set toRealisedPaths(Store & store) const; + /** + * Get the store path this is ultimately derived from (by realising + * and projecting outputs). + * + * Note that this is *not* a property of the store object being + * referred to, but just of this path --- how we happened to be + * referring to that store object. In other words, this means this + * function breaks "referential transparency". It should therefore + * be used only with great care. + */ + const StorePath & getBaseStorePath() const; + /** + * Uses `^` as the separator + */ + std::string to_string(const Store & store) const; + /** + * Uses `!` as the separator + */ + std::string to_string_legacy(const Store & store) const; + /** + * Uses `^` as the separator + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static DerivedPath parse( + const Store & store, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + /** + * Uses `!` as the separator + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static DerivedPath parseLegacy( + const Store & store, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + + /** + * Convert a `SingleDerivedPath` to a `DerivedPath`. + */ + static DerivedPath fromSingle(const SingleDerivedPath &); + + nlohmann::json toJSON(Store & store) const; }; typedef std::vector DerivedPaths; -typedef std::vector BuiltPaths; +/** + * Used by various parser functions to require experimental features as + * needed. + * + * Somewhat unfortunate this cannot just be an implementation detail for + * this module. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ +void drvRequireExperiment( + const SingleDerivedPath & drv, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); } diff --git a/src/libstore/downstream-placeholder.cc b/src/libstore/downstream-placeholder.cc index 1752738f2fd7..ca9f7476e7b2 100644 --- a/src/libstore/downstream-placeholder.cc +++ b/src/libstore/downstream-placeholder.cc @@ -5,14 +5,16 @@ namespace nix { std::string DownstreamPlaceholder::render() const { - return "/" + hash.to_string(Base32, false); + return "/" + hash.to_string(HashFormat::Base32, false); } DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput( const StorePath & drvPath, - std::string_view outputName) + OutputNameView outputName, + const ExperimentalFeatureSettings & xpSettings) { + xpSettings.require(Xp::CaDerivations); auto drvNameWithExtension = drvPath.name(); auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4); auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName); @@ -23,17 +25,34 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput( DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation( const DownstreamPlaceholder & placeholder, - std::string_view outputName, + OutputNameView outputName, const ExperimentalFeatureSettings & xpSettings) { xpSettings.require(Xp::DynamicDerivations); auto compressed = compressHash(placeholder.hash, 20); auto clearText = "nix-computed-output:" - + compressed.to_string(Base32, false) + + compressed.to_string(HashFormat::Base32, false) + ":" + std::string { outputName }; return DownstreamPlaceholder { hashString(htSHA256, clearText) }; } +DownstreamPlaceholder DownstreamPlaceholder::fromSingleDerivedPathBuilt( + const SingleDerivedPath::Built & b, + const ExperimentalFeatureSettings & xpSettings) +{ + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) { + return DownstreamPlaceholder::unknownCaOutput(o.path, b.output, xpSettings); + }, + [&](const SingleDerivedPath::Built & b2) { + return DownstreamPlaceholder::unknownDerivation( + DownstreamPlaceholder::fromSingleDerivedPathBuilt(b2, xpSettings), + b.output, + xpSettings); + }, + }, b.drvPath->raw()); +} + } diff --git a/src/libstore/downstream-placeholder.hh b/src/libstore/downstream-placeholder.hh index f0c0dee773ee..c911ecea2ed7 100644 --- a/src/libstore/downstream-placeholder.hh +++ b/src/libstore/downstream-placeholder.hh @@ -3,6 +3,7 @@ #include "hash.hh" #include "path.hh" +#include "derived-path.hh" namespace nix { @@ -52,10 +53,13 @@ public: * * The derivation itself is known (we have a store path for it), but * the output doesn't yet have a known store path. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ static DownstreamPlaceholder unknownCaOutput( const StorePath & drvPath, - std::string_view outputName); + OutputNameView outputName, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); /** * Create a placehold for the output of an unknown derivation. @@ -68,7 +72,19 @@ public: */ static DownstreamPlaceholder unknownDerivation( const DownstreamPlaceholder & drvPlaceholder, - std::string_view outputName, + OutputNameView outputName, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + + /** + * Convenience constructor that handles both cases (unknown + * content-addressed output and unknown derivation), delegating as + * needed to `unknownCaOutput` and `unknownDerivation`. + * + * Recursively builds up a placeholder from a + * `SingleDerivedPath::Built.drvPath` chain. + */ + static DownstreamPlaceholder fromSingleDerivedPathBuilt( + const SingleDerivedPath::Built & built, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); }; diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 74d6ed3b518a..821cda399c9e 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -72,7 +72,7 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store Callback> callback) noexcept override { callback(nullptr); } - virtual ref getFSAccessor() override + virtual ref getFSAccessor(bool requireValidPath) override { unsupported("getFSAccessor"); } }; diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index e866aeb42d27..52130f8f6e47 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -1,8 +1,8 @@ #include "serialise.hh" #include "store-api.hh" #include "archive.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" +#include "common-protocol.hh" +#include "common-protocol-impl.hh" #include @@ -41,13 +41,13 @@ void Store::exportPath(const StorePath & path, Sink & sink) Hash hash = hashSink.currentHash().first; if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) throw Error("hash of path '%s' has changed from '%s' to '%s'!", - printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true)); + printStorePath(path), info->narHash.to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true)); teeSink << exportMagic << printStorePath(path); - WorkerProto::write(*this, - WorkerProto::WriteConn { .to = teeSink }, + CommonProto::write(*this, + CommonProto::WriteConn { .to = teeSink }, info->references); teeSink << (info->deriver ? printStorePath(*info->deriver) : "") @@ -65,7 +65,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) /* Extract the NAR from the source. */ StringSink saved; TeeSource tee { source, saved }; - ParseSink ether; + NullParseSink ether; parseDump(ether, tee); uint32_t magic = readInt(source); @@ -76,8 +76,8 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) //Activity act(*logger, lvlInfo, "importing path '%s'", info.path); - auto references = WorkerProto::Serialise::read(*this, - WorkerProto::ReadConn { .from = source }); + auto references = CommonProto::Serialise::read(*this, + CommonProto::ReadConn { .from = source }); auto deriver = readString(source); auto narHash = hashString(htSHA256, saved.s); diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 38b691279b35..dcbec4acd848 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -1,11 +1,12 @@ #include "filetransfer.hh" -#include "util.hh" +#include "namespaces.hh" #include "globals.hh" #include "store-api.hh" #include "s3.hh" #include "compression.hh" #include "finally.hh" #include "callback.hh" +#include "signals.hh" #if ENABLE_S3 #include @@ -863,6 +864,8 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink) } chunk = std::move(state->data); + /* Reset state->data after the move, since we check data.empty() */ + state->data = ""; state->request.notify_one(); } diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh deleted file mode 100644 index 1df19e647094..000000000000 --- a/src/libstore/fs-accessor.hh +++ /dev/null @@ -1,52 +0,0 @@ -#pragma once -///@file - -#include "types.hh" - -namespace nix { - -/** - * An abstract class for accessing a filesystem-like structure, such - * as a (possibly remote) Nix store or the contents of a NAR file. - */ -class FSAccessor -{ -public: - enum Type { tMissing, tRegular, tSymlink, tDirectory }; - - struct Stat - { - Type type = tMissing; - /** - * regular files only - */ - uint64_t fileSize = 0; - /** - * regular files only - */ - bool isExecutable = false; // regular files only - /** - * regular files only - */ - uint64_t narOffset = 0; // regular files only - }; - - virtual ~FSAccessor() { } - - virtual Stat stat(const Path & path) = 0; - - virtual StringSet readDirectory(const Path & path) = 0; - - /** - * Read a file inside the store. - * - * If `requireValidPath` is set to `true` (the default), the path must be - * inside a valid store path, otherwise it just needs to be physically - * present (but not necessarily properly registered) - */ - virtual std::string readFile(const Path & path, bool requireValidPath = true) = 0; - - virtual std::string readLink(const Path & path) = 0; -}; - -} diff --git a/src/libstore/gc-store.hh b/src/libstore/gc-store.hh index 2c26c65c42ca..ab1059fb1ece 100644 --- a/src/libstore/gc-store.hh +++ b/src/libstore/gc-store.hh @@ -71,19 +71,36 @@ struct GCResults }; +/** + * Mix-in class for \ref Store "stores" which expose a notion of garbage + * collection. + * + * Garbage collection will allow deleting paths which are not + * transitively "rooted". + * + * The notion of GC roots actually not part of this class. + * + * - The base `Store` class has `Store::addTempRoot()` because for a store + * that doesn't support garbage collection at all, a temporary GC root is + * safely implementable as no-op. + * + * @todo actually this is not so good because stores are *views*. + * Some views have only a no-op temp roots even though others to the + * same store allow triggering GC. For instance one can't add a root + * over ssh, but that doesn't prevent someone from gc-ing that store + * accesed via SSH locally). + * + * - The derived `LocalFSStore` class has `LocalFSStore::addPermRoot`, + * which is not part of this class because it relies on the notion of + * an ambient file system. There are stores (`ssh-ng://`, for one), + * that *do* support garbage collection but *don't* expose any file + * system, and `LocalFSStore::addPermRoot` thus does not make sense + * for them. + */ struct GcStore : public virtual Store { inline static std::string operationName = "Garbage collection"; - /** - * Add an indirect root, which is merely a symlink to `path` from - * `/nix/var/nix/gcroots/auto/`. `path` is supposed - * to be a symlink to a store path. The garbage collector will - * automatically remove the indirect root when it finds that - * `path` has disappeared. - */ - virtual void addIndirectRoot(const Path & path) = 0; - /** * Find the roots of the garbage collector. Each root is a pair * `(link, storepath)` where `link` is the path of the symlink diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 20720fb99ab6..93fa60682ce9 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -1,8 +1,14 @@ #include "derivations.hh" #include "globals.hh" #include "local-store.hh" -#include "local-fs-store.hh" #include "finally.hh" +#include "unix-domain-socket.hh" +#include "signals.hh" + +#if !defined(__linux__) +// For shelling out to lsof +# include "processes.hh" +#endif #include #include @@ -44,13 +50,13 @@ static void makeSymlink(const Path & link, const Path & target) void LocalStore::addIndirectRoot(const Path & path) { - std::string hash = hashString(htSHA1, path).to_string(Base32, false); + std::string hash = hashString(htSHA1, path).to_string(HashFormat::Base32, false); Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", stateDir, gcRootsDir, hash)); makeSymlink(realRoot, path); } -Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot) +Path IndirectRootStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot) { Path gcRoot(canonPath(_gcRoot)); @@ -324,9 +330,7 @@ typedef std::unordered_map> UncheckedRoots static void readProcLink(const std::string & file, UncheckedRoots & roots) { - /* 64 is the starting buffer size gnu readlink uses... */ - auto bufsiz = ssize_t{64}; -try_again: + constexpr auto bufsiz = PATH_MAX; char buf[bufsiz]; auto res = readlink(file.c_str(), buf, bufsiz); if (res == -1) { @@ -335,10 +339,7 @@ static void readProcLink(const std::string & file, UncheckedRoots & roots) throw SysError("reading symlink"); } if (res == bufsiz) { - if (SSIZE_MAX / 2 < bufsiz) - throw Error("stupidly long symlink"); - bufsiz *= 2; - goto try_again; + throw Error("overly long symlink starting with '%1%'", std::string_view(buf, bufsiz)); } if (res > 0 && buf[0] == '/') roots[std::string(static_cast(buf), res)] @@ -777,7 +778,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } }; - /* Synchronisation point for testing, see tests/gc-concurrent.sh. */ + /* Synchronisation point for testing, see tests/functional/gc-concurrent.sh. */ if (auto p = getEnv("_NIX_TEST_GC_SYNC")) readFile(*p); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 73fd1f8da83a..ac83d907eb30 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -1,7 +1,8 @@ #include "globals.hh" -#include "util.hh" +#include "current-process.hh" #include "archive.hh" #include "args.hh" +#include "users.hh" #include "abstract-setting-to-json.hh" #include "compute-levels.hh" @@ -17,13 +18,20 @@ #include #ifdef __GLIBC__ -#include -#include -#include +# include +# include +# include +#endif + +#if __APPLE__ +# include "processes.hh" #endif #include "config-impl.hh" +#ifdef __APPLE__ +#include +#endif namespace nix { @@ -154,6 +162,29 @@ unsigned int Settings::getDefaultCores() return concurrency; } +#if __APPLE__ +static bool hasVirt() { + + int hasVMM; + int hvSupport; + size_t size; + + size = sizeof(hasVMM); + if (sysctlbyname("kern.hv_vmm_present", &hasVMM, &size, NULL, 0) == 0) { + if (hasVMM) + return false; + } + + // whether the kernel and hardware supports virt + size = sizeof(hvSupport); + if (sysctlbyname("kern.hv_support", &hvSupport, &size, NULL, 0) == 0) { + return hvSupport == 1; + } else { + return false; + } +} +#endif + StringSet Settings::getDefaultSystemFeatures() { /* For backwards compatibility, accept some "features" that are @@ -170,6 +201,11 @@ StringSet Settings::getDefaultSystemFeatures() features.insert("kvm"); #endif + #if __APPLE__ + if (hasVirt()) + features.insert("apple-virt"); + #endif + return features; } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 4015a2040be3..c4ab38922056 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -3,7 +3,8 @@ #include "types.hh" #include "config.hh" -#include "util.hh" +#include "environment-variables.hh" +#include "experimental-features.hh" #include #include @@ -116,10 +117,11 @@ public: Setting storeUri{this, getEnv("NIX_REMOTE").value_or("auto"), "store", R"( - The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + The [URL of the Nix store](@docroot@/store/types/index.md#store-url-format) to use for most operations. - See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md) - for supported store types and settings. + See the + [Store Types](@docroot@/store/types/index.md) + section of the manual for supported store types and settings. )"}; Setting keepFailed{this, false, "keep-failed", @@ -182,7 +184,9 @@ public: command line switch and defaults to `1`. The value `0` means that the builder should use all available CPU cores in the system. )", - {"build-cores"}, false}; + {"build-cores"}, + // Don't document the machine-specific default value + false}; /** * Read-only mode. Don't copy stuff to the store, don't change @@ -193,18 +197,24 @@ public: Setting thisSystem{ this, SYSTEM, "system", R"( - This option specifies the canonical Nix system name of the current - installation, such as `i686-linux` or `x86_64-darwin`. Nix can only - build derivations whose `system` attribute equals the value - specified here. In general, it never makes sense to modify this - value from its default, since you can use it to ‘lie’ about the - platform you are building on (e.g., perform a Mac OS build on a - Linux machine; the result would obviously be wrong). It only makes - sense if the Nix binaries can run on multiple platforms, e.g., - ‘universal binaries’ that run on `x86_64-linux` and `i686-linux`. - - It defaults to the canonical Nix system name detected by `configure` - at build time. + The system type of the current Nix installation. + Nix will only build a given [derivation](@docroot@/language/derivations.md) locally when its `system` attribute equals any of the values specified here or in [`extra-platforms`](#conf-extra-platforms). + + The default value is set when Nix itself is compiled for the system it will run on. + The following system types are widely used, as [Nix is actively supported on these platforms](@docroot@/contributing/hacking.md#platforms): + + - `x86_64-linux` + - `x86_64-darwin` + - `i686-linux` + - `aarch64-linux` + - `aarch64-darwin` + - `armv6l-linux` + - `armv7l-linux` + + In general, you do not have to modify this setting. + While you can force Nix to run a Darwin-specific `builder` executable on a Linux machine, the result would obviously be wrong. + + This value is available in the Nix language as [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem). )"}; Setting maxSilentTime{ @@ -255,6 +265,14 @@ public: For the exact format and examples, see [the manual chapter on remote builds](../advanced-topics/distributed-builds.md) )"}; + Setting alwaysAllowSubstitutes{ + this, false, "always-allow-substitutes", + R"( + If set to `true`, Nix will ignore the `allowSubstitutes` attribute in + derivations and always attempt to use available substituters. + For more information on `allowSubstitutes`, see [the manual chapter on advanced attributes](../language/advanced-attributes.md). + )"}; + Setting buildersUseSubstitutes{ this, false, "builders-use-substitutes", R"( @@ -337,7 +355,7 @@ public: users in `build-users-group`. UIDs are allocated starting at 872415232 (0x34000000) on Linux and 56930 on macOS. - )"}; + )", {}, true, Xp::AutoAllocateUids}; Setting startId{this, #if __linux__ @@ -524,13 +542,31 @@ public: Setting sandboxFallback{this, true, "sandbox-fallback", "Whether to disable sandboxing when the kernel doesn't allow it."}; + Setting requireDropSupplementaryGroups{this, getuid() == 0, "require-drop-supplementary-groups", + R"( + Following the principle of least privilege, + Nix will attempt to drop supplementary groups when building with sandboxing. + + However this can fail under some circumstances. + For example, if the user lacks the `CAP_SETGID` capability. + Search `setgroups(2)` for `EPERM` to find more detailed information on this. + + If you encounter such a failure, setting this option to `false` will let you ignore it and continue. + But before doing so, you should consider the security implications carefully. + Not dropping supplementary groups means the build sandbox will be less restricted than intended. + + This option defaults to `true` when the user is root + (since `root` usually has permissions to call setgroups) + and `false` otherwise. + )"}; + #if __linux__ Setting sandboxShmSize{ this, "50%", "sandbox-dev-shm-size", R"( This option determines the maximum size of the `tmpfs` filesystem mounted on `/dev/shm` in Linux sandboxes. For the format, see the - description of the `size` option of `tmpfs` in mount8. The default + description of the `size` option of `tmpfs` in mount(8). The default is `50%`. )"}; @@ -652,46 +688,79 @@ public: getDefaultExtraPlatforms(), "extra-platforms", R"( - Platforms other than the native one which this machine is capable of - building for. This can be useful for supporting additional - architectures on compatible machines: i686-linux can be built on - x86\_64-linux machines (and the default for this setting reflects - this); armv7 is backwards-compatible with armv6 and armv5tel; some - aarch64 machines can also natively run 32-bit ARM code; and - qemu-user may be used to support non-native platforms (though this - may be slow and buggy). Most values for this are not enabled by - default because build systems will often misdetect the target - platform and generate incompatible code, so you may wish to - cross-check the results of using this option against proper - natively-built versions of your derivations. - )", {}, false}; + System types of executables that can be run on this machine. + + Nix will only build a given [derivation](@docroot@/language/derivations.md) locally when its `system` attribute equals any of the values specified here or in the [`system` option](#conf-system). + + Setting this can be useful to build derivations locally on compatible machines: + - `i686-linux` executables can be run on `x86_64-linux` machines (set by default) + - `x86_64-darwin` executables can be run on macOS `aarch64-darwin` with Rosetta 2 (set by default where applicable) + - `armv6` and `armv5tel` executables can be run on `armv7` + - some `aarch64` machines can also natively run 32-bit ARM code + - `qemu-user` may be used to support non-native platforms (though this + may be slow and buggy) + + Build systems will usually detect the target platform to be the current physical system and therefore produce machine code incompatible with what may be intended in the derivation. + You should design your derivation's `builder` accordingly and cross-check the results when using this option against natively-built versions of your derivation. + )", + {}, + // Don't document the machine-specific default value + false}; Setting systemFeatures{ this, getDefaultSystemFeatures(), "system-features", R"( - A set of system “features” supported by this machine, e.g. `kvm`. - Derivations can express a dependency on such features through the - derivation attribute `requiredSystemFeatures`. For example, the - attribute + A set of system “features” supported by this machine. + + This complements the [`system`](#conf-system) and [`extra-platforms`](#conf-extra-platforms) configuration options and the corresponding [`system`](@docroot@/language/derivations.md#attr-system) attribute on derivations. + + A derivation can require system features in the [`requiredSystemFeatures` attribute](@docroot@/language/advanced-attributes.md#adv-attr-requiredSystemFeatures), and the machine to build the derivation must have them. + + System features are user-defined, but Nix sets the following defaults: + + - `apple-virt` + + Included on Darwin if virtualization is available. - requiredSystemFeatures = [ "kvm" ]; + - `kvm` - ensures that the derivation can only be built on a machine with the - `kvm` feature. + Included on Linux if `/dev/kvm` is accessible. - This setting by default includes `kvm` if `/dev/kvm` is accessible, - and the pseudo-features `nixos-test`, `benchmark` and `big-parallel` - that are used in Nixpkgs to route builds to specific machines. - )", {}, false}; + - `nixos-test`, `benchmark`, `big-parallel` + + These historical pseudo-features are always enabled for backwards compatibility, as they are used in Nixpkgs to route Hydra builds to specific machines. + + - `ca-derivations` + + Included by default if the [`ca-derivations` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) is enabled. + + This system feature is implicitly required by derivations with the [`__contentAddressed` attribute](@docroot@/language/advanced-attributes.md#adv-attr-__contentAddressed). + + - `recursive-nix` + + Included by default if the [`recursive-nix` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-recursive-nix) is enabled. + + - `uid-range` + + On Linux, Nix can run builds in a user namespace where they run as root (UID 0) and have 65,536 UIDs available. + This is primarily useful for running containers such as `systemd-nspawn` inside a Nix build. For an example, see [`tests/systemd-nspawn/nix`][nspawn]. + + [nspawn]: https://github.com/NixOS/nix/blob/67bcb99700a0da1395fa063d7c6586740b304598/tests/systemd-nspawn.nix. + + Included by default on Linux if the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting is enabled. + )", + {}, + // Don't document the machine-specific default value + false}; Setting substituters{ this, Strings{"https://cache.nixos.org/"}, "substituters", R"( - A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) to be used as substituters, separated by whitespace. + A list of [URLs of Nix stores](@docroot@/store/types/index.md#store-url-format) to be used as substituters, separated by whitespace. A substituter is an additional [store]{@docroot@/glossary.md##gloss-store} from which Nix can obtain [store objects](@docroot@/glossary.md#gloss-store-object) instead of building them. Substituters are tried based on their priority value, which each substituter can set independently. @@ -710,7 +779,7 @@ public: Setting trustedSubstituters{ this, {}, "trusted-substituters", R"( - A list of [Nix store URLs](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), separated by whitespace. + A list of [Nix store URLs](@docroot@/store/types/index.md#store-url-format), separated by whitespace. These are not used by default, but users of the Nix daemon can enable them by specifying [`substituters`](#conf-substituters). Unprivileged users (those set in only [`allowed-users`](#conf-allowed-users) but not [`trusted-users`](#conf-trusted-users)) can pass as `substituters` only those URLs listed in `trusted-substituters`. @@ -992,7 +1061,7 @@ public: | `~/.nix-defexpr` | `$XDG_STATE_HOME/nix/defexpr` | | `~/.nix-channels` | `$XDG_STATE_HOME/nix/channels` | - If you already have Nix installed and are using [profiles](@docroot@/package-management/profiles.md) or [channels](@docroot@/package-management/channels.md), you should migrate manually when you enable this option. + If you already have Nix installed and are using [profiles](@docroot@/package-management/profiles.md) or [channels](@docroot@/command-ref/nix-channel.md), you should migrate manually when you enable this option. If `$XDG_STATE_HOME` is not set, use `$HOME/.local/state/nix` instead of `$XDG_STATE_HOME/nix`. This can be achieved with the following shell commands: @@ -1014,6 +1083,35 @@ public: Requires the `acls` experimental feature. )" }; + + Setting impureEnv {this, {}, "impure-env", + R"( + A list of items, each in the format of: + + - `name=value`: Set environment variable `name` to `value`. + + If the user is trusted (see `trusted-users` option), when building + a fixed-output derivation, environment variables set in this option + will be passed to the builder if they are listed in [`impureEnvVars`](@docroot@/language/advanced-attributes.md##adv-attr-impureEnvVars). + + This option is useful for, e.g., setting `https_proxy` for + fixed-output derivations and in a multi-user Nix installation, or + setting private access tokens when fetching a private repository. + )", + {}, // aliases + true, // document default + Xp::ConfigurableImpureEnv + }; + + Setting upgradeNixStorePathUrl{ + this, + "https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix", + "upgrade-nix-store-path-url", + R"( + Used by `nix upgrade-nix`, the URL of the file that contains the + store paths of the latest Nix release. + )" + }; }; diff --git a/src/libstore/granular-access-store.hh b/src/libstore/granular-access-store.hh index 51dcacc6e4b9..530e767672b1 100644 --- a/src/libstore/granular-access-store.hh +++ b/src/libstore/granular-access-store.hh @@ -14,16 +14,17 @@ struct StoreObjectDerivationOutput StorePath drvPath; std::string output; - StoreObjectDerivationOutput(DerivedPath::Built p) : drvPath(p.drvPath) + StoreObjectDerivationOutput(DerivedPath::Built p) : drvPath(p.drvPath->getBaseStorePath()) { - if (auto names = std::get_if(&p.outputs)) + if (auto names = std::get_if(&p.outputs.raw)) if (names->size() == 1) { output = *names->begin(); return; } throw Error("StoreObjectDerivationOutput requires a DerivedPathBuilt with just one named output"); - } - StoreObjectDerivationOutput(StorePath drvPath, std::string output = "out") : drvPath(drvPath), output(output) { }; + }; + StoreObjectDerivationOutput(SingleDerivedPathBuilt p) : drvPath(p.drvPath->getBaseStorePath()), output(p.output) { }; + StoreObjectDerivationOutput(StorePath drvPath, std::string output) : drvPath(drvPath), output(output) { }; GENERATE_CMP(StoreObjectDerivationOutput, me->drvPath, me->output); }; diff --git a/src/libstore/indirect-root-store.hh b/src/libstore/indirect-root-store.hh new file mode 100644 index 000000000000..c11679fe8b8e --- /dev/null +++ b/src/libstore/indirect-root-store.hh @@ -0,0 +1,72 @@ +#pragma once +///@file + +#include "local-fs-store.hh" + +namespace nix { + +/** + * Mix-in class for implementing permanent roots as a pair of a direct + * (strong) reference and indirect weak reference to the first + * reference. + * + * See methods for details on the operations it represents. + * + * @note + * To understand the purpose of this class, it might help to do some + * "closed-world" rather than "open-world" reasoning, and consider the + * problem it solved for us. This class was factored out from + * `LocalFSStore` in order to support the following table, which + * contains 4 concrete store types (non-abstract classes, exposed to the + * user), and how they implemented the two GC root methods: + * + * @note + * | | `addPermRoot()` | `addIndirectRoot()` | + * |-------------------|-----------------|---------------------| + * | `LocalStore` | local | local | + * | `UDSRemoteStore` | local | remote | + * | `SSHStore` | doesn't have | doesn't have | + * | `MountedSSHStore` | remote | doesn't have | + * + * @note + * Note how only the local implementations of `addPermRoot()` need + * `addIndirectRoot()`; that is what this class enforces. Without it, + * and with `addPermRoot()` and `addIndirectRoot()` both `virtual`, we + * would accidentally be allowing for a combinatorial explosion of + * possible implementations many of which make no sense. Having this and + * that invariant enforced cuts down that space. + */ +struct IndirectRootStore : public virtual LocalFSStore +{ + inline static std::string operationName = "Indirect GC roots registration"; + + /** + * Implementation of `LocalFSStore::addPermRoot` where the permanent + * root is a pair of + * + * - The user-facing symlink which all implementations must create + * + * - An additional weak reference known as the "indirect root" that + * points to that symlink. + * + * The garbage collector will automatically remove the indirect root + * when it finds that the symlink has disappeared. + * + * The implementation of this method is concrete, but it delegates + * to `addIndirectRoot()` which is abstract. + */ + Path addPermRoot(const StorePath & storePath, const Path & gcRoot) override final; + + /** + * Add an indirect root, which is a weak reference to the + * user-facing symlink created by `addPermRoot()`. + * + * @param path user-facing and user-controlled symlink to a store + * path. + * + * The form this weak-reference takes is implementation-specific. + */ + virtual void addIndirectRoot(const Path & path) = 0; +}; + +} diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index fa17d606d430..7314573547f2 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -3,11 +3,10 @@ #include "pool.hh" #include "remote-store.hh" #include "serve-protocol.hh" +#include "serve-protocol-impl.hh" #include "build-result.hh" #include "store-api.hh" #include "path-with-outputs.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" #include "ssh.hh" #include "derivations.hh" #include "callback.hh" @@ -18,10 +17,10 @@ struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig { using CommonSSHStoreConfig::CommonSSHStoreConfig; - const Setting remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", + const Setting remoteProgram{this, "nix-store", "remote-program", "Path to the `nix-store` executable on the remote machine."}; - const Setting maxConnections{(StoreConfig*) this, 1, "max-connections", + const Setting maxConnections{this, 1, "max-connections", "Maximum number of concurrent SSH connections."}; const std::string name() override { return "SSH Store"; } @@ -39,49 +38,45 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor // Hack for getting remote build log output. // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in // the documentation - const Setting logFD{(StoreConfig*) this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; + const Setting logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; struct Connection { std::unique_ptr sshConn; FdSink to; FdSource from; - int remoteVersion; + ServeProto::Version remoteVersion; bool good = true; /** - * Coercion to `WorkerProto::ReadConn`. This makes it easy to use the - * factored out worker protocol searlizers with a + * Coercion to `ServeProto::ReadConn`. This makes it easy to use the + * factored out serve protocol searlizers with a * `LegacySSHStore::Connection`. * - * The worker protocol connection types are unidirectional, unlike + * The serve protocol connection types are unidirectional, unlike * this type. - * - * @todo Use server protocol serializers, not worker protocol - * serializers, once we have made that distiction. */ - operator WorkerProto::ReadConn () + operator ServeProto::ReadConn () { - return WorkerProto::ReadConn { + return ServeProto::ReadConn { .from = from, + .version = remoteVersion, }; } /* - * Coercion to `WorkerProto::WriteConn`. This makes it easy to use the - * factored out worker protocol searlizers with a + * Coercion to `ServeProto::WriteConn`. This makes it easy to use the + * factored out serve protocol searlizers with a * `LegacySSHStore::Connection`. * - * The worker protocol connection types are unidirectional, unlike + * The serve protocol connection types are unidirectional, unlike * this type. - * - * @todo Use server protocol serializers, not worker protocol - * serializers, once we have made that distiction. */ - operator WorkerProto::WriteConn () + operator ServeProto::WriteConn () { - return WorkerProto::WriteConn { + return ServeProto::WriteConn { .to = to, + .version = remoteVersion, }; } }; @@ -183,7 +178,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor auto deriver = readString(conn->from); if (deriver != "") info->deriver = parseStorePath(deriver); - info->references = WorkerProto::Serialise::read(*this, *conn); + info->references = ServeProto::Serialise::read(*this, *conn); readLongLong(conn->from); // download size info->narSize = readLongLong(conn->from); @@ -216,8 +211,8 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor << ServeProto::Command::AddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base16, false); - WorkerProto::write(*this, *conn, info.references); + << info.narHash.to_string(HashFormat::Base16, false); + ServeProto::write(*this, *conn, info.references); conn->to << info.registrationTime << info.narSize @@ -246,7 +241,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor conn->to << exportMagic << printStorePath(info.path); - WorkerProto::write(*this, *conn, info.references); + ServeProto::write(*this, *conn, info.references); conn->to << (info.deriver ? printStorePath(*info.deriver) : "") << 0 @@ -324,20 +319,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor conn->to.flush(); - BuildResult status; - status.status = (BuildResult::Status) readInt(conn->from); - conn->from >> status.errorMsg; - - if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) - conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; - if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) { - auto builtOutputs = WorkerProto::Serialise::read(*this, *conn); - for (auto && [output, realisation] : builtOutputs) - status.builtOutputs.insert_or_assign( - std::move(output.outputName), - std::move(realisation)); - } - return status; + return ServeProto::Serialise::read(*this, *conn); } void buildPaths(const std::vector & drvPaths, BuildMode buildMode, std::shared_ptr evalStore) override @@ -358,6 +340,9 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor [&](const StorePath & drvPath) { throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath)); }, + [&](std::monostate) { + throw Error("wanted build derivation that is itself a build product, but the legacy ssh protocol doesn't support that. Try using ssh-ng://"); + }, }, sOrDrvPath); } conn->to << ss; @@ -378,7 +363,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor void ensurePath(const StorePath & path) override { unsupported("ensurePath"); } - virtual ref getFSAccessor() override + virtual ref getFSAccessor(bool requireValidPath) override { unsupported("getFSAccessor"); } /** @@ -406,10 +391,10 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor conn->to << ServeProto::Command::QueryClosure << includeOutputs; - WorkerProto::write(*this, *conn, paths); + ServeProto::write(*this, *conn, paths); conn->to.flush(); - for (auto & i : WorkerProto::Serialise::read(*this, *conn)) + for (auto & i : ServeProto::Serialise::read(*this, *conn)) out.insert(i); } @@ -422,10 +407,10 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor << ServeProto::Command::QueryValidPaths << false // lock << maybeSubstitute; - WorkerProto::write(*this, *conn, paths); + ServeProto::write(*this, *conn, paths); conn->to.flush(); - return WorkerProto::Serialise::read(*this, *conn); + return ServeProto::Serialise::read(*this, *conn); } void connect() override diff --git a/src/libstore/length-prefixed-protocol-helper.hh b/src/libstore/length-prefixed-protocol-helper.hh new file mode 100644 index 000000000000..4061b0cd6de0 --- /dev/null +++ b/src/libstore/length-prefixed-protocol-helper.hh @@ -0,0 +1,162 @@ +#pragma once +/** + * @file Reusable serialisers for serialization container types in a + * length-prefixed manner. + * + * Used by both the Worker and Serve protocols. + */ + +#include "types.hh" + +namespace nix { + +class Store; + +/** + * Reusable serialisers for serialization container types in a + * length-prefixed manner. + * + * @param T The type of the collection being serialised + * + * @param Inner This the most important parameter; this is the "inner" + * protocol. The user of this will substitute `MyProtocol` or similar + * when making a `MyProtocol::Serialiser>`. Note that the + * inside is allowed to call to call `Inner::Serialiser` on different + * types. This is especially important for `std::map` which doesn't have + * a single `T` but one `K` and one `V`. + */ +template +struct LengthPrefixedProtoHelper; + +/*! + * \typedef LengthPrefixedProtoHelper::S + * + * Read this as simply `using S = Inner::Serialise;`. + * + * It would be nice to use that directly, but C++ doesn't seem to allow + * it. The `typename` keyword needed to refer to `Inner` seems to greedy + * (low precedence), and then C++ complains that `Serialise` is not a + * type parameter but a real type. + * + * Making this `S` alias seems to be the only way to avoid these issues. + */ + +#define LENGTH_PREFIXED_PROTO_HELPER(Inner, T) \ + struct LengthPrefixedProtoHelper< Inner, T > \ + { \ + static T read(const Store & store, typename Inner::ReadConn conn); \ + static void write(const Store & store, typename Inner::WriteConn conn, const T & str); \ + private: \ + template using S = typename Inner::template Serialise; \ + } + +template +LENGTH_PREFIXED_PROTO_HELPER(Inner, std::vector); + +template +LENGTH_PREFIXED_PROTO_HELPER(Inner, std::set); + +template +LENGTH_PREFIXED_PROTO_HELPER(Inner, std::tuple); + +template +#define _X std::map +LENGTH_PREFIXED_PROTO_HELPER(Inner, _X); +#undef _X + +template +std::vector +LengthPrefixedProtoHelper>::read( + const Store & store, typename Inner::ReadConn conn) +{ + std::vector resSet; + auto size = readNum(conn.from); + while (size--) { + resSet.push_back(S::read(store, conn)); + } + return resSet; +} + +template +void +LengthPrefixedProtoHelper>::write( + const Store & store, typename Inner::WriteConn conn, const std::vector & resSet) +{ + conn.to << resSet.size(); + for (auto & key : resSet) { + S::write(store, conn, key); + } +} + +template +std::set +LengthPrefixedProtoHelper>::read( + const Store & store, typename Inner::ReadConn conn) +{ + std::set resSet; + auto size = readNum(conn.from); + while (size--) { + resSet.insert(S::read(store, conn)); + } + return resSet; +} + +template +void +LengthPrefixedProtoHelper>::write( + const Store & store, typename Inner::WriteConn conn, const std::set & resSet) +{ + conn.to << resSet.size(); + for (auto & key : resSet) { + S::write(store, conn, key); + } +} + +template +std::map +LengthPrefixedProtoHelper>::read( + const Store & store, typename Inner::ReadConn conn) +{ + std::map resMap; + auto size = readNum(conn.from); + while (size--) { + auto k = S::read(store, conn); + auto v = S::read(store, conn); + resMap.insert_or_assign(std::move(k), std::move(v)); + } + return resMap; +} + +template +void +LengthPrefixedProtoHelper>::write( + const Store & store, typename Inner::WriteConn conn, const std::map & resMap) +{ + conn.to << resMap.size(); + for (auto & i : resMap) { + S::write(store, conn, i.first); + S::write(store, conn, i.second); + } +} + +template +std::tuple +LengthPrefixedProtoHelper>::read( + const Store & store, typename Inner::ReadConn conn) +{ + return std::tuple { + S::read(store, conn)..., + }; +} + +template +void +LengthPrefixedProtoHelper>::write( + const Store & store, typename Inner::WriteConn conn, const std::tuple & res) +{ + std::apply([&](const Us &... args) { + (S::write(store, conn, args), ...); + }, res); +} + +} diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index b224fc3e989b..953f3a264eb0 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -1,5 +1,5 @@ #include "archive.hh" -#include "fs-accessor.hh" +#include "posix-source-accessor.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "globals.hh" @@ -13,69 +13,53 @@ LocalFSStore::LocalFSStore(const Params & params) { } -struct LocalStoreAccessor : public FSAccessor +struct LocalStoreAccessor : PosixSourceAccessor { ref store; + bool requireValidPath; - LocalStoreAccessor(ref store) : store(store) { } + LocalStoreAccessor(ref store, bool requireValidPath) + : store(store) + , requireValidPath(requireValidPath) + { } - Path toRealPath(const Path & path, bool requireValidPath = true) + CanonPath toRealPath(const CanonPath & path) { - auto storePath = store->toStorePath(path).first; + auto [storePath, rest] = store->toStorePath(path.abs()); if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); - return store->getRealStoreDir() + std::string(path, store->storeDir.size()); + return CanonPath(store->getRealStoreDir()) + storePath.to_string() + CanonPath(rest); } - FSAccessor::Stat stat(const Path & path) override + std::optional maybeLstat(const CanonPath & path) override { - auto realPath = toRealPath(path); - - struct stat st; - if (lstat(realPath.c_str(), &st)) { - if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false}; - throw SysError("getting status of '%1%'", path); - } - - if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode)) - throw Error("file '%1%' has unsupported type", path); - - return { - S_ISREG(st.st_mode) ? Type::tRegular : - S_ISLNK(st.st_mode) ? Type::tSymlink : - Type::tDirectory, - S_ISREG(st.st_mode) ? (uint64_t) st.st_size : 0, - S_ISREG(st.st_mode) && st.st_mode & S_IXUSR}; + return PosixSourceAccessor::maybeLstat(toRealPath(path)); } - StringSet readDirectory(const Path & path) override + DirEntries readDirectory(const CanonPath & path) override { - auto realPath = toRealPath(path); - - auto entries = nix::readDirectory(realPath); - - StringSet res; - for (auto & entry : entries) - res.insert(entry.name); - - return res; + return PosixSourceAccessor::readDirectory(toRealPath(path)); } - std::string readFile(const Path & path, bool requireValidPath = true) override + void readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) override { - return nix::readFile(toRealPath(path, requireValidPath)); + return PosixSourceAccessor::readFile(toRealPath(path), sink, sizeCallback); } - std::string readLink(const Path & path) override + std::string readLink(const CanonPath & path) override { - return nix::readLink(toRealPath(path)); + return PosixSourceAccessor::readLink(toRealPath(path)); } }; -ref LocalFSStore::getFSAccessor() +ref LocalFSStore::getFSAccessor(bool requireValidPath) { return make_ref(ref( - std::dynamic_pointer_cast(shared_from_this()))); + std::dynamic_pointer_cast(shared_from_this())), + requireValidPath); } void LocalFSStore::narFromPath(const StorePath & path, Sink & sink) diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index e4a96c5f4b8e..f730518362c4 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -12,25 +12,21 @@ struct LocalFSStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - // FIXME: the (StoreConfig*) cast works around a bug in gcc that causes - // it to omit the call to the Setting constructor. Clang works fine - // either way. - - const OptionalPathSetting rootDir{(StoreConfig*) this, std::nullopt, + const OptionalPathSetting rootDir{this, std::nullopt, "root", "Directory prefixed to all other paths."}; - const PathSetting stateDir{(StoreConfig*) this, + const PathSetting stateDir{this, rootDir.get() ? *rootDir.get() + "/nix/var/nix" : settings.nixStateDir, "state", "Directory where Nix will store state."}; - const PathSetting logDir{(StoreConfig*) this, + const PathSetting logDir{this, rootDir.get() ? *rootDir.get() + "/nix/var/log/nix" : settings.nixLogDir, "log", "directory where Nix will store log files."}; - const PathSetting realStoreDir{(StoreConfig*) this, + const PathSetting realStoreDir{this, rootDir.get() ? *rootDir.get() + "/nix/store" : storeDir, "real", "Physical path of the Nix store."}; }; @@ -41,18 +37,30 @@ class LocalFSStore : public virtual LocalFSStoreConfig, public virtual LogStore { public: + inline static std::string operationName = "Local Filesystem Store"; const static std::string drvsLogDir; LocalFSStore(const Params & params); void narFromPath(const StorePath & path, Sink & sink) override; - ref getFSAccessor() override; + ref getFSAccessor(bool requireValidPath) override; /** - * Register a permanent GC root. + * Creates symlink from the `gcRoot` to the `storePath` and + * registers the `gcRoot` as a permanent GC root. The `gcRoot` + * symlink lives outside the store and is created and owned by the + * user. + * + * @param gcRoot The location of the symlink. + * + * @param storePath The store object being rooted. The symlink will + * point to `toRealPath(store.printStorePath(storePath))`. + * + * How the permanent GC root corresponding to this symlink is + * managed is implementation-specific. */ - Path addPermRoot(const StorePath & storePath, const Path & gcRoot); + virtual Path addPermRoot(const StorePath & storePath, const Path & gcRoot) = 0; virtual Path getRealStoreDir() { return realStoreDir; } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 312b09d46da2..7164a923e2fa 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -14,6 +14,8 @@ #include "topo-sort.hh" #include "finally.hh" #include "compression.hh" +#include "signals.hh" +#include "posix-fs-canonicalise.hh" #include #include @@ -40,7 +42,6 @@ #include #include #include -#include #endif #ifdef __CYGWIN__ @@ -601,179 +602,6 @@ void LocalStore::makeStoreWritable() } -const time_t mtimeStore = 1; /* 1 second into the epoch */ - - -static void canonicaliseTimestampAndPermissions(const Path & path, const struct stat & st) -{ - if (!S_ISLNK(st.st_mode)) { - - if (experimentalFeatureSettings.isEnabled(Xp::ACLs)) { - /* Mask out all type related bits. */ - mode_t mode = st.st_mode & ~S_IFMT; - - if (mode != 0440 && mode != 0550 && mode != 0444 && mode != 0555) { - mode = (st.st_mode & S_IFMT) - | 0444 - | (st.st_mode & S_IXUSR ? 0111 : 0); - if (! (st.st_mode & S_IRWXO)) - mode &= ~S_IRWXO; - if (chmod(path.c_str(), mode) == -1) - throw SysError("changing mode of '%1%' to %2$o", path, mode); - } - } else { - /* Mask out all type related bits. */ - mode_t mode = st.st_mode & ~S_IFMT; - - if (mode != 0444 && mode != 0555) { - mode = (st.st_mode & S_IFMT) - | 0444 - | (st.st_mode & S_IXUSR ? 0111 : 0); - if (chmod(path.c_str(), mode) == -1) - throw SysError("changing mode of '%1%' to %2$o", path, mode); - } - } - - } - - if (st.st_mtime != mtimeStore) { - struct timeval times[2]; - times[0].tv_sec = st.st_atime; - times[0].tv_usec = 0; - times[1].tv_sec = mtimeStore; - times[1].tv_usec = 0; -#if HAVE_LUTIMES - if (lutimes(path.c_str(), times) == -1) - if (errno != ENOSYS || - (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)) -#else - if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) -#endif - throw SysError("changing modification time of '%1%'", path); - } -} - - -void canonicaliseTimestampAndPermissions(const Path & path) -{ - canonicaliseTimestampAndPermissions(path, lstat(path)); -} - - -static void canonicalisePathMetaData_( - const Path & path, - std::optional> uidRange, - InodesSeen & inodesSeen) -{ - checkInterrupt(); - -#if __APPLE__ - /* Remove flags, in particular UF_IMMUTABLE which would prevent - the file from being garbage-collected. FIXME: Use - setattrlist() to remove other attributes as well. */ - if (lchflags(path.c_str(), 0)) { - if (errno != ENOTSUP) - throw SysError("clearing flags of path '%1%'", path); - } -#endif - - auto st = lstat(path); - - /* Really make sure that the path is of a supported type. */ - if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) - throw Error("file '%1%' has an unsupported type", path); - -#if __linux__ - /* Remove extended attributes / ACLs. */ - ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0); - - if (eaSize < 0) { - if (errno != ENOTSUP && errno != ENODATA) - throw SysError("querying extended attributes of '%s'", path); - } else if (eaSize > 0) { - std::vector eaBuf(eaSize); - - if ((eaSize = llistxattr(path.c_str(), eaBuf.data(), eaBuf.size())) < 0) - throw SysError("querying extended attributes of '%s'", path); - - for (auto & eaName: tokenizeString(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) { - if (settings.ignoredAcls.get().count(eaName)) continue; - if (lremovexattr(path.c_str(), eaName.c_str()) == -1) - throw SysError("removing extended attribute '%s' from '%s'", eaName, path); - } - } -#endif - - /* Fail if the file is not owned by the build user. This prevents - us from messing up the ownership/permissions of files - hard-linked into the output (e.g. "ln /etc/shadow $out/foo"). - However, ignore files that we chown'ed ourselves previously to - ensure that we don't fail on hard links within the same build - (i.e. "touch $out/foo; ln $out/foo $out/bar"). */ - if (uidRange && (st.st_uid < uidRange->first || st.st_uid > uidRange->second)) { - if (S_ISDIR(st.st_mode) || !inodesSeen.count(Inode(st.st_dev, st.st_ino))) - throw BuildError("invalid ownership on file '%1%'", path); - mode_t mode = st.st_mode & ~S_IFMT; - assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore)); - return; - } - - inodesSeen.insert(Inode(st.st_dev, st.st_ino)); - - canonicaliseTimestampAndPermissions(path, st); - - /* Change ownership to the current uid. If it's a symlink, use - lchown if available, otherwise don't bother. Wrong ownership - of a symlink doesn't matter, since the owning user can't change - the symlink and can't delete it because the directory is not - writable. The only exception is top-level paths in the Nix - store (since that directory is group-writable for the Nix build - users group); we check for this case below. */ - if (st.st_uid != geteuid()) { -#if HAVE_LCHOWN - if (lchown(path.c_str(), geteuid(), getegid()) == -1) -#else - if (!S_ISLNK(st.st_mode) && - chown(path.c_str(), geteuid(), getegid()) == -1) -#endif - throw SysError("changing owner of '%1%' to %2%", - path, geteuid()); - } - - if (S_ISDIR(st.st_mode)) { - DirEntries entries = readDirectory(path); - for (auto & i : entries) - canonicalisePathMetaData_(path + "/" + i.name, uidRange, inodesSeen); - } -} - - -void canonicalisePathMetaData( - const Path & path, - std::optional> uidRange, - InodesSeen & inodesSeen) -{ - canonicalisePathMetaData_(path, uidRange, inodesSeen); - - /* On platforms that don't have lchown(), the top-level path can't - be a symlink, since we can't change its ownership. */ - auto st = lstat(path); - - if (st.st_uid != geteuid()) { - assert(S_ISLNK(st.st_mode)); - throw Error("wrong ownership of top-level store path '%1%'", path); - } -} - - -void canonicalisePathMetaData(const Path & path, - std::optional> uidRange) -{ - InodesSeen inodesSeen; - canonicalisePathMetaData(path, uidRange, inodesSeen); -} - - void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) { experimentalFeatureSettings.require(Xp::CaDerivations); @@ -863,7 +691,7 @@ uint64_t LocalStore::addValidPath(State & state, state.stmts->RegisterValidPath.use() (printStorePath(info.path)) - (info.narHash.to_string(Base16, true)) + (info.narHash.to_string(HashFormat::Base16, true)) (info.registrationTime == 0 ? time(0) : info.registrationTime) (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver) (info.narSize, info.narSize != 0) @@ -976,7 +804,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) { state.stmts->UpdatePathInfo.use() (info.narSize, info.narSize != 0) - (info.narHash.to_string(Base16, true)) + (info.narHash.to_string(HashFormat::Base16, true)) (info.ultimate ? 1 : 0, info.ultimate) (concatStringsSep(" ", info.sigs), !info.sigs.empty()) (renderContentAddress(info.ca), (bool) info.ca) @@ -1068,10 +896,9 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) std::map> -LocalStore::queryPartialDerivationOutputMap(const StorePath & path_) +LocalStore::queryStaticPartialDerivationOutputMap(const StorePath & path) { - auto path = path_; - auto outputs = retrySQLite>>([&]() { + return retrySQLite>>([&]() { auto state(_state.lock()); std::map> outputs; uint64_t drvId; @@ -1083,21 +910,6 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath & path_) return outputs; }); - - if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) - return outputs; - - auto drv = readInvalidDerivation(path); - auto drvHashes = staticOutputHashes(*this, drv); - for (auto& [outputName, hash] : drvHashes) { - auto realisation = queryRealisation(DrvOutput{hash, outputName}); - if (realisation) - outputs.insert_or_assign(outputName, realisation->outPath); - else - outputs.insert({outputName, std::nullopt}); - } - - return outputs; } std::optional LocalStore::queryPathFromHashPart(const std::string & hashPart) @@ -1584,6 +1396,15 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (checkSigs && pathInfoIsUntrusted(info)) throw Error("cannot add path '%s' because it lacks a signature by a trusted key", printStorePath(info.path)); + /* In case we are not interested in reading the NAR: discard it. */ + bool narRead = false; + Finally cleanup = [&]() { + if (!narRead) { + NullParseSink sink; + parseDump(sink, source); + } + }; + addTempRoot(info.path); if (repair || !isValidPath(info.path) || !canAccess(info.path)) { @@ -1598,42 +1419,6 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (!locksHeld.count(printStorePath(info.path))) outputLock.lockPaths({realPath}); - auto checkInfoValidity = [&](HashResult hashResult){ - if (hashResult.first != info.narHash) - throw Error("hash mismatch importing path '%s';\n specified: %s\n got: %s", - printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true)); - - if (hashResult.second != info.narSize) - throw Error("size mismatch importing path '%s';\n specified: %s\n got: %s", - printStorePath(info.path), info.narSize, hashResult.second); - - if (info.ca) { - if (auto foHash = std::get_if(&info.ca->raw)) { - auto actualFoHash = hashCAPath( - foHash->method, - foHash->hash.type, - info.path - ); - if (foHash->hash != actualFoHash.hash) { - throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", - printStorePath(info.path), - foHash->hash.to_string(Base32, true), - actualFoHash.hash.to_string(Base32, true)); - } - } - if (auto textHash = std::get_if(&info.ca->raw)) { - auto actualTextHash = hashString(htSHA256, readFile(realPath)); - if (textHash->hash != actualTextHash) { - throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", - printStorePath(info.path), - textHash->hash.to_string(Base32, true), - actualTextHash.to_string(Base32, true)); - } - } - } - }; - - if (repair || !isValidPath(info.path)) { deletePath(realPath); @@ -1644,28 +1429,34 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, TeeSource wrapperSource { source, hashSink }; - restorePath(realPath, wrapperSource, experimentalFeatureSettings.isEnabled(Xp::ACLs)); - - checkInfoValidity(hashSink.finish()); + narRead = true; + restorePath(realPath, wrapperSource); - autoGC(); - - canonicalisePathMetaData(realPath, {}); - - optimisePath(realPath, repair); // FIXME: combine with hashPath() + auto hashResult = hashSink.finish(); - registerValidPath(info); - } else if (effectiveUser && !canAccess(info.path)) { - auto curInfo = queryPathInfo(info.path); - HashSink hashSink(htSHA256); - source.drainInto(hashSink); + if (hashResult.first != info.narHash) + throw Error("hash mismatch importing path '%s';\n specified: %s\n got: %s", + printStorePath(info.path), info.narHash.to_string(HashFormat::Base32, true), hashResult.first.to_string(HashFormat::Base32, true)); - /* Check that both new and old info matches */ - checkInfoValidity(hashSink.finish()); - checkInfoValidity({curInfo->narHash, curInfo->narSize}); + if (hashResult.second != info.narSize) + throw Error("size mismatch importing path '%s';\n specified: %s\n got: %s", + printStorePath(info.path), info.narSize, hashResult.second); - addAllowedEntities(info.path, {*effectiveUser}); - } + if (info.ca) { + auto & specified = *info.ca; + auto actualHash = hashCAPath( + specified.method, + specified.hash.type, + info.path + ); + if (specified.hash != actualHash.hash) { + throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", + printStorePath(info.path), + specified.hash.to_string(HashFormat::Base32, true), + actualHash.hash.to_string(HashFormat::Base32, true)); + } + } + }; outputLock.setDeletion(true); } @@ -1736,10 +1527,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name auto [hash, size] = hashSink->finish(); ContentAddressWithReferences desc = FixedOutputInfo { - .hash = { - .method = method, - .hash = hash, - }, + .method = method, + .hash = hash, .references = { .others = references, // caller is not capable of creating a self-reference, because this is content-addressed without modulus @@ -1815,8 +1604,8 @@ StorePath LocalStore::addTextToStore( { auto hash = hashString(htSHA256, s); auto dstPath = makeTextPath(name, TextInfo { - { .hash = hash }, - references, + .hash = hash, + .references = references, }); addTempRoot(dstPath); @@ -1848,7 +1637,10 @@ StorePath LocalStore::addTextToStore( ValidPathInfo info { dstPath, narHash }; info.narSize = sink.s.size(); info.references = references; - info.ca = TextHash { .hash = hash }; + info.ca = { + .method = TextIngestionMethod {}, + .hash = hash, + }; registerValidPath(info); } @@ -1913,17 +1705,33 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) auto fdGCLock = openGCLock(); FdLock gcLock(fdGCLock.get(), ltRead, true, "waiting for the big garbage collector lock..."); - StringSet store; - for (auto & i : readDirectory(realStoreDir)) store.insert(i.name); + StorePathSet validPaths; + + { + StorePathSet storePathsInStoreDir; + /* Why aren't we using `queryAllValidPaths`? Because that would + tell us about all the paths than the database knows about. Here we + want to know about all the store paths in the store directory, + regardless of what the database thinks. + + We will end up cross-referencing these two sources of truth (the + database and the filesystem) in the loop below, in order to catch + invalid states. + */ + for (auto & i : readDirectory(realStoreDir)) { + try { + storePathsInStoreDir.insert({i.name}); + } catch (BadStorePath &) { } + } - /* Check whether all valid paths actually exist. */ - printInfo("checking path existence..."); + /* Check whether all valid paths actually exist. */ + printInfo("checking path existence..."); - StorePathSet validPaths; - PathSet done; + StorePathSet done; - for (auto & i : queryAllValidPaths()) - verifyPath(printStorePath(i), store, done, validPaths, repair, errors); + for (auto & i : queryAllValidPaths()) + verifyPath(i, storePathsInStoreDir, done, validPaths, repair, errors); + } /* Optionally, check the content hashes (slow). */ if (checkContents) { @@ -1933,7 +1741,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) for (auto & link : readDirectory(linksDir)) { printMsg(lvlTalkative, "checking contents of '%s'", link.name); Path linkPath = linksDir + "/" + link.name; - std::string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false); + std::string hash = hashPath(htSHA256, linkPath).first.to_string(HashFormat::Base32, false); if (hash != link.name) { printError("link '%s' was modified! expected hash '%s', got '%s'", linkPath, link.name, hash); @@ -1966,7 +1774,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) if (info->narHash != nullHash && info->narHash != current.first) { printError("path '%s' was modified! expected hash '%s', got '%s'", - printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)); + printStorePath(i), info->narHash.to_string(HashFormat::Base32, true), current.first.to_string(HashFormat::Base32, true)); if (repair) repairPath(i); else errors = true; } else { @@ -2009,32 +1817,27 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) } -void LocalStore::verifyPath(const Path & pathS, const StringSet & store, - PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors) +void LocalStore::verifyPath(const StorePath & path, const StorePathSet & storePathsInStoreDir, + StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors) { checkInterrupt(); - if (!done.insert(pathS).second) return; - - if (!isStorePath(pathS)) { - printError("path '%s' is not in the Nix store", pathS); - return; - } - - auto path = parseStorePath(pathS); + if (!done.insert(path).second) return; - if (!store.count(std::string(path.to_string()))) { + if (!storePathsInStoreDir.count(path)) { /* Check any referrers first. If we can invalidate them first, then we can invalidate this path as well. */ bool canInvalidate = true; StorePathSet referrers; queryReferrers(path, referrers); for (auto & i : referrers) if (i != path) { - verifyPath(printStorePath(i), store, done, validPaths, repair, errors); + verifyPath(i, storePathsInStoreDir, done, validPaths, repair, errors); if (validPaths.count(i)) canInvalidate = false; } + auto pathS = printStorePath(path); + if (canInvalidate) { printInfo("path '%s' disappeared, removing from database...", pathS); auto state(_state.lock()); @@ -2245,33 +2048,39 @@ void LocalStore::queryRealisationUncached(const DrvOutput & id, } } -FixedOutputHash LocalStore::hashCAPath( - const FileIngestionMethod & method, const HashType & hashType, +ContentAddress LocalStore::hashCAPath( + const ContentAddressMethod & method, const HashType & hashType, const StorePath & path) { return hashCAPath(method, hashType, Store::toRealPath(path), path.hashPart()); } -FixedOutputHash LocalStore::hashCAPath( - const FileIngestionMethod & method, +ContentAddress LocalStore::hashCAPath( + const ContentAddressMethod & method, const HashType & hashType, const Path & path, const std::string_view pathHash ) { HashModuloSink caSink ( hashType, std::string(pathHash) ); - switch (method) { - case FileIngestionMethod::Recursive: - dumpPath(path, caSink); - break; - case FileIngestionMethod::Flat: - readFile(path, caSink); - break; - } - auto hash = caSink.finish().first; - return FixedOutputHash{ + std::visit(overloaded { + [&](const TextIngestionMethod &) { + readFile(path, caSink); + }, + [&](const FileIngestionMethod & m2) { + switch (m2) { + case FileIngestionMethod::Recursive: + dumpPath(path, caSink); + break; + case FileIngestionMethod::Flat: + readFile(path, caSink); + break; + } + }, + }, method.raw); + return ContentAddress { .method = method, - .hash = hash, + .hash = caSink.finish().first, }; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index bdf2bbd244b0..63e2f6eb6602 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -5,10 +5,8 @@ #include "pathlocks.hh" #include "store-api.hh" -#include "local-fs-store.hh" -#include "gc-store.hh" +#include "indirect-root-store.hh" #include "sync.hh" -#include "util.hh" #include #include @@ -41,12 +39,12 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig { using LocalFSStoreConfig::LocalFSStoreConfig; - Setting requireSigs{(StoreConfig*) this, + Setting requireSigs{this, settings.requireSigs, "require-sigs", "Whether store paths copied into this store should have a trusted signature."}; - Setting readOnly{(StoreConfig*) this, + Setting readOnly{this, false, "read-only", R"( @@ -68,7 +66,11 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig std::string doc() override; }; -class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore, public virtual GcStore, public virtual LocalGranularAccessStore +class LocalStore : public virtual LocalStoreConfig + , public virtual LocalFSStore + , public virtual GcStore + , public virtual LocalGranularAccessStore + , public virtual IndirectRootStore { private: @@ -175,7 +177,7 @@ public: StorePathSet queryValidDerivers(const StorePath & path) override; - std::map> queryPartialDerivationOutputMap(const StorePath & path) override; + std::map> queryStaticPartialDerivationOutputMap(const StorePath & path) override; std::optional queryPathFromHashPart(const std::string & hashPart) override; @@ -219,6 +221,12 @@ private: public: + /** + * Implementation of IndirectRootStore::addIndirectRoot(). + * + * The weak reference merely is a symlink to `path' from + * /nix/var/nix/gcroots/auto/. + */ void addIndirectRoot(const Path & path) override; private: @@ -330,8 +338,8 @@ private: */ void invalidatePathChecked(const StorePath & path); - void verifyPath(const Path & path, const StringSet & store, - PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); + void verifyPath(const StorePath & path, const StorePathSet & store, + StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); std::shared_ptr queryPathInfoInternal(State & state, const StorePath & path); @@ -368,13 +376,13 @@ private: void signRealisation(Realisation &); // XXX: Make a generic `Store` method - FixedOutputHash hashCAPath( - const FileIngestionMethod & method, + ContentAddress hashCAPath( + const ContentAddressMethod & method, const HashType & hashType, const StorePath & path); - FixedOutputHash hashCAPath( - const FileIngestionMethod & method, + ContentAddress hashCAPath( + const ContentAddressMethod & method, const HashType & hashType, const Path & path, const std::string_view pathHash @@ -388,38 +396,4 @@ private: friend struct DerivationGoal; }; - -typedef std::pair Inode; -typedef std::set InodesSeen; - - -/** - * "Fix", or canonicalise, the meta-data of the files in a store path - * after it has been built. In particular: - * - * - the last modification date on each file is set to 1 (i.e., - * 00:00:01 1/1/1970 UTC) - * - * - the permissions are set of 444 or 555 (i.e., read-only with or - * without execute permission; setuid bits etc. are cleared) - * - * - the owner and group are set to the Nix user and group, if we're - * running as root. - * - * If uidRange is not empty, this function will throw an error if it - * encounters files owned by a user outside of the closed interval - * [uidRange->first, uidRange->second]. - */ -void canonicalisePathMetaData( - const Path & path, - std::optional> uidRange, - InodesSeen & inodesSeen); -void canonicalisePathMetaData( - const Path & path, - std::optional> uidRange); - -void canonicaliseTimestampAndPermissions(const Path & path); - -MakeError(PathInUse, Error); - } diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 438af83259c7..b39d349bdad6 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -63,7 +63,7 @@ $(d)/build.cc: clean-files += $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh -$(eval $(call install-file-in, $(d)/nix-store.pc, $(libdir)/pkgconfig, 0644)) +$(eval $(call install-file-in, $(buildprefix)$(d)/nix-store.pc, $(libdir)/pkgconfig, 0644)) $(foreach i, $(wildcard src/libstore/builtins/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/builtins, 0644))) diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index 83b46d1058b3..41602d9e55ca 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -1,4 +1,5 @@ #include "lock.hh" +#include "file-system.hh" #include "globals.hh" #include "pathlocks.hh" @@ -7,6 +8,31 @@ namespace nix { +#if __linux__ + +static std::vector get_group_list(const char *username, gid_t group_id) +{ + std::vector gids; + gids.resize(32); // Initial guess + + auto getgroupl_failed {[&] { + int ngroups = gids.size(); + int err = getgrouplist(username, group_id, gids.data(), &ngroups); + gids.resize(ngroups); + return err == -1; + }}; + + // The first error means that the vector was not big enough. + // If it happens again, there is some different problem. + if (getgroupl_failed() && getgroupl_failed()) { + throw SysError("failed to get list of supplementary groups for '%s'", username); + } + + return gids; +} +#endif + + struct SimpleUserLock : UserLock { AutoCloseFD fdUserLock; @@ -67,37 +93,14 @@ struct SimpleUserLock : UserLock throw Error("the Nix user should not be a member of '%s'", settings.buildUsersGroup); #if __linux__ - /* Get the list of supplementary groups of this build - user. This is usually either empty or contains a - group such as "kvm". */ - int ngroups = 32; // arbitrary initial guess - std::vector gids; - gids.resize(ngroups); - - int err = getgrouplist( - pw->pw_name, pw->pw_gid, - gids.data(), - &ngroups); - - /* Our initial size of 32 wasn't sufficient, the - correct size has been stored in ngroups, so we try - again. */ - if (err == -1) { - gids.resize(ngroups); - err = getgrouplist( - pw->pw_name, pw->pw_gid, - gids.data(), - &ngroups); - } - - // If it failed once more, then something must be broken. - if (err == -1) - throw Error("failed to get list of supplementary groups for '%s'", pw->pw_name); + /* Get the list of supplementary groups of this user. This is + * usually either empty or contains a group such as "kvm". */ // Finally, trim back the GID list to its real size. - for (auto i = 0; i < ngroups; i++) - if (gids[i] != lock->gid) - lock->supplementaryGIDs.push_back(gids[i]); + for (auto gid : get_group_list(pw->pw_name, pw->pw_gid)) { + if (gid != lock->gid) + lock->supplementaryGIDs.push_back(gid); + } #endif return lock; diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index e87f469800dc..512115893edc 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -1,5 +1,4 @@ #include "machines.hh" -#include "util.hh" #include "globals.hh" #include "store-api.hh" diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 53fe04704215..253609ed2a9d 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -52,10 +52,8 @@ std::map makeContentAddressed( dstStore, path.name(), FixedOutputInfo { - .hash = { - .method = FileIngestionMethod::Recursive, - .hash = narModuloHash, - }, + .method = FileIngestionMethod::Recursive, + .hash = narModuloHash, .references = std::move(refs), }, Hash::dummy, @@ -80,4 +78,15 @@ std::map makeContentAddressed( return remappings; } +StorePath makeContentAddressed( + Store & srcStore, + Store & dstStore, + const StorePath & fromPath) +{ + auto remappings = makeContentAddressed(srcStore, dstStore, StorePathSet { fromPath }); + auto i = remappings.find(fromPath); + assert(i != remappings.end()); + return i->second; +} + } diff --git a/src/libstore/make-content-addressed.hh b/src/libstore/make-content-addressed.hh index 2ce6ec7bc715..60bb2b477db0 100644 --- a/src/libstore/make-content-addressed.hh +++ b/src/libstore/make-content-addressed.hh @@ -5,9 +5,20 @@ namespace nix { +/** Rewrite a closure of store paths to be completely content addressed. + */ std::map makeContentAddressed( Store & srcStore, Store & dstStore, - const StorePathSet & storePaths); + const StorePathSet & rootPaths); + +/** Rewrite a closure of a store path to be completely content addressed. + * + * This is a convenience function for the case where you only have one root path. + */ +StorePath makeContentAddressed( + Store & srcStore, + Store & dstStore, + const StorePath & rootPath); } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index c9caa8722872..bf44aa78e289 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -88,7 +88,7 @@ const ContentAddress * getDerivationCA(const BasicDerivation & drv) auto out = drv.outputs.find("out"); if (out == drv.outputs.end()) return nullptr; - if (auto dof = std::get_if(&out->second)) { + if (auto dof = std::get_if(&out->second.raw)) { return &dof->ca; } return nullptr; @@ -125,14 +125,26 @@ void Store::queryMissing(const std::vector & targets, std::function doPath; + std::function, const DerivedPathMap::ChildNode &)> enqueueDerivedPaths; + + enqueueDerivedPaths = [&](ref inputDrv, const DerivedPathMap::ChildNode & inputNode) { + if (!inputNode.value.empty()) + pool.enqueue(std::bind(doPath, DerivedPath::Built { inputDrv, inputNode.value })); + for (const auto & [outputName, childNode] : inputNode.childMap) + enqueueDerivedPaths( + make_ref(SingleDerivedPath::Built { inputDrv, outputName }), + childNode); + }; + auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) { { auto state(state_.lock()); state->willBuild.insert(drvPath); } - for (auto & i : drv.inputDrvs) - pool.enqueue(std::bind(doPath, DerivedPath::Built { i.first, i.second })); + for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) { + enqueueDerivedPaths(makeConstantStorePathRef(inputDrv), inputNode); + } }; auto checkOutput = [&]( @@ -176,10 +188,18 @@ void Store::queryMissing(const std::vector & targets, std::visit(overloaded { [&](const DerivedPath::Built & bfd) { - if (!isValidPath(bfd.drvPath)) { + auto drvPathP = std::get_if(&*bfd.drvPath); + if (!drvPathP) { + // TODO make work in this case. + warn("Ignoring dynamic derivation %s while querying missing paths; not yet implemented", bfd.drvPath->to_string(*this)); + return; + } + auto & drvPath = drvPathP->path; + + if (!isValidPath(drvPath)) { // FIXME: we could try to substitute the derivation. auto state(state_.lock()); - state->unknown.insert(bfd.drvPath); + state->unknown.insert(drvPath); return; } @@ -187,7 +207,7 @@ void Store::queryMissing(const std::vector & targets, /* true for regular derivations, and CA derivations for which we have a trust mapping for all wanted outputs. */ auto knownOutputPaths = true; - for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(bfd.drvPath)) { + for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(drvPath)) { if (!pathOpt) { knownOutputPaths = false; break; @@ -197,15 +217,45 @@ void Store::queryMissing(const std::vector & targets, } if (knownOutputPaths && invalid.empty()) return; - auto drv = make_ref(derivationFromPath(bfd.drvPath)); - ParsedDerivation parsedDrv(StorePath(bfd.drvPath), *drv); + auto drv = make_ref(derivationFromPath(drvPath)); + ParsedDerivation parsedDrv(StorePath(drvPath), *drv); + + if (!knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) { + experimentalFeatureSettings.require(Xp::CaDerivations); + + // If there are unknown output paths, attempt to find if the + // paths are known to substituters through a realisation. + auto outputHashes = staticOutputHashes(*this, *drv); + knownOutputPaths = true; + + for (auto [outputName, hash] : outputHashes) { + if (!bfd.outputs.contains(outputName)) + continue; + + bool found = false; + for (auto &sub : getDefaultSubstituters()) { + auto realisation = sub->queryRealisation({hash, outputName}); + if (!realisation) + continue; + found = true; + if (!isValidPath(realisation->outPath)) + invalid.insert(realisation->outPath); + break; + } + if (!found) { + // Some paths did not have a realisation, this must be built. + knownOutputPaths = false; + break; + } + } + } if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) { auto drvState = make_ref>(DrvState(invalid.size())); for (auto & output : invalid) - pool.enqueue(std::bind(checkOutput, bfd.drvPath, drv, output, drvState)); + pool.enqueue(std::bind(checkOutput, drvPath, drv, output, drvState)); } else - mustBuildDrv(bfd.drvPath, *drv); + mustBuildDrv(drvPath, *drv); }, [&](const DerivedPath::Opaque & bo) { @@ -284,38 +334,47 @@ std::map drvOutputReferences( { std::set inputRealisations; - for (const auto & [inputDrv, outputNames] : drv.inputDrvs) { - const auto outputHashes = - staticOutputHashes(store, store.readDerivation(inputDrv)); - for (const auto & outputName : outputNames) { - auto outputHash = get(outputHashes, outputName); - if (!outputHash) - throw Error( - "output '%s' of derivation '%s' isn't realised", outputName, - store.printStorePath(inputDrv)); - auto thisRealisation = store.queryRealisation( - DrvOutput{*outputHash, outputName}); - if (!thisRealisation) - throw Error( - "output '%s' of derivation '%s' isn't built", outputName, - store.printStorePath(inputDrv)); - inputRealisations.insert(*thisRealisation); + std::function::ChildNode &)> accumRealisations; + + accumRealisations = [&](const StorePath & inputDrv, const DerivedPathMap::ChildNode & inputNode) { + if (!inputNode.value.empty()) { + auto outputHashes = + staticOutputHashes(store, store.readDerivation(inputDrv)); + for (const auto & outputName : inputNode.value) { + auto outputHash = get(outputHashes, outputName); + if (!outputHash) + throw Error( + "output '%s' of derivation '%s' isn't realised", outputName, + store.printStorePath(inputDrv)); + auto thisRealisation = store.queryRealisation( + DrvOutput{*outputHash, outputName}); + if (!thisRealisation) + throw Error( + "output '%s' of derivation '%s' isn’t built", outputName, + store.printStorePath(inputDrv)); + inputRealisations.insert(*thisRealisation); + } } - } + if (!inputNode.value.empty()) { + auto d = makeConstantStorePathRef(inputDrv); + for (const auto & [outputName, childNode] : inputNode.childMap) { + SingleDerivedPath next = SingleDerivedPath::Built { d, outputName }; + accumRealisations( + // TODO deep resolutions for dynamic derivations, issue #8947, would go here. + resolveDerivedPath(store, next), + childNode); + } + } + }; + + for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) + accumRealisations(inputDrv, inputNode); auto info = store.queryPathInfo(outputPath); return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references); } -OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_) -{ - auto [outputs, missing] = resolveDerivedPathAll(store, bfd, evalStore_); - if (!missing.empty()) - throw MissingRealisation(*missing.begin()); - return outputs; -} - // FIXME refactor with resolveDerivedPath to remove repetition std::pair> resolveDerivedPathAll(Store & store, const DerivedPath::Built & bfd, Store * evalStore_) { @@ -323,7 +382,7 @@ std::pair> resolveDerivedPathAll(Store & stor OutputPathMap outputs; std::set missingOutputs; - auto drv = evalStore.readDerivation(bfd.drvPath); + auto drv = evalStore.readDerivation(bfd.drvPath->getBaseStorePath()); auto outputHashes = staticOutputHashes(store, drv); auto drvOutputs = drv.outputsAndOptPaths(store); auto outputNames = std::visit(overloaded { @@ -336,13 +395,13 @@ std::pair> resolveDerivedPathAll(Store & stor [&](const OutputsSpec::Names & names) { return static_cast>(names); }, - }, bfd.outputs.raw()); + }, bfd.outputs.raw); for (auto & output : outputNames) { auto outputHash = get(outputHashes, output); if (!outputHash) throw Error( "the derivation '%s' doesn't have an output named '%s'", - store.printStorePath(bfd.drvPath), output); + store.printStorePath(bfd.drvPath->getBaseStorePath()), output); if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { DrvOutput outputId { *outputHash, output }; auto realisation = store.queryRealisation(outputId); @@ -362,4 +421,91 @@ std::pair> resolveDerivedPathAll(Store & stor return {outputs, missingOutputs}; } +OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_) +{ + auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_); + + auto outputsOpt_ = store.queryPartialDerivationOutputMap(drvPath, evalStore_); + + auto outputsOpt = std::visit(overloaded { + [&](const OutputsSpec::All &) { + // Keep all outputs + return std::move(outputsOpt_); + }, + [&](const OutputsSpec::Names & names) { + // Get just those mentioned by name + std::map> outputsOpt; + for (auto & output : names) { + auto * pOutputPathOpt = get(outputsOpt_, output); + if (!pOutputPathOpt) + throw Error( + "the derivation '%s' doesn't have an output named '%s'", + bfd.drvPath->to_string(store), output); + outputsOpt.insert_or_assign(output, std::move(*pOutputPathOpt)); + } + return outputsOpt; + }, + }, bfd.outputs.raw); + + OutputPathMap outputs; + for (auto & [outputName, outputPathOpt] : outputsOpt) { + if (!outputPathOpt) + throw MissingRealisation(bfd.drvPath->to_string(store), outputName); + auto & outputPath = *outputPathOpt; + outputs.insert_or_assign(outputName, outputPath); + } + return outputs; +} + +StorePath resolveDerivedPath(Store & store, const SingleDerivedPath & req, Store * evalStore_) +{ + auto & evalStore = evalStore_ ? *evalStore_ : store; + + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) { + return bo.path; + }, + [&](const SingleDerivedPath::Built & bfd) { + auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_); + auto outputPaths = evalStore.queryPartialDerivationOutputMap(drvPath, evalStore_); + if (outputPaths.count(bfd.output) == 0) + throw Error("derivation '%s' does not have an output named '%s'", + store.printStorePath(drvPath), bfd.output); + auto & optPath = outputPaths.at(bfd.output); + if (!optPath) + throw MissingRealisation(bfd.drvPath->to_string(store), bfd.output); + return *optPath; + }, + }, req.raw()); +} + + +OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd) +{ + auto drvPath = resolveDerivedPath(store, *bfd.drvPath); + auto outputMap = store.queryDerivationOutputMap(drvPath); + auto outputsLeft = std::visit(overloaded { + [&](const OutputsSpec::All &) { + return StringSet {}; + }, + [&](const OutputsSpec::Names & names) { + return static_cast(names); + }, + }, bfd.outputs.raw); + for (auto iter = outputMap.begin(); iter != outputMap.end();) { + auto & outputName = iter->first; + if (bfd.outputs.contains(outputName)) { + outputsLeft.erase(outputName); + ++iter; + } else { + iter = outputMap.erase(iter); + } + } + if (!outputsLeft.empty()) + throw Error("derivation '%s' does not have an outputs %s", + store.printStorePath(drvPath), + concatStringsSep(", ", quoteStrings(std::get(bfd.outputs.raw)))); + return outputMap; +} + } diff --git a/src/libstore/mounted-ssh-store.md b/src/libstore/mounted-ssh-store.md new file mode 100644 index 000000000000..1ebfe3081dcc --- /dev/null +++ b/src/libstore/mounted-ssh-store.md @@ -0,0 +1,18 @@ +R"( + +**Store URL format**: `mounted-ssh-ng://[username@]hostname` + +Experimental store type that allows full access to a Nix store on a remote machine, +and additionally requires that store be mounted in the local file system. + +The mounting of that store is not managed by Nix, and must by managed manually. +It could be accomplished with SSHFS or NFS, for example. + +The local file system is used to optimize certain operations. +For example, rather than serializing Nix archives and sending over the Nix channel, +we can directly access the file system data via the mount-point. + +The local file system is also used to make certain operations possible that wouldn't otherwise be. +For example, persistent GC roots can be created if they reside on the same file system as the remote store: +the remote side will create the symlinks necessary to avoid race conditions. +)" diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index f0dfcb19b77e..15b05fe25fe2 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -11,13 +11,7 @@ namespace nix { struct NarMember { - FSAccessor::Type type = FSAccessor::Type::tMissing; - - bool isExecutable = false; - - /* If this is a regular file, position of the contents of this - file in the NAR. */ - uint64_t start = 0, size = 0; + SourceAccessor::Stat stat; std::string target; @@ -25,7 +19,7 @@ struct NarMember std::map children; }; -struct NarAccessor : public FSAccessor +struct NarAccessor : public SourceAccessor { std::optional nar; @@ -57,7 +51,7 @@ struct NarAccessor : public FSAccessor acc.root = std::move(member); parents.push(&acc.root); } else { - if (parents.top()->type != FSAccessor::Type::tDirectory) + if (parents.top()->stat.type != Type::tDirectory) throw Error("NAR file missing parent directory of path '%s'", path); auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member)); parents.push(&result.first->second); @@ -66,12 +60,22 @@ struct NarAccessor : public FSAccessor void createDirectory(const Path & path) override { - createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0}); + createMember(path, NarMember{ .stat = { + .type = Type::tDirectory, + .fileSize = 0, + .isExecutable = false, + .narOffset = 0 + } }); } void createRegularFile(const Path & path) override { - createMember(path, {FSAccessor::Type::tRegular, false, 0, 0}); + createMember(path, NarMember{ .stat = { + .type = Type::tRegular, + .fileSize = 0, + .isExecutable = false, + .narOffset = 0 + } }); } void closeRegularFile() override @@ -79,14 +83,14 @@ struct NarAccessor : public FSAccessor void isExecutable() override { - parents.top()->isExecutable = true; + parents.top()->stat.isExecutable = true; } void preallocateContents(uint64_t size) override { - assert(size <= std::numeric_limits::max()); - parents.top()->size = (uint64_t) size; - parents.top()->start = pos; + auto & st = parents.top()->stat; + st.fileSize = size; + st.narOffset = pos; } void receiveContents(std::string_view data) override @@ -95,7 +99,9 @@ struct NarAccessor : public FSAccessor void createSymlink(const Path & path, const std::string & target) override { createMember(path, - NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); + NarMember{ + .stat = {.type = Type::tSymlink}, + .target = target}); } size_t read(char * data, size_t len) override @@ -130,18 +136,19 @@ struct NarAccessor : public FSAccessor std::string type = v["type"]; if (type == "directory") { - member.type = FSAccessor::Type::tDirectory; - for (auto i = v["entries"].begin(); i != v["entries"].end(); ++i) { - std::string name = i.key(); - recurse(member.children[name], i.value()); + member.stat = {.type = Type::tDirectory}; + for (const auto &[name, function] : v["entries"].items()) { + recurse(member.children[name], function); } } else if (type == "regular") { - member.type = FSAccessor::Type::tRegular; - member.size = v["size"]; - member.isExecutable = v.value("executable", false); - member.start = v["narOffset"]; + member.stat = { + .type = Type::tRegular, + .fileSize = v["size"], + .isExecutable = v.value("executable", false), + .narOffset = v["narOffset"] + }; } else if (type == "symlink") { - member.type = FSAccessor::Type::tSymlink; + member.stat = {.type = Type::tSymlink}; member.target = v.value("target", ""); } else return; }; @@ -150,134 +157,122 @@ struct NarAccessor : public FSAccessor recurse(root, v); } - NarMember * find(const Path & path) + NarMember * find(const CanonPath & path) { - Path canon = path == "" ? "" : canonPath(path); NarMember * current = &root; - auto end = path.end(); - for (auto it = path.begin(); it != end; ) { - // because it != end, the remaining component is non-empty so we need - // a directory - if (current->type != FSAccessor::Type::tDirectory) return nullptr; - - // skip slash (canonPath above ensures that this is always a slash) - assert(*it == '/'); - it += 1; - - // lookup current component - auto next = std::find(it, end, '/'); - auto child = current->children.find(std::string(it, next)); + + for (const auto & i : path) { + if (current->stat.type != Type::tDirectory) return nullptr; + auto child = current->children.find(std::string(i)); if (child == current->children.end()) return nullptr; current = &child->second; - - it = next; } return current; } - NarMember & get(const Path & path) { + NarMember & get(const CanonPath & path) { auto result = find(path); - if (result == nullptr) + if (!result) throw Error("NAR file does not contain path '%1%'", path); return *result; } - Stat stat(const Path & path) override + std::optional maybeLstat(const CanonPath & path) override { auto i = find(path); - if (i == nullptr) - return {FSAccessor::Type::tMissing, 0, false}; - return {i->type, i->size, i->isExecutable, i->start}; + if (!i) + return std::nullopt; + return i->stat; } - StringSet readDirectory(const Path & path) override + DirEntries readDirectory(const CanonPath & path) override { auto i = get(path); - if (i.type != FSAccessor::Type::tDirectory) + if (i.stat.type != Type::tDirectory) throw Error("path '%1%' inside NAR file is not a directory", path); - StringSet res; - for (auto & child : i.children) - res.insert(child.first); + DirEntries res; + for (const auto & child : i.children) + res.insert_or_assign(child.first, std::nullopt); return res; } - std::string readFile(const Path & path, bool requireValidPath = true) override + std::string readFile(const CanonPath & path) override { auto i = get(path); - if (i.type != FSAccessor::Type::tRegular) + if (i.stat.type != Type::tRegular) throw Error("path '%1%' inside NAR file is not a regular file", path); - if (getNarBytes) return getNarBytes(i.start, i.size); + if (getNarBytes) return getNarBytes(*i.stat.narOffset, *i.stat.fileSize); assert(nar); - return std::string(*nar, i.start, i.size); + return std::string(*nar, *i.stat.narOffset, *i.stat.fileSize); } - std::string readLink(const Path & path) override + std::string readLink(const CanonPath & path) override { auto i = get(path); - if (i.type != FSAccessor::Type::tSymlink) + if (i.stat.type != Type::tSymlink) throw Error("path '%1%' inside NAR file is not a symlink", path); return i.target; } }; -ref makeNarAccessor(std::string && nar) +ref makeNarAccessor(std::string && nar) { return make_ref(std::move(nar)); } -ref makeNarAccessor(Source & source) +ref makeNarAccessor(Source & source) { return make_ref(source); } -ref makeLazyNarAccessor(const std::string & listing, +ref makeLazyNarAccessor(const std::string & listing, GetNarBytes getNarBytes) { return make_ref(listing, getNarBytes); } using nlohmann::json; -json listNar(ref accessor, const Path & path, bool recurse) +json listNar(ref accessor, const CanonPath & path, bool recurse) { - auto st = accessor->stat(path); + auto st = accessor->lstat(path); json obj = json::object(); switch (st.type) { - case FSAccessor::Type::tRegular: + case SourceAccessor::Type::tRegular: obj["type"] = "regular"; - obj["size"] = st.fileSize; + if (st.fileSize) + obj["size"] = *st.fileSize; if (st.isExecutable) obj["executable"] = true; - if (st.narOffset) - obj["narOffset"] = st.narOffset; + if (st.narOffset && *st.narOffset) + obj["narOffset"] = *st.narOffset; break; - case FSAccessor::Type::tDirectory: + case SourceAccessor::Type::tDirectory: obj["type"] = "directory"; { obj["entries"] = json::object(); json &res2 = obj["entries"]; - for (auto & name : accessor->readDirectory(path)) { + for (const auto & [name, type] : accessor->readDirectory(path)) { if (recurse) { - res2[name] = listNar(accessor, path + "/" + name, true); + res2[name] = listNar(accessor, path + name, true); } else res2[name] = json::object(); } } break; - case FSAccessor::Type::tSymlink: + case SourceAccessor::Type::tSymlink: obj["type"] = "symlink"; obj["target"] = accessor->readLink(path); break; - case FSAccessor::Type::tMissing: - default: - throw Error("path '%s' does not exist in NAR", path); + case SourceAccessor::Type::tMisc: + assert(false); // cannot happen for NARs } return obj; } diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh index 5e19bd3c755e..0043897c658c 100644 --- a/src/libstore/nar-accessor.hh +++ b/src/libstore/nar-accessor.hh @@ -1,10 +1,11 @@ #pragma once ///@file +#include "source-accessor.hh" + #include #include -#include "fs-accessor.hh" namespace nix { @@ -14,9 +15,9 @@ struct Source; * Return an object that provides access to the contents of a NAR * file. */ -ref makeNarAccessor(std::string && nar); +ref makeNarAccessor(std::string && nar); -ref makeNarAccessor(Source & source); +ref makeNarAccessor(Source & source); /** * Create a NAR accessor from a NAR listing (in the format produced by @@ -24,9 +25,9 @@ ref makeNarAccessor(Source & source); * readFile() method of the accessor to get the contents of files * inside the NAR. */ -typedef std::function GetNarBytes; +using GetNarBytes = std::function; -ref makeLazyNarAccessor( +ref makeLazyNarAccessor( const std::string & listing, GetNarBytes getNarBytes); @@ -34,6 +35,6 @@ ref makeLazyNarAccessor( * Write a JSON representation of the contents of a NAR (except file * contents). */ -nlohmann::json listNar(ref accessor, const Path & path, bool recurse); +nlohmann::json listNar(ref accessor, const CanonPath & path, bool recurse); } diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index c7176d30f7af..e50c15939dce 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -1,4 +1,5 @@ #include "nar-info-disk-cache.hh" +#include "users.hh" #include "sync.hh" #include "sqlite.hh" #include "globals.hh" @@ -332,9 +333,9 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache (std::string(info->path.name())) (narInfo ? narInfo->url : "", narInfo != 0) (narInfo ? narInfo->compression : "", narInfo != 0) - (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(Base32, true) : "", narInfo && narInfo->fileHash) + (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(HashFormat::Base32, true) : "", narInfo && narInfo->fileHash) (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) - (info->narHash.to_string(Base32, true)) + (info->narHash.to_string(HashFormat::Base32, true)) (info->narSize) (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 5bd9328a5a52..6528e667a487 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -6,6 +6,15 @@ namespace nix { +GENERATE_CMP_EXT( + , + NarInfo, + me->url, + me->compression, + me->fileHash, + me->fileSize, + static_cast(*me)); + NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence) : ValidPathInfo(StorePath(StorePath::dummy), Hash(Hash::dummy)) // FIXME: hack { @@ -31,12 +40,12 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & while (pos < s.size()) { size_t colon = s.find(':', pos); - if (colon == std::string::npos) throw corrupt("expecting ':'"); + if (colon == s.npos) throw corrupt("expecting ':'"); std::string name(s, pos, colon - pos); size_t eol = s.find('\n', colon + 2); - if (eol == std::string::npos) throw corrupt("expecting '\\n'"); + if (eol == s.npos) throw corrupt("expecting '\\n'"); std::string value(s, colon + 2, eol - colon - 2); @@ -125,10 +134,10 @@ std::string NarInfo::to_string(const Store & store) const assert(compression != ""); res += "Compression: " + compression + "\n"; assert(fileHash && fileHash->type == htSHA256); - res += "FileHash: " + fileHash->to_string(Base32, true) + "\n"; + res += "FileHash: " + fileHash->to_string(HashFormat::Base32, true) + "\n"; res += "FileSize: " + std::to_string(fileSize) + "\n"; assert(narHash.type == htSHA256); - res += "NarHash: " + narHash.to_string(Base32, true) + "\n"; + res += "NarHash: " + narHash.to_string(HashFormat::Base32, true) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n"; res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; @@ -154,4 +163,59 @@ std::string NarInfo::to_string(const Store & store) const return res; } +nlohmann::json NarInfo::toJSON( + const Store & store, + bool includeImpureInfo, + HashFormat hashFormat) const +{ + using nlohmann::json; + + auto jsonObject = ValidPathInfo::toJSON(store, includeImpureInfo, hashFormat); + + if (includeImpureInfo) { + if (!url.empty()) + jsonObject["url"] = url; + if (!compression.empty()) + jsonObject["compression"] = compression; + if (fileHash) + jsonObject["downloadHash"] = fileHash->to_string(hashFormat, true); + if (fileSize) + jsonObject["downloadSize"] = fileSize; + } + + return jsonObject; +} + +NarInfo NarInfo::fromJSON( + const Store & store, + const StorePath & path, + const nlohmann::json & json) +{ + using nlohmann::detail::value_t; + + NarInfo res { + ValidPathInfo { + path, + UnkeyedValidPathInfo::fromJSON(store, json), + } + }; + + if (json.contains("url")) + res.url = ensureType(valueAt(json, "url"), value_t::string); + + if (json.contains("compression")) + res.compression = ensureType(valueAt(json, "compression"), value_t::string); + + if (json.contains("downloadHash")) + res.fileHash = Hash::parseAny( + static_cast( + ensureType(valueAt(json, "downloadHash"), value_t::string)), + std::nullopt); + + if (json.contains("downloadSize")) + res.fileSize = ensureType(valueAt(json, "downloadSize"), value_t::number_integer); + + return res; +} + } diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 5dbdafac3ebd..fd538a7cd9fc 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -17,14 +17,25 @@ struct NarInfo : ValidPathInfo uint64_t fileSize = 0; NarInfo() = delete; - NarInfo(const Store & store, std::string && name, ContentAddressWithReferences && ca, Hash narHash) + NarInfo(const Store & store, std::string name, ContentAddressWithReferences ca, Hash narHash) : ValidPathInfo(store, std::move(name), std::move(ca), narHash) { } - NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } + NarInfo(StorePath path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const Store & store, const std::string & s, const std::string & whence); + DECLARE_CMP(NarInfo); + std::string to_string(const Store & store) const; + + nlohmann::json toJSON( + const Store & store, + bool includeImpureInfo, + HashFormat hashFormat) const override; + static NarInfo fromJSON( + const Store & store, + const StorePath & path, + const nlohmann::json & json); }; } diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 4a79cf4a1976..0fa977545c7b 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -1,6 +1,7 @@ -#include "util.hh" #include "local-store.hh" #include "globals.hh" +#include "signals.hh" +#include "posix-fs-canonicalise.hh" #include #include @@ -146,10 +147,10 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ Hash hash = hashPath(htSHA256, path).first; - debug("'%1%' has hash '%2%'", path, hash.to_string(Base32, true)); + debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Base32, true)); /* Check if this is a known hash. */ - Path linkPath = linksDir + "/" + hash.to_string(Base32, false); + Path linkPath = linksDir + "/" + hash.to_string(HashFormat::Base32, false); /* Maybe delete the link, if it has been corrupted. */ if (pathExists(linkPath)) { diff --git a/src/libstore/outputs-spec.cc b/src/libstore/outputs-spec.cc index e26c3813822f..21c06922379f 100644 --- a/src/libstore/outputs-spec.cc +++ b/src/libstore/outputs-spec.cc @@ -17,7 +17,7 @@ bool OutputsSpec::contains(const std::string & outputName) const [&](const OutputsSpec::Names & outputNames) { return outputNames.count(outputName) > 0; }, - }, raw()); + }, raw); } static std::string outputSpecRegexStr = @@ -49,7 +49,7 @@ OutputsSpec OutputsSpec::parse(std::string_view s) std::optional spec = parseOpt(s); if (!spec) throw Error("invalid outputs specifier '%s'", s); - return *spec; + return std::move(*spec); } @@ -63,7 +63,7 @@ std::optional> ExtendedOutputsS auto specOpt = OutputsSpec::parseOpt(s.substr(found + 1)); if (!specOpt) return std::nullopt; - return std::pair { s.substr(0, found), ExtendedOutputsSpec::Explicit { *std::move(specOpt) } }; + return std::pair { s.substr(0, found), ExtendedOutputsSpec::Explicit { std::move(*specOpt) } }; } @@ -85,7 +85,7 @@ std::string OutputsSpec::to_string() const [&](const OutputsSpec::Names & outputNames) -> std::string { return concatStringsSep(",", outputNames); }, - }, raw()); + }, raw); } @@ -98,7 +98,7 @@ std::string ExtendedOutputsSpec::to_string() const [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> std::string { return "^" + outputSpec.to_string(); }, - }, raw()); + }, raw); } @@ -118,9 +118,9 @@ OutputsSpec OutputsSpec::union_(const OutputsSpec & that) const ret.insert(thoseNames.begin(), thoseNames.end()); return ret; }, - }, that.raw()); + }, that.raw); }, - }, raw()); + }, raw); } @@ -142,9 +142,9 @@ bool OutputsSpec::isSubsetOf(const OutputsSpec & that) const ret = false; return ret; }, - }, raw()); + }, raw); }, - }, that.raw()); + }, that.raw); } } @@ -169,7 +169,7 @@ void adl_serializer::to_json(json & json, OutputsSpec t) { [&](const OutputsSpec::Names & names) { json = names; }, - }, t.raw()); + }, t.raw); } @@ -189,7 +189,7 @@ void adl_serializer::to_json(json & json, ExtendedOutputsSp [&](const ExtendedOutputsSpec::Explicit & e) { adl_serializer::to_json(json, e); }, - }, t.raw()); + }, t.raw); } } diff --git a/src/libstore/outputs-spec.hh b/src/libstore/outputs-spec.hh index 5a726fe907af..1ef99a5fc67d 100644 --- a/src/libstore/outputs-spec.hh +++ b/src/libstore/outputs-spec.hh @@ -6,63 +6,70 @@ #include #include +#include "comparator.hh" #include "json-impls.hh" +#include "comparator.hh" +#include "variant-wrapper.hh" namespace nix { /** - * A non-empty set of outputs, specified by name + * An (owned) output name. Just a type alias used to make code more + * readible. */ -struct OutputNames : std::set { - using std::set::set; +typedef std::string OutputName; - /* These need to be "inherited manually" */ - - OutputNames(const std::set & s) - : std::set(s) - { assert(!empty()); } +/** + * A borrowed output name. Just a type alias used to make code more + * readible. + */ +typedef std::string_view OutputNameView; +struct OutputsSpec { /** - * Needs to be "inherited manually" + * A non-empty set of outputs, specified by name */ - OutputNames(std::set && s) - : std::set(s) - { assert(!empty()); } - - /* This set should always be non-empty, so we delete this - constructor in order make creating empty ones by mistake harder. - */ - OutputNames() = delete; -}; + struct Names : std::set { + using std::set::set; -/** - * The set of all outputs, without needing to name them explicitly - */ -struct AllOutputs : std::monostate { }; + /* These need to be "inherited manually" */ -typedef std::variant _OutputsSpecRaw; + Names(const std::set & s) + : std::set(s) + { assert(!empty()); } -struct OutputsSpec : _OutputsSpecRaw { - using Raw = _OutputsSpecRaw; - using Raw::Raw; + /** + * Needs to be "inherited manually" + */ + Names(std::set && s) + : std::set(s) + { assert(!empty()); } + + /* This set should always be non-empty, so we delete this + constructor in order make creating empty ones by mistake harder. + */ + Names() = delete; + }; /** - * Force choosing a variant + * The set of all outputs, without needing to name them explicitly */ - OutputsSpec() = delete; + struct All : std::monostate { }; - using Names = OutputNames; - using All = AllOutputs; + typedef std::variant Raw; - inline const Raw & raw() const { - return static_cast(*this); - } + Raw raw; - inline Raw & raw() { - return static_cast(*this); - } + GENERATE_CMP(OutputsSpec, me->raw); - bool contains(const std::string & output) const; + MAKE_WRAPPER_CONSTRUCTOR(OutputsSpec); + + /** + * Force choosing a variant + */ + OutputsSpec() = delete; + + bool contains(const OutputName & output) const; /** * Create a new OutputsSpec which is the union of this and that. @@ -84,20 +91,22 @@ struct OutputsSpec : _OutputsSpecRaw { std::string to_string() const; }; -struct DefaultOutputs : std::monostate { }; +struct ExtendedOutputsSpec { + struct Default : std::monostate { }; + using Explicit = OutputsSpec; + + typedef std::variant Raw; -typedef std::variant _ExtendedOutputsSpecRaw; + Raw raw; -struct ExtendedOutputsSpec : _ExtendedOutputsSpecRaw { - using Raw = _ExtendedOutputsSpecRaw; - using Raw::Raw; + GENERATE_CMP(ExtendedOutputsSpec, me->raw); - using Default = DefaultOutputs; - using Explicit = OutputsSpec; + MAKE_WRAPPER_CONSTRUCTOR(ExtendedOutputsSpec); - inline const Raw & raw() const { - return static_cast(*this); - } + /** + * Force choosing a variant + */ + ExtendedOutputsSpec() = delete; /** * Parse a string of the form 'prefix^output1,...outputN' or diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index cc4a94fab8d5..73e55a96ca50 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -122,7 +122,7 @@ bool ParsedDerivation::willBuildLocally(Store & localStore) const bool ParsedDerivation::substitutesAllowed() const { - return getBoolAttr("allowSubstitutes", true); + return settings.alwaysAllowSubstitutes ? true : getBoolAttr("allowSubstitutes", true); } bool ParsedDerivation::useUidRange() const @@ -132,6 +132,41 @@ bool ParsedDerivation::useUidRange() const static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); +/** + * Write a JSON representation of store object metadata, such as the + * hash and the references. + */ +static nlohmann::json pathInfoToJSON( + Store & store, + const StorePathSet & storePaths) +{ + nlohmann::json::array_t jsonList = nlohmann::json::array(); + + for (auto & storePath : storePaths) { + auto info = store.queryPathInfo(storePath); + + auto & jsonPath = jsonList.emplace_back( + info->toJSON(store, false, HashFormat::Base32)); + + // Add the path to the object whose metadata we are including. + jsonPath["path"] = store.printStorePath(storePath); + + jsonPath["valid"] = true; + + jsonPath["closureSize"] = ({ + uint64_t totalNarSize = 0; + StorePathSet closure; + store.computeFSClosure(info->path, closure, false, false); + for (auto & p : closure) { + auto info = store.queryPathInfo(p); + totalNarSize += info->narSize; + } + totalNarSize; + }); + } + return jsonList; +} + std::optional ParsedDerivation::prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths) { auto structuredAttrs = getStructuredAttrs(); @@ -152,8 +187,8 @@ std::optional ParsedDerivation::prepareStructuredAttrs(Store & s StorePathSet storePaths; for (auto & p : *i) storePaths.insert(store.parseStorePath(p.get())); - json[i.key()] = store.pathInfoToJSON( - store.exportReferences(storePaths, inputPaths), false, true); + json[i.key()] = pathInfoToJSON(store, + store.exportReferences(storePaths, inputPaths)); } } diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index dccd6e87d1da..25851e0a90d9 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -1,10 +1,30 @@ +#include + #include "path-info.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" #include "store-api.hh" +#include "json-utils.hh" namespace nix { +GENERATE_CMP_EXT( + , + UnkeyedValidPathInfo, + me->deriver, + me->narHash, + me->references, + me->registrationTime, + me->narSize, + //me->id, + me->ultimate, + me->sigs, + me->ca); + +GENERATE_CMP_EXT( + , + ValidPathInfo, + me->path, + static_cast(*me)); + std::string ValidPathInfo::fingerprint(const Store & store) const { if (narSize == 0) @@ -12,7 +32,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const store.printStorePath(path)); return "1;" + store.printStorePath(path) + ";" - + narHash.to_string(Base32, true) + ";" + + narHash.to_string(HashFormat::Base32, true) + ";" + std::to_string(narSize) + ";" + concatStringsSep(",", store.printStorePathSet(references)); } @@ -29,14 +49,14 @@ std::optional ValidPathInfo::contentAddressWithRef return std::nullopt; return std::visit(overloaded { - [&](const TextHash & th) -> ContentAddressWithReferences { + [&](const TextIngestionMethod &) -> ContentAddressWithReferences { assert(references.count(path) == 0); return TextInfo { - .hash = th, + .hash = ca->hash, .references = references, }; }, - [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { + [&](const FileIngestionMethod & m2) -> ContentAddressWithReferences { auto refs = references; bool hasSelfReference = false; if (refs.count(path)) { @@ -44,14 +64,15 @@ std::optional ValidPathInfo::contentAddressWithRef refs.erase(path); } return FixedOutputInfo { - .hash = foh, + .method = m2, + .hash = ca->hash, .references = { .others = std::move(refs), .self = hasSelfReference, }, }; }, - }, ca->raw); + }, ca->method.raw); } bool ValidPathInfo::isContentAddressed(const Store & store) const @@ -98,86 +119,131 @@ Strings ValidPathInfo::shortRefs() const return refs; } - ValidPathInfo::ValidPathInfo( const Store & store, std::string_view name, ContentAddressWithReferences && ca, Hash narHash) - : path(store.makeFixedOutputPathFromCA(name, ca)) - , narHash(narHash) + : UnkeyedValidPathInfo(narHash) + , path(store.makeFixedOutputPathFromCA(name, ca)) { std::visit(overloaded { [this](TextInfo && ti) { this->references = std::move(ti.references); - this->ca = std::move((TextHash &&) ti); + this->ca = ContentAddress { + .method = TextIngestionMethod {}, + .hash = std::move(ti.hash), + }; }, [this](FixedOutputInfo && foi) { this->references = std::move(foi.references.others); if (foi.references.self) this->references.insert(path); - this->ca = std::move((FixedOutputHash &&) foi); + this->ca = ContentAddress { + .method = std::move(foi.method), + .hash = std::move(foi.hash), + }; }, }, std::move(ca).raw); } -ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format) +nlohmann::json UnkeyedValidPathInfo::toJSON( + const Store & store, + bool includeImpureInfo, + HashFormat hashFormat) const { - return read(source, store, format, store.parseStorePath(readString(source))); -} + using nlohmann::json; -ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format, StorePath && path) -{ - auto deriver = readString(source); - auto narHash = Hash::parseAny(readString(source), htSHA256); - ValidPathInfo info(path, narHash); - if (deriver != "") info.deriver = store.parseStorePath(deriver); - info.references = WorkerProto::Serialise::read(store, - WorkerProto::ReadConn { .from = source }); - source >> info.registrationTime >> info.narSize; - if (format >= 16) { - source >> info.ultimate; - info.sigs = readStrings(source); - info.ca = ContentAddress::parseOpt(readString(source)); + auto jsonObject = json::object(); + + jsonObject["narHash"] = narHash.to_string(hashFormat, true); + jsonObject["narSize"] = narSize; + + { + auto& jsonRefs = (jsonObject["references"] = json::array()); + for (auto & ref : references) + jsonRefs.emplace_back(store.printStorePath(ref)); } - if (format >= 36) { - bool hasAccessStatus; - source >> hasAccessStatus; - if (hasAccessStatus) - info.accessStatus = WorkerProto::Serialise::read(store, WorkerProto::ReadConn {.from = source}); + + if (ca) + jsonObject["ca"] = renderContentAddress(ca); + + if (accessStatus) { + jsonObject["protected"] = accessStatus->isProtected; + for (auto & entity : accessStatus->entities) { + std::visit(overloaded { + [&](ACL::User u) { jsonObject["allowedUsers"].push_back(getpwuid(u.uid)->pw_name); }, + [&](ACL::Group g) { jsonObject["allowedGroups"].push_back(getgrgid(g.gid)->gr_name); }, + }, entity); + } } - return info; -} + if (includeImpureInfo) { + if (deriver) + jsonObject["deriver"] = store.printStorePath(*deriver); + + if (registrationTime) + jsonObject["registrationTime"] = registrationTime; + if (ultimate) + jsonObject["ultimate"] = ultimate; -void ValidPathInfo::write( - Sink & sink, + if (!sigs.empty()) { + for (auto & sig : sigs) + jsonObject["signatures"].push_back(sig); + } + } + + return jsonObject; +} + +UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON( const Store & store, - unsigned int format, - bool includePath) const + const nlohmann::json & json) { - if (includePath) - sink << store.printStorePath(path); - sink << (deriver ? store.printStorePath(*deriver) : "") - << narHash.to_string(Base16, false); - WorkerProto::write(store, - WorkerProto::WriteConn { .to = sink }, - references); - sink << registrationTime << narSize; - if (format >= 16) { - sink << ultimate - << sigs - << renderContentAddress(ca); - } - if (format >= 36) { - if (accessStatus) { - sink << true; - WorkerProto::Serialise::write(store, WorkerProto::WriteConn {.to = sink}, *accessStatus); - } else { - sink << false; - } + using nlohmann::detail::value_t; + + UnkeyedValidPathInfo res { + Hash(Hash::dummy), + }; + + ensureType(json, value_t::object); + res.narHash = Hash::parseAny( + static_cast( + ensureType(valueAt(json, "narHash"), value_t::string)), + std::nullopt); + res.narSize = ensureType(valueAt(json, "narSize"), value_t::number_integer); + + try { + auto & references = ensureType(valueAt(json, "references"), value_t::array); + for (auto & input : references) + res.references.insert(store.parseStorePath(static_cast +(input))); + } catch (Error & e) { + e.addTrace({}, "while reading key 'references'"); + throw; } + + if (json.contains("ca")) + res.ca = ContentAddress::parse( + static_cast( + ensureType(valueAt(json, "ca"), value_t::string))); + + if (json.contains("deriver")) + res.deriver = store.parseStorePath( + static_cast( + ensureType(valueAt(json, "deriver"), value_t::string))); + + if (json.contains("registrationTime")) + res.registrationTime = ensureType(valueAt(json, "registrationTime"), value_t::number_integer); + + if (json.contains("ultimate")) + res.ultimate = ensureType(valueAt(json, "ultimate"), value_t::boolean); + + if (json.contains("signatures")) + res.sigs = valueAt(json, "signatures"); + + return res; } } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index da472e4ab365..144a6fc1165c 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -30,12 +30,11 @@ struct SubstitutablePathInfo uint64_t narSize; }; -typedef std::map SubstitutablePathInfos; +using SubstitutablePathInfos = std::map; -struct ValidPathInfo +struct UnkeyedValidPathInfo { - StorePath path; std::optional deriver; /** * \todo document this @@ -44,7 +43,7 @@ struct ValidPathInfo StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown - uint64_t id; // internal use only + uint64_t id = 0; // internal use only /** * Whether the path is ultimately trusted, that is, it's a @@ -73,13 +72,34 @@ struct ValidPathInfo */ std::optional ca; - bool operator == (const ValidPathInfo & i) const - { - return - path == i.path - && narHash == i.narHash - && references == i.references; - } + using AccessStatus = AccessStatusFor>; + std::optional accessStatus; + + UnkeyedValidPathInfo(const UnkeyedValidPathInfo & other) = default; + + UnkeyedValidPathInfo(Hash narHash) : narHash(narHash) { }; + + DECLARE_CMP(UnkeyedValidPathInfo); + + virtual ~UnkeyedValidPathInfo() { } + + /** + * @param includeImpureInfo If true, variable elements such as the + * registration time are included. + */ + virtual nlohmann::json toJSON( + const Store & store, + bool includeImpureInfo, + HashFormat hashFormat) const; + static UnkeyedValidPathInfo fromJSON( + const Store & store, + const nlohmann::json & json); +}; + +struct ValidPathInfo : UnkeyedValidPathInfo { + StorePath path; + + DECLARE_CMP(ValidPathInfo); /** * Return a fingerprint of the store path to be used in binary @@ -93,11 +113,11 @@ struct ValidPathInfo void sign(const Store & store, const SecretKey & secretKey); - /** - * @return The `ContentAddressWithReferences` that determines the - * store path for a content-addressed store object, `std::nullopt` - * for an input-addressed store object. - */ + /** + * @return The `ContentAddressWithReferences` that determines the + * store path for a content-addressed store object, `std::nullopt` + * for an input-addressed store object. + */ std::optional contentAddressWithReferences() const; /** @@ -118,27 +138,19 @@ struct ValidPathInfo * Verify a single signature. */ bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const; - using AccessStatus = AccessStatusFor>; - std::optional accessStatus; - Strings shortRefs() const; ValidPathInfo(const ValidPathInfo & other) = default; - ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { }; - ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; + ValidPathInfo(StorePath && path, UnkeyedValidPathInfo info) : UnkeyedValidPathInfo(info), path(std::move(path)) { }; + ValidPathInfo(const StorePath & path, UnkeyedValidPathInfo info) : UnkeyedValidPathInfo(info), path(path) { }; ValidPathInfo(const Store & store, std::string_view name, ContentAddressWithReferences && ca, Hash narHash); virtual ~ValidPathInfo() { } - - static ValidPathInfo read(Source & source, const Store & store, unsigned int format); - static ValidPathInfo read(Source & source, const Store & store, unsigned int format, StorePath && path); - - void write(Sink & sink, const Store & store, unsigned int format, bool includePath = true) const; }; -typedef std::map ValidPathInfos; +using ValidPathInfos = std::map; } diff --git a/src/libstore/path-references.cc b/src/libstore/path-references.cc index 33cf66ce366f..274b596c00ec 100644 --- a/src/libstore/path-references.cc +++ b/src/libstore/path-references.cc @@ -1,6 +1,5 @@ #include "path-references.hh" #include "hash.hh" -#include "util.hh" #include "archive.hh" #include diff --git a/src/libstore/path-references.hh b/src/libstore/path-references.hh index 7b44e32617f7..0553003f83ac 100644 --- a/src/libstore/path-references.hh +++ b/src/libstore/path-references.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "references.hh" #include "path.hh" diff --git a/src/libstore/path-regex.hh b/src/libstore/path-regex.hh index 4f8dc4c1faa5..a44e6a2eb525 100644 --- a/src/libstore/path-regex.hh +++ b/src/libstore/path-regex.hh @@ -3,6 +3,6 @@ namespace nix { -static constexpr std::string_view nameRegexStr = R"([0-9a-zA-Z\+\-\._\?=]+)"; +static constexpr std::string_view nameRegexStr = R"([0-9a-zA-Z\+\-_\?=][0-9a-zA-Z\+\-\._\?=]*)"; } diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index 869b490ade74..af6837370aaa 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -16,10 +16,16 @@ std::string StorePathWithOutputs::to_string(const Store & store) const DerivedPath StorePathWithOutputs::toDerivedPath() const { if (!outputs.empty()) { - return DerivedPath::Built { path, OutputsSpec::Names { outputs } }; + return DerivedPath::Built { + .drvPath = makeConstantStorePathRef(path), + .outputs = OutputsSpec::Names { outputs }, + }; } else if (path.isDerivation()) { assert(outputs.empty()); - return DerivedPath::Built { path, OutputsSpec::All { } }; + return DerivedPath::Built { + .drvPath = makeConstantStorePathRef(path), + .outputs = OutputsSpec::All { }, + }; } else { return DerivedPath::Opaque { path }; } @@ -34,29 +40,36 @@ std::vector toDerivedPaths(const std::vector } -std::variant StorePathWithOutputs::tryFromDerivedPath(const DerivedPath & p) +StorePathWithOutputs::ParseResult StorePathWithOutputs::tryFromDerivedPath(const DerivedPath & p) { return std::visit(overloaded { - [&](const DerivedPath::Opaque & bo) -> std::variant { + [&](const DerivedPath::Opaque & bo) -> StorePathWithOutputs::ParseResult { if (bo.path.isDerivation()) { // drv path gets interpreted as "build", not "get drv file itself" return bo.path; } return StorePathWithOutputs { bo.path }; }, - [&](const DerivedPath::Built & bfd) -> std::variant { - return StorePathWithOutputs { - .path = bfd.drvPath, - // Use legacy encoding of wildcard as empty set - .outputs = std::visit(overloaded { - [&](const OutputsSpec::All &) -> StringSet { - return {}; - }, - [&](const OutputsSpec::Names & outputs) { - return static_cast(outputs); - }, - }, bfd.outputs.raw()), - }; + [&](const DerivedPath::Built & bfd) -> StorePathWithOutputs::ParseResult { + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) -> StorePathWithOutputs::ParseResult { + return StorePathWithOutputs { + .path = bo.path, + // Use legacy encoding of wildcard as empty set + .outputs = std::visit(overloaded { + [&](const OutputsSpec::All &) -> StringSet { + return {}; + }, + [&](const OutputsSpec::Names & outputs) { + return static_cast(outputs); + }, + }, bfd.outputs.raw), + }; + }, + [&](const SingleDerivedPath::Built &) -> StorePathWithOutputs::ParseResult { + return std::monostate {}; + }, + }, bfd.drvPath->raw()); }, }, p.raw()); } diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index d75850868104..57e03252df70 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -23,7 +23,9 @@ struct StorePathWithOutputs DerivedPath toDerivedPath() const; - static std::variant tryFromDerivedPath(const DerivedPath &); + typedef std::variant ParseResult; + + static StorePathWithOutputs::ParseResult tryFromDerivedPath(const DerivedPath &); }; std::vector toDerivedPaths(const std::vector); diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 552e831146a5..ec3e53232b3e 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -11,6 +11,8 @@ static void checkName(std::string_view path, std::string_view name) if (name.size() > StorePath::MaxPathLen) throw BadStorePath("store path '%s' has a name longer than %d characters", path, StorePath::MaxPathLen); + if (name[0] == '.') + throw BadStorePath("store path '%s' starts with illegal character '.'", path); // See nameRegexStr for the definition for (auto c : name) if (!((c >= '0' && c <= '9') @@ -33,7 +35,7 @@ StorePath::StorePath(std::string_view _baseName) } StorePath::StorePath(const Hash & hash, std::string_view _name) - : baseName((hash.to_string(Base32, false) + "-").append(std::string(_name))) + : baseName((hash.to_string(HashFormat::Base32, false) + "-").append(std::string(_name))) { checkName(baseName, name()); } diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index adc763e6a3cb..2b5b8dfe7355 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -1,6 +1,7 @@ #include "pathlocks.hh" #include "util.hh" #include "sync.hh" +#include "signals.hh" #include #include diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index 4921df352fc9..7fcfa2e40870 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "util.hh" +#include "file-descriptor.hh" namespace nix { diff --git a/src/libstore/posix-fs-canonicalise.cc b/src/libstore/posix-fs-canonicalise.cc new file mode 100644 index 000000000000..853d46cc6f38 --- /dev/null +++ b/src/libstore/posix-fs-canonicalise.cc @@ -0,0 +1,186 @@ +#if HAVE_SYS_XATTR_H +# include +#endif + +#include "posix-fs-canonicalise.hh" +#include "file-system.hh" +#include "signals.hh" +#include "util.hh" +#include "globals.hh" +#include "store-api.hh" + +namespace nix { + +const time_t mtimeStore = 1; /* 1 second into the epoch */ + + +static void canonicaliseTimestampAndPermissions(const Path & path, const struct stat & st) +{ + if (!S_ISLNK(st.st_mode)) { + + if (experimentalFeatureSettings.isEnabled(Xp::ACLs)) { + /* Mask out all type related bits. */ + mode_t mode = st.st_mode & ~S_IFMT; + + if (mode != 0440 && mode != 0550 && mode != 0444 && mode != 0555) { + mode = (st.st_mode & S_IFMT) + | 0444 + | (st.st_mode & S_IXUSR ? 0111 : 0); + if (! (st.st_mode & S_IRWXO)) + mode &= ~S_IRWXO; + if (chmod(path.c_str(), mode) == -1) + throw SysError("changing mode of '%1%' to %2$o", path, mode); + } + } else { + /* Mask out all type related bits. */ + mode_t mode = st.st_mode & ~S_IFMT; + + if (mode != 0444 && mode != 0555) { + mode = (st.st_mode & S_IFMT) + | 0444 + | (st.st_mode & S_IXUSR ? 0111 : 0); + if (chmod(path.c_str(), mode) == -1) + throw SysError("changing mode of '%1%' to %2$o", path, mode); + } + } + + } + + if (st.st_mtime != mtimeStore) { + struct timeval times[2]; + times[0].tv_sec = st.st_atime; + times[0].tv_usec = 0; + times[1].tv_sec = mtimeStore; + times[1].tv_usec = 0; +#if HAVE_LUTIMES + if (lutimes(path.c_str(), times) == -1) + if (errno != ENOSYS || + (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)) +#else + if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) +#endif + throw SysError("changing modification time of '%1%'", path); + } +} + + +void canonicaliseTimestampAndPermissions(const Path & path) +{ + canonicaliseTimestampAndPermissions(path, lstat(path)); +} + + +static void canonicalisePathMetaData_( + const Path & path, + std::optional> uidRange, + InodesSeen & inodesSeen) +{ + checkInterrupt(); + +#if __APPLE__ + /* Remove flags, in particular UF_IMMUTABLE which would prevent + the file from being garbage-collected. FIXME: Use + setattrlist() to remove other attributes as well. */ + if (lchflags(path.c_str(), 0)) { + if (errno != ENOTSUP) + throw SysError("clearing flags of path '%1%'", path); + } +#endif + + auto st = lstat(path); + + /* Really make sure that the path is of a supported type. */ + if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) + throw Error("file '%1%' has an unsupported type", path); + +#ifdef HAVE_SYS_XATTR_H + /* Remove extended attributes / ACLs. */ + ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0); + + if (eaSize < 0) { + if (errno != ENOTSUP && errno != ENODATA) + throw SysError("querying extended attributes of '%s'", path); + } else if (eaSize > 0) { + std::vector eaBuf(eaSize); + + if ((eaSize = llistxattr(path.c_str(), eaBuf.data(), eaBuf.size())) < 0) + throw SysError("querying extended attributes of '%s'", path); + + for (auto & eaName: tokenizeString(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) { + if (settings.ignoredAcls.get().count(eaName)) continue; + if (lremovexattr(path.c_str(), eaName.c_str()) == -1) + throw SysError("removing extended attribute '%s' from '%s'", eaName, path); + } + } +#endif + + /* Fail if the file is not owned by the build user. This prevents + us from messing up the ownership/permissions of files + hard-linked into the output (e.g. "ln /etc/shadow $out/foo"). + However, ignore files that we chown'ed ourselves previously to + ensure that we don't fail on hard links within the same build + (i.e. "touch $out/foo; ln $out/foo $out/bar"). */ + if (uidRange && (st.st_uid < uidRange->first || st.st_uid > uidRange->second)) { + if (S_ISDIR(st.st_mode) || !inodesSeen.count(Inode(st.st_dev, st.st_ino))) + throw BuildError("invalid ownership on file '%1%'", path); + mode_t mode = st.st_mode & ~S_IFMT; + assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore)); + return; + } + + inodesSeen.insert(Inode(st.st_dev, st.st_ino)); + + canonicaliseTimestampAndPermissions(path, st); + + /* Change ownership to the current uid. If it's a symlink, use + lchown if available, otherwise don't bother. Wrong ownership + of a symlink doesn't matter, since the owning user can't change + the symlink and can't delete it because the directory is not + writable. The only exception is top-level paths in the Nix + store (since that directory is group-writable for the Nix build + users group); we check for this case below. */ + if (st.st_uid != geteuid()) { +#if HAVE_LCHOWN + if (lchown(path.c_str(), geteuid(), getegid()) == -1) +#else + if (!S_ISLNK(st.st_mode) && + chown(path.c_str(), geteuid(), getegid()) == -1) +#endif + throw SysError("changing owner of '%1%' to %2%", + path, geteuid()); + } + + if (S_ISDIR(st.st_mode)) { + DirEntries entries = readDirectory(path); + for (auto & i : entries) + canonicalisePathMetaData_(path + "/" + i.name, uidRange, inodesSeen); + } +} + + +void canonicalisePathMetaData( + const Path & path, + std::optional> uidRange, + InodesSeen & inodesSeen) +{ + canonicalisePathMetaData_(path, uidRange, inodesSeen); + + /* On platforms that don't have lchown(), the top-level path can't + be a symlink, since we can't change its ownership. */ + auto st = lstat(path); + + if (st.st_uid != geteuid()) { + assert(S_ISLNK(st.st_mode)); + throw Error("wrong ownership of top-level store path '%1%'", path); + } +} + + +void canonicalisePathMetaData(const Path & path, + std::optional> uidRange) +{ + InodesSeen inodesSeen; + canonicalisePathMetaData(path, uidRange, inodesSeen); +} + +} diff --git a/src/libstore/posix-fs-canonicalise.hh b/src/libstore/posix-fs-canonicalise.hh new file mode 100644 index 000000000000..35644af125fb --- /dev/null +++ b/src/libstore/posix-fs-canonicalise.hh @@ -0,0 +1,45 @@ +#pragma once +///@file + +#include +#include + +#include "types.hh" +#include "error.hh" + +namespace nix { + +typedef std::pair Inode; +typedef std::set InodesSeen; + + +/** + * "Fix", or canonicalise, the meta-data of the files in a store path + * after it has been built. In particular: + * + * - the last modification date on each file is set to 1 (i.e., + * 00:00:01 1/1/1970 UTC) + * + * - the permissions are set of 444 or 555 (i.e., read-only with or + * without execute permission; setuid bits etc. are cleared) + * + * - the owner and group are set to the Nix user and group, if we're + * running as root. + * + * If uidRange is not empty, this function will throw an error if it + * encounters files owned by a user outside of the closed interval + * [uidRange->first, uidRange->second]. + */ +void canonicalisePathMetaData( + const Path & path, + std::optional> uidRange, + InodesSeen & inodesSeen); +void canonicalisePathMetaData( + const Path & path, + std::optional> uidRange); + +void canonicaliseTimestampAndPermissions(const Path & path); + +MakeError(PathInUse, Error); + +} diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 4e99559481c1..e8b88693d262 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -1,7 +1,7 @@ #include "profiles.hh" #include "store-api.hh" #include "local-fs-store.hh" -#include "util.hh" +#include "users.hh" #include #include @@ -183,7 +183,7 @@ void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bo iterDropUntil(gens, i, [&](auto & g) { return g.number == curGen; }); // Skip over `max` generations, preserving them - for (auto keep = 0; i != gens.rend() && keep < max; ++i, ++keep); + for (GenerationNumber keep = 0; i != gens.rend() && keep < max; ++i, ++keep); // Delete the rest for (; i != gens.rend(); ++i) diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 5aa865162716..4ba2123d81ce 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -5,6 +5,7 @@ #include "hash.hh" #include "path.hh" +#include "derived-path.hh" #include #include "comparator.hh" #include "crypto.hh" @@ -33,12 +34,12 @@ struct DrvOutput { /** * The name of the output. */ - std::string outputName; + OutputName outputName; std::string to_string() const; std::string strHash() const - { return drvHash.to_string(Base16, true); } + { return drvHash.to_string(HashFormat::Base16, true); } static DrvOutput parse(const std::string &); @@ -83,7 +84,7 @@ struct Realisation { * Since these are the outputs of a single derivation, we know the * output names are unique so we can use them as the map key. */ -typedef std::map SingleDrvOutputs; +typedef std::map SingleDrvOutputs; /** * Collection type for multiple derivations' outputs' `Realisation`s. @@ -142,10 +143,14 @@ struct RealisedPath { class MissingRealisation : public Error { public: - MissingRealisation(const DrvOutput & outputId) - : Error( "cannot operate on an output of the " + MissingRealisation(DrvOutput & outputId) + : MissingRealisation(outputId.outputName, outputId.strHash()) + {} + MissingRealisation(std::string_view drv, OutputName outputName) + : Error( "cannot operate on output '%s' of the " "unbuilt derivation '%s'", - outputId.to_string()) + outputName, + drv) {} }; diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index fcfb527f50e1..03e57a565d09 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -8,8 +8,9 @@ namespace nix { -RemoteFSAccessor::RemoteFSAccessor(ref store, const Path & cacheDir) +RemoteFSAccessor::RemoteFSAccessor(ref store, bool requireValidPath, const Path & cacheDir) : store(store) + , requireValidPath(requireValidPath) , cacheDir(cacheDir) { if (cacheDir != "") @@ -22,7 +23,7 @@ Path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::strin return fmt("%s/%s.%s", cacheDir, hashPart, ext); } -ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::string && nar) +ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::string && nar) { if (cacheDir != "") { try { @@ -38,7 +39,7 @@ ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::str if (cacheDir != "") { try { - nlohmann::json j = listNar(narAccessor, "", true); + nlohmann::json j = listNar(narAccessor, CanonPath::root, true); writeFile(makeCacheFile(hashPart, "ls"), j.dump()); } catch (...) { ignoreException(); @@ -48,11 +49,10 @@ ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::str return narAccessor; } -std::pair, Path> RemoteFSAccessor::fetch(const Path & path_, bool requireValidPath) +std::pair, CanonPath> RemoteFSAccessor::fetch(const CanonPath & path) { - auto path = canonPath(path_); - - auto [storePath, restPath] = store->toStorePath(path); + auto [storePath, restPath_] = store->toStorePath(path.abs()); + auto restPath = CanonPath(restPath_); if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); @@ -63,7 +63,7 @@ std::pair, Path> RemoteFSAccessor::fetch(const Path & path_, boo std::string listing; Path cacheFile; - if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) { + if (cacheDir != "" && nix::pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) { try { listing = nix::readFile(makeCacheFile(storePath.hashPart(), "ls")); @@ -101,25 +101,25 @@ std::pair, Path> RemoteFSAccessor::fetch(const Path & path_, boo return {addToCache(storePath.hashPart(), std::move(sink.s)), restPath}; } -FSAccessor::Stat RemoteFSAccessor::stat(const Path & path) +std::optional RemoteFSAccessor::maybeLstat(const CanonPath & path) { auto res = fetch(path); - return res.first->stat(res.second); + return res.first->maybeLstat(res.second); } -StringSet RemoteFSAccessor::readDirectory(const Path & path) +SourceAccessor::DirEntries RemoteFSAccessor::readDirectory(const CanonPath & path) { auto res = fetch(path); return res.first->readDirectory(res.second); } -std::string RemoteFSAccessor::readFile(const Path & path, bool requireValidPath) +std::string RemoteFSAccessor::readFile(const CanonPath & path) { - auto res = fetch(path, requireValidPath); + auto res = fetch(path); return res.first->readFile(res.second); } -std::string RemoteFSAccessor::readLink(const Path & path) +std::string RemoteFSAccessor::readLink(const CanonPath & path) { auto res = fetch(path); return res.first->readLink(res.second); diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh index e2673b6f6d98..d09762a53c4e 100644 --- a/src/libstore/remote-fs-accessor.hh +++ b/src/libstore/remote-fs-accessor.hh @@ -1,40 +1,43 @@ #pragma once ///@file -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "ref.hh" #include "store-api.hh" namespace nix { -class RemoteFSAccessor : public FSAccessor +class RemoteFSAccessor : public SourceAccessor { ref store; - std::map> nars; + std::map> nars; + + bool requireValidPath; Path cacheDir; - std::pair, Path> fetch(const Path & path_, bool requireValidPath = true); + std::pair, CanonPath> fetch(const CanonPath & path); friend class BinaryCacheStore; Path makeCacheFile(std::string_view hashPart, const std::string & ext); - ref addToCache(std::string_view hashPart, std::string && nar); + ref addToCache(std::string_view hashPart, std::string && nar); public: RemoteFSAccessor(ref store, + bool requireValidPath = true, const /* FIXME: use std::optional */ Path & cacheDir = ""); - Stat stat(const Path & path) override; + std::optional maybeLstat(const CanonPath & path) override; - StringSet readDirectory(const Path & path) override; + DirEntries readDirectory(const CanonPath & path) override; - std::string readFile(const Path & path, bool requireValidPath = true) override; + std::string readFile(const CanonPath & path) override; - std::string readLink(const Path & path) override; + std::string readLink(const CanonPath & path) override; }; } diff --git a/src/libstore/remote-store-connection.hh b/src/libstore/remote-store-connection.hh index d32d91a60b08..44328b06b556 100644 --- a/src/libstore/remote-store-connection.hh +++ b/src/libstore/remote-store-connection.hh @@ -1,5 +1,9 @@ +#pragma once +///@file + #include "remote-store.hh" #include "worker-protocol.hh" +#include "pool.hh" namespace nix { @@ -29,7 +33,7 @@ struct RemoteStore::Connection * sides support. (If the maximum doesn't exist, we would fail to * establish a connection and produce a value of this type.) */ - unsigned int daemonVersion; + WorkerProto::Version daemonVersion; /** * Whether the remote side trusts us or not. @@ -69,6 +73,7 @@ struct RemoteStore::Connection { return WorkerProto::ReadConn { .from = from, + .version = daemonVersion, }; } @@ -84,6 +89,7 @@ struct RemoteStore::Connection { return WorkerProto::WriteConn { .to = to, + .version = daemonVersion, }; } @@ -94,4 +100,34 @@ struct RemoteStore::Connection std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0, bool flush = true); }; +/** + * A wrapper around Pool::Handle that marks + * the connection as bad (causing it to be closed) if a non-daemon + * exception is thrown before the handle is closed. Such an exception + * causes a deviation from the expected protocol and therefore a + * desynchronization between the client and daemon. + */ +struct RemoteStore::ConnectionHandle +{ + Pool::Handle handle; + bool daemonException = false; + + ConnectionHandle(Pool::Handle && handle) + : handle(std::move(handle)) + { } + + ConnectionHandle(ConnectionHandle && h) + : handle(std::move(h.handle)) + { } + + ~ConnectionHandle(); + + RemoteStore::Connection & operator * () { return *handle; } + RemoteStore::Connection * operator -> () { return &*handle; } + + void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true); + + void withFramedSink(std::function fun); +}; + } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index eb5d1c4b5692..55c6af3f0184 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -21,7 +21,7 @@ namespace nix { -/* TODO: Separate these store impls into different files, give them better names */ +/* TODO: Separate these store types into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) : RemoteStoreConfig(params) , Store(params) @@ -162,49 +162,42 @@ void RemoteStore::setOptions(Connection & conn) } -/* A wrapper around Pool::Handle that marks - the connection as bad (causing it to be closed) if a non-daemon - exception is thrown before the handle is closed. Such an exception - causes a deviation from the expected protocol and therefore a - desynchronization between the client and daemon. */ -struct ConnectionHandle +RemoteStore::ConnectionHandle::~ConnectionHandle() { - Pool::Handle handle; - bool daemonException = false; - - ConnectionHandle(Pool::Handle && handle) - : handle(std::move(handle)) - { } - - ConnectionHandle(ConnectionHandle && h) - : handle(std::move(h.handle)) - { } - - ~ConnectionHandle() - { - if (!daemonException && std::uncaught_exceptions()) { - handle.markBad(); - debug("closing daemon connection because of an exception"); - } + if (!daemonException && std::uncaught_exceptions()) { + handle.markBad(); + debug("closing daemon connection because of an exception"); } +} - RemoteStore::Connection * operator -> () { return &*handle; } - RemoteStore::Connection & operator * () { return *handle; } - - void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true) - { - auto ex = handle->processStderr(sink, source, flush); - if (ex) { - daemonException = true; +void RemoteStore::ConnectionHandle::processStderr(Sink * sink, Source * source, bool flush) +{ + auto ex = handle->processStderr(sink, source, flush); + if (ex) { + daemonException = true; + try { std::rethrow_exception(ex); + } catch (const Error & e) { + // Nix versions before #4628 did not have an adequate behavior for reporting that the derivation format was upgraded. + // To avoid having to add compatibility logic in many places, we expect to catch almost all occurrences of the + // old incomprehensible error here, so that we can explain to users what's going on when their daemon is + // older than #4628 (2023). + if (experimentalFeatureSettings.isEnabled(Xp::DynamicDerivations) && + GET_PROTOCOL_MINOR(handle->daemonVersion) <= 35) + { + auto m = e.msg(); + if (m.find("parsing derivation") != std::string::npos && + m.find("expected string") != std::string::npos && + m.find("Derive([") != std::string::npos) + throw Error("%s, this might be because the daemon is too old to understand dependencies on dynamic derivations. Check to see if the raw dervation is in the form '%s'", std::move(m), "DrvWithVersion(..)"); + } + throw; } } - - void withFramedSink(std::function fun); -}; +} -ConnectionHandle RemoteStore::getConnection() +RemoteStore::ConnectionHandle RemoteStore::getConnection() { return ConnectionHandle(connections->get()); } @@ -342,7 +335,8 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path)); } info = std::make_shared( - ValidPathInfo::read(conn->from, *this, GET_PROTOCOL_MINOR(conn->daemonVersion), StorePath{path})); + StorePath{path}, + WorkerProto::Serialise::read(*this, *conn)); } callback(std::move(info)); } catch (...) { callback.rethrow(); } @@ -381,27 +375,36 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) } -std::map> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path) +std::map> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore_) { if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) { - auto conn(getConnection()); - conn->to << WorkerProto::Op::QueryDerivationOutputMap << printStorePath(path); - conn.processStderr(); - return WorkerProto::Serialise>>::read(*this, *conn); + if (!evalStore_) { + auto conn(getConnection()); + conn->to << WorkerProto::Op::QueryDerivationOutputMap << printStorePath(path); + conn.processStderr(); + return WorkerProto::Serialise>>::read(*this, *conn); + } else { + auto & evalStore = *evalStore_; + auto outputs = evalStore.queryStaticPartialDerivationOutputMap(path); + // union with the first branch overriding the statically-known ones + // when non-`std::nullopt`. + for (auto && [outputName, optPath] : queryPartialDerivationOutputMap(path, nullptr)) { + if (optPath) + outputs.insert_or_assign(std::move(outputName), std::move(optPath)); + else + outputs.insert({std::move(outputName), std::nullopt}); + } + return outputs; + } } else { + auto & evalStore = evalStore_ ? *evalStore_ : *this; // Fallback for old daemon versions. // For floating-CA derivations (and their co-dependencies) this is an // under-approximation as it only returns the paths that can be inferred // from the derivation itself (and not the ones that are known because // the have been built), but as old stores don't handle floating-CA // derivations this shouldn't matter - auto derivation = readDerivation(path); - auto outputsWithOptPaths = derivation.outputsAndOptPaths(*this); - std::map> ret; - for (auto & [outputName, outputAndPath] : outputsWithOptPaths) { - ret.emplace(outputName, outputAndPath.second); - } - return ret; + return evalStore.queryStaticPartialDerivationOutputMap(path); } } @@ -446,7 +449,7 @@ ref RemoteStore::addCAToStore( } return make_ref( - ValidPathInfo::read(conn->from, *this, GET_PROTOCOL_MINOR(conn->daemonVersion))); + WorkerProto::Serialise::read(*this, *conn)); } else { if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25"); @@ -542,7 +545,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, conn->to << WorkerProto::Op::AddToStoreNar << printStorePath(info.path) << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(Base16, false); + << info.narHash.to_string(HashFormat::Base16, false); WorkerProto::write(*this, *conn, info.references); conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) @@ -571,7 +574,12 @@ void RemoteStore::addMultipleToStore( auto source = sinkToSource([&](Sink & sink) { sink << pathsToCopy.size(); for (auto & [pathInfo, pathSource] : pathsToCopy) { - pathInfo.write(sink, *this, 16); + WorkerProto::Serialise::write(*this, + WorkerProto::WriteConn { + .to = sink, + .version = 16, + }, + pathInfo); pathSource->drainInto(sink); } }); @@ -656,30 +664,6 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id, } catch (...) { return callback.rethrow(); } } -static void writeDerivedPaths(RemoteStore & store, RemoteStore::Connection & conn, const std::vector & reqs) -{ - if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 30) { - WorkerProto::write(store, conn, reqs); - } else { - Strings ss; - for (auto & p : reqs) { - auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p); - std::visit(overloaded { - [&](const StorePathWithOutputs & s) { - ss.push_back(s.to_string(store)); - }, - [&](const StorePath & drvPath) { - throw Error("trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file", - store.printStorePath(drvPath), - GET_PROTOCOL_MAJOR(conn.daemonVersion), - GET_PROTOCOL_MINOR(conn.daemonVersion)); - }, - }, sOrDrvPath); - } - conn.to << ss; - } -} - void RemoteStore::copyDrvsFromEvalStore( const std::vector & paths, std::shared_ptr evalStore) @@ -688,9 +672,16 @@ void RemoteStore::copyDrvsFromEvalStore( /* The remote doesn't have a way to access evalStore, so copy the .drvs. */ RealisedPath::Set drvPaths2; - for (auto & i : paths) - if (auto p = std::get_if(&i)) - drvPaths2.insert(p->drvPath); + for (const auto & i : paths) { + std::visit(overloaded { + [&](const DerivedPath::Opaque & bp) { + // Do nothing, path is hopefully there already + }, + [&](const DerivedPath::Built & bp) { + drvPaths2.insert(bp.drvPath->getBaseStorePath()); + }, + }, i.raw()); + } copyClosure(*evalStore, *this, drvPaths2); } } @@ -702,7 +693,7 @@ void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMod auto conn(getConnection()); conn->to << WorkerProto::Op::BuildPaths; assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13); - writeDerivedPaths(*this, *conn, drvPaths); + WorkerProto::write(*this, *conn, drvPaths); if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) conn->to << buildMode; else @@ -726,7 +717,7 @@ std::vector RemoteStore::buildPathsWithResults( if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 34) { conn->to << WorkerProto::Op::BuildPathsWithResults; - writeDerivedPaths(*this, *conn, paths); + WorkerProto::write(*this, *conn, paths); conn->to << buildMode; conn.processStderr(); return WorkerProto::Serialise>::read(*this, *conn); @@ -760,7 +751,8 @@ std::vector RemoteStore::buildPathsWithResults( }; OutputPathMap outputs; - auto drv = evalStore->readDerivation(bfd.drvPath); + auto drvPath = resolveDerivedPath(*evalStore, *bfd.drvPath); + auto drv = evalStore->readDerivation(drvPath); const auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive auto built = resolveDerivedPath(*this, bfd, &*evalStore); for (auto & [output, outputPath] : built) { @@ -768,7 +760,7 @@ std::vector RemoteStore::buildPathsWithResults( if (!outputHash) throw Error( "the derivation '%s' doesn't have an output named '%s'", - printStorePath(bfd.drvPath), output); + printStorePath(drvPath), output); auto outputId = DrvOutput{ *outputHash, output }; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { auto realisation = @@ -805,20 +797,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD writeDerivation(conn->to, *this, drv); conn->to << buildMode; conn.processStderr(); - BuildResult res; - res.status = (BuildResult::Status) readInt(conn->from); - conn->from >> res.errorMsg; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) { - conn->from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime; - } - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) { - auto builtOutputs = WorkerProto::Serialise::read(*this, *conn); - for (auto && [output, realisation] : builtOutputs) - res.builtOutputs.insert_or_assign( - std::move(output.outputName), - std::move(realisation)); - } - return res; + return WorkerProto::Serialise::read(*this, *conn); } @@ -840,15 +819,6 @@ void RemoteStore::addTempRoot(const StorePath & path) } -void RemoteStore::addIndirectRoot(const Path & path) -{ - auto conn(getConnection()); - conn->to << WorkerProto::Op::AddIndirectRoot << path; - conn.processStderr(); - readInt(conn->from); -} - - Roots RemoteStore::findRoots(bool censor) { auto conn(getConnection()); @@ -928,7 +898,7 @@ void RemoteStore::queryMissing(const std::vector & targets, // to prevent a deadlock. goto fallback; conn->to << WorkerProto::Op::QueryMissing; - writeDerivedPaths(*this, *conn, targets); + WorkerProto::write(*this, *conn, targets); conn.processStderr(); willBuild = WorkerProto::Serialise::read(*this, *conn); willSubstitute = WorkerProto::Serialise::read(*this, *conn); @@ -1033,7 +1003,7 @@ void RemoteStore::narFromPath(const StorePath & path, Sink & sink) copyNAR(conn->from, sink); } -ref RemoteStore::getFSAccessor() +ref RemoteStore::getFSAccessor(bool requireValidPath) { return make_ref(ref(shared_from_this())); } @@ -1123,7 +1093,7 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * return nullptr; } -void ConnectionHandle::withFramedSink(std::function fun) +void RemoteStore::ConnectionHandle::withFramedSink(std::function fun) { (*this)->to.flush(); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 3bd5f155d57b..c7381a28fcdd 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -18,16 +18,15 @@ class Pid; struct FdSink; struct FdSource; template class Pool; -struct ConnectionHandle; struct RemoteStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - const Setting maxConnections{(StoreConfig*) this, 1, "max-connections", + const Setting maxConnections{this, 1, "max-connections", "Maximum number of concurrent connections to the Nix daemon."}; - const Setting maxConnectionAge{(StoreConfig*) this, + const Setting maxConnectionAge{this, std::numeric_limits::max(), "max-connection-age", "Maximum age of a connection before it is closed."}; @@ -65,7 +64,7 @@ public: StorePathSet queryDerivationOutputs(const StorePath & path) override; - std::map> queryPartialDerivationOutputMap(const StorePath & path) override; + std::map> queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore = nullptr) override; std::optional queryPathFromHashPart(const std::string & hashPart) override; StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; @@ -129,8 +128,6 @@ public: void addTempRoot(const StorePath & path) override; - void addIndirectRoot(const Path & path) override; - Roots findRoots(bool censor) override; void collectGarbage(const GCOptions & options, GCResults & results) override; @@ -189,11 +186,13 @@ protected: void setOptions() override; + struct ConnectionHandle; + ConnectionHandle getConnection(); friend struct ConnectionHandle; - virtual ref getFSAccessor() override; + virtual ref getFSAccessor(bool requireValidPath) override; virtual void narFromPath(const StorePath & path, Sink & sink) override; @@ -206,5 +205,4 @@ private: std::shared_ptr evalStore); }; - } diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index d2fc6abafe46..1a62d92d44ab 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -193,20 +193,20 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig { using BinaryCacheStoreConfig::BinaryCacheStoreConfig; - const Setting profile{(StoreConfig*) this, "", "profile", + const Setting profile{this, "", "profile", R"( The name of the AWS configuration profile to use. By default Nix will use the `default` profile. )"}; - const Setting region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region", + const Setting region{this, Aws::Region::US_EAST_1, "region", R"( The region of the S3 bucket. If your bucket is not in `us–east-1`, you should always explicitly specify the region parameter. )"}; - const Setting scheme{(StoreConfig*) this, "", "scheme", + const Setting scheme{this, "", "scheme", R"( The scheme used for S3 requests, `https` (default) or `http`. This option allows you to disable HTTPS for binary caches which don't @@ -218,7 +218,7 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig > information. )"}; - const Setting endpoint{(StoreConfig*) this, "", "endpoint", + const Setting endpoint{this, "", "endpoint", R"( The URL of the endpoint of an S3-compatible service such as MinIO. Do not specify this setting if you're using Amazon S3. @@ -229,13 +229,13 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig > addressing instead of virtual host based addressing. )"}; - const Setting narinfoCompression{(StoreConfig*) this, "", "narinfo-compression", + const Setting narinfoCompression{this, "", "narinfo-compression", "Compression method for `.narinfo` files."}; - const Setting lsCompression{(StoreConfig*) this, "", "ls-compression", + const Setting lsCompression{this, "", "ls-compression", "Compression method for `.ls` files."}; - const Setting logCompression{(StoreConfig*) this, "", "log-compression", + const Setting logCompression{this, "", "log-compression", R"( Compression method for `log/*` files. It is recommended to use a compression method supported by most web browsers @@ -243,11 +243,11 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig )"}; const Setting multipartUpload{ - (StoreConfig*) this, false, "multipart-upload", + this, false, "multipart-upload", "Whether to use multi-part uploads."}; const Setting bufferSize{ - (StoreConfig*) this, 5 * 1024 * 1024, "buffer-size", + this, 5 * 1024 * 1024, "buffer-size", "Size (in bytes) of each part in multi-part uploads."}; const std::string name() override { return "S3 Binary Cache Store"; } diff --git a/src/libstore/s3-binary-cache-store.md b/src/libstore/s3-binary-cache-store.md index 70fe0eb09660..675470261ec3 100644 --- a/src/libstore/s3-binary-cache-store.md +++ b/src/libstore/s3-binary-cache-store.md @@ -2,7 +2,103 @@ R"( **Store URL format**: `s3://`*bucket-name* -This store allows reading and writing a binary cache stored in an AWS -S3 bucket. +This store allows reading and writing a binary cache stored in an AWS S3 (or S3-compatible service) bucket. +This store shares many idioms with the [HTTP Binary Cache Store](#http-binary-cache-store). + +For AWS S3, the binary cache URL for a bucket named `example-nix-cache` will be exactly . +For S3 compatible binary caches, consult that cache's documentation. + +### Anonymous reads to your S3-compatible binary cache + +> If your binary cache is publicly accessible and does not require authentication, +> it is simplest to use the [HTTP Binary Cache Store] rather than S3 Binary Cache Store with +> instead of . + +Your bucket will need a +[bucket policy](https://docs.aws.amazon.com/AmazonS3/v1/userguide/bucket-policies.html) +like the following to be accessible: + +```json +{ + "Id": "DirectReads", + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowDirectReads", + "Action": [ + "s3:GetObject", + "s3:GetBucketLocation" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::example-nix-cache", + "arn:aws:s3:::example-nix-cache/*" + ], + "Principal": "*" + } + ] +} +``` + +### Authentication + +Nix will use the +[default credential provider chain](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html) +for authenticating requests to Amazon S3. + +Note that this means Nix will read environment variables and files with different idioms than with Nix's own settings, as implemented by the AWS SDK. +Consult the documentation linked above for further details. + +### Authenticated reads to your S3 binary cache + +Your bucket will need a bucket policy allowing the desired users to perform the `s3:GetObject` and `s3:GetBucketLocation` action on all objects in the bucket. +The [anonymous policy given above](#anonymous-reads-to-your-s3-compatible-binary-cache) can be updated to have a restricted `Principal` to support this. + +### Authenticated writes to your S3-compatible binary cache + +Your account will need an IAM policy to support uploading to the bucket: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "UploadToCache", + "Effect": "Allow", + "Action": [ + "s3:AbortMultipartUpload", + "s3:GetBucketLocation", + "s3:GetObject", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:ListMultipartUploadParts", + "s3:PutObject" + ], + "Resource": [ + "arn:aws:s3:::example-nix-cache", + "arn:aws:s3:::example-nix-cache/*" + ] + } + ] +} +``` + +### Examples + +With bucket policies and authentication set up as described above, uploading works via [`nix copy`](@docroot@/command-ref/new-cli/nix3-copy.md) (experimental). + +- To upload with a specific credential profile for Amazon S3: + + ```console + $ nix copy nixpkgs.hello \ + --to 's3://example-nix-cache?profile=cache-upload®ion=eu-west-2' + ``` + +- To upload to an S3-compatible binary cache: + + ```console + $ nix copy nixpkgs.hello --to \ + 's3://example-nix-cache?profile=cache-upload&scheme=https&endpoint=minio.example.com' + ``` )" diff --git a/src/libstore/serve-protocol-impl.hh b/src/libstore/serve-protocol-impl.hh new file mode 100644 index 000000000000..a3ce81026095 --- /dev/null +++ b/src/libstore/serve-protocol-impl.hh @@ -0,0 +1,59 @@ +#pragma once +/** + * @file + * + * Template implementations (as opposed to mere declarations). + * + * This file is an exmample of the "impl.hh" pattern. See the + * contributing guide. + */ + +#include "serve-protocol.hh" +#include "length-prefixed-protocol-helper.hh" + +namespace nix { + +/* protocol-agnostic templates */ + +#define SERVE_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \ + TEMPLATE T ServeProto::Serialise< T >::read(const Store & store, ServeProto::ReadConn conn) \ + { \ + return LengthPrefixedProtoHelper::read(store, conn); \ + } \ + TEMPLATE void ServeProto::Serialise< T >::write(const Store & store, ServeProto::WriteConn conn, const T & t) \ + { \ + LengthPrefixedProtoHelper::write(store, conn, t); \ + } + +SERVE_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) +SERVE_USE_LENGTH_PREFIX_SERIALISER(template, std::set) +SERVE_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) + +#define COMMA_ , +SERVE_USE_LENGTH_PREFIX_SERIALISER( + template, + std::map) +#undef COMMA_ + +/** + * Use `CommonProto` where possible. + */ +template +struct ServeProto::Serialise +{ + static T read(const Store & store, ServeProto::ReadConn conn) + { + return CommonProto::Serialise::read(store, + CommonProto::ReadConn { .from = conn.from }); + } + static void write(const Store & store, ServeProto::WriteConn conn, const T & t) + { + CommonProto::Serialise::write(store, + CommonProto::WriteConn { .to = conn.to }, + t); + } +}; + +/* protocol-specific templates */ + +} diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc new file mode 100644 index 000000000000..9bfcc279cbac --- /dev/null +++ b/src/libstore/serve-protocol.cc @@ -0,0 +1,57 @@ +#include "serialise.hh" +#include "path-with-outputs.hh" +#include "store-api.hh" +#include "build-result.hh" +#include "serve-protocol.hh" +#include "serve-protocol-impl.hh" +#include "archive.hh" + +#include + +namespace nix { + +/* protocol-specific definitions */ + +BuildResult ServeProto::Serialise::read(const Store & store, ServeProto::ReadConn conn) +{ + BuildResult status; + status.status = (BuildResult::Status) readInt(conn.from); + conn.from >> status.errorMsg; + + if (GET_PROTOCOL_MINOR(conn.version) >= 3) + conn.from + >> status.timesBuilt + >> status.isNonDeterministic + >> status.startTime + >> status.stopTime; + if (GET_PROTOCOL_MINOR(conn.version) >= 6) { + auto builtOutputs = ServeProto::Serialise::read(store, conn); + for (auto && [output, realisation] : builtOutputs) + status.builtOutputs.insert_or_assign( + std::move(output.outputName), + std::move(realisation)); + } + return status; +} + +void ServeProto::Serialise::write(const Store & store, ServeProto::WriteConn conn, const BuildResult & status) +{ + conn.to + << status.status + << status.errorMsg; + + if (GET_PROTOCOL_MINOR(conn.version) >= 3) + conn.to + << status.timesBuilt + << status.isNonDeterministic + << status.startTime + << status.stopTime; + if (GET_PROTOCOL_MINOR(conn.version) >= 6) { + DrvOutputs builtOutputs; + for (auto & [output, realisation] : status.builtOutputs) + builtOutputs.insert_or_assign(realisation.id, realisation); + ServeProto::write(store, conn, builtOutputs); + } +} + +} diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index 7e43b3969917..ba159f6e9ab1 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -1,6 +1,8 @@ #pragma once ///@file +#include "common-protocol.hh" + namespace nix { #define SERVE_MAGIC_1 0x390c9deb @@ -10,6 +12,14 @@ namespace nix { #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) + +class Store; +struct Source; + +// items being serialised +struct BuildResult; + + /** * The "serve protocol", used by ssh:// stores. * @@ -22,6 +32,60 @@ struct ServeProto * Enumeration of all the request types for the protocol. */ enum struct Command : uint64_t; + + /** + * Version type for the protocol. + * + * @todo Convert to struct with separate major vs minor fields. + */ + using Version = unsigned int; + + /** + * A unidirectional read connection, to be used by the read half of the + * canonical serializers below. + */ + struct ReadConn { + Source & from; + Version version; + }; + + /** + * A unidirectional write connection, to be used by the write half of the + * canonical serializers below. + */ + struct WriteConn { + Sink & to; + Version version; + }; + + /** + * Data type for canonical pairs of serialisers for the serve protocol. + * + * See https://en.cppreference.com/w/cpp/language/adl for the broader + * concept of what is going on here. + */ + template + struct Serialise; + // This is the definition of `Serialise` we *want* to put here, but + // do not do so. + // + // See `worker-protocol.hh` for a longer explanation. +#if 0 + { + static T read(const Store & store, ReadConn conn); + static void write(const Store & store, WriteConn conn, const T & t); + }; +#endif + + /** + * Wrapper function around `ServeProto::Serialise::write` that allows us to + * infer the type instead of having to write it down explicitly. + */ + template + static void write(const Store & store, WriteConn conn, const T & t) + { + ServeProto::Serialise::write(store, conn, t); + } }; enum struct ServeProto::Command : uint64_t @@ -58,4 +122,36 @@ inline std::ostream & operator << (std::ostream & s, ServeProto::Command op) return s << (uint64_t) op; } +/** + * Declare a canonical serialiser pair for the worker protocol. + * + * We specialise the struct merely to indicate that we are implementing + * the function for the given type. + * + * Some sort of `template<...>` must be used with the caller for this to + * be legal specialization syntax. See below for what that looks like in + * practice. + */ +#define DECLARE_SERVE_SERIALISER(T) \ + struct ServeProto::Serialise< T > \ + { \ + static T read(const Store & store, ServeProto::ReadConn conn); \ + static void write(const Store & store, ServeProto::WriteConn conn, const T & t); \ + }; + +template<> +DECLARE_SERVE_SERIALISER(BuildResult); + +template +DECLARE_SERVE_SERIALISER(std::vector); +template +DECLARE_SERVE_SERIALISER(std::set); +template +DECLARE_SERVE_SERIALISER(std::tuple); + +#define COMMA_ , +template +DECLARE_SERVE_SERIALISER(std::map); +#undef COMMA_ + } diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 7c8decb74305..d7432a3059e5 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -2,6 +2,7 @@ #include "globals.hh" #include "util.hh" #include "url.hh" +#include "signals.hh" #include diff --git a/src/libstore/ssh-store-config.hh b/src/libstore/ssh-store-config.hh index c27a5d00fc99..4ce4ffc4ca1c 100644 --- a/src/libstore/ssh-store-config.hh +++ b/src/libstore/ssh-store-config.hh @@ -9,18 +9,18 @@ struct CommonSSHStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - const Setting sshKey{(StoreConfig*) this, "", "ssh-key", + const Setting sshKey{this, "", "ssh-key", "Path to the SSH private key used to authenticate to the remote machine."}; - const Setting sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", + const Setting sshPublicHostKey{this, "", "base64-ssh-public-host-key", "The public host key of the remote machine."}; - const Setting compress{(StoreConfig*) this, false, "compress", + const Setting compress{this, false, "compress", "Whether to enable SSH compression."}; - const Setting remoteStore{(StoreConfig*) this, "", "remote-store", + const Setting remoteStore{this, "", "remote-store", R"( - [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + [Store URL](@docroot@/store/types/index.md#store-url-format) to be used on the remote machine. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). )"}; diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 0200076c08b4..d4c8ab5b2f0f 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -1,10 +1,12 @@ #include "ssh-store-config.hh" #include "store-api.hh" +#include "local-fs-store.hh" #include "remote-store.hh" #include "remote-store-connection.hh" -#include "remote-fs-accessor.hh" +#include "source-accessor.hh" #include "archive.hh" #include "worker-protocol.hh" +#include "worker-protocol-impl.hh" #include "pool.hh" #include "ssh.hh" @@ -15,7 +17,7 @@ struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig using RemoteStoreConfig::RemoteStoreConfig; using CommonSSHStoreConfig::CommonSSHStoreConfig; - const Setting remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", + const Setting remoteProgram{this, "nix-daemon", "remote-program", "Path to the `nix-daemon` executable on the remote machine."}; const std::string name() override { return "Experimental SSH Store"; } @@ -61,7 +63,7 @@ class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore std::optional getBuildLogExact(const StorePath & path) override { unsupported("getBuildLogExact"); } -private: +protected: struct Connection : RemoteStore::Connection { @@ -77,6 +79,8 @@ class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore std::string host; + std::vector extraRemoteProgramArgs; + SSHMaster master; void setOptions(RemoteStore::Connection & conn) override @@ -90,17 +94,138 @@ class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore }; }; +struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig +{ + using SSHStoreConfig::SSHStoreConfig; + using LocalFSStoreConfig::LocalFSStoreConfig; + + MountedSSHStoreConfig(StringMap params) + : StoreConfig(params) + , RemoteStoreConfig(params) + , CommonSSHStoreConfig(params) + , SSHStoreConfig(params) + , LocalFSStoreConfig(params) + { + } + + const std::string name() override { return "Experimental SSH Store with filesytem mounted"; } + + std::string doc() override + { + return + #include "mounted-ssh-store.md" + ; + } + + std::optional experimentalFeature() const override + { + return ExperimentalFeature::MountedSSHStore; + } +}; + +/** + * The mounted ssh store assumes that filesystems on the remote host are + * shared with the local host. This means that the remote nix store is + * available locally and is therefore treated as a local filesystem + * store. + * + * MountedSSHStore is very similar to UDSRemoteStore --- ignoring the + * superficial differnce of SSH vs Unix domain sockets, they both are + * accessing remote stores, and they both assume the store will be + * mounted in the local filesystem. + * + * The difference lies in how they manage GC roots. See addPermRoot + * below for details. + */ +class MountedSSHStore : public virtual MountedSSHStoreConfig, public virtual SSHStore, public virtual LocalFSStore +{ +public: + + MountedSSHStore(const std::string & scheme, const std::string & host, const Params & params) + : StoreConfig(params) + , RemoteStoreConfig(params) + , CommonSSHStoreConfig(params) + , SSHStoreConfig(params) + , LocalFSStoreConfig(params) + , MountedSSHStoreConfig(params) + , Store(params) + , RemoteStore(params) + , SSHStore(scheme, host, params) + , LocalFSStore(params) + { + extraRemoteProgramArgs = { + "--process-ops", + }; + } + + static std::set uriSchemes() + { + return {"mounted-ssh-ng"}; + } + + std::string getUri() override + { + return *uriSchemes().begin() + "://" + host; + } + + void narFromPath(const StorePath & path, Sink & sink) override + { + return LocalFSStore::narFromPath(path, sink); + } + + ref getFSAccessor(bool requireValidPath) override + { + return LocalFSStore::getFSAccessor(requireValidPath); + } + + std::optional getBuildLogExact(const StorePath & path) override + { + return LocalFSStore::getBuildLogExact(path); + } + + /** + * This is the key difference from UDSRemoteStore: UDSRemote store + * has the client create the direct root, and the remote side create + * the indirect root. + * + * We could also do that, but the race conditions (will the remote + * side see the direct root the client made?) seems bigger. + * + * In addition, the remote-side will have a process associated with + * the authenticating user handling the connection (even if there + * is a system-wide daemon or similar). This process can safely make + * the direct and indirect roots without there being such a risk of + * privilege escalation / symlinks in directories owned by the + * originating requester that they cannot delete. + */ + Path addPermRoot(const StorePath & path, const Path & gcRoot) override + { + auto conn(getConnection()); + conn->to << WorkerProto::Op::AddPermRoot; + WorkerProto::write(*this, *conn, path); + WorkerProto::write(*this, *conn, gcRoot); + conn.processStderr(); + return readString(conn->from); + } +}; + ref SSHStore::openConnection() { auto conn = make_ref(); - conn->sshConn = master.startCommand( - fmt("%s --stdio", remoteProgram) - + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get()))); + + std::string command = remoteProgram + " --stdio"; + if (remoteStore.get() != "") + command += " --store " + shellEscape(remoteStore.get()); + for (auto & arg : extraRemoteProgramArgs) + command += " " + shellEscape(arg); + + conn->sshConn = master.startCommand(command); conn->to = FdSink(conn->sshConn->in.get()); conn->from = FdSource(conn->sshConn->out.get()); return conn; } static RegisterStoreImplementation regSSHStore; +static RegisterStoreImplementation regMountedSSHStore; } diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index fae99d75b92b..5c8d6a5042b1 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -1,5 +1,8 @@ #include "ssh.hh" #include "finally.hh" +#include "current-process.hh" +#include "environment-variables.hh" +#include "util.hh" namespace nix { @@ -42,7 +45,10 @@ void SSHMaster::addCommonSSHOpts(Strings & args) } bool SSHMaster::isMasterRunning() { - auto res = runProgram(RunOptions {.program = "ssh", .args = {"-O", "check", host}, .mergeStderrToStdout = true}); + Strings args = {"-O", "check", host}; + addCommonSSHOpts(args); + + auto res = runProgram(RunOptions {.program = "ssh", .args = args, .mergeStderrToStdout = true}); return res.first == 0; } @@ -108,8 +114,10 @@ std::unique_ptr SSHMaster::startCommand(const std::string reply = readLine(out.readSide.get()); } catch (EndOfFile & e) { } - if (reply != "started") + if (reply != "started") { + printTalkative("SSH stdout first line: %s", reply); throw Error("failed to start SSH connection to '%s'", host); + } } conn->out = std::move(out.readSide); @@ -126,7 +134,6 @@ Path SSHMaster::startMaster() if (state->sshMaster != -1) return state->socketPath; - state->socketPath = (Path) *state->tmpDir + "/ssh.sock"; Pipe out; @@ -138,7 +145,8 @@ Path SSHMaster::startMaster() logger->pause(); Finally cleanup = [&]() { logger->resume(); }; - bool wasMasterRunning = isMasterRunning(); + if (isMasterRunning()) + return state->socketPath; state->sshMaster = startProcess([&]() { restoreProcessContext(); @@ -159,14 +167,14 @@ Path SSHMaster::startMaster() out.writeSide = -1; - if (!wasMasterRunning) { - std::string reply; - try { - reply = readLine(out.readSide.get()); - } catch (EndOfFile & e) { } + std::string reply; + try { + reply = readLine(out.readSide.get()); + } catch (EndOfFile & e) { } - if (reply != "started") - throw Error("failed to start SSH master connection to '%s'", host); + if (reply != "started") { + printTalkative("SSH master stdout first line: %s", reply); + throw Error("failed to start SSH master connection to '%s'", host); } return state->socketPath; diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh index 94b952af9c07..bfcd6f21cea8 100644 --- a/src/libstore/ssh.hh +++ b/src/libstore/ssh.hh @@ -1,8 +1,9 @@ #pragma once ///@file -#include "util.hh" #include "sync.hh" +#include "processes.hh" +#include "file-system.hh" namespace nix { diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index ad39f7bbded5..f53f6da9dae9 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1,5 +1,5 @@ #include "crypto.hh" -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "globals.hh" #include "derivations.hh" #include "store-api.hh" @@ -11,6 +11,11 @@ #include "archive.hh" #include "callback.hh" #include "remote-store.hh" +// FIXME this should not be here, see TODO below on +// `addMultipleToStore`. +#include "worker-protocol.hh" +#include "signals.hh" +#include "users.hh" #include #include @@ -154,7 +159,7 @@ StorePath Store::makeStorePath(std::string_view type, StorePath Store::makeStorePath(std::string_view type, const Hash & hash, std::string_view name) const { - return makeStorePath(type, hash.to_string(Base16, true), name); + return makeStorePath(type, hash.to_string(HashFormat::Base16, true), name); } @@ -184,15 +189,15 @@ static std::string makeType( StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { - if (info.hash.hash.type == htSHA256 && info.hash.method == FileIngestionMethod::Recursive) { - return makeStorePath(makeType(*this, "source", info.references), info.hash.hash, name); + if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) { + return makeStorePath(makeType(*this, "source", info.references), info.hash, name); } else { assert(info.references.size() == 0); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" - + makeFileIngestionPrefix(info.hash.method) - + info.hash.hash.to_string(Base16, true) + ":"), + + makeFileIngestionPrefix(info.method) + + info.hash.to_string(HashFormat::Base16, true) + ":"), name); } } @@ -200,13 +205,13 @@ StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInf StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const { - assert(info.hash.hash.type == htSHA256); + assert(info.hash.type == htSHA256); return makeStorePath( makeType(*this, "text", StoreReferences { .others = info.references, .self = false, }), - info.hash.hash, + info.hash, name); } @@ -225,17 +230,19 @@ StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentA } -std::pair Store::computeStorePathForPath(std::string_view name, - const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter) const +std::pair Store::computeStorePathFromDump( + Source & dump, + std::string_view name, + FileIngestionMethod method, + HashType hashAlgo, + const StorePathSet & references) const { - Hash h = method == FileIngestionMethod::Recursive - ? hashPath(hashAlgo, srcPath, filter).first - : hashFile(hashAlgo, srcPath); + HashSink sink(hashAlgo); + dump.drainInto(sink); + auto h = sink.finish().first; FixedOutputInfo caInfo { - .hash = { - .method = method, - .hash = h, - }, + .method = method, + .hash = h, .references = {}, }; return std::make_pair(makeFixedOutputPath(name, caInfo), h); @@ -248,8 +255,8 @@ StorePath Store::computeStorePathForText( const StorePathSet & references) const { return makeTextPath(name, TextInfo { - { .hash = hashString(htSHA256, s) }, - references, + .hash = hashString(htSHA256, s), + .references = references, }); } @@ -359,7 +366,13 @@ void Store::addMultipleToStore( { auto expected = readNum(source); for (uint64_t i = 0; i < expected; ++i) { - auto info = ValidPathInfo::read(source, *this, 16); + // FIXME we should not be using the worker protocol here, let + // alone the worker protocol with a hard-coded version! + auto info = WorkerProto::Serialise::read(*this, + WorkerProto::ReadConn { + .from = source, + .version = 16, + }); info.ultimate = false; addToStore(info, source, repair, checkSigs); } @@ -399,7 +412,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, /* Note that fileSink and unusualHashTee must be mutually exclusive, since they both write to caHashSink. Note that that requisite is currently true because the former is only used in the flat case. */ - RetrieveRegularNARSink fileSink { caHashSink }; + RegularFileSink fileSink { caHashSink }; TeeSink unusualHashTee { narHashSink, caHashSink }; auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != htSHA256 @@ -417,10 +430,10 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, information to narSink. */ TeeSource tapped { *fileSource, narSink }; - ParseSink blank; + NullParseSink blank; auto & parseSink = method == FileIngestionMethod::Flat - ? fileSink - : blank; + ? (ParseSink &) fileSink + : (ParseSink &) blank; /* The information that flows from tapped (besides being replicated in narSink), is now put in parseSink. */ @@ -441,10 +454,8 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, *this, name, FixedOutputInfo { - .hash = { - .method = method, - .hash = hash, - }, + .method = method, + .hash = hash, .references = {}, }, narHash, @@ -496,22 +507,50 @@ bool Store::PathInfoCacheValue::isKnownNow() return std::chrono::steady_clock::now() < time_point + ttl; } -std::map> Store::queryPartialDerivationOutputMap(const StorePath & path) +std::map> Store::queryStaticPartialDerivationOutputMap(const StorePath & path) { std::map> outputs; auto drv = readInvalidDerivation(path); - for (auto& [outputName, output] : drv.outputsAndOptPaths(*this)) { + for (auto & [outputName, output] : drv.outputsAndOptPaths(*this)) { outputs.emplace(outputName, output.second); } return outputs; } +std::map> Store::queryPartialDerivationOutputMap( + const StorePath & path, + Store * evalStore_) +{ + auto & evalStore = evalStore_ ? *evalStore_ : *this; + + auto outputs = evalStore.queryStaticPartialDerivationOutputMap(path); + + if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) + return outputs; + + auto drv = evalStore.readInvalidDerivation(path); + auto drvHashes = staticOutputHashes(*this, drv); + for (auto & [outputName, hash] : drvHashes) { + auto realisation = queryRealisation(DrvOutput{hash, outputName}); + if (realisation) { + outputs.insert_or_assign(outputName, realisation->outPath); + } else { + // queryStaticPartialDerivationOutputMap is not guaranteed + // to return std::nullopt for outputs which are not + // statically known. + outputs.insert({outputName, std::nullopt}); + } + } + + return outputs; +} + OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) { auto resp = queryPartialDerivationOutputMap(path); OutputPathMap result; for (auto & [outName, optOutPath] : resp) { if (!optOutPath) - throw Error("output '%s' of derivation '%s' has no store path mapped to it", outName, printStorePath(path)); + throw MissingRealisation(printStorePath(path), outName); result.insert_or_assign(outName, *optOutPath); } return result; @@ -782,7 +821,7 @@ void Store::substitutePaths(const StorePathSet & paths) std::vector paths2; for (auto & path : paths) if (!path.isDerivation()) - paths2.push_back(DerivedPath::Opaque{path}); + paths2.emplace_back(DerivedPath::Opaque{path}); uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; queryMissing(paths2, @@ -860,7 +899,7 @@ std::string Store::makeValidityRegistration(const StorePathSet & paths, auto info = queryPathInfo(i); if (showHash) { - s += info->narHash.to_string(Base16, false) + "\n"; + s += info->narHash.to_string(HashFormat::Base16, false) + "\n"; s += fmt("%1%\n", info->narSize); } @@ -912,107 +951,6 @@ StorePathSet Store::exportReferences(const StorePathSet & storePaths, const Stor return paths; } -json Store::pathInfoToJSON(const StorePathSet & storePaths, - bool includeImpureInfo, bool showClosureSize, - Base hashBase, - AllowInvalidFlag allowInvalid) -{ - json::array_t jsonList = json::array(); - - for (auto & storePath : storePaths) { - auto& jsonPath = jsonList.emplace_back(json::object()); - - try { - auto info = queryPathInfo(storePath); - - jsonPath["path"] = printStorePath(info->path); - jsonPath["valid"] = true; - jsonPath["narHash"] = info->narHash.to_string(hashBase, true); - jsonPath["narSize"] = info->narSize; - - { - auto& jsonRefs = (jsonPath["references"] = json::array()); - for (auto & ref : info->references) - jsonRefs.emplace_back(printStorePath(ref)); - } - - if (info->ca) - jsonPath["ca"] = renderContentAddress(info->ca); - - std::pair closureSizes; - - if (showClosureSize) { - closureSizes = getClosureSize(info->path); - jsonPath["closureSize"] = closureSizes.first; - } - - if (includeImpureInfo) { - - if (info->deriver) - jsonPath["deriver"] = printStorePath(*info->deriver); - - if (info->registrationTime) - jsonPath["registrationTime"] = info->registrationTime; - - if (info->ultimate) - jsonPath["ultimate"] = info->ultimate; - - if (!info->sigs.empty()) { - for (auto & sig : info->sigs) - jsonPath["signatures"].push_back(sig); - } - - auto narInfo = std::dynamic_pointer_cast( - std::shared_ptr(info)); - - if (narInfo) { - if (!narInfo->url.empty()) - jsonPath["url"] = narInfo->url; - if (narInfo->fileHash) - jsonPath["downloadHash"] = narInfo->fileHash->to_string(hashBase, true); - if (narInfo->fileSize) - jsonPath["downloadSize"] = narInfo->fileSize; - if (showClosureSize) - jsonPath["closureDownloadSize"] = closureSizes.second; - } - - if (info->accessStatus) { - jsonPath["protected"] = info->accessStatus->isProtected; - for (auto & entity : info->accessStatus->entities) { - std::visit(overloaded { - [&](ACL::User u) { jsonPath["allowedUsers"].push_back(getpwuid(u.uid)->pw_name); }, - [&](ACL::Group g) { jsonPath["allowedGroups"].push_back(getgrgid(g.gid)->gr_name); }, - }, entity); - } - } - } - - } catch (InvalidPath &) { - jsonPath["path"] = printStorePath(storePath); - jsonPath["valid"] = false; - } - } - return jsonList; -} - - -std::pair Store::getClosureSize(const StorePath & storePath) -{ - uint64_t totalNarSize = 0, totalDownloadSize = 0; - StorePathSet closure; - computeFSClosure(storePath, closure, false, false); - for (auto & p : closure) { - auto info = queryPathInfo(p); - totalNarSize += info->narSize; - auto narInfo = std::dynamic_pointer_cast( - std::shared_ptr(info)); - if (narInfo) - totalDownloadSize += narInfo->fileSize; - } - return {totalNarSize, totalDownloadSize}; -} - - const Store::Stats & Store::getStats() { { @@ -1311,12 +1249,12 @@ Derivation Store::derivationFromPath(const StorePath & drvPath) return readDerivation(drvPath); } -Derivation readDerivationCommon(Store& store, const StorePath& drvPath, bool requireValidPath) +static Derivation readDerivationCommon(Store & store, const StorePath & drvPath, bool requireValidPath) { - auto accessor = store.getFSAccessor(); + auto accessor = store.getFSAccessor(requireValidPath); try { return parseDerivation(store, - accessor->readFile(store.printStorePath(drvPath), requireValidPath), + accessor->readFile(CanonPath(store.printStorePath(drvPath))), Derivation::nameFromPath(drvPath)); } catch (FormatError & e) { throw Error("error parsing derivation '%s': %s", store.printStorePath(drvPath), e.msg()); @@ -1482,6 +1420,7 @@ ref openStore(const std::string & uri_, if (implem.uriSchemes.count(parsedUri.scheme)) { auto store = implem.create(parsedUri.scheme, baseURI, params); if (store) { + experimentalFeatureSettings.require(store->experimentalFeature()); store->init(); store->warnUnknownSettings(); return ref(store); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 8029f4db1ff0..1e448efe4276 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -32,7 +32,7 @@ namespace nix { /** - * About the class hierarchy of the store implementations: + * About the class hierarchy of the store types: * * Each store type `Foo` consists of two classes: * @@ -74,7 +74,7 @@ MakeError(AccessDenied, Error); struct BasicDerivation; struct Derivation; -class FSAccessor; +struct SourceAccessor; class NarInfoDiskCache; class Store; @@ -84,7 +84,6 @@ typedef std::map OutputPathMap; enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true }; enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true }; -enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true }; /** @@ -103,6 +102,8 @@ typedef std::map> StorePathCAMap; struct StoreConfig : public Config { + typedef std::map Params; + using Config::Config; StoreConfig() = delete; @@ -111,13 +112,28 @@ struct StoreConfig : public Config virtual ~StoreConfig() { } + /** + * The name of this type of store. + */ virtual const std::string name() = 0; + /** + * Documentation for this type of store. + */ virtual std::string doc() { return ""; } + /** + * An experimental feature this type store is gated, if it is to be + * experimental. + */ + virtual std::optional experimentalFeature() const + { + return std::nullopt; + } + const PathSetting storeDir_{this, settings.nixStore, "store", R"( @@ -140,27 +156,29 @@ struct StoreConfig : public Config Setting priority{this, 0, "priority", R"( - Priority of this store when used as a substituter. A lower value means a higher priority. + Priority of this store when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). + A lower value means a higher priority. )"}; Setting wantMassQuery{this, false, "want-mass-query", R"( - Whether this store (when used as a substituter) can be - queried efficiently for path validity. + Whether this store can be queried efficiently for path validity when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). )"}; Setting systemFeatures{this, getDefaultSystemFeatures(), "system-features", - "Optional features that the system this store builds on implements (like \"kvm\")."}; + R"( + Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations. + Example: `"kvm"` + )", + {}, + // Don't document the machine-specific default value + false}; }; class Store : public std::enable_shared_from_this, public virtual StoreConfig { -public: - - typedef std::map Params; - protected: struct PathInfoCacheValue { @@ -280,14 +298,15 @@ public: StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const; /** - * Preparatory part of addToStore(). - * - * @return the store path to which srcPath is to be copied - * and the cryptographic hash of the contents of srcPath. + * Read-only variant of addToStoreFromDump(). It returns the store + * path to which a NAR or flat file would be written. */ - std::pair computeStorePathForPath(std::string_view name, - const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, - HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const; + std::pair computeStorePathFromDump( + Source & dump, + std::string_view name, + FileIngestionMethod method = FileIngestionMethod::Recursive, + HashType hashAlgo = htSHA256, + const StorePathSet & references = {}) const; /** * Preparatory part of addTextToStore(). @@ -429,7 +448,20 @@ public: * derivation. All outputs are mentioned so ones mising the mapping * are mapped to `std::nullopt`. */ - virtual std::map> queryPartialDerivationOutputMap(const StorePath & path); + virtual std::map> queryPartialDerivationOutputMap( + const StorePath & path, + Store * evalStore = nullptr); + + /** + * Like `queryPartialDerivationOutputMap` but only considers + * statically known output paths (i.e. those that can be gotten from + * the derivation itself. + * + * Just a helper function for implementing + * `queryPartialDerivationOutputMap`. + */ + virtual std::map> queryStaticPartialDerivationOutputMap( + const StorePath & path); /** * Query the mapping outputName=>outputPath for the given derivation. @@ -639,28 +671,6 @@ public: std::string makeValidityRegistration(const StorePathSet & paths, bool showDerivers, bool showHash); - /** - * Write a JSON representation of store path metadata, such as the - * hash and the references. - * - * @param includeImpureInfo If true, variable elements such as the - * registration time are included. - * - * @param showClosureSize If true, the closure size of each path is - * included. - */ - nlohmann::json pathInfoToJSON(const StorePathSet & storePaths, - bool includeImpureInfo, bool showClosureSize, - Base hashBase = Base32, - AllowInvalidFlag allowInvalid = DisallowInvalid); - - /** - * @return the size of the closure of the specified path, that is, - * the sum of the size of the NAR serialisation of each path in the - * closure. - */ - std::pair getClosureSize(const StorePath & storePath); - /** * Optimise the disk space usage of the Nix store by hard-linking files * with the same contents. @@ -677,7 +687,7 @@ public: /** * @return An object to access files in the Nix store. */ - virtual ref getFSAccessor() = 0; + virtual ref getFSAccessor(bool requireValidPath = true) = 0; /** * Repair the contents of the given path by redownloading it using @@ -923,6 +933,7 @@ void removeTempRoots(); * Resolve the derived path completely, failing if any derivation output * is unknown. */ +StorePath resolveDerivedPath(Store &, const SingleDerivedPath &, Store * evalStore = nullptr); OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * evalStore = nullptr); /** * Resolve the derived path, splitting it into known and unknown outputs. @@ -959,7 +970,7 @@ std::pair> resolveDerivedPathAll(Store & stor * - ‘ssh://[user@]’: A remote Nix store accessed by running * ‘nix-store --serve’ via SSH. * - * You can pass parameters to the store implementation by appending + * You can pass parameters to the store type by appending * ‘?key=value&key=value&...’ to the URI. */ ref openStore(const std::string & uri = settings.storeUri.get(), diff --git a/src/libstore/tests/common-protocol.cc b/src/libstore/tests/common-protocol.cc new file mode 100644 index 000000000000..c09ac6a3eb03 --- /dev/null +++ b/src/libstore/tests/common-protocol.cc @@ -0,0 +1,187 @@ +#include + +#include +#include + +#include "common-protocol.hh" +#include "common-protocol-impl.hh" +#include "build-result.hh" +#include "tests/protocol.hh" +#include "tests/characterization.hh" + +namespace nix { + +const char commonProtoDir[] = "common-protocol"; + +class CommonProtoTest : public ProtoTest +{ +public: + /** + * Golden test for `T` reading + */ + template + void readProtoTest(PathView testStem, const T & expected) + { + CharacterizationTest::readTest(testStem, [&](const auto & encoded) { + T got = ({ + StringSource from { encoded }; + CommonProto::Serialise::read( + *store, + CommonProto::ReadConn { .from = from }); + }); + + ASSERT_EQ(got, expected); + }); + } + + /** + * Golden test for `T` write + */ + template + void writeProtoTest(PathView testStem, const T & decoded) + { + CharacterizationTest::writeTest(testStem, [&]() -> std::string { + StringSink to; + CommonProto::Serialise::write( + *store, + CommonProto::WriteConn { .to = to }, + decoded); + return to.s; + }); + } +}; + +#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ + TEST_F(CommonProtoTest, NAME ## _read) { \ + readProtoTest(STEM, VALUE); \ + } \ + TEST_F(CommonProtoTest, NAME ## _write) { \ + writeProtoTest(STEM, VALUE); \ + } + +CHARACTERIZATION_TEST( + string, + "string", + (std::tuple { + "", + "hi", + "white rabbit", + "大白兔", + "oh no \0\0\0 what was that!", + })) + +CHARACTERIZATION_TEST( + storePath, + "store-path", + (std::tuple { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + })) + +CHARACTERIZATION_TEST( + contentAddress, + "content-address", + (std::tuple { + ContentAddress { + .method = TextIngestionMethod {}, + .hash = hashString(HashType::htSHA256, "Derive(...)"), + }, + ContentAddress { + .method = FileIngestionMethod::Flat, + .hash = hashString(HashType::htSHA1, "blob blob..."), + }, + ContentAddress { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashType::htSHA256, "(...)"), + }, + })) + +CHARACTERIZATION_TEST( + drvOutput, + "drv-output", + (std::tuple { + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + })) + +CHARACTERIZATION_TEST( + realisation, + "realisation", + (std::tuple { + Realisation { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + }, + Realisation { + .id = { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + .dependentRealisations = { + { + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + }, + })) + +CHARACTERIZATION_TEST( + vector, + "vector", + (std::tuple, std::vector, std::vector, std::vector>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +CHARACTERIZATION_TEST( + set, + "set", + (std::tuple, std::set, std::set, std::set>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +CHARACTERIZATION_TEST( + optionalStorePath, + "optional-store-path", + (std::tuple, std::optional> { + std::nullopt, + std::optional { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + }, + })) + +CHARACTERIZATION_TEST( + optionalContentAddress, + "optional-content-address", + (std::tuple, std::optional> { + std::nullopt, + std::optional { + ContentAddress { + .method = FileIngestionMethod::Flat, + .hash = hashString(HashType::htSHA1, "blob blob..."), + }, + }, + })) + +} diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc index 6328ad3700e2..7becfa5aba1e 100644 --- a/src/libstore/tests/derivation.cc +++ b/src/libstore/tests/derivation.cc @@ -5,12 +5,21 @@ #include "derivations.hh" #include "tests/libstore.hh" +#include "tests/characterization.hh" namespace nix { -class DerivationTest : public LibStoreTest +using nlohmann::json; + +class DerivationTest : public CharacterizationTest, public LibStoreTest { + Path unitTestData = getUnitTestData() + "/libstore/derivation"; + public: + Path goldenMaster(std::string_view testStem) const override { + return unitTestData + "/" + testStem; + } + /** * We set these in tests rather than the regular globals so we don't have * to worry about race conditions if the tests run concurrently. @@ -42,46 +51,63 @@ class ImpureDerivationTest : public DerivationTest } }; -#define TEST_JSON(FIXTURE, NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \ - TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - STR ## _json, \ - (DerivationOutput { VAL }).toJSON( \ - *store, \ - DRV_NAME, \ - OUTPUT_NAME)); \ - } \ - \ - TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _from_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - DerivationOutput { VAL }, \ - DerivationOutput::fromJSON( \ - *store, \ - DRV_NAME, \ - OUTPUT_NAME, \ - STR ## _json, \ - mockXpSettings)); \ +TEST_F(DerivationTest, BadATerm_version) { + ASSERT_THROW( + parseDerivation( + *store, + readFile(goldenMaster("bad-version.drv")), + "whatever", + mockXpSettings), + FormatError); +} + +TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) { + ASSERT_THROW( + parseDerivation( + *store, + readFile(goldenMaster("bad-old-version-dyn-deps.drv")), + "dyn-dep-derivation", + mockXpSettings), + FormatError); +} + +#define TEST_JSON(FIXTURE, NAME, VAL, DRV_NAME, OUTPUT_NAME) \ + TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _from_json) { \ + readTest("output-" #NAME ".json", [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + DerivationOutput got = DerivationOutput::fromJSON( \ + *store, \ + DRV_NAME, \ + OUTPUT_NAME, \ + encoded, \ + mockXpSettings); \ + DerivationOutput expected { VAL }; \ + ASSERT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) { \ + writeTest("output-" #NAME ".json", [&]() -> json { \ + return DerivationOutput { (VAL) }.toJSON( \ + *store, \ + (DRV_NAME), \ + (OUTPUT_NAME)); \ + }, [](const auto & file) { \ + return json::parse(readFile(file)); \ + }, [](const auto & file, const auto & got) { \ + return writeFile(file, got.dump(2) + "\n"); \ + }); \ } TEST_JSON(DerivationTest, inputAddressed, - R"({ - "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" - })", (DerivationOutput::InputAddressed { .path = store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"), }), "drv-name", "output-name") TEST_JSON(DerivationTest, caFixedFlat, - R"({ - "hashAlgo": "sha256", - "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", - "path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name" - })", (DerivationOutput::CAFixed { - .ca = FixedOutputHash { + .ca = { .method = FileIngestionMethod::Flat, .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), }, @@ -89,13 +115,8 @@ TEST_JSON(DerivationTest, caFixedFlat, "drv-name", "output-name") TEST_JSON(DerivationTest, caFixedNAR, - R"({ - "hashAlgo": "r:sha256", - "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", - "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" - })", (DerivationOutput::CAFixed { - .ca = FixedOutputHash { + .ca = { .method = FileIngestionMethod::Recursive, .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), }, @@ -103,22 +124,14 @@ TEST_JSON(DerivationTest, caFixedNAR, "drv-name", "output-name") TEST_JSON(DynDerivationTest, caFixedText, - R"({ - "hashAlgo": "text:sha256", - "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", - "path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name" - })", (DerivationOutput::CAFixed { - .ca = TextHash { + .ca = { .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), }, }), "drv-name", "output-name") TEST_JSON(CaDerivationTest, caFloating, - R"({ - "hashAlgo": "r:sha256" - })", (DerivationOutput::CAFloating { .method = FileIngestionMethod::Recursive, .hashType = htSHA256, @@ -126,15 +139,10 @@ TEST_JSON(CaDerivationTest, caFloating, "drv-name", "output-name") TEST_JSON(DerivationTest, deferred, - R"({ })", DerivationOutput::Deferred { }, "drv-name", "output-name") TEST_JSON(ImpureDerivationTest, impure, - R"({ - "hashAlgo": "r:sha256", - "impure": true - })", (DerivationOutput::Impure { .method = FileIngestionMethod::Recursive, .hashType = htSHA256, @@ -143,77 +151,148 @@ TEST_JSON(ImpureDerivationTest, impure, #undef TEST_JSON -#define TEST_JSON(NAME, STR, VAL, DRV_NAME) \ - TEST_F(DerivationTest, Derivation_ ## NAME ## _to_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - STR ## _json, \ - (Derivation { VAL }).toJSON(*store)); \ - } \ - \ - TEST_F(DerivationTest, Derivation_ ## NAME ## _from_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - Derivation { VAL }, \ - Derivation::fromJSON( \ - *store, \ - STR ## _json)); \ +#define TEST_JSON(FIXTURE, NAME, VAL) \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _from_json) { \ + readTest(#NAME ".json", [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + Derivation expected { VAL }; \ + Derivation got = Derivation::fromJSON( \ + *store, \ + encoded, \ + mockXpSettings); \ + ASSERT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _to_json) { \ + writeTest(#NAME ".json", [&]() -> json { \ + return Derivation { VAL }.toJSON(*store); \ + }, [](const auto & file) { \ + return json::parse(readFile(file)); \ + }, [](const auto & file, const auto & got) { \ + return writeFile(file, got.dump(2) + "\n"); \ + }); \ } -TEST_JSON(simple, - R"({ - "name": "my-derivation", - "inputSrcs": [ - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" - ], - "inputDrvs": { - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": [ - "cat", - "dog" - ] - }, - "system": "wasm-sel4", - "builder": "foo", - "args": [ - "bar", - "baz" - ], - "env": { - "BIG_BAD": "WOLF" - }, - "outputs": {} - })", - ({ - Derivation drv; - drv.name = "my-derivation"; - drv.inputSrcs = { - store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), - }; - drv.inputDrvs = { +#define TEST_ATERM(FIXTURE, NAME, VAL, DRV_NAME) \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _from_aterm) { \ + readTest(#NAME ".drv", [&](auto encoded) { \ + Derivation expected { VAL }; \ + auto got = parseDerivation( \ + *store, \ + std::move(encoded), \ + DRV_NAME, \ + mockXpSettings); \ + ASSERT_EQ(got.toJSON(*store), expected.toJSON(*store)) ; \ + ASSERT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _to_aterm) { \ + writeTest(#NAME ".drv", [&]() -> std::string { \ + return (VAL).unparse(*store, false); \ + }); \ + } + +Derivation makeSimpleDrv(const Store & store) { + Derivation drv; + drv.name = "simple-derivation"; + drv.inputSrcs = { + store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), + }; + drv.inputDrvs = { + .map = { { - store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), + store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), { - "cat", - "dog", + .value = { + "cat", + "dog", + }, }, - } - }; - drv.platform = "wasm-sel4"; - drv.builder = "foo"; - drv.args = { - "bar", - "baz", - }; - drv.env = { + }, + }, + }; + drv.platform = "wasm-sel4"; + drv.builder = "foo"; + drv.args = { + "bar", + "baz", + }; + drv.env = { + { + "BIG_BAD", + "WOLF", + }, + }; + return drv; +} + +TEST_JSON(DerivationTest, simple, makeSimpleDrv(*store)) + +TEST_ATERM(DerivationTest, simple, + makeSimpleDrv(*store), + "simple-derivation") + +Derivation makeDynDepDerivation(const Store & store) { + Derivation drv; + drv.name = "dyn-dep-derivation"; + drv.inputSrcs = { + store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), + }; + drv.inputDrvs = { + .map = { { - "BIG_BAD", - "WOLF", + store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), + DerivedPathMap::ChildNode { + .value = { + "cat", + "dog", + }, + .childMap = { + { + "cat", + DerivedPathMap::ChildNode { + .value = { + "kitten", + }, + }, + }, + { + "goose", + DerivedPathMap::ChildNode { + .value = { + "gosling", + }, + }, + }, + }, + }, }, - }; - drv; - }), - "drv-name") + }, + }; + drv.platform = "wasm-sel4"; + drv.builder = "foo"; + drv.args = { + "bar", + "baz", + }; + drv.env = { + { + "BIG_BAD", + "WOLF", + }, + }; + return drv; +} + +TEST_JSON(DynDerivationTest, dynDerivationDeps, makeDynDepDerivation(*store)) + +TEST_ATERM(DynDerivationTest, dynDerivationDeps, + makeDynDepDerivation(*store), + "dyn-dep-derivation") #undef TEST_JSON +#undef TEST_ATERM } diff --git a/src/libstore/tests/derived-path.cc b/src/libstore/tests/derived-path.cc index 160443ec1d6a..3fa3c080181d 100644 --- a/src/libstore/tests/derived-path.cc +++ b/src/libstore/tests/derived-path.cc @@ -17,14 +17,34 @@ Gen Arbitrary::arbitrary() }); } +Gen Arbitrary::arbitrary() +{ + return gen::just(SingleDerivedPath::Built { + .drvPath = make_ref(*gen::arbitrary()), + .output = (*gen::arbitrary()).name, + }); +} + Gen Arbitrary::arbitrary() { return gen::just(DerivedPath::Built { - .drvPath = *gen::arbitrary(), + .drvPath = make_ref(*gen::arbitrary()), .outputs = *gen::arbitrary(), }); } +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just(*gen::arbitrary()); + case 1: + return gen::just(*gen::arbitrary()); + default: + assert(false); + } +} + Gen Arbitrary::arbitrary() { switch (*gen::inRange(0, std::variant_size_v)) { @@ -45,14 +65,73 @@ class DerivedPathTest : public LibStoreTest { }; -// FIXME: `RC_GTEST_FIXTURE_PROP` isn't calling `SetUpTestSuite` because it is -// no a real fixture. -// -// See https://github.com/emil-e/rapidcheck/blob/master/doc/gtest.md#rc_gtest_fixture_propfixture-name-args -TEST_F(DerivedPathTest, force_init) -{ +/** + * Round trip (string <-> data structure) test for + * `DerivedPath::Opaque`. + */ +TEST_F(DerivedPathTest, opaque) { + std::string_view opaque = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x"; + auto elem = DerivedPath::parse(*store, opaque); + auto * p = std::get_if(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->path, store->parseStorePath(opaque)); + ASSERT_EQ(elem.to_string(*store), opaque); +} + +/** + * Round trip (string <-> data structure) test for a simpler + * `DerivedPath::Built`. + */ +TEST_F(DerivedPathTest, built_opaque) { + std::string_view built = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv^bar,foo"; + auto elem = DerivedPath::parse(*store, built); + auto * p = std::get_if(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->outputs, ((OutputsSpec) OutputsSpec::Names { "foo", "bar" })); + ASSERT_EQ(*p->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { + .path = store->parseStorePath(built.substr(0, 49)), + })); + ASSERT_EQ(elem.to_string(*store), built); } +/** + * Round trip (string <-> data structure) test for a more complex, + * inductive `DerivedPath::Built`. + */ +TEST_F(DerivedPathTest, built_built) { + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations"); + + std::string_view built = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv^foo^bar,baz"; + auto elem = DerivedPath::parse(*store, built, mockXpSettings); + auto * p = std::get_if(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->outputs, ((OutputsSpec) OutputsSpec::Names { "bar", "baz" })); + auto * drvPath = std::get_if(&*p->drvPath); + ASSERT_TRUE(drvPath); + ASSERT_EQ(drvPath->output, "foo"); + ASSERT_EQ(*drvPath->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { + .path = store->parseStorePath(built.substr(0, 49)), + })); + ASSERT_EQ(elem.to_string(*store), built); +} + +/** + * Without the right experimental features enabled, we cannot parse a + * complex inductive derived path. + */ +TEST_F(DerivedPathTest, built_built_xp) { + ASSERT_THROW( + DerivedPath::parse(*store, "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv^foo^bar,baz"), + MissingExperimentalFeature); +} + +#ifndef COVERAGE + RC_GTEST_FIXTURE_PROP( DerivedPathTest, prop_legacy_round_rip, @@ -69,4 +148,6 @@ RC_GTEST_FIXTURE_PROP( RC_ASSERT(o == DerivedPath::parse(*store, o.to_string(*store))); } +#endif + } diff --git a/src/libstore/tests/derived-path.hh b/src/libstore/tests/derived-path.hh index 506f3ccb1060..98d61f2283d2 100644 --- a/src/libstore/tests/derived-path.hh +++ b/src/libstore/tests/derived-path.hh @@ -12,8 +12,18 @@ namespace rc { using namespace nix; template<> -struct Arbitrary { - static Gen arbitrary(); +struct Arbitrary { + static Gen arbitrary(); +}; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +template<> +struct Arbitrary { + static Gen arbitrary(); }; template<> diff --git a/src/libstore/tests/downstream-placeholder.cc b/src/libstore/tests/downstream-placeholder.cc index ec3e1000f86b..fd29530acfc9 100644 --- a/src/libstore/tests/downstream-placeholder.cc +++ b/src/libstore/tests/downstream-placeholder.cc @@ -5,17 +5,24 @@ namespace nix { TEST(DownstreamPlaceholder, unknownCaOutput) { + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "ca-derivations"); + ASSERT_EQ( DownstreamPlaceholder::unknownCaOutput( StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" }, - "out").render(), + "out", + mockXpSettings).render(), "/0c6rn30q4frawknapgwq386zq358m8r6msvywcvc89n6m5p2dgbz"); } TEST(DownstreamPlaceholder, unknownDerivation) { /** - * We set these in tests rather than the regular globals so we don't have - * to worry about race conditions if the tests run concurrently. + * Same reason as above */ ExperimentalFeatureSettings mockXpSettings; mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations"); @@ -24,7 +31,8 @@ TEST(DownstreamPlaceholder, unknownDerivation) { DownstreamPlaceholder::unknownDerivation( DownstreamPlaceholder::unknownCaOutput( StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv.drv" }, - "out"), + "out", + mockXpSettings), "out", mockXpSettings).render(), "/0gn6agqxjyyalf0dpihgyf49xq5hqxgw100f0wydnj6yqrhqsb3w"); diff --git a/src/libstore/tests/libstore.hh b/src/libstore/tests/libstore.hh index ef93457b53c9..78b162b95687 100644 --- a/src/libstore/tests/libstore.hh +++ b/src/libstore/tests/libstore.hh @@ -8,7 +8,7 @@ namespace nix { -class LibStoreTest : public ::testing::Test { +class LibStoreTest : public virtual ::testing::Test { public: static void SetUpTestSuite() { initLibStore(); diff --git a/src/libstore/tests/local.mk b/src/libstore/tests/local.mk index 03becc7d11b6..e9b8b4f9901f 100644 --- a/src/libstore/tests/local.mk +++ b/src/libstore/tests/local.mk @@ -6,7 +6,11 @@ libstore-tests-exe_NAME = libnixstore-tests libstore-tests-exe_DIR := $(d) -libstore-tests-exe_INSTALL_DIR := +ifeq ($(INSTALL_UNIT_TESTS), yes) + libstore-tests-exe_INSTALL_DIR := $(checkbindir) +else + libstore-tests-exe_INSTALL_DIR := +endif libstore-tests-exe_LIBS = libstore-tests @@ -18,7 +22,11 @@ libstore-tests_NAME = libnixstore-tests libstore-tests_DIR := $(d) -libstore-tests_INSTALL_DIR := +ifeq ($(INSTALL_UNIT_TESTS), yes) + libstore-tests_INSTALL_DIR := $(checklibdir) +else + libstore-tests_INSTALL_DIR := +endif libstore-tests_SOURCES := $(wildcard $(d)/*.cc) diff --git a/src/libstore/tests/machines.cc b/src/libstore/tests/machines.cc index f51052b14626..fede328ea3f3 100644 --- a/src/libstore/tests/machines.cc +++ b/src/libstore/tests/machines.cc @@ -1,5 +1,7 @@ #include "machines.hh" #include "globals.hh" +#include "file-system.hh" +#include "util.hh" #include diff --git a/src/libstore/tests/nar-info.cc b/src/libstore/tests/nar-info.cc new file mode 100644 index 000000000000..c5b21d56b400 --- /dev/null +++ b/src/libstore/tests/nar-info.cc @@ -0,0 +1,85 @@ +#include +#include + +#include "path-info.hh" + +#include "tests/characterization.hh" +#include "tests/libstore.hh" + +namespace nix { + +using nlohmann::json; + +class NarInfoTest : public CharacterizationTest, public LibStoreTest +{ + Path unitTestData = getUnitTestData() + "/libstore/nar-info"; + + Path goldenMaster(PathView testStem) const override { + return unitTestData + "/" + testStem + ".json"; + } +}; + +static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) { + NarInfo info = ValidPathInfo { + store, + "foo", + FixedOutputInfo { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashType::htSHA256, "(...)"), + + .references = { + .others = { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }, + .self = true, + }, + }, + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.narSize = 34878; + if (includeImpureInfo) { + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.registrationTime = 23423; + info.ultimate = true; + info.sigs = { "asdf", "qwer" }; + + info.url = "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz"; + info.compression = "xz"; + info.fileHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="); + info.fileSize = 4029176; + } + return info; +} + +#define JSON_TEST(STEM, PURE) \ + TEST_F(NarInfoTest, NarInfo_ ## STEM ## _from_json) { \ + readTest(#STEM, [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + auto expected = makeNarInfo(*store, PURE); \ + NarInfo got = NarInfo::fromJSON( \ + *store, \ + expected.path, \ + encoded); \ + ASSERT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(NarInfoTest, NarInfo_ ## STEM ## _to_json) { \ + writeTest(#STEM, [&]() -> json { \ + return makeNarInfo(*store, PURE) \ + .toJSON(*store, PURE, HashFormat::SRI); \ + }, [](const auto & file) { \ + return json::parse(readFile(file)); \ + }, [](const auto & file, const auto & got) { \ + return writeFile(file, got.dump(2) + "\n"); \ + }); \ + } + +JSON_TEST(pure, false) +JSON_TEST(impure, true) + +} diff --git a/src/libstore/tests/outputs-spec.cc b/src/libstore/tests/outputs-spec.cc index bf8deaa9d991..95294518544b 100644 --- a/src/libstore/tests/outputs-spec.cc +++ b/src/libstore/tests/outputs-spec.cc @@ -224,6 +224,8 @@ Gen Arbitrary::arbitrary() namespace nix { +#ifndef COVERAGE + RC_GTEST_PROP( OutputsSpec, prop_round_rip, @@ -232,4 +234,6 @@ RC_GTEST_PROP( RC_ASSERT(o == OutputsSpec::parse(o.to_string())); } +#endif + } diff --git a/src/libstore/tests/path-info.cc b/src/libstore/tests/path-info.cc new file mode 100644 index 000000000000..49bf623bdc08 --- /dev/null +++ b/src/libstore/tests/path-info.cc @@ -0,0 +1,79 @@ +#include +#include + +#include "path-info.hh" + +#include "tests/characterization.hh" +#include "tests/libstore.hh" + +namespace nix { + +using nlohmann::json; + +class PathInfoTest : public CharacterizationTest, public LibStoreTest +{ + Path unitTestData = getUnitTestData() + "/libstore/path-info"; + + Path goldenMaster(PathView testStem) const override { + return unitTestData + "/" + testStem + ".json"; + } +}; + +static UnkeyedValidPathInfo makePathInfo(const Store & store, bool includeImpureInfo) { + UnkeyedValidPathInfo info = ValidPathInfo { + store, + "foo", + FixedOutputInfo { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashType::htSHA256, "(...)"), + + .references = { + .others = { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }, + .self = true, + }, + }, + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.narSize = 34878; + if (includeImpureInfo) { + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.registrationTime = 23423; + info.ultimate = true; + info.sigs = { "asdf", "qwer" }; + } + return info; +} + +#define JSON_TEST(STEM, PURE) \ + TEST_F(PathInfoTest, PathInfo_ ## STEM ## _from_json) { \ + readTest(#STEM, [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + UnkeyedValidPathInfo got = UnkeyedValidPathInfo::fromJSON( \ + *store, \ + encoded); \ + auto expected = makePathInfo(*store, PURE); \ + ASSERT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(PathInfoTest, PathInfo_ ## STEM ## _to_json) { \ + writeTest(#STEM, [&]() -> json { \ + return makePathInfo(*store, PURE) \ + .toJSON(*store, PURE, HashFormat::SRI); \ + }, [](const auto & file) { \ + return json::parse(readFile(file)); \ + }, [](const auto & file, const auto & got) { \ + return writeFile(file, got.dump(2) + "\n"); \ + }); \ + } + +JSON_TEST(pure, false) +JSON_TEST(impure, true) + +} diff --git a/src/libstore/tests/path.cc b/src/libstore/tests/path.cc index 430aa009937c..5a84d646c25b 100644 --- a/src/libstore/tests/path.cc +++ b/src/libstore/tests/path.cc @@ -39,6 +39,7 @@ TEST_DONT_PARSE(double_star, "**") TEST_DONT_PARSE(star_first, "*,foo") TEST_DONT_PARSE(star_second, "foo,*") TEST_DONT_PARSE(bang, "foo!o") +TEST_DONT_PARSE(dotfile, ".gitignore") #undef TEST_DONT_PARSE @@ -101,8 +102,12 @@ Gen Arbitrary::arbitrary() pre += '-'; break; case 64: - pre += '.'; - break; + // names aren't permitted to start with a period, + // so just fall through to the next case here + if (c != 0) { + pre += '.'; + break; + } case 65: pre += '_'; break; @@ -134,6 +139,8 @@ Gen Arbitrary::arbitrary() namespace nix { +#ifndef COVERAGE + RC_GTEST_FIXTURE_PROP( StorePathTest, prop_regex_accept, @@ -150,4 +157,6 @@ RC_GTEST_FIXTURE_PROP( RC_ASSERT(p == store->parseStorePath(store->printStorePath(p))); } +#endif + } diff --git a/src/libstore/tests/protocol.hh b/src/libstore/tests/protocol.hh new file mode 100644 index 000000000000..466032a79926 --- /dev/null +++ b/src/libstore/tests/protocol.hh @@ -0,0 +1,75 @@ +#pragma once +///@file + +#include +#include + +#include "tests/libstore.hh" +#include "tests/characterization.hh" + +namespace nix { + +template +class ProtoTest : public CharacterizationTest, public LibStoreTest +{ + Path unitTestData = getUnitTestData() + "/libstore/" + protocolDir; + + Path goldenMaster(std::string_view testStem) const override { + return unitTestData + "/" + testStem + ".bin"; + } +}; + +template +class VersionedProtoTest : public ProtoTest +{ +public: + /** + * Golden test for `T` reading + */ + template + void readProtoTest(PathView testStem, typename Proto::Version version, T expected) + { + CharacterizationTest::readTest(testStem, [&](const auto & encoded) { + T got = ({ + StringSource from { encoded }; + Proto::template Serialise::read( + *LibStoreTest::store, + typename Proto::ReadConn { + .from = from, + .version = version, + }); + }); + + ASSERT_EQ(got, expected); + }); + } + + /** + * Golden test for `T` write + */ + template + void writeProtoTest(PathView testStem, typename Proto::Version version, const T & decoded) + { + CharacterizationTest::writeTest(testStem, [&]() { + StringSink to; + Proto::template Serialise::write( + *LibStoreTest::store, + typename Proto::WriteConn { + .to = to, + .version = version, + }, + decoded); + return std::move(to.s); + }); + } +}; + +#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \ + TEST_F(FIXTURE, NAME ## _read) { \ + readProtoTest(STEM, VERSION, VALUE); \ + } \ + TEST_F(FIXTURE, NAME ## _write) { \ + writeProtoTest(STEM, VERSION, VALUE); \ + } + +} diff --git a/src/libstore/tests/serve-protocol.cc b/src/libstore/tests/serve-protocol.cc new file mode 100644 index 000000000000..c8ac87a04ced --- /dev/null +++ b/src/libstore/tests/serve-protocol.cc @@ -0,0 +1,279 @@ +#include + +#include +#include + +#include "serve-protocol.hh" +#include "serve-protocol-impl.hh" +#include "build-result.hh" +#include "tests/protocol.hh" +#include "tests/characterization.hh" + +namespace nix { + +const char serveProtoDir[] = "serve-protocol"; + +struct ServeProtoTest : VersionedProtoTest +{ + /** + * For serializers that don't care about the minimum version, we + * used the oldest one: 1.0. + */ + ServeProto::Version defaultVersion = 2 << 8 | 0; +}; + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + string, + "string", + defaultVersion, + (std::tuple { + "", + "hi", + "white rabbit", + "大白兔", + "oh no \0\0\0 what was that!", + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + storePath, + "store-path", + defaultVersion, + (std::tuple { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + contentAddress, + "content-address", + defaultVersion, + (std::tuple { + ContentAddress { + .method = TextIngestionMethod {}, + .hash = hashString(HashType::htSHA256, "Derive(...)"), + }, + ContentAddress { + .method = FileIngestionMethod::Flat, + .hash = hashString(HashType::htSHA1, "blob blob..."), + }, + ContentAddress { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashType::htSHA256, "(...)"), + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + drvOutput, + "drv-output", + defaultVersion, + (std::tuple { + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + realisation, + "realisation", + defaultVersion, + (std::tuple { + Realisation { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + }, + Realisation { + .id = { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + .dependentRealisations = { + { + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + buildResult_2_2, + "build-result-2.2", + 2 << 8 | 2, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::Built, + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + buildResult_2_3, + "build-result-2.3", + 2 << 8 | 3, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + .timesBuilt = 3, + .isNonDeterministic = true, + .startTime = 30, + .stopTime = 50, + }, + BuildResult { + .status = BuildResult::Built, + .startTime = 30, + .stopTime = 50, + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + buildResult_2_6, + "build-result-2.6", + 2 << 8 | 6, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + .timesBuilt = 3, + .isNonDeterministic = true, + .startTime = 30, + .stopTime = 50, + }, + BuildResult { + .status = BuildResult::Built, + .timesBuilt = 1, + .builtOutputs = { + { + "foo", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + { + "bar", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" }, + }, + }, + }, + .startTime = 30, + .stopTime = 50, +#if 0 + // These fields are not yet serialized. + // FIXME Include in next version of protocol or document + // why they are skipped. + .cpuUser = std::chrono::milliseconds(500s), + .cpuSystem = std::chrono::milliseconds(604s), +#endif + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + vector, + "vector", + defaultVersion, + (std::tuple, std::vector, std::vector, std::vector>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + set, + "set", + defaultVersion, + (std::tuple, std::set, std::set, std::set>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + optionalStorePath, + "optional-store-path", + defaultVersion, + (std::tuple, std::optional> { + std::nullopt, + std::optional { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + optionalContentAddress, + "optional-content-address", + defaultVersion, + (std::tuple, std::optional> { + std::nullopt, + std::optional { + ContentAddress { + .method = FileIngestionMethod::Flat, + .hash = hashString(HashType::htSHA1, "blob blob..."), + }, + }, + })) + +} diff --git a/src/libstore/tests/worker-protocol.cc b/src/libstore/tests/worker-protocol.cc new file mode 100644 index 000000000000..ad5943c69bc4 --- /dev/null +++ b/src/libstore/tests/worker-protocol.cc @@ -0,0 +1,547 @@ +#include + +#include +#include + +#include "worker-protocol.hh" +#include "worker-protocol-impl.hh" +#include "derived-path.hh" +#include "build-result.hh" +#include "tests/protocol.hh" +#include "tests/characterization.hh" + +namespace nix { + +const char workerProtoDir[] = "worker-protocol"; + +struct WorkerProtoTest : VersionedProtoTest +{ + /** + * For serializers that don't care about the minimum version, we + * used the oldest one: 1.0. + */ + WorkerProto::Version defaultVersion = 1 << 8 | 0; +}; + + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + string, + "string", + defaultVersion, + (std::tuple { + "", + "hi", + "white rabbit", + "大白兔", + "oh no \0\0\0 what was that!", + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + storePath, + "store-path", + defaultVersion, + (std::tuple { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + contentAddress, + "content-address", + defaultVersion, + (std::tuple { + ContentAddress { + .method = TextIngestionMethod {}, + .hash = hashString(HashType::htSHA256, "Derive(...)"), + }, + ContentAddress { + .method = FileIngestionMethod::Flat, + .hash = hashString(HashType::htSHA1, "blob blob..."), + }, + ContentAddress { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashType::htSHA256, "(...)"), + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + derivedPath_1_29, + "derived-path-1.29", + 1 << 8 | 29, + (std::tuple { + DerivedPath::Opaque { + .path = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }), + .outputs = OutputsSpec::All { }, + }, + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }), + .outputs = OutputsSpec::Names { "x", "y" }, + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + derivedPath_1_30, + "derived-path-1.30", + 1 << 8 | 30, + (std::tuple { + DerivedPath::Opaque { + .path = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + DerivedPath::Opaque { + .path = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" }, + }, + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }), + .outputs = OutputsSpec::All { }, + }, + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }), + .outputs = OutputsSpec::Names { "x", "y" }, + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + drvOutput, + "drv-output", + defaultVersion, + (std::tuple { + { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + realisation, + "realisation", + defaultVersion, + (std::tuple { + Realisation { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + }, + Realisation { + .id = { + .drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + .outputName = "baz", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + .signatures = { "asdf", "qwer" }, + .dependentRealisations = { + { + DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "quux", + }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + buildResult_1_27, + "build-result-1.27", + 1 << 8 | 27, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::Built, + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + buildResult_1_28, + "build-result-1.28", + 1 << 8 | 28, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::Built, + .builtOutputs = { + { + "foo", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + { + "bar", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" }, + }, + }, + }, + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + buildResult_1_29, + "build-result-1.29", + 1 << 8 | 29, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + .timesBuilt = 3, + .isNonDeterministic = true, + .startTime = 30, + .stopTime = 50, + }, + BuildResult { + .status = BuildResult::Built, + .timesBuilt = 1, + .builtOutputs = { + { + "foo", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + { + "bar", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" }, + }, + }, + }, + .startTime = 30, + .stopTime = 50, +#if 0 + // These fields are not yet serialized. + // FIXME Include in next version of protocol or document + // why they are skipped. + .cpuUser = std::chrono::milliseconds(500s), + .cpuSystem = std::chrono::milliseconds(604s), +#endif + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + keyedBuildResult_1_29, + "keyed-build-result-1.29", + 1 << 8 | 29, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + KeyedBuildResult { + { + .status = KeyedBuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + /* .path = */ DerivedPath::Opaque { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-xxx" }, + }, + }, + KeyedBuildResult { + { + .status = KeyedBuildResult::NotDeterministic, + .errorMsg = "no idea why", + .timesBuilt = 3, + .isNonDeterministic = true, + .startTime = 30, + .stopTime = 50, + }, + /* .path = */ DerivedPath::Built { + .drvPath = makeConstantStorePathRef(StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }), + .outputs = OutputsSpec::Names { "out" }, + }, + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + unkeyedValidPathInfo_1_15, + "unkeyed-valid-path-info-1.15", + 1 << 8 | 15, + (std::tuple { + ({ + UnkeyedValidPathInfo info { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.registrationTime = 23423; + info.narSize = 34878; + info; + }), + ({ + UnkeyedValidPathInfo info { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.references = { + StorePath { + "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo.drv", + }, + }; + info.registrationTime = 23423; + info.narSize = 34878; + info; + }), + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + validPathInfo_1_15, + "valid-path-info-1.15", + 1 << 8 | 15, + (std::tuple { + ({ + ValidPathInfo info { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + UnkeyedValidPathInfo { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }, + }; + info.registrationTime = 23423; + info.narSize = 34878; + info; + }), + ({ + ValidPathInfo info { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + UnkeyedValidPathInfo { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }, + }; + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.references = { + // other reference + StorePath { + "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo", + }, + // self reference + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }; + info.registrationTime = 23423; + info.narSize = 34878; + info; + }), + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + validPathInfo_1_16, + "valid-path-info-1.16", + 1 << 8 | 16, + (std::tuple { + ({ + ValidPathInfo info { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + UnkeyedValidPathInfo { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }, + }; + info.registrationTime = 23423; + info.narSize = 34878; + info.ultimate = true; + info; + }), + ({ + ValidPathInfo info { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + UnkeyedValidPathInfo { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }, + }; + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.references = { + // other reference + StorePath { + "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo", + }, + // self reference + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }; + info.registrationTime = 23423; + info.narSize = 34878; + info.sigs = { + "fake-sig-1", + "fake-sig-2", + }, + info; + }), + ({ + ValidPathInfo info { + *LibStoreTest::store, + "foo", + FixedOutputInfo { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashType::htSHA256, "(...)"), + .references = { + .others = { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }, + .self = true, + }, + }, + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.registrationTime = 23423; + info.narSize = 34878; + info; + }), + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + optionalTrustedFlag, + "optional-trusted-flag", + defaultVersion, + (std::tuple, std::optional, std::optional> { + std::nullopt, + std::optional { Trusted }, + std::optional { NotTrusted }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + vector, + "vector", + defaultVersion, + (std::tuple, std::vector, std::vector, std::vector>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + set, + "set", + defaultVersion, + (std::tuple, std::set, std::set, std::set>> { + { }, + { "" }, + { "", "foo", "bar" }, + { {}, { "" }, { "", "1", "2" } }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + optionalStorePath, + "optional-store-path", + defaultVersion, + (std::tuple, std::optional> { + std::nullopt, + std::optional { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + optionalContentAddress, + "optional-content-address", + defaultVersion, + (std::tuple, std::optional> { + std::nullopt, + std::optional { + ContentAddress { + .method = FileIngestionMethod::Flat, + .hash = hashString(HashType::htSHA1, "blob blob..."), + }, + }, + })) + +} diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 69dae2da5608..226cdf7175c6 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -1,4 +1,6 @@ #include "uds-remote-store.hh" +#include "unix-domain-socket.hh" +#include "worker-protocol.hh" #include #include @@ -77,6 +79,15 @@ ref UDSRemoteStore::openConnection() } +void UDSRemoteStore::addIndirectRoot(const Path & path) +{ + auto conn(getConnection()); + conn->to << WorkerProto::Op::AddIndirectRoot << path; + conn.processStderr(); + readInt(conn->from); +} + + static RegisterStoreImplementation regUDSRemoteStore; } diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index 2bd6517fa2ca..a5ac9080ad19 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -3,13 +3,13 @@ #include "remote-store.hh" #include "remote-store-connection.hh" -#include "local-fs-store.hh" +#include "indirect-root-store.hh" namespace nix { struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig { - UDSRemoteStoreConfig(const Store::Params & params) + UDSRemoteStoreConfig(const Params & params) : StoreConfig(params) , LocalFSStoreConfig(params) , RemoteStoreConfig(params) @@ -21,7 +21,9 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon std::string doc() override; }; -class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual LocalFSStore, public virtual RemoteStore +class UDSRemoteStore : public virtual UDSRemoteStoreConfig + , public virtual IndirectRootStore + , public virtual RemoteStore { public: @@ -33,12 +35,22 @@ public: static std::set uriSchemes() { return {"unix"}; } - ref getFSAccessor() override - { return LocalFSStore::getFSAccessor(); } + ref getFSAccessor(bool requireValidPath) override + { return LocalFSStore::getFSAccessor(requireValidPath); } void narFromPath(const StorePath & path, Sink & sink) override { LocalFSStore::narFromPath(path, sink); } + /** + * Implementation of `IndirectRootStore::addIndirectRoot()` which + * delegates to the remote store. + * + * The idea is that the client makes the direct symlink, so it is + * owned managed by the client's user account, and the server makes + * the indirect symlink. + */ + void addIndirectRoot(const Path & path) override; + private: struct Connection : RemoteStore::Connection diff --git a/src/libstore/worker-protocol-impl.hh b/src/libstore/worker-protocol-impl.hh index 7167d6ad2aa5..e464cdf13af7 100644 --- a/src/libstore/worker-protocol-impl.hh +++ b/src/libstore/worker-protocol-impl.hh @@ -10,9 +10,53 @@ #include "worker-protocol.hh" #include "granular-access-store.hh" +#include "length-prefixed-protocol-helper.hh" namespace nix { +/* protocol-agnostic templates */ + +#define WORKER_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \ + TEMPLATE T WorkerProto::Serialise< T >::read(const Store & store, WorkerProto::ReadConn conn) \ + { \ + return LengthPrefixedProtoHelper::read(store, conn); \ + } \ + TEMPLATE void WorkerProto::Serialise< T >::write(const Store & store, WorkerProto::WriteConn conn, const T & t) \ + { \ + LengthPrefixedProtoHelper::write(store, conn, t); \ + } + +WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::vector) +WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::set) +WORKER_USE_LENGTH_PREFIX_SERIALISER(template, std::tuple) + +#define COMMA_ , +WORKER_USE_LENGTH_PREFIX_SERIALISER( + template, + std::map) +#undef COMMA_ + +/** + * Use `CommonProto` where possible. + */ +template +struct WorkerProto::Serialise +{ + static T read(const Store & store, WorkerProto::ReadConn conn) + { + return CommonProto::Serialise::read(store, + CommonProto::ReadConn { .from = conn.from }); + } + static void write(const Store & store, WorkerProto::WriteConn conn, const T & t) + { + CommonProto::Serialise::write(store, + CommonProto::WriteConn { .to = conn.to }, + t); + } +}; + +/* protocol-specific templates */ + template AccessStatusFor WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) { AccessStatusFor status; @@ -28,46 +72,6 @@ void WorkerProto::Serialise>::write(const Store & store, Work WorkerProto::Serialise>::write(store, conn, status.entities); } -template -std::vector WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) -{ - std::vector resSet; - auto size = readNum(conn.from); - while (size--) { - resSet.push_back(WorkerProto::Serialise::read(store, conn)); - } - return resSet; -} - -template -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::vector & resSet) -{ - conn.to << resSet.size(); - for (auto & key : resSet) { - WorkerProto::Serialise::write(store, conn, key); - } -} - -template -std::set WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) -{ - std::set resSet; - auto size = readNum(conn.from); - while (size--) { - resSet.insert(WorkerProto::Serialise::read(store, conn)); - } - return resSet; -} - -template -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::set & resSet) -{ - conn.to << resSet.size(); - for (auto & key : resSet) { - WorkerProto::Serialise::write(store, conn, key); - } -} - template std::variant WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) { @@ -138,42 +142,4 @@ void WorkerProto::Serialise>::write(const Store & store, W } } -template -std::map WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) -{ - std::map resMap; - auto size = readNum(conn.from); - while (size--) { - auto k = WorkerProto::Serialise::read(store, conn); - auto v = WorkerProto::Serialise::read(store, conn); - resMap.insert_or_assign(std::move(k), std::move(v)); - } - return resMap; -} - -template -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::map & resMap) -{ - conn.to << resMap.size(); - for (auto & i : resMap) { - WorkerProto::Serialise::write(store, conn, i.first); - WorkerProto::Serialise::write(store, conn, i.second); - } -} - -template -std::pair WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) -{ - auto a = WorkerProto::Serialise::read(store, conn); - auto b = WorkerProto::Serialise::read(store, conn); - return {a, b}; -} - -template -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::pair & p) -{ - WorkerProto::Serialise::write(store, conn, p.first); - WorkerProto::Serialise::write(store, conn, p.second); -} - } diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index ea0fa888cfef..4dde903a16a2 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -1,40 +1,19 @@ #include "globals.hh" #include "granular-access-store.hh" #include "serialise.hh" -#include "util.hh" #include "path-with-outputs.hh" #include "store-api.hh" #include "build-result.hh" #include "worker-protocol.hh" #include "worker-protocol-impl.hh" #include "archive.hh" -#include "derivations.hh" +#include "path-info.hh" #include namespace nix { -std::string WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) -{ - return readString(conn.from); -} - -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const std::string & str) -{ - conn.to << str; -} - - -StorePath WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) -{ - return store.parseStorePath(readString(conn.from)); -} - -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const StorePath & storePath) -{ - conn.to << store.printStorePath(storePath); -} - +/* protocol-specific definitions */ std::optional WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) { @@ -54,14 +33,14 @@ std::optional WorkerProto::Serialise>::r void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional & optTrusted) { if (!optTrusted) - conn.to << (uint8_t)0; + conn.to << uint8_t{0}; else { switch (*optTrusted) { case Trusted: - conn.to << (uint8_t)1; + conn.to << uint8_t{1}; break; case NotTrusted: - conn.to << (uint8_t)2; + conn.to << uint8_t{2}; break; default: assert(false); @@ -104,52 +83,37 @@ void WorkerProto::Serialise::write(const Store & store, WorkerProto: conn.to << group.gid; } -ContentAddress WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) -{ - return ContentAddress::parse(readString(conn.from)); -} - -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const ContentAddress & ca) -{ - conn.to << renderContentAddress(ca); -} - - DerivedPath WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) { auto s = readString(conn.from); - return DerivedPath::parseLegacy(store, s); + if (GET_PROTOCOL_MINOR(conn.version) >= 30) { + return DerivedPath::parseLegacy(store, s); + } else { + return parsePathWithOutputs(store, s).toDerivedPath(); + } } void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const DerivedPath & req) { - conn.to << req.to_string_legacy(store); -} - - -Realisation WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) -{ - std::string rawInput = readString(conn.from); - return Realisation::fromJSON( - nlohmann::json::parse(rawInput), - "remote-protocol" - ); -} - -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const Realisation & realisation) -{ - conn.to << realisation.toJSON().dump(); -} - - -DrvOutput WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) -{ - return DrvOutput::parse(readString(conn.from)); -} - -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const DrvOutput & drvOutput) -{ - conn.to << drvOutput.to_string(); + if (GET_PROTOCOL_MINOR(conn.version) >= 30) { + conn.to << req.to_string_legacy(store); + } else { + auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(req); + std::visit(overloaded { + [&](const StorePathWithOutputs & s) { + conn.to << s.to_string(store); + }, + [&](const StorePath & drvPath) { + throw Error("trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file", + store.printStorePath(drvPath), + GET_PROTOCOL_MAJOR(conn.version), + GET_PROTOCOL_MINOR(conn.version)); + }, + [&](std::monostate) { + throw Error("wanted to build a derivation that is itself a build product, but protocols do not support that. Try upgrading the Nix on the other end of this connection"); + }, + }, sOrDrvPath); + } } StoreObjectDerivationOutput WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) @@ -195,18 +159,22 @@ void WorkerProto::Serialise::write(const Store & store, Worker BuildResult WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) { BuildResult res; - res.status = (BuildResult::Status) readInt(conn.from); - conn.from - >> res.errorMsg - >> res.timesBuilt - >> res.isNonDeterministic - >> res.startTime - >> res.stopTime; - auto builtOutputs = WorkerProto::Serialise::read(store, conn); - for (auto && [output, realisation] : builtOutputs) - res.builtOutputs.insert_or_assign( - std::move(output.outputName), - std::move(realisation)); + res.status = static_cast(readInt(conn.from)); + conn.from >> res.errorMsg; + if (GET_PROTOCOL_MINOR(conn.version) >= 29) { + conn.from + >> res.timesBuilt + >> res.isNonDeterministic + >> res.startTime + >> res.stopTime; + } + if (GET_PROTOCOL_MINOR(conn.version) >= 28) { + auto builtOutputs = WorkerProto::Serialise::read(store, conn); + for (auto && [output, realisation] : builtOutputs) + res.builtOutputs.insert_or_assign( + std::move(output.outputName), + std::move(realisation)); + } return res; } @@ -214,38 +182,68 @@ void WorkerProto::Serialise::write(const Store & store, WorkerProto { conn.to << res.status - << res.errorMsg - << res.timesBuilt - << res.isNonDeterministic - << res.startTime - << res.stopTime; - DrvOutputs builtOutputs; - for (auto & [output, realisation] : res.builtOutputs) - builtOutputs.insert_or_assign(realisation.id, realisation); - WorkerProto::write(store, conn, builtOutputs); + << res.errorMsg; + if (GET_PROTOCOL_MINOR(conn.version) >= 29) { + conn.to + << res.timesBuilt + << res.isNonDeterministic + << res.startTime + << res.stopTime; + } + if (GET_PROTOCOL_MINOR(conn.version) >= 28) { + DrvOutputs builtOutputs; + for (auto & [output, realisation] : res.builtOutputs) + builtOutputs.insert_or_assign(realisation.id, realisation); + WorkerProto::write(store, conn, builtOutputs); + } } -std::optional WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) +ValidPathInfo WorkerProto::Serialise::read(const Store & store, ReadConn conn) { - auto s = readString(conn.from); - return s == "" ? std::optional {} : store.parseStorePath(s); + auto path = WorkerProto::Serialise::read(store, conn); + return ValidPathInfo { + std::move(path), + WorkerProto::Serialise::read(store, conn), + }; } -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional & storePathOpt) +void WorkerProto::Serialise::write(const Store & store, WriteConn conn, const ValidPathInfo & pathInfo) { - conn.to << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); + WorkerProto::write(store, conn, pathInfo.path); + WorkerProto::write(store, conn, static_cast(pathInfo)); } -std::optional WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) +UnkeyedValidPathInfo WorkerProto::Serialise::read(const Store & store, ReadConn conn) { - return ContentAddress::parseOpt(readString(conn.from)); + auto deriver = readString(conn.from); + auto narHash = Hash::parseAny(readString(conn.from), htSHA256); + UnkeyedValidPathInfo info(narHash); + if (deriver != "") info.deriver = store.parseStorePath(deriver); + info.references = WorkerProto::Serialise::read(store, conn); + conn.from >> info.registrationTime >> info.narSize; + if (GET_PROTOCOL_MINOR(conn.version) >= 16) { + conn.from >> info.ultimate; + info.sigs = readStrings(conn.from); + info.ca = ContentAddress::parseOpt(readString(conn.from)); + } + return info; } -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional & caOpt) +void WorkerProto::Serialise::write(const Store & store, WriteConn conn, const UnkeyedValidPathInfo & pathInfo) { - conn.to << (caOpt ? renderContentAddress(*caOpt) : ""); + conn.to + << (pathInfo.deriver ? store.printStorePath(*pathInfo.deriver) : "") + << pathInfo.narHash.to_string(HashFormat::Base16, false); + WorkerProto::write(store, conn, pathInfo.references); + conn.to << pathInfo.registrationTime << pathInfo.narSize; + if (GET_PROTOCOL_MINOR(conn.version) >= 16) { + conn.to + << pathInfo.ultimate + << pathInfo.sigs + << renderContentAddress(pathInfo.ca); + } } } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 06fa69becbf0..7087e7f24c91 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -3,6 +3,7 @@ #include "serialise.hh" #include "acl.hh" +#include "common-protocol.hh" namespace nix { @@ -36,6 +37,8 @@ struct DrvOutput; struct Realisation; struct BuildResult; struct KeyedBuildResult; +struct ValidPathInfo; +struct UnkeyedValidPathInfo; enum TrustedFlag : bool; struct AuthenticatedUser; namespace acl { struct User; struct Group; }; @@ -54,26 +57,29 @@ struct WorkerProto */ enum struct Op : uint64_t; + /** + * Version type for the protocol. + * + * @todo Convert to struct with separate major vs minor fields. + */ + using Version = unsigned int; + /** * A unidirectional read connection, to be used by the read half of the * canonical serializers below. - * - * This currently is just a `Source &`, but more fields will be added - * later. */ struct ReadConn { Source & from; + Version version; }; /** * A unidirectional write connection, to be used by the write half of the * canonical serializers below. - * - * This currently is just a `Sink &`, but more fields will be added - * later. */ struct WriteConn { Sink & to; + Version version; }; /** @@ -163,8 +169,9 @@ enum struct WorkerProto::Op : uint64_t AddMultipleToStore = 44, AddBuildLog = 45, BuildPathsWithResults = 46, - GetAccessStatus = 47, - SetAccessStatus = 48, + AddPermRoot = 47, + GetAccessStatus = 48, + SetAccessStatus = 49, }; /** @@ -175,7 +182,7 @@ enum struct WorkerProto::Op : uint64_t */ inline Sink & operator << (Sink & sink, WorkerProto::Op op) { - return sink << (uint64_t) op; + return sink << static_cast(op); } /** @@ -185,7 +192,7 @@ inline Sink & operator << (Sink & sink, WorkerProto::Op op) */ inline std::ostream & operator << (std::ostream & s, WorkerProto::Op op) { - return s << (uint64_t) op; + return s << static_cast(op); } /** @@ -198,84 +205,57 @@ inline std::ostream & operator << (std::ostream & s, WorkerProto::Op op) * be legal specialization syntax. See below for what that looks like in * practice. */ -#define MAKE_WORKER_PROTO(T) \ - struct WorkerProto::Serialise< T > { \ +#define DECLARE_WORKER_SERIALISER(T) \ + struct WorkerProto::Serialise< T > \ + { \ static T read(const Store & store, WorkerProto::ReadConn conn); \ static void write(const Store & store, WorkerProto::WriteConn conn, const T & t); \ }; template<> -MAKE_WORKER_PROTO(std::string); +DECLARE_WORKER_SERIALISER(DerivedPath); template<> -MAKE_WORKER_PROTO(StorePath); +DECLARE_WORKER_SERIALISER(BuildResult); template<> -MAKE_WORKER_PROTO(ContentAddress); +DECLARE_WORKER_SERIALISER(KeyedBuildResult); template<> -MAKE_WORKER_PROTO(DerivedPath); +DECLARE_WORKER_SERIALISER(ValidPathInfo); template<> -MAKE_WORKER_PROTO(StoreObjectDerivationOutput); +DECLARE_WORKER_SERIALISER(StoreObjectDerivationOutput); template<> -MAKE_WORKER_PROTO(StoreObjectDerivationLog); +DECLARE_WORKER_SERIALISER(StoreObjectDerivationLog); template<> -MAKE_WORKER_PROTO(Realisation); +DECLARE_WORKER_SERIALISER(std::optional); template<> -MAKE_WORKER_PROTO(DrvOutput); +DECLARE_WORKER_SERIALISER(AuthenticatedUser); template<> -MAKE_WORKER_PROTO(BuildResult); +DECLARE_WORKER_SERIALISER(ACL::User); template<> -MAKE_WORKER_PROTO(KeyedBuildResult); -template<> -MAKE_WORKER_PROTO(std::optional); -template<> -MAKE_WORKER_PROTO(AuthenticatedUser); -template<> -MAKE_WORKER_PROTO(ACL::User); -template<> -MAKE_WORKER_PROTO(ACL::Group); +DECLARE_WORKER_SERIALISER(ACL::Group); template -MAKE_WORKER_PROTO(AccessStatusFor); +DECLARE_WORKER_SERIALISER(AccessStatusFor); +template<> +DECLARE_WORKER_SERIALISER(UnkeyedValidPathInfo); template -MAKE_WORKER_PROTO(std::vector); +DECLARE_WORKER_SERIALISER(std::vector); template -MAKE_WORKER_PROTO(std::set); +DECLARE_WORKER_SERIALISER(std::set); +template +DECLARE_WORKER_SERIALISER(std::tuple); template #define X_ std::variant -MAKE_WORKER_PROTO(X_); +DECLARE_WORKER_SERIALISER(X_); #undef X_ template #define X_ std::variant -MAKE_WORKER_PROTO(X_); +DECLARE_WORKER_SERIALISER(X_); #undef X_ +#define COMMA_ , template -#define X_ std::map -MAKE_WORKER_PROTO(X_); -#undef X_ - -template -#define X_ std::pair -MAKE_WORKER_PROTO(X_); -#undef X_ - -/** - * These use the empty string for the null case, relying on the fact - * that the underlying types never serialise to the empty string. - * - * We do this instead of a generic std::optional instance because - * ordinal tags (0 or 1, here) are a bit of a compatability hazard. For - * the same reason, we don't have a std::variant instances (ordinal - * tags 0...n). - * - * We could the generic instances and then these as specializations for - * compatability, but that's proven a bit finnicky, and also makes the - * worker protocol harder to implement in other languages where such - * specializations may not be allowed. - */ -template<> -MAKE_WORKER_PROTO(std::optional); -template<> -MAKE_WORKER_PROTO(std::optional); +DECLARE_WORKER_SERIALISER(std::map); +#undef COMMA_ } diff --git a/src/libutil/abstract-setting-to-json.hh b/src/libutil/abstract-setting-to-json.hh index d506dfb743f1..eea687d8a4ad 100644 --- a/src/libutil/abstract-setting-to-json.hh +++ b/src/libutil/abstract-setting-to-json.hh @@ -7,7 +7,7 @@ namespace nix { template -std::map BaseSetting::toJSONObject() +std::map BaseSetting::toJSONObject() const { auto obj = AbstractSetting::toJSONObject(); obj.emplace("value", value); diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 4feb424e5b0e..c1953bf9c512 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -5,15 +5,11 @@ #include // for strcasecmp -#include -#include -#include -#include -#include - #include "archive.hh" -#include "util.hh" #include "config.hh" +#include "posix-source-accessor.hh" +#include "file-system.hh" +#include "signals.hh" namespace nix { @@ -27,8 +23,6 @@ struct ArchiveSettings : Config #endif "use-case-hack", "Whether to enable a Darwin-specific hack for dealing with file name collisions."}; - Setting preallocateContents{this, false, "preallocate-contents", - "Whether to preallocate files when writing objects with known size."}; }; static ArchiveSettings archiveSettings; @@ -38,91 +32,87 @@ static GlobalConfig::Register rArchiveSettings(&archiveSettings); PathFilter defaultPathFilter = [](const Path &) { return true; }; -static void dumpContents(const Path & path, off_t size, - Sink & sink) +void SourceAccessor::dumpPath( + const CanonPath & path, + Sink & sink, + PathFilter & filter) { - sink << "contents" << size; - - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) throw SysError("opening file '%1%'", path); - - std::vector buf(65536); - size_t left = size; - - while (left > 0) { - auto n = std::min(left, buf.size()); - readFull(fd.get(), buf.data(), n); - left -= n; - sink({buf.data(), n}); - } + auto dumpContents = [&](const CanonPath & path) + { + sink << "contents"; + std::optional size; + readFile(path, sink, [&](uint64_t _size) + { + size = _size; + sink << _size; + }); + assert(size); + writePadding(*size, sink); + }; - writePadding(size, sink); -} + std::function dump; + dump = [&](const CanonPath & path) { + checkInterrupt(); -static time_t dump(const Path & path, Sink & sink, PathFilter & filter) -{ - checkInterrupt(); + auto st = lstat(path); - auto st = lstat(path); - time_t result = st.st_mtime; + sink << "("; - sink << "("; + if (st.type == tRegular) { + sink << "type" << "regular"; + if (st.isExecutable) + sink << "executable" << ""; + dumpContents(path); + } - if (S_ISREG(st.st_mode)) { - sink << "type" << "regular"; - if (st.st_mode & S_IXUSR) - sink << "executable" << ""; - dumpContents(path, st.st_size, sink); - } + else if (st.type == tDirectory) { + sink << "type" << "directory"; + + /* If we're on a case-insensitive system like macOS, undo + the case hack applied by restorePath(). */ + std::map unhacked; + for (auto & i : readDirectory(path)) + if (archiveSettings.useCaseHack) { + std::string name(i.first); + size_t pos = i.first.find(caseHackSuffix); + if (pos != std::string::npos) { + debug("removing case hack suffix from '%s'", path + i.first); + name.erase(pos); + } + if (!unhacked.emplace(name, i.first).second) + throw Error("file name collision in between '%s' and '%s'", + (path + unhacked[name]), + (path + i.first)); + } else + unhacked.emplace(i.first, i.first); - else if (S_ISDIR(st.st_mode)) { - sink << "type" << "directory"; - - /* If we're on a case-insensitive system like macOS, undo - the case hack applied by restorePath(). */ - std::map unhacked; - for (auto & i : readDirectory(path)) - if (archiveSettings.useCaseHack) { - std::string name(i.name); - size_t pos = i.name.find(caseHackSuffix); - if (pos != std::string::npos) { - debug("removing case hack suffix from '%1%'", path + "/" + i.name); - name.erase(pos); + for (auto & i : unhacked) + if (filter((path + i.first).abs())) { + sink << "entry" << "(" << "name" << i.first << "node"; + dump(path + i.second); + sink << ")"; } - if (!unhacked.emplace(name, i.name).second) - throw Error("file name collision in between '%1%' and '%2%'", - (path + "/" + unhacked[name]), - (path + "/" + i.name)); - } else - unhacked.emplace(i.name, i.name); - - for (auto & i : unhacked) - if (filter(path + "/" + i.first)) { - sink << "entry" << "(" << "name" << i.first << "node"; - auto tmp_mtime = dump(path + "/" + i.second, sink, filter); - if (tmp_mtime > result) { - result = tmp_mtime; - } - sink << ")"; - } - } + } - else if (S_ISLNK(st.st_mode)) - sink << "type" << "symlink" << "target" << readLink(path); + else if (st.type == tSymlink) + sink << "type" << "symlink" << "target" << readLink(path); - else throw Error("file '%1%' has an unsupported type", path); + else throw Error("file '%s' has an unsupported type", path); - sink << ")"; + sink << ")"; + }; - return result; + sink << narVersionMagic1; + dump(path); } time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter) { - sink << narVersionMagic1; - return dump(path, sink, filter); + PosixSourceAccessor accessor; + accessor.dumpPath(CanonPath::fromCwd(path), sink, filter); + return accessor.mtime; } void dumpPath(const Path & path, Sink & sink, PathFilter & filter) @@ -143,17 +133,6 @@ static SerialisationError badArchive(const std::string & s) } -#if 0 -static void skipGeneric(Source & source) -{ - if (readString(source) == "(") { - while (readString(source) != ")") - skipGeneric(source); - } -} -#endif - - static void parseContents(ParseSink & sink, Source & source, const Path & path) { uint64_t size = readLongLong(source); @@ -302,74 +281,6 @@ void parseDump(ParseSink & sink, Source & source) } -struct RestoreSink : ParseSink -{ - Path dstPath; - AutoCloseFD fd; - bool protect = false; - - void createDirectory(const Path & path) override - { - Path p = dstPath + path; - auto mode = (protect && (path == "" || path == "/")) ? 0770 : 0777; - if (mkdir(p.c_str(), mode) == -1) - throw SysError("creating directory '%1%'", p); - }; - - void createRegularFile(const Path & path) override - { - Path p = dstPath + path; - auto mode = (protect && (path == "" || path == "/")) ? 0660 : 0666; - fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, mode); - if (!fd) throw SysError("creating file '%1%'", p); - } - - void closeRegularFile() override - { - /* Call close explicitly to make sure the error is checked */ - fd.close(); - } - - void isExecutable() override - { - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError("fstat"); - if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) - throw SysError("fchmod"); - } - - void preallocateContents(uint64_t len) override - { - if (!archiveSettings.preallocateContents) - return; - -#if HAVE_POSIX_FALLOCATE - if (len) { - errno = posix_fallocate(fd.get(), 0, len); - /* Note that EINVAL may indicate that the underlying - filesystem doesn't support preallocation (e.g. on - OpenSolaris). Since preallocation is just an - optimisation, ignore it. */ - if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) - throw SysError("preallocating file of %1% bytes", len); - } -#endif - } - - void receiveContents(std::string_view data) override - { - writeFull(fd.get(), data); - } - - void createSymlink(const Path & path, const std::string & target) override - { - Path p = dstPath + path; - nix::createSymlink(target, p); - } -}; - - void restorePath(const Path & path, Source & source, bool protect) { RestoreSink sink; @@ -384,7 +295,7 @@ void copyNAR(Source & source, Sink & sink) // FIXME: if 'source' is the output of dumpPath() followed by EOF, // we should just forward all data directly without parsing. - ParseSink parseSink; /* null sink; just parse the NAR */ + NullParseSink parseSink; /* just parse the NAR */ TeeSource wrapper { source, sink }; diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index bb7e711e980d..ededb72488bf 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -3,6 +3,7 @@ #include "types.hh" #include "serialise.hh" +#include "fs-sink.hh" namespace nix { @@ -72,49 +73,6 @@ time_t dumpPathAndGetMtime(const Path & path, Sink & sink, */ void dumpString(std::string_view s, Sink & sink); -/** - * \todo Fix this API, it sucks. - */ -struct ParseSink -{ - virtual void createDirectory(const Path & path) { }; - - virtual void createRegularFile(const Path & path) { }; - virtual void closeRegularFile() { }; - virtual void isExecutable() { }; - virtual void preallocateContents(uint64_t size) { }; - virtual void receiveContents(std::string_view data) { }; - - virtual void createSymlink(const Path & path, const std::string & target) { }; -}; - -/** - * If the NAR archive contains a single file at top-level, then save - * the contents of the file to `s`. Otherwise barf. - */ -struct RetrieveRegularNARSink : ParseSink -{ - bool regular = true; - Sink & sink; - - RetrieveRegularNARSink(Sink & sink) : sink(sink) { } - - void createDirectory(const Path & path) override - { - regular = false; - } - - void receiveContents(std::string_view data) override - { - sink(data); - } - - void createSymlink(const Path & path, const std::string & target) override - { - regular = false; - } -}; - void parseDump(ParseSink & sink, Source & source); void restorePath(const Path & path, Source & source, bool protect = false); diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 3cf3ed9ca10c..4480a03f5c67 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -1,7 +1,14 @@ #include "args.hh" +#include "args/root.hh" #include "hash.hh" +#include "environment-variables.hh" +#include "signals.hh" +#include "users.hh" #include "json-utils.hh" +#include +#include +#include #include namespace nix { @@ -26,6 +33,11 @@ void Args::removeFlag(const std::string & longName) longFlags.erase(flag); } +void Completions::setType(AddCompletions::Type t) +{ + type = t; +} + void Completions::add(std::string completion, std::string description) { description = trim(description); @@ -37,7 +49,7 @@ void Completions::add(std::string completion, std::string description) if (needs_ellipsis) description.append(" [...]"); } - insert(Completion { + completions.insert(Completion { .completion = completion, .description = description }); @@ -46,12 +58,20 @@ void Completions::add(std::string completion, std::string description) bool Completion::operator<(const Completion & other) const { return completion < other.completion || (completion == other.completion && description < other.description); } -CompletionType completionType = ctNormal; -std::shared_ptr completions; - std::string completionMarker = "___COMPLETE___"; -static std::optional needsCompletion(std::string_view s) +RootArgs & Args::getRoot() +{ + Args * p = this; + while (p->parent) + p = p->parent; + + auto * res = dynamic_cast(p); + assert(res); + return *res; +} + +std::optional RootArgs::needsCompletion(std::string_view s) { if (!completions) return {}; auto i = s.find(completionMarker); @@ -60,7 +80,178 @@ static std::optional needsCompletion(std::string_view s) return {}; } -void Args::parseCmdline(const Strings & _cmdline) +/** + * Basically this is `typedef std::optional Parser(std::string_view s, Strings & r);` + * + * Except we can't recursively reference the Parser typedef, so we have to write a class. + */ +struct Parser { + std::string_view remaining; + + /** + * @brief Parse the next character(s) + * + * @param r + * @return std::shared_ptr + */ + virtual void operator()(std::shared_ptr & state, Strings & r) = 0; + + Parser(std::string_view s) : remaining(s) {}; + + virtual ~Parser() { }; +}; + +struct ParseQuoted : public Parser { + /** + * @brief Accumulated string + * + * Parsed argument up to this point. + */ + std::string acc; + + ParseQuoted(std::string_view s) : Parser(s) {}; + + virtual void operator()(std::shared_ptr & state, Strings & r) override; +}; + + +struct ParseUnquoted : public Parser { + /** + * @brief Accumulated string + * + * Parsed argument up to this point. Empty string is not representable in + * unquoted syntax, so we use it for the initial state. + */ + std::string acc; + + ParseUnquoted(std::string_view s) : Parser(s) {}; + + virtual void operator()(std::shared_ptr & state, Strings & r) override { + if (remaining.empty()) { + if (!acc.empty()) + r.push_back(acc); + state = nullptr; // done + return; + } + switch (remaining[0]) { + case ' ': case '\t': case '\n': case '\r': + if (!acc.empty()) + r.push_back(acc); + state = std::make_shared(ParseUnquoted(remaining.substr(1))); + return; + case '`': + if (remaining.size() > 1 && remaining[1] == '`') { + state = std::make_shared(ParseQuoted(remaining.substr(2))); + return; + } + else + throw Error("single backtick is not a supported syntax in the nix shebang."); + + // reserved characters + // meaning to be determined, or may be reserved indefinitely so that + // #!nix syntax looks unambiguous + case '$': + case '*': + case '~': + case '<': + case '>': + case '|': + case ';': + case '(': + case ')': + case '[': + case ']': + case '{': + case '}': + case '\'': + case '"': + case '\\': + throw Error("unsupported unquoted character in nix shebang: " + std::string(1, remaining[0]) + ". Use double backticks to escape?"); + + case '#': + if (acc.empty()) { + throw Error ("unquoted nix shebang argument cannot start with #. Use double backticks to escape?"); + } else { + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } + + default: + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } + assert(false); + } +}; + +void ParseQuoted::operator()(std::shared_ptr &state, Strings & r) { + if (remaining.empty()) { + throw Error("unterminated quoted string in nix shebang"); + } + switch (remaining[0]) { + case ' ': + if ((remaining.size() == 3 && remaining[1] == '`' && remaining[2] == '`') + || (remaining.size() > 3 && remaining[1] == '`' && remaining[2] == '`' && remaining[3] != '`')) { + // exactly two backticks mark the end of a quoted string, but a preceding space is ignored if present. + state = std::make_shared(ParseUnquoted(remaining.substr(3))); + r.push_back(acc); + return; + } + else { + // just a normal space + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } + case '`': + // exactly two backticks mark the end of a quoted string + if ((remaining.size() == 2 && remaining[1] == '`') + || (remaining.size() > 2 && remaining[1] == '`' && remaining[2] != '`')) { + state = std::make_shared(ParseUnquoted(remaining.substr(2))); + r.push_back(acc); + return; + } + + // a sequence of at least 3 backticks is one escape-backtick which is ignored, followed by any number of backticks, which are verbatim + else if (remaining.size() >= 3 && remaining[1] == '`' && remaining[2] == '`') { + // ignore "escape" backtick + remaining = remaining.substr(1); + // add the rest + while (remaining.size() > 0 && remaining[0] == '`') { + acc += '`'; + remaining = remaining.substr(1); + } + return; + } + else { + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } + default: + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } + assert(false); +} + +Strings parseShebangContent(std::string_view s) { + Strings result; + std::shared_ptr parserState(std::make_shared(ParseUnquoted(s))); + + // trampoline == iterated strategy pattern + while (parserState) { + auto currentState = parserState; + (*currentState)(parserState, result); + } + + return result; +} + +void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang) { Strings pendingArgs; bool dashDash = false; @@ -71,11 +262,50 @@ void Args::parseCmdline(const Strings & _cmdline) size_t n = std::stoi(*s); assert(n > 0 && n <= cmdline.size()); *std::next(cmdline.begin(), n - 1) += completionMarker; - completions = std::make_shared(); + completions = std::make_shared(); verbosity = lvlError; } bool argsSeen = false; + + // Heuristic to see if we're invoked as a shebang script, namely, + // if we have at least one argument, it's the name of an + // executable file, and it starts with "#!". + Strings savedArgs; + if (allowShebang){ + auto script = *cmdline.begin(); + try { + std::ifstream stream(script); + char shebang[3]={0,0,0}; + stream.get(shebang,3); + if (strncmp(shebang,"#!",2) == 0){ + for (auto pos = std::next(cmdline.begin()); pos != cmdline.end();pos++) + savedArgs.push_back(*pos); + cmdline.clear(); + + std::string line; + std::getline(stream,line); + static const std::string commentChars("#/\\%@*-"); + std::string shebangContent; + while (std::getline(stream,line) && !line.empty() && commentChars.find(line[0]) != std::string::npos){ + line = chomp(line); + + std::smatch match; + // We match one space after `nix` so that we preserve indentation. + // No space is necessary for an empty line. An empty line has basically no effect. + if (std::regex_match(line, match, std::regex("^#!\\s*nix(:? |$)(.*)$"))) + shebangContent += match[2].str() + "\n"; + } + for (const auto & word : parseShebangContent(shebangContent)) { + cmdline.push_back(word); + } + cmdline.push_back(script); + commandBaseDir = dirOf(script); + for (auto pos = savedArgs.begin(); pos != savedArgs.end();pos++) + cmdline.push_back(*pos); + } + } catch (SysError &) { } + } for (auto pos = cmdline.begin(); pos != cmdline.end(); ) { auto arg = *pos; @@ -125,17 +355,34 @@ void Args::parseCmdline(const Strings & _cmdline) for (auto & f : flagExperimentalFeatures) experimentalFeatureSettings.require(f); + /* Now that all the other args are processed, run the deferred completions. + */ + for (auto d : deferredCompletions) + d.completer(*completions, d.n, d.prefix); +} + +Path Args::getCommandBaseDir() const +{ + assert(parent); + return parent->getCommandBaseDir(); +} + +Path RootArgs::getCommandBaseDir() const +{ + return commandBaseDir; } bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) { assert(pos != end); + auto & rootArgs = getRoot(); + auto process = [&](const std::string & name, const Flag & flag) -> bool { ++pos; if (auto & f = flag.experimentalFeature) - flagExperimentalFeatures.insert(*f); + rootArgs.flagExperimentalFeatures.insert(*f); std::vector args; bool anyCompleted = false; @@ -146,10 +393,15 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) "flag '%s' requires %d argument(s), but only %d were given", name, flag.handler.arity, n); } - if (auto prefix = needsCompletion(*pos)) { + if (auto prefix = rootArgs.needsCompletion(*pos)) { anyCompleted = true; - if (flag.completer) - flag.completer(n, *prefix); + if (flag.completer) { + rootArgs.deferredCompletions.push_back({ + .completer = flag.completer, + .n = n, + .prefix = *prefix, + }); + } } args.push_back(*pos++); } @@ -159,14 +411,14 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) }; if (std::string(*pos, 0, 2) == "--") { - if (auto prefix = needsCompletion(*pos)) { + if (auto prefix = rootArgs.needsCompletion(*pos)) { for (auto & [name, flag] : longFlags) { if (!hiddenCategories.count(flag->category) && hasPrefix(name, std::string(*prefix, 2))) { if (auto & f = flag->experimentalFeature) - flagExperimentalFeatures.insert(*f); - completions->add("--" + name, flag->description); + rootArgs.flagExperimentalFeatures.insert(*f); + rootArgs.completions->add("--" + name, flag->description); } } return false; @@ -183,12 +435,12 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) return process(std::string("-") + c, *i->second); } - if (auto prefix = needsCompletion(*pos)) { + if (auto prefix = rootArgs.needsCompletion(*pos)) { if (prefix == "-") { - completions->add("--"); + rootArgs.completions->add("--"); for (auto & [flagName, flag] : shortFlags) if (experimentalFeatureSettings.isEnabled(flag->experimentalFeature)) - completions->add(std::string("-") + flagName, flag->description); + rootArgs.completions->add(std::string("-") + flagName, flag->description); } } @@ -203,6 +455,8 @@ bool Args::processArgs(const Strings & args, bool finish) return true; } + auto & rootArgs = getRoot(); + auto & exp = expectedArgs.front(); bool res = false; @@ -211,16 +465,35 @@ bool Args::processArgs(const Strings & args, bool finish) (exp.handler.arity != ArityAny && args.size() == exp.handler.arity)) { std::vector ss; + bool anyCompleted = false; for (const auto &[n, s] : enumerate(args)) { - if (auto prefix = needsCompletion(s)) { + if (auto prefix = rootArgs.needsCompletion(s)) { + anyCompleted = true; ss.push_back(*prefix); - if (exp.completer) - exp.completer(n, *prefix); + if (exp.completer) { + rootArgs.deferredCompletions.push_back({ + .completer = exp.completer, + .n = n, + .prefix = *prefix, + }); + } } else ss.push_back(s); } - exp.handler.fun(ss); - expectedArgs.pop_front(); + if (!anyCompleted) + exp.handler.fun(ss); + + /* Move the list element to the processedArgs. This is almost the same as + `processedArgs.push_back(expectedArgs.front()); expectedArgs.pop_front()`, + except that it will only adjust the next and prev pointers of the list + elements, meaning the actual contents don't move in memory. This is + critical to prevent invalidating internal pointers! */ + processedArgs.splice( + processedArgs.end(), + expectedArgs, + expectedArgs.begin(), + ++expectedArgs.begin()); + res = true; } @@ -236,6 +509,7 @@ nlohmann::json Args::toJSON() for (auto & [name, flag] : longFlags) { auto j = nlohmann::json::object(); + j["hiddenCategory"] = hiddenCategories.count(flag->category) > 0; if (flag->aliases.count(name)) continue; if (flag->shortName) j["shortName"] = std::string(1, flag->shortName); @@ -270,11 +544,11 @@ nlohmann::json Args::toJSON() return res; } -static void hashTypeCompleter(size_t index, std::string_view prefix) +static void hashTypeCompleter(AddCompletions & completions, size_t index, std::string_view prefix) { for (auto & type : hashTypes) if (hasPrefix(type, prefix)) - completions->add(type); + completions.add(type); } Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht) @@ -286,7 +560,7 @@ Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht) .handler = {[ht](std::string s) { *ht = parseHashType(s); }}, - .completer = hashTypeCompleter + .completer = hashTypeCompleter, }; } @@ -299,13 +573,13 @@ Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional< .handler = {[oht](std::string s) { *oht = std::optional { parseHashType(s) }; }}, - .completer = hashTypeCompleter + .completer = hashTypeCompleter, }; } -static void _completePath(std::string_view prefix, bool onlyDirs) +static void _completePath(AddCompletions & completions, std::string_view prefix, bool onlyDirs) { - completionType = ctFilenames; + completions.setType(Completions::Type::Filenames); glob_t globbuf; int flags = GLOB_NOESCAPE; #ifdef GLOB_ONLYDIR @@ -319,20 +593,20 @@ static void _completePath(std::string_view prefix, bool onlyDirs) auto st = stat(globbuf.gl_pathv[i]); if (!S_ISDIR(st.st_mode)) continue; } - completions->add(globbuf.gl_pathv[i]); + completions.add(globbuf.gl_pathv[i]); } } globfree(&globbuf); } -void completePath(size_t, std::string_view prefix) +void Args::completePath(AddCompletions & completions, size_t, std::string_view prefix) { - _completePath(prefix, false); + _completePath(completions, prefix, false); } -void completeDir(size_t, std::string_view prefix) +void Args::completeDir(AddCompletions & completions, size_t, std::string_view prefix) { - _completePath(prefix, true); + _completePath(completions, prefix, true); } Strings argvToStrings(int argc, char * * argv) @@ -367,10 +641,10 @@ MultiCommand::MultiCommand(const Commands & commands_) command = {s, i->second()}; command->second->parent = this; }}, - .completer = {[&](size_t, std::string_view prefix) { + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { for (auto & [name, command] : commands) if (hasPrefix(name, prefix)) - completions->add(name); + completions.add(name); }} }); @@ -392,14 +666,6 @@ bool MultiCommand::processArgs(const Strings & args, bool finish) return Args::processArgs(args, finish); } -void MultiCommand::completionHook() -{ - if (command) - return command->second->completionHook(); - else - return Args::completionHook(); -} - nlohmann::json MultiCommand::toJSON() { auto cmds = nlohmann::json::object(); @@ -410,8 +676,8 @@ nlohmann::json MultiCommand::toJSON() auto cat = nlohmann::json::object(); cat["id"] = command->category(); cat["description"] = trim(categories[command->category()]); - j["category"] = std::move(cat); cat["experimental-feature"] = command->experimentalFeature(); + j["category"] = std::move(cat); cmds[name] = std::move(j); } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index d90129796ff6..7af82b17884c 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -2,12 +2,15 @@ ///@file #include +#include #include #include +#include #include -#include "util.hh" +#include "types.hh" +#include "experimental-features.hh" namespace nix { @@ -15,19 +18,18 @@ enum HashType : char; class MultiCommand; +class RootArgs; + +class AddCompletions; + class Args { -public: - /** - * Parse the command line, throwing a UsageError if something goes - * wrong. - */ - void parseCmdline(const Strings & cmdline); +public: /** * Return a short one-line description of the command. - */ + */ virtual std::string description() { return ""; } virtual bool forceImpureByDefault() { return false; } @@ -37,16 +39,39 @@ public: */ virtual std::string doc() { return ""; } + /** + * @brief Get the base directory for the command. + * + * @return Generally the working directory, but in case of a shebang + * interpreter, returns the directory of the script. + * + * This only returns the correct value after parseCmdline() has run. + */ + virtual Path getCommandBaseDir() const; + protected: + /** + * The largest `size_t` is used to indicate the "any" arity, for + * handlers/flags/arguments that accept an arbitrary number of + * arguments. + */ static const size_t ArityAny = std::numeric_limits::max(); + /** + * Arguments (flags/options and positional) have a "handler" which is + * caused when the argument is parsed. The handler has an arbitrary side + * effect, including possible affect further command-line parsing. + * + * There are many constructors in order to support many shorthand + * initializations, and this is used a lot. + */ struct Handler { std::function)> fun; size_t arity; - Handler() {} + Handler() = default; Handler(std::function)> && fun) : fun(std::move(fun)) @@ -73,29 +98,29 @@ protected: { } Handler(std::vector * dest) - : fun([=](std::vector ss) { *dest = ss; }) + : fun([dest](std::vector ss) { *dest = ss; }) , arity(ArityAny) { } Handler(std::string * dest) - : fun([=](std::vector ss) { *dest = ss[0]; }) + : fun([dest](std::vector ss) { *dest = ss[0]; }) , arity(1) { } Handler(std::optional * dest) - : fun([=](std::vector ss) { *dest = ss[0]; }) + : fun([dest](std::vector ss) { *dest = ss[0]; }) , arity(1) { } template Handler(T * dest, const T & val) - : fun([=](std::vector ss) { *dest = val; }) + : fun([dest, val](std::vector ss) { *dest = val; }) , arity(0) { } template Handler(I * dest) - : fun([=](std::vector ss) { + : fun([dest](std::vector ss) { *dest = string2IntWithUnitPrefix(ss[0]); }) , arity(1) @@ -103,17 +128,41 @@ protected: template Handler(std::optional * dest) - : fun([=](std::vector ss) { + : fun([dest](std::vector ss) { *dest = string2IntWithUnitPrefix(ss[0]); }) , arity(1) { } }; - /* Options. */ + /** + * The basic function type of the completion callback. + * + * Used to define `CompleterClosure` and some common case completers + * that individual flags/arguments can use. + * + * The `AddCompletions` that is passed is an interface to the state + * stored as part of the root command + */ + using CompleterFun = void(AddCompletions &, size_t, std::string_view); + + /** + * The closure type of the completion callback. + * + * This is what is actually stored as part of each Flag / Expected + * Arg. + */ + using CompleterClosure = std::function; + + /** + * Description of flags / options + * + * These are arguments like `-s` or `--long` that can (mostly) + * appear in any order. + */ struct Flag { - typedef std::shared_ptr ptr; + using ptr = std::shared_ptr; std::string longName; std::set aliases; @@ -122,7 +171,7 @@ protected: std::string category; Strings labels; Handler handler; - std::function completer; + CompleterClosure completer; std::optional experimentalFeature; @@ -130,22 +179,68 @@ protected: static Flag mkHashTypeOptFlag(std::string && longName, std::optional * oht); }; + /** + * Index of all registered "long" flag descriptions (flags like + * `--long`). + */ std::map longFlags; + + /** + * Index of all registered "short" flag descriptions (flags like + * `-s`). + */ std::map shortFlags; + /** + * Process a single flag and its arguments, pulling from an iterator + * of raw CLI args as needed. + */ virtual bool processFlag(Strings::iterator & pos, Strings::iterator end); - /* Positional arguments. */ + /** + * Description of positional arguments + * + * These are arguments that do not start with a `-`, and for which + * the order does matter. + */ struct ExpectedArg { std::string label; bool optional = false; Handler handler; - std::function completer; + CompleterClosure completer; }; + /** + * Queue of expected positional argument forms. + * + * Positional argument descriptions are inserted on the back. + * + * As positional arguments are passed, these are popped from the + * front, until there are hopefully none left as all args that were + * expected in fact were passed. + */ std::list expectedArgs; + /** + * List of processed positional argument forms. + * + * All items removed from `expectedArgs` are added here. After all + * arguments were processed, this list should be exactly the same as + * `expectedArgs` was before. + * + * This list is used to extend the lifetime of the argument forms. + * If this is not done, some closures that reference the command + * itself will segfault. + */ + std::list processedArgs; + /** + * Process some positional arugments + * + * @param finish: We have parsed everything else, and these are the only + * arguments left. Used because we accumulate some "pending args" we might + * have left over. + */ virtual bool processArgs(const Strings & args, bool finish); virtual Strings::iterator rewriteArgs(Strings & args, Strings::iterator pos) @@ -159,13 +254,6 @@ protected: */ virtual void initialFlagsProcessed() {} - /** - * Called after the command line has been processed if we need to generate - * completions. Useful for commands that need to know the whole command line - * in order to know what completions to generate. - */ - virtual void completionHook() { } - public: void addFlag(Flag && flag); @@ -200,21 +288,30 @@ public: }); } + static CompleterFun completePath; + + static CompleterFun completeDir; + virtual nlohmann::json toJSON(); friend class MultiCommand; + /** + * The parent command, used if this is a subcommand. + * + * Invariant: An Args with a null parent must also be a RootArgs + * + * \todo this would probably be better in the CommandClass. + * getRoot() could be an abstract method that peels off at most one + * layer before recuring. + */ MultiCommand * parent = nullptr; -private: - /** - * Experimental features needed when parsing args. These are checked - * after flag parsing is completed in order to support enabling - * experimental features coming after the flag that needs the - * experimental feature. + * Traverse parent pointers until we find the \ref RootArgs "root + * arguments" object. */ - std::set flagExperimentalFeatures; + RootArgs & getRoot(); }; /** @@ -225,23 +322,23 @@ struct Command : virtual public Args { friend class MultiCommand; - virtual ~Command() { } + virtual ~Command() = default; /** * Entry point to the command */ virtual void run() = 0; - typedef int Category; + using Category = int; static constexpr Category catDefault = 0; - virtual std::optional experimentalFeature (); + virtual std::optional experimentalFeature(); virtual Category category() { return catDefault; } }; -typedef std::map()>> Commands; +using Commands = std::map()>>; /** * An argument parser that supports multiple subcommands, @@ -265,8 +362,6 @@ public: bool processArgs(const Strings & args, bool finish) override; - void completionHook() override; - nlohmann::json toJSON() override; }; @@ -278,21 +373,42 @@ struct Completion { bool operator<(const Completion & other) const; }; -class Completions : public std::set { + +/** + * The abstract interface for completions callbacks + * + * The idea is to restrict the callback so it can only add additional + * completions to the collection, or set the completion type. By making + * it go through this interface, the callback cannot make any other + * changes, or even view the completions / completion type that have + * been set so far. + */ +class AddCompletions +{ public: - void add(std::string completion, std::string description = ""); -}; -extern std::shared_ptr completions; -enum CompletionType { - ctNormal, - ctFilenames, - ctAttrs -}; -extern CompletionType completionType; + /** + * The type of completion we are collecting. + */ + enum class Type { + Normal, + Filenames, + Attrs, + }; + + /** + * Set the type of the completions being collected + * + * \todo it should not be possible to change the type after it has been set. + */ + virtual void setType(Type type) = 0; -void completePath(size_t, std::string_view prefix); + /** + * Add a single completion to the collection + */ + virtual void add(std::string completion, std::string description = "") = 0; +}; -void completeDir(size_t, std::string_view prefix); +Strings parseShebangContent(std::string_view s); } diff --git a/src/libutil/args/root.hh b/src/libutil/args/root.hh new file mode 100644 index 000000000000..5c55c37a55d8 --- /dev/null +++ b/src/libutil/args/root.hh @@ -0,0 +1,84 @@ +#pragma once + +#include "args.hh" + +namespace nix { + +/** + * The concrete implementation of a collection of completions. + * + * This is exposed so that the main entry point can print out the + * collected completions. + */ +struct Completions final : AddCompletions +{ + std::set completions; + Type type = Type::Normal; + + void setType(Type type) override; + void add(std::string completion, std::string description = "") override; +}; + +/** + * The outermost Args object. This is the one we will actually parse a command + * line with, whereas the inner ones (if they exists) are subcommands (and this + * is also a MultiCommand or something like it). + * + * This Args contains completions state shared between it and all of its + * descendent Args. + */ +class RootArgs : virtual public Args +{ + /** + * @brief The command's "working directory", but only set when top level. + * + * Use getCommandBaseDir() to get the directory regardless of whether this + * is a top-level command or subcommand. + * + * @see getCommandBaseDir() + */ + Path commandBaseDir = "."; + +public: + /** Parse the command line, throwing a UsageError if something goes + * wrong. + */ + void parseCmdline(const Strings & cmdline, bool allowShebang = false); + + std::shared_ptr completions; + + Path getCommandBaseDir() const override; + +protected: + + friend class Args; + + /** + * A pointer to the completion and its two arguments; a thunk; + */ + struct DeferredCompletion { + const CompleterClosure & completer; + size_t n; + std::string prefix; + }; + + /** + * Completions are run after all args and flags are parsed, so completions + * of earlier arguments can benefit from later arguments. + */ + std::vector deferredCompletions; + + /** + * Experimental features needed when parsing args. These are checked + * after flag parsing is completed in order to support enabling + * experimental features coming after the flag that needs the + * experimental feature. + */ + std::set flagExperimentalFeatures; + +private: + + std::optional needsCompletion(std::string_view s); +}; + +} diff --git a/src/libutil/canon-path.cc b/src/libutil/canon-path.cc index 040464532b82..1e465f1f6999 100644 --- a/src/libutil/canon-path.cc +++ b/src/libutil/canon-path.cc @@ -1,5 +1,5 @@ #include "canon-path.hh" -#include "util.hh" +#include "file-system.hh" namespace nix { @@ -13,6 +13,13 @@ CanonPath::CanonPath(std::string_view raw, const CanonPath & root) : path(absPath((Path) raw, root.abs())) { } +CanonPath::CanonPath(const std::vector & elems) + : path("/") +{ + for (auto & s : elems) + push(s); +} + CanonPath CanonPath::fromCwd(std::string_view path) { return CanonPath(unchecked_t(), absPath((Path) path)); diff --git a/src/libutil/canon-path.hh b/src/libutil/canon-path.hh index eefe05ed5872..6d0519f4f168 100644 --- a/src/libutil/canon-path.hh +++ b/src/libutil/canon-path.hh @@ -6,6 +6,7 @@ #include #include #include +#include namespace nix { @@ -46,6 +47,11 @@ public: : path(std::move(path)) { } + /** + * Construct a canon path from a vector of elements. + */ + CanonPath(const std::vector & elems); + static CanonPath fromCwd(std::string_view path = "."); static CanonPath root; diff --git a/src/libutil/cgroup.cc b/src/libutil/cgroup.cc index a008481caa90..4c2bf31ffe1a 100644 --- a/src/libutil/cgroup.cc +++ b/src/libutil/cgroup.cc @@ -2,6 +2,7 @@ #include "cgroup.hh" #include "util.hh" +#include "file-system.hh" #include "finally.hh" #include diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh index 9f661c5c3986..a4d20a675a16 100644 --- a/src/libutil/comparator.hh +++ b/src/libutil/comparator.hh @@ -1,6 +1,49 @@ #pragma once ///@file +#define DECLARE_ONE_CMP(PRE, QUAL, COMPARATOR, MY_TYPE) \ + PRE bool QUAL operator COMPARATOR(const MY_TYPE & other) const; +#define DECLARE_EQUAL(prefix, qualification, my_type) \ + DECLARE_ONE_CMP(prefix, qualification, ==, my_type) +#define DECLARE_LEQ(prefix, qualification, my_type) \ + DECLARE_ONE_CMP(prefix, qualification, <, my_type) +#define DECLARE_NEQ(prefix, qualification, my_type) \ + DECLARE_ONE_CMP(prefix, qualification, !=, my_type) + +#define GENERATE_ONE_CMP(PRE, QUAL, COMPARATOR, MY_TYPE, ...) \ + PRE bool QUAL operator COMPARATOR(const MY_TYPE & other) const { \ + __VA_OPT__(const MY_TYPE * me = this;) \ + auto fields1 = std::make_tuple( __VA_ARGS__ ); \ + __VA_OPT__(me = &other;) \ + auto fields2 = std::make_tuple( __VA_ARGS__ ); \ + return fields1 COMPARATOR fields2; \ + } +#define GENERATE_EQUAL(prefix, qualification, my_type, args...) \ + GENERATE_ONE_CMP(prefix, qualification, ==, my_type, args) +#define GENERATE_LEQ(prefix, qualification, my_type, args...) \ + GENERATE_ONE_CMP(prefix, qualification, <, my_type, args) +#define GENERATE_NEQ(prefix, qualification, my_type, args...) \ + GENERATE_ONE_CMP(prefix, qualification, !=, my_type, args) + +/** + * Declare comparison methods without defining them. + */ +#define DECLARE_CMP(my_type) \ + DECLARE_EQUAL(,,my_type) \ + DECLARE_LEQ(,,my_type) \ + DECLARE_NEQ(,,my_type) + +/** + * @param prefix This is for something before each declaration like + * `template`. + * + * @param my_type the type are defining operators for. + */ +#define DECLARE_CMP_EXT(prefix, qualification, my_type) \ + DECLARE_EQUAL(prefix, qualification, my_type) \ + DECLARE_LEQ(prefix, qualification, my_type) \ + DECLARE_NEQ(prefix, qualification, my_type) + /** * Awful hacky generation of the comparison operators by doing a lexicographic * comparison between the choosen fields. @@ -17,18 +60,18 @@ * } * ``` */ -#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, ...) \ - bool operator COMPARATOR(const MY_TYPE& other) const { \ - __VA_OPT__(const MY_TYPE* me = this;) \ - auto fields1 = std::make_tuple( __VA_ARGS__ ); \ - __VA_OPT__(me = &other;) \ - auto fields2 = std::make_tuple( __VA_ARGS__ ); \ - return fields1 COMPARATOR fields2; \ - } -#define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args) -#define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args) -#define GENERATE_NEQ(args...) GENERATE_ONE_CMP(!=, args) #define GENERATE_CMP(args...) \ - GENERATE_EQUAL(args) \ - GENERATE_LEQ(args) \ - GENERATE_NEQ(args) + GENERATE_EQUAL(,,args) \ + GENERATE_LEQ(,,args) \ + GENERATE_NEQ(,,args) + +/** + * @param prefix This is for something before each declaration like + * `template`. + * + * @param my_type the type are defining operators for. + */ +#define GENERATE_CMP_EXT(prefix, my_type, args...) \ + GENERATE_EQUAL(prefix, my_type ::, my_type, args) \ + GENERATE_LEQ(prefix, my_type ::, my_type, args) \ + GENERATE_NEQ(prefix, my_type ::, my_type, args) diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index ba0847cded20..d06f1f87bd88 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -1,6 +1,6 @@ #include "compression.hh" +#include "signals.hh" #include "tarfile.hh" -#include "util.hh" #include "finally.hh" #include "logging.hh" diff --git a/src/libutil/config-impl.hh b/src/libutil/config-impl.hh index b9639e761791..9f69e8444176 100644 --- a/src/libutil/config-impl.hh +++ b/src/libutil/config-impl.hh @@ -45,13 +45,13 @@ bool BaseSetting::isAppendable() return trait::appendable; } -template<> void BaseSetting::appendOrSet(Strings && newValue, bool append); -template<> void BaseSetting::appendOrSet(StringSet && newValue, bool append); -template<> void BaseSetting::appendOrSet(StringMap && newValue, bool append); -template<> void BaseSetting>::appendOrSet(std::set && newValue, bool append); +template<> void BaseSetting::appendOrSet(Strings newValue, bool append); +template<> void BaseSetting::appendOrSet(StringSet newValue, bool append); +template<> void BaseSetting::appendOrSet(StringMap newValue, bool append); +template<> void BaseSetting>::appendOrSet(std::set newValue, bool append); template -void BaseSetting::appendOrSet(T && newValue, bool append) +void BaseSetting::appendOrSet(T newValue, bool append) { static_assert( !trait::appendable, diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 38d406e8a3c4..96a0a4df87f0 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -2,6 +2,8 @@ #include "args.hh" #include "abstract-setting-to-json.hh" #include "experimental-features.hh" +#include "util.hh" +#include "file-system.hh" #include "config-impl.hh" @@ -9,6 +11,10 @@ namespace nix { +Config::Config(StringMap initials) + : AbstractConfig(std::move(initials)) +{ } + bool Config::set(const std::string & name, const std::string & value) { bool append = false; @@ -29,28 +35,26 @@ bool Config::set(const std::string & name, const std::string & value) void Config::addSetting(AbstractSetting * setting) { - _settings.emplace(setting->name, Config::SettingData(false, setting)); - for (auto & alias : setting->aliases) - _settings.emplace(alias, Config::SettingData(true, setting)); + _settings.emplace(setting->name, Config::SettingData{false, setting}); + for (const auto & alias : setting->aliases) + _settings.emplace(alias, Config::SettingData{true, setting}); bool set = false; - auto i = unknownSettings.find(setting->name); - if (i != unknownSettings.end()) { - setting->set(i->second); + if (auto i = unknownSettings.find(setting->name); i != unknownSettings.end()) { + setting->set(std::move(i->second)); setting->overridden = true; unknownSettings.erase(i); set = true; } for (auto & alias : setting->aliases) { - auto i = unknownSettings.find(alias); - if (i != unknownSettings.end()) { + if (auto i = unknownSettings.find(alias); i != unknownSettings.end()) { if (set) warn("setting '%s' is set, but it's an alias of '%s' which is also set", alias, setting->name); else { - setting->set(i->second); + setting->set(std::move(i->second)); setting->overridden = true; unknownSettings.erase(i); set = true; @@ -59,22 +63,27 @@ void Config::addSetting(AbstractSetting * setting) } } +AbstractConfig::AbstractConfig(StringMap initials) + : unknownSettings(std::move(initials)) +{ } + void AbstractConfig::warnUnknownSettings() { - for (auto & s : unknownSettings) + for (const auto & s : unknownSettings) warn("unknown setting '%s'", s.first); } void AbstractConfig::reapplyUnknownSettings() { auto unknownSettings2 = std::move(unknownSettings); + unknownSettings = {}; for (auto & s : unknownSettings2) set(s.first, s.second); } void Config::getSettings(std::map & res, bool overriddenOnly) { - for (auto & opt : _settings) + for (const auto & opt : _settings) if (!opt.second.isAlias && (!overriddenOnly || opt.second.setting->overridden)) res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description}); } @@ -90,8 +99,7 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string line += contents[pos++]; pos++; - auto hash = line.find('#'); - if (hash != std::string::npos) + if (auto hash = line.find('#'); hash != line.npos) line = std::string(line, 0, hash); auto tokens = tokenizeString>(line); @@ -124,24 +132,24 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string if (tokens[1] != "=") throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); - std::string name = tokens[0]; + std::string name = std::move(tokens[0]); auto i = tokens.begin(); advance(i, 2); parsedContents.push_back({ - name, + std::move(name), concatStringsSep(" ", Strings(i, tokens.end())), }); }; // First apply experimental-feature related settings - for (auto & [name, value] : parsedContents) + for (const auto & [name, value] : parsedContents) if (name == "experimental-features" || name == "extra-experimental-features") set(name, value); // Then apply other settings - for (auto & [name, value] : parsedContents) + for (const auto & [name, value] : parsedContents) if (name != "experimental-features" && name != "extra-experimental-features") set(name, value); } @@ -163,7 +171,7 @@ void Config::resetOverridden() nlohmann::json Config::toJSON() { auto res = nlohmann::json::object(); - for (auto & s : _settings) + for (const auto & s : _settings) if (!s.second.isAlias) res.emplace(s.first, s.second.setting->toJSON()); return res; @@ -171,8 +179,8 @@ nlohmann::json Config::toJSON() std::string Config::toKeyValue() { - auto res = std::string(); - for (auto & s : _settings) + std::string res; + for (const auto & s : _settings) if (s.second.isAlias) res += fmt("%s = %s\n", s.first, s.second.setting->to_string()); return res; @@ -194,8 +202,15 @@ AbstractSetting::AbstractSetting( : name(name) , description(stripIndentation(description)) , aliases(aliases) - , experimentalFeature(experimentalFeature) + , experimentalFeature(std::move(experimentalFeature)) +{ +} + +AbstractSetting::~AbstractSetting() { + // Check against a gcc miscompilation causing our constructor + // not to run (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431). + assert(created == 123); } nlohmann::json AbstractSetting::toJSON() @@ -203,7 +218,7 @@ nlohmann::json AbstractSetting::toJSON() return nlohmann::json(toJSONObject()); } -std::map AbstractSetting::toJSONObject() +std::map AbstractSetting::toJSONObject() const { std::map obj; obj.emplace("description", description); @@ -219,6 +234,9 @@ void AbstractSetting::convertToArg(Args & args, const std::string & category) { } + +bool AbstractSetting::isOverridden() const { return overridden; } + template<> std::string BaseSetting::parse(const std::string & str) const { return str; @@ -263,14 +281,14 @@ template<> void BaseSetting::convertToArg(Args & args, const std::string & .longName = name, .description = fmt("Enable the `%s` setting.", name), .category = category, - .handler = {[this]() { override(true); }}, + .handler = {[this] { override(true); }}, .experimentalFeature = experimentalFeature, }); args.addFlag({ .longName = "no-" + name, .description = fmt("Disable the `%s` setting.", name), .category = category, - .handler = {[this]() { override(false); }}, + .handler = {[this] { override(false); }}, .experimentalFeature = experimentalFeature, }); } @@ -280,10 +298,11 @@ template<> Strings BaseSetting::parse(const std::string & str) const return tokenizeString(str); } -template<> void BaseSetting::appendOrSet(Strings && newValue, bool append) +template<> void BaseSetting::appendOrSet(Strings newValue, bool append) { if (!append) value.clear(); - for (auto && s : std::move(newValue)) value.push_back(std::move(s)); + value.insert(value.end(), std::make_move_iterator(newValue.begin()), + std::make_move_iterator(newValue.end())); } template<> std::string BaseSetting::to_string() const @@ -296,11 +315,10 @@ template<> StringSet BaseSetting::parse(const std::string & str) cons return tokenizeString(str); } -template<> void BaseSetting::appendOrSet(StringSet && newValue, bool append) +template<> void BaseSetting::appendOrSet(StringSet newValue, bool append) { if (!append) value.clear(); - for (auto && s : std::move(newValue)) - value.insert(s); + value.insert(std::make_move_iterator(newValue.begin()), std::make_move_iterator(newValue.end())); } template<> std::string BaseSetting::to_string() const @@ -312,26 +330,26 @@ template<> std::set BaseSetting res; for (auto & s : tokenizeString(str)) { - auto thisXpFeature = parseExperimentalFeature(s); - if (thisXpFeature) + if (auto thisXpFeature = parseExperimentalFeature(s); thisXpFeature) { res.insert(thisXpFeature.value()); - else + if (thisXpFeature.value() == Xp::Flakes) + res.insert(Xp::FetchTree); + } else warn("unknown experimental feature '%s'", s); } return res; } -template<> void BaseSetting>::appendOrSet(std::set && newValue, bool append) +template<> void BaseSetting>::appendOrSet(std::set newValue, bool append) { if (!append) value.clear(); - for (auto && s : std::move(newValue)) - value.insert(s); + value.insert(std::make_move_iterator(newValue.begin()), std::make_move_iterator(newValue.end())); } template<> std::string BaseSetting>::to_string() const { StringSet stringifiedXpFeatures; - for (auto & feature : value) + for (const auto & feature : value) stringifiedXpFeatures.insert(std::string(showExperimentalFeature(feature))); return concatStringsSep(" ", stringifiedXpFeatures); } @@ -339,28 +357,25 @@ template<> std::string BaseSetting>::to_string() c template<> StringMap BaseSetting::parse(const std::string & str) const { StringMap res; - for (auto & s : tokenizeString(str)) { - auto eq = s.find_first_of('='); - if (std::string::npos != eq) + for (const auto & s : tokenizeString(str)) { + if (auto eq = s.find_first_of('='); s.npos != eq) res.emplace(std::string(s, 0, eq), std::string(s, eq + 1)); // else ignored } return res; } -template<> void BaseSetting::appendOrSet(StringMap && newValue, bool append) +template<> void BaseSetting::appendOrSet(StringMap newValue, bool append) { if (!append) value.clear(); - for (auto && [k, v] : std::move(newValue)) - value.emplace(std::move(k), std::move(v)); + value.insert(std::make_move_iterator(newValue.begin()), std::make_move_iterator(newValue.end())); } template<> std::string BaseSetting::to_string() const { - Strings kvstrs; - std::transform(value.begin(), value.end(), back_inserter(kvstrs), - [&](auto kvpair){ return kvpair.first + "=" + kvpair.second; }); - return concatStringsSep(" ", kvstrs); + return std::transform_reduce(value.cbegin(), value.cend(), std::string{}, + [](const auto & l, const auto &r) { return l + " " + r; }, + [](const auto & kvpair){ return kvpair.first + "=" + kvpair.second; }); } template class BaseSetting; @@ -384,11 +399,33 @@ static Path parsePath(const AbstractSetting & s, const std::string & str) return canonPath(str); } +PathSetting::PathSetting(Config * options, + const Path & def, + const std::string & name, + const std::string & description, + const std::set & aliases) + : BaseSetting(def, true, name, description, aliases) +{ + options->addSetting(this); +} + Path PathSetting::parse(const std::string & str) const { return parsePath(*this, str); } + +OptionalPathSetting::OptionalPathSetting(Config * options, + const std::optional & def, + const std::string & name, + const std::string & description, + const std::set & aliases) + : BaseSetting>(def, true, name, description, aliases) +{ + options->addSetting(this); +} + + std::optional OptionalPathSetting::parse(const std::string & str) const { if (str == "") @@ -397,6 +434,11 @@ std::optional OptionalPathSetting::parse(const std::string & str) const return parsePath(*this, str); } +void OptionalPathSetting::operator =(const std::optional & v) +{ + this->assign(v); +} + bool GlobalConfig::set(const std::string & name, const std::string & value) { for (auto & config : *configRegistrations) @@ -422,7 +464,7 @@ void GlobalConfig::resetOverridden() nlohmann::json GlobalConfig::toJSON() { auto res = nlohmann::json::object(); - for (auto & config : *configRegistrations) + for (const auto & config : *configRegistrations) res.update(config->toJSON()); return res; } @@ -432,7 +474,7 @@ std::string GlobalConfig::toKeyValue() std::string res; std::map settings; globalConfig.getSettings(settings); - for (auto & s : settings) + for (const auto & s : settings) res += fmt("%s = %s\n", s.first, s.second.value); return res; } diff --git a/src/libutil/config.hh b/src/libutil/config.hh index cc8532587cc7..5d7bd8e0cdbb 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -36,8 +36,8 @@ namespace nix { * * std::map settings; * config.getSettings(settings); - * config["system"].description == "the current system" - * config["system"].value == "x86_64-linux" + * settings["system"].description == "the current system" + * settings["system"].value == "x86_64-linux" * * * The above retrieves all currently known settings from the `Config` object @@ -52,9 +52,7 @@ class AbstractConfig protected: StringMap unknownSettings; - AbstractConfig(const StringMap & initials = {}) - : unknownSettings(initials) - { } + AbstractConfig(StringMap initials = {}); public: @@ -150,12 +148,9 @@ public: { bool isAlias; AbstractSetting * setting; - SettingData(bool isAlias, AbstractSetting * setting) - : isAlias(isAlias), setting(setting) - { } }; - typedef std::map Settings; + using Settings = std::map; private: @@ -163,9 +158,7 @@ private: public: - Config(const StringMap & initials = {}) - : AbstractConfig(initials) - { } + Config(StringMap initials = {}); bool set(const std::string & name, const std::string & value) override; @@ -206,12 +199,7 @@ protected: const std::set & aliases, std::optional experimentalFeature = std::nullopt); - virtual ~AbstractSetting() - { - // Check against a gcc miscompilation causing our constructor - // not to run (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431). - assert(created == 123); - } + virtual ~AbstractSetting(); virtual void set(const std::string & value, bool append = false) = 0; @@ -225,11 +213,11 @@ protected: nlohmann::json toJSON(); - virtual std::map toJSONObject(); + virtual std::map toJSONObject() const; virtual void convertToArg(Args & args, const std::string & category); - bool isOverridden() const { return overridden; } + bool isOverridden() const; }; /** @@ -259,7 +247,7 @@ protected: * * @param append Whether to append or overwrite. */ - virtual void appendOrSet(T && newValue, bool append); + virtual void appendOrSet(T newValue, bool append); public: @@ -318,18 +306,17 @@ public: void convertToArg(Args & args, const std::string & category) override; - std::map toJSONObject() override; + std::map toJSONObject() const override; }; template std::ostream & operator <<(std::ostream & str, const BaseSetting & opt) { - str << (const T &) opt; - return str; + return str << static_cast(opt); } template -bool operator ==(const T & v1, const BaseSetting & v2) { return v1 == (const T &) v2; } +bool operator ==(const T & v1, const BaseSetting & v2) { return v1 == static_cast(v2); } template class Setting : public BaseSetting @@ -342,7 +329,7 @@ public: const std::set & aliases = {}, const bool documentDefault = true, std::optional experimentalFeature = std::nullopt) - : BaseSetting(def, documentDefault, name, description, aliases, experimentalFeature) + : BaseSetting(def, documentDefault, name, description, aliases, std::move(experimentalFeature)) { options->addSetting(this); } @@ -365,11 +352,7 @@ public: const Path & def, const std::string & name, const std::string & description, - const std::set & aliases = {}) - : BaseSetting(def, true, name, description, aliases) - { - options->addSetting(this); - } + const std::set & aliases = {}); Path parse(const std::string & str) const override; @@ -391,15 +374,11 @@ public: const std::optional & def, const std::string & name, const std::string & description, - const std::set & aliases = {}) - : BaseSetting>(def, true, name, description, aliases) - { - options->addSetting(this); - } + const std::set & aliases = {}); std::optional parse(const std::string & str) const override; - void operator =(const std::optional & v) { this->assign(v); } + void operator =(const std::optional & v); }; struct GlobalConfig : public AbstractConfig diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc new file mode 100644 index 000000000000..352a6a0fb36e --- /dev/null +++ b/src/libutil/current-process.cc @@ -0,0 +1,110 @@ +#include "current-process.hh" +#include "namespaces.hh" +#include "util.hh" +#include "finally.hh" +#include "file-system.hh" +#include "processes.hh" +#include "signals.hh" + +#ifdef __APPLE__ +# include +#endif + +#if __linux__ +# include +# include +# include "cgroup.hh" +#endif + +#include + +namespace nix { + +unsigned int getMaxCPU() +{ + #if __linux__ + try { + auto cgroupFS = getCgroupFS(); + if (!cgroupFS) return 0; + + auto cgroups = getCgroups("/proc/self/cgroup"); + auto cgroup = cgroups[""]; + if (cgroup == "") return 0; + + auto cpuFile = *cgroupFS + "/" + cgroup + "/cpu.max"; + + auto cpuMax = readFile(cpuFile); + auto cpuMaxParts = tokenizeString>(cpuMax, " \n"); + auto quota = cpuMaxParts[0]; + auto period = cpuMaxParts[1]; + if (quota != "max") + return std::ceil(std::stoi(quota) / std::stof(period)); + } catch (Error &) { ignoreException(lvlDebug); } + #endif + + return 0; +} + + +////////////////////////////////////////////////////////////////////// + + +#if __linux__ +rlim_t savedStackSize = 0; +#endif + +void setStackSize(size_t stackSize) +{ + #if __linux__ + struct rlimit limit; + if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) { + savedStackSize = limit.rlim_cur; + limit.rlim_cur = stackSize; + setrlimit(RLIMIT_STACK, &limit); + } + #endif +} + +void restoreProcessContext(bool restoreMounts) +{ + restoreSignals(); + if (restoreMounts) { + restoreMountNamespace(); + } + + #if __linux__ + if (savedStackSize) { + struct rlimit limit; + if (getrlimit(RLIMIT_STACK, &limit) == 0) { + limit.rlim_cur = savedStackSize; + setrlimit(RLIMIT_STACK, &limit); + } + } + #endif +} + + +////////////////////////////////////////////////////////////////////// + + +std::optional getSelfExe() +{ + static auto cached = []() -> std::optional + { + #if __linux__ + return readLink("/proc/self/exe"); + #elif __APPLE__ + char buf[1024]; + uint32_t size = sizeof(buf); + if (_NSGetExecutablePath(buf, &size) == 0) + return buf; + else + return std::nullopt; + #else + return std::nullopt; + #endif + }(); + return cached; +} + +} diff --git a/src/libutil/current-process.hh b/src/libutil/current-process.hh new file mode 100644 index 000000000000..826d6fe2006d --- /dev/null +++ b/src/libutil/current-process.hh @@ -0,0 +1,34 @@ +#pragma once +///@file + +#include + +#include "types.hh" + +namespace nix { + +/** + * If cgroups are active, attempt to calculate the number of CPUs available. + * If cgroups are unavailable or if cpu.max is set to "max", return 0. + */ +unsigned int getMaxCPU(); + +/** + * Change the stack size. + */ +void setStackSize(size_t stackSize); + +/** + * Restore the original inherited Unix process context (such as signal + * masks, stack size). + + * See startSignalHandlerThread(), saveSignalMask(). + */ +void restoreProcessContext(bool restoreMounts = true); + +/** + * @return the path of the current executable. + */ +std::optional getSelfExe(); + +} diff --git a/src/libutil/environment-variables.cc b/src/libutil/environment-variables.cc new file mode 100644 index 000000000000..6618d787271c --- /dev/null +++ b/src/libutil/environment-variables.cc @@ -0,0 +1,49 @@ +#include "util.hh" +#include "environment-variables.hh" + +extern char * * environ __attribute__((weak)); + +namespace nix { + +std::optional getEnv(const std::string & key) +{ + char * value = getenv(key.c_str()); + if (!value) return {}; + return std::string(value); +} + +std::optional getEnvNonEmpty(const std::string & key) { + auto value = getEnv(key); + if (value == "") return {}; + return value; +} + +std::map getEnv() +{ + std::map env; + for (size_t i = 0; environ[i]; ++i) { + auto s = environ[i]; + auto eq = strchr(s, '='); + if (!eq) + // invalid env, just keep going + continue; + env.emplace(std::string(s, eq), std::string(eq + 1)); + } + return env; +} + + +void clearEnv() +{ + for (auto & name : getEnv()) + unsetenv(name.first.c_str()); +} + +void replaceEnv(const std::map & newEnv) +{ + clearEnv(); + for (auto & newEnvVar : newEnv) + setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); +} + +} diff --git a/src/libutil/environment-variables.hh b/src/libutil/environment-variables.hh new file mode 100644 index 000000000000..21eb4619b284 --- /dev/null +++ b/src/libutil/environment-variables.hh @@ -0,0 +1,41 @@ +#pragma once +/** + * @file + * + * Utilities for working with the current process's environment + * variables. + */ + +#include + +#include "types.hh" + +namespace nix { + +/** + * @return an environment variable. + */ +std::optional getEnv(const std::string & key); + +/** + * @return a non empty environment variable. Returns nullopt if the env + * variable is set to "" + */ +std::optional getEnvNonEmpty(const std::string & key); + +/** + * Get the entire environment. + */ +std::map getEnv(); + +/** + * Clear the environment. + */ +void clearEnv(); + +/** + * Replace the entire environment with the given one. + */ +void replaceEnv(const std::map & newEnv); + +} diff --git a/src/libutil/error.cc b/src/libutil/error.cc index c9d61942aedd..72c346cb583f 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -1,4 +1,7 @@ #include "error.hh" +#include "environment-variables.hh" +#include "signals.hh" +#include "terminal.hh" #include #include @@ -7,13 +10,16 @@ namespace nix { -const std::string nativeSystem = SYSTEM; - void BaseError::addTrace(std::shared_ptr && e, hintformat hint, bool frame) { err.traces.push_front(Trace { .pos = std::move(e), .hint = hint, .frame = frame }); } +void throwExceptionSelfCheck(){ + // This is meant to be caught in initLibUtil() + throw SysError("C++ exception handling is broken. This would appear to be a problem with the way Nix was compiled and/or linked and/or loaded."); +} + // c++ std::exception descendants must have a 'const char* what()' function. // This stringifies the error and caches it for use by what(), or similarly by msg(). const std::string & BaseError::calcWhat() const @@ -150,6 +156,36 @@ static std::string indent(std::string_view indentFirst, std::string_view indentR return res; } +/** + * A development aid for finding missing positions, to improve error messages. Example use: + * + * _NIX_EVAL_SHOW_UNKNOWN_LOCATIONS=1 _NIX_TEST_ACCEPT=1 make tests/lang.sh.test + * git diff -U20 tests + * + */ +static bool printUnknownLocations = getEnv("_NIX_EVAL_SHOW_UNKNOWN_LOCATIONS").has_value(); + +/** + * Print a position, if it is known. + * + * @return true if a position was printed. + */ +static bool printPosMaybe(std::ostream & oss, std::string_view indent, const std::shared_ptr & pos) { + bool hasPos = pos && *pos; + if (hasPos) { + oss << "\n" << indent << ANSI_BLUE << "at " ANSI_WARNING << *pos << ANSI_NORMAL << ":"; + + if (auto loc = pos->getCodeLines()) { + oss << "\n"; + printCodeLines(oss, "", *pos, *loc); + oss << "\n"; + } + } else if (printUnknownLocations) { + oss << "\n" << indent << ANSI_BLUE << "at " ANSI_RED << "UNKNOWN LOCATION" << ANSI_NORMAL << "\n"; + } + return hasPos; +} + std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool showTrace) { std::string prefix; @@ -198,8 +234,6 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s std::ostringstream oss; - auto noSource = ANSI_ITALIC " (source not available)" ANSI_NORMAL "\n"; - /* * Traces * ------ @@ -315,34 +349,15 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s oss << "\n" << "… " << trace.hint.str() << "\n"; - if (trace.pos) { + if (printPosMaybe(oss, ellipsisIndent, trace.pos)) count++; - - oss << "\n" << ellipsisIndent << ANSI_BLUE << "at " ANSI_WARNING << *trace.pos << ANSI_NORMAL << ":"; - - if (auto loc = trace.pos->getCodeLines()) { - oss << "\n"; - printCodeLines(oss, "", *trace.pos, *loc); - oss << "\n"; - } else - oss << noSource; - } } oss << "\n" << prefix; } oss << einfo.msg << "\n"; - if (einfo.errPos) { - oss << "\n" << ANSI_BLUE << "at " ANSI_WARNING << *einfo.errPos << ANSI_NORMAL << ":"; - - if (auto loc = einfo.errPos->getCodeLines()) { - oss << "\n"; - printCodeLines(oss, "", *einfo.errPos, *loc); - oss << "\n"; - } else - oss << noSource; - } + printPosMaybe(oss, "", einfo.errPos); auto suggestions = einfo.suggestions.trim(); if (!suggestions.suggestions.empty()) { diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 6a09230812ec..c04dcbd77b2f 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -70,6 +70,13 @@ struct AbstractPos uint32_t line = 0; uint32_t column = 0; + /** + * An AbstractPos may be a "null object", representing an unknown position. + * + * Return true if this position is known. + */ + inline operator bool() const { return line != 0; }; + /** * Return the contents of the source file. */ @@ -214,4 +221,8 @@ public: } }; +/** Throw an exception for the purpose of checking that exception handling works; see 'initLibUtil()'. + */ +void throwExceptionSelfCheck(); + } diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 03d8f1ef011f..913885a7fbd4 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -12,7 +12,26 @@ struct ExperimentalFeatureDetails std::string_view description; }; -constexpr std::array xpFeatureDetails = {{ +/** + * If two different PRs both add an experimental feature, and we just + * used a number for this, we *woudln't* get merge conflict and the + * counter will be incremented once instead of twice, causing a build + * failure. + * + * By instead defining this instead as 1 + the bottom experimental + * feature, we either have no issue at all if few features are not added + * at the end of the list, or a proper merge conflict if they are. + */ +constexpr size_t numXpFeatures = 1 + static_cast(Xp::VerifiedFetches); + +constexpr std::array xpFeatureDetails = {{ + { + .tag = Xp::ACLs, + .name = "acls", + .description = R"( + Allow protection of store paths with the use of [POSIX ACLs](https://man7.org/linux/man-pages/man5/acl.5.html). + )", + }, { .tag = Xp::CaDerivations, .name = "ca-derivations", @@ -62,6 +81,20 @@ constexpr std::array xpFeatureDetails = {{ flake`](@docroot@/command-ref/new-cli/nix3-flake.md) for details. )", }, + { + .tag = Xp::FetchTree, + .name = "fetch-tree", + .description = R"( + Enable the use of the [`fetchTree`](@docroot@/language/builtins.md#builtins-fetchTree) built-in function in the Nix language. + + `fetchTree` exposes a large suite of fetching functionality in a more systematic way. + The [`flakes`](#xp-feature-flakes) feature flag always enables `fetch-tree`. + + This built-in was previously guarded by the `flakes` experimental feature because of that overlap, + but since the plan is to work on stabilizing this first (due 2024 Q1), we are putting it underneath a separate feature. + Once we've made the changes we want to make, enabling just this feature will serve as a "release candidate" --- allowing users to try out the functionality we want to stabilize and not any other functionality we don't yet want to, in isolation. + )", + }, { .tag = Xp::NixCommand, .name = "nix-command", @@ -70,6 +103,14 @@ constexpr std::array xpFeatureDetails = {{ [`nix`](@docroot@/command-ref/new-cli/nix.md) for details. )", }, + { + .tag = Xp::GitHashing, + .name = "git-hashing", + .description = R"( + Allow creating (content-addressed) store objects which are hashed via Git's hashing algorithm. + These store objects will not be understandable by older versions of Nix. + )", + }, { .tag = Xp::RecursiveNix, .name = "recursive-nix", @@ -163,6 +204,8 @@ constexpr std::array xpFeatureDetails = {{ .tag = Xp::ReplFlake, .name = "repl-flake", .description = R"( + *Enabled with [`flakes`](#xp-feature-flakes) since 2.19* + Allow passing [installables](@docroot@/command-ref/new-cli/nix.md#installables) to `nix repl`, making its interface consistent with the other experimental commands. )", }, @@ -171,7 +214,7 @@ constexpr std::array xpFeatureDetails = {{ .name = "auto-allocate-uids", .description = R"( Allows Nix to automatically pick UIDs for builds, rather than creating - `nixbld*` user accounts. See the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting for details. + `nixbld*` user accounts. See the [`auto-allocate-uids`](@docroot@/command-ref/conf-file.md#conf-auto-allocate-uids) setting for details. )", }, { @@ -179,16 +222,7 @@ constexpr std::array xpFeatureDetails = {{ .name = "cgroups", .description = R"( Allows Nix to execute builds inside cgroups. See - the [`use-cgroups`](#conf-use-cgroups) setting for details. - )", - }, - { - .tag = Xp::DiscardReferences, - .name = "discard-references", - .description = R"( - Allow the use of the [`unsafeDiscardReferences`](@docroot@/language/advanced-attributes.html#adv-attr-unsafeDiscardReferences) attribute in derivations - that use [structured attributes](@docroot@/language/advanced-attributes.html#adv-attr-structuredAttrs). This disables scanning of outputs for - runtime dependencies. + the [`use-cgroups`](@docroot@/command-ref/conf-file.md#conf-use-cgroups) setting for details. )", }, { @@ -225,14 +259,28 @@ constexpr std::array xpFeatureDetails = {{ .tag = Xp::ReadOnlyLocalStore, .name = "read-only-local-store", .description = R"( - Allow the use of the `read-only` parameter in [local store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) URIs. + Allow the use of the `read-only` parameter in [local store](@docroot@/store/types/local-store.md) URIs. )", }, { - .tag = Xp::ACLs, - .name = "acls", + .tag = Xp::ConfigurableImpureEnv, + .name = "configurable-impure-env", .description = R"( - Allow protection of store paths with the use of [POSIX ACLs](https://man7.org/linux/man-pages/man5/acl.5.html). + Allow the use of the [impure-env](@docroot@/command-ref/conf-file.md#conf-impure-env) setting. + )", + }, + { + .tag = Xp::MountedSSHStore, + .name = "mounted-ssh-store", + .description = R"( + Allow the use of the [`mounted SSH store`](@docroot@/command-ref/new-cli/nix3-help-stores.html#experimental-ssh-store-with-filesytem-mounted). + )", + }, + { + .tag = Xp::VerifiedFetches, + .name = "verified-fetches", + .description = R"( + Enables verification of git commit signatures through the [`fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) built-in. )", }, }}; @@ -288,7 +336,7 @@ std::set parseFeatures(const std::set & rawFea } MissingExperimentalFeature::MissingExperimentalFeature(ExperimentalFeature feature) - : Error("experimental Nix feature '%1%' is disabled; use '--extra-experimental-features %1%' to override", showExperimentalFeature(feature)) + : Error("experimental Nix feature '%1%' is disabled; add '--extra-experimental-features %1%' to enable it", showExperimentalFeature(feature)) , missingFeature(feature) {} diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 8ba09ab321a1..7f5d156121b8 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -17,22 +17,26 @@ namespace nix { */ enum struct ExperimentalFeature { + ACLs, CaDerivations, ImpureDerivations, Flakes, + FetchTree, NixCommand, + GitHashing, RecursiveNix, NoUrlLiterals, FetchClosure, ReplFlake, AutoAllocateUids, Cgroups, - DiscardReferences, DaemonTrustOverride, DynamicDerivations, ParseTomlTimestamps, ReadOnlyLocalStore, - ACLs, + ConfigurableImpureEnv, + MountedSSHStore, + VerifiedFetches, }; /** diff --git a/src/libutil/file-descriptor.cc b/src/libutil/file-descriptor.cc new file mode 100644 index 000000000000..38dd70c8e4c3 --- /dev/null +++ b/src/libutil/file-descriptor.cc @@ -0,0 +1,254 @@ +#include "file-system.hh" +#include "signals.hh" +#include "finally.hh" +#include "serialise.hh" + +#include +#include + +namespace nix { + +std::string readFile(int fd) +{ + struct stat st; + if (fstat(fd, &st) == -1) + throw SysError("statting file"); + + return drainFD(fd, true, st.st_size); +} + + +void readFull(int fd, char * buf, size_t count) +{ + while (count) { + checkInterrupt(); + ssize_t res = read(fd, buf, count); + if (res == -1) { + if (errno == EINTR) continue; + throw SysError("reading from file"); + } + if (res == 0) throw EndOfFile("unexpected end-of-file"); + count -= res; + buf += res; + } +} + + +void writeFull(int fd, std::string_view s, bool allowInterrupts) +{ + while (!s.empty()) { + if (allowInterrupts) checkInterrupt(); + ssize_t res = write(fd, s.data(), s.size()); + if (res == -1 && errno != EINTR) + throw SysError("writing to file"); + if (res > 0) + s.remove_prefix(res); + } +} + + +std::string readLine(int fd) +{ + std::string s; + while (1) { + checkInterrupt(); + char ch; + // FIXME: inefficient + ssize_t rd = read(fd, &ch, 1); + if (rd == -1) { + if (errno != EINTR) + throw SysError("reading a line"); + } else if (rd == 0) + throw EndOfFile("unexpected EOF reading a line"); + else { + if (ch == '\n') return s; + s += ch; + } + } +} + + +void writeLine(int fd, std::string s) +{ + s += '\n'; + writeFull(fd, s); +} + + +std::string drainFD(int fd, bool block, const size_t reserveSize) +{ + // the parser needs two extra bytes to append terminating characters, other users will + // not care very much about the extra memory. + StringSink sink(reserveSize + 2); + drainFD(fd, sink, block); + return std::move(sink.s); +} + + +void drainFD(int fd, Sink & sink, bool block) +{ + // silence GCC maybe-uninitialized warning in finally + int saved = 0; + + if (!block) { + saved = fcntl(fd, F_GETFL); + if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) + throw SysError("making file descriptor non-blocking"); + } + + Finally finally([&]() { + if (!block) { + if (fcntl(fd, F_SETFL, saved) == -1) + throw SysError("making file descriptor blocking"); + } + }); + + std::vector buf(64 * 1024); + while (1) { + checkInterrupt(); + ssize_t rd = read(fd, buf.data(), buf.size()); + if (rd == -1) { + if (!block && (errno == EAGAIN || errno == EWOULDBLOCK)) + break; + if (errno != EINTR) + throw SysError("reading from file"); + } + else if (rd == 0) break; + else sink({(char *) buf.data(), (size_t) rd}); + } +} + +////////////////////////////////////////////////////////////////////// + +AutoCloseFD::AutoCloseFD() : fd{-1} {} + + +AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {} + + +AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd} +{ + that.fd = -1; +} + + +AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that) +{ + close(); + fd = that.fd; + that.fd = -1; + return *this; +} + + +AutoCloseFD::~AutoCloseFD() +{ + try { + close(); + } catch (...) { + ignoreException(); + } +} + + +int AutoCloseFD::get() const +{ + return fd; +} + + +void AutoCloseFD::close() +{ + if (fd != -1) { + if (::close(fd) == -1) + /* This should never happen. */ + throw SysError("closing file descriptor %1%", fd); + fd = -1; + } +} + +void AutoCloseFD::fsync() +{ + if (fd != -1) { + int result; +#if __APPLE__ + result = ::fcntl(fd, F_FULLFSYNC); +#else + result = ::fsync(fd); +#endif + if (result == -1) + throw SysError("fsync file descriptor %1%", fd); + } +} + + +AutoCloseFD::operator bool() const +{ + return fd != -1; +} + + +int AutoCloseFD::release() +{ + int oldFD = fd; + fd = -1; + return oldFD; +} + + +void Pipe::create() +{ + int fds[2]; +#if HAVE_PIPE2 + if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe"); +#else + if (pipe(fds) != 0) throw SysError("creating pipe"); + closeOnExec(fds[0]); + closeOnExec(fds[1]); +#endif + readSide = fds[0]; + writeSide = fds[1]; +} + + +void Pipe::close() +{ + readSide.close(); + writeSide.close(); +} + +////////////////////////////////////////////////////////////////////// + +void closeMostFDs(const std::set & exceptions) +{ +#if __linux__ + try { + for (auto & s : readDirectory("/proc/self/fd")) { + auto fd = std::stoi(s.name); + if (!exceptions.count(fd)) { + debug("closing leaked FD %d", fd); + close(fd); + } + } + return; + } catch (SysError &) { + } +#endif + + int maxFD = 0; + maxFD = sysconf(_SC_OPEN_MAX); + for (int fd = 0; fd < maxFD; ++fd) + if (!exceptions.count(fd)) + close(fd); /* ignore result */ +} + + +void closeOnExec(int fd) +{ + int prev; + if ((prev = fcntl(fd, F_GETFD, 0)) == -1 || + fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1) + throw SysError("setting close-on-exec flag"); +} + +} diff --git a/src/libutil/file-descriptor.hh b/src/libutil/file-descriptor.hh new file mode 100644 index 000000000000..80ec86135920 --- /dev/null +++ b/src/libutil/file-descriptor.hh @@ -0,0 +1,84 @@ +#pragma once +///@file + +#include "types.hh" +#include "error.hh" + +namespace nix { + +struct Sink; +struct Source; + +/** + * Read the contents of a resource into a string. + */ +std::string readFile(int fd); + +/** + * Wrappers arount read()/write() that read/write exactly the + * requested number of bytes. + */ +void readFull(int fd, char * buf, size_t count); + +void writeFull(int fd, std::string_view s, bool allowInterrupts = true); + +/** + * Read a line from a file descriptor. + */ +std::string readLine(int fd); + +/** + * Write a line to a file descriptor. + */ +void writeLine(int fd, std::string s); + +/** + * Read a file descriptor until EOF occurs. + */ +std::string drainFD(int fd, bool block = true, const size_t reserveSize=0); + +void drainFD(int fd, Sink & sink, bool block = true); + +/** + * Automatic cleanup of resources. + */ +class AutoCloseFD +{ + int fd; +public: + AutoCloseFD(); + AutoCloseFD(int fd); + AutoCloseFD(const AutoCloseFD & fd) = delete; + AutoCloseFD(AutoCloseFD&& fd); + ~AutoCloseFD(); + AutoCloseFD& operator =(const AutoCloseFD & fd) = delete; + AutoCloseFD& operator =(AutoCloseFD&& fd); + int get() const; + explicit operator bool() const; + int release(); + void close(); + void fsync(); +}; + +class Pipe +{ +public: + AutoCloseFD readSide, writeSide; + void create(); + void close(); +}; + +/** + * Close all file descriptors except those listed in the given set. + * Good practice in child processes. + */ +void closeMostFDs(const std::set & exceptions); + +/** + * Set the close-on-exec flag for the given file descriptor. + */ +void closeOnExec(int fd); + +MakeError(EndOfFile, Error); + +} diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc new file mode 100644 index 000000000000..c96effff9199 --- /dev/null +++ b/src/libutil/file-system.cc @@ -0,0 +1,647 @@ +#include "environment-variables.hh" +#include "file-system.hh" +#include "signals.hh" +#include "finally.hh" +#include "serialise.hh" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace nix { + +Path absPath(Path path, std::optional dir, bool resolveSymlinks) +{ + if (path[0] != '/') { + if (!dir) { +#ifdef __GNU__ + /* GNU (aka. GNU/Hurd) doesn't have any limitation on path + lengths and doesn't define `PATH_MAX'. */ + char *buf = getcwd(NULL, 0); + if (buf == NULL) +#else + char buf[PATH_MAX]; + if (!getcwd(buf, sizeof(buf))) +#endif + throw SysError("cannot get cwd"); + path = concatStrings(buf, "/", path); +#ifdef __GNU__ + free(buf); +#endif + } else + path = concatStrings(*dir, "/", path); + } + return canonPath(path, resolveSymlinks); +} + + +Path canonPath(PathView path, bool resolveSymlinks) +{ + assert(path != ""); + + std::string s; + s.reserve(256); + + if (path[0] != '/') + throw Error("not an absolute path: '%1%'", path); + + std::string temp; + + /* Count the number of times we follow a symlink and stop at some + arbitrary (but high) limit to prevent infinite loops. */ + unsigned int followCount = 0, maxFollow = 1024; + + while (1) { + + /* Skip slashes. */ + while (!path.empty() && path[0] == '/') path.remove_prefix(1); + if (path.empty()) break; + + /* Ignore `.'. */ + if (path == "." || path.substr(0, 2) == "./") + path.remove_prefix(1); + + /* If `..', delete the last component. */ + else if (path == ".." || path.substr(0, 3) == "../") + { + if (!s.empty()) s.erase(s.rfind('/')); + path.remove_prefix(2); + } + + /* Normal component; copy it. */ + else { + s += '/'; + if (const auto slash = path.find('/'); slash == std::string::npos) { + s += path; + path = {}; + } else { + s += path.substr(0, slash); + path = path.substr(slash); + } + + /* If s points to a symlink, resolve it and continue from there */ + if (resolveSymlinks && isLink(s)) { + if (++followCount >= maxFollow) + throw Error("infinite symlink recursion in path '%1%'", path); + temp = concatStrings(readLink(s), path); + path = temp; + if (!temp.empty() && temp[0] == '/') { + s.clear(); /* restart for symlinks pointing to absolute path */ + } else { + s = dirOf(s); + if (s == "/") { // we don’t want trailing slashes here, which dirOf only produces if s = / + s.clear(); + } + } + } + } + } + + return s.empty() ? "/" : std::move(s); +} + + +Path dirOf(const PathView path) +{ + Path::size_type pos = path.rfind('/'); + if (pos == std::string::npos) + return "."; + return pos == 0 ? "/" : Path(path, 0, pos); +} + + +std::string_view baseNameOf(std::string_view path) +{ + if (path.empty()) + return ""; + + auto last = path.size() - 1; + if (path[last] == '/' && last > 0) + last -= 1; + + auto pos = path.rfind('/', last); + if (pos == std::string::npos) + pos = 0; + else + pos += 1; + + return path.substr(pos, last - pos + 1); +} + + +bool isInDir(std::string_view path, std::string_view dir) +{ + return path.substr(0, 1) == "/" + && path.substr(0, dir.size()) == dir + && path.size() >= dir.size() + 2 + && path[dir.size()] == '/'; +} + + +bool isDirOrInDir(std::string_view path, std::string_view dir) +{ + return path == dir || isInDir(path, dir); +} + + +struct stat stat(const Path & path) +{ + struct stat st; + if (stat(path.c_str(), &st)) + throw SysError("getting status of '%1%'", path); + return st; +} + + +struct stat lstat(const Path & path) +{ + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError("getting status of '%1%'", path); + return st; +} + + +bool pathExists(const Path & path) +{ + int res; + struct stat st; + res = lstat(path.c_str(), &st); + if (!res) return true; + if (errno != ENOENT && errno != ENOTDIR) + throw SysError("getting status of %1%", path); + return false; +} + +bool pathAccessible(const Path & path) +{ + try { + return pathExists(path); + } catch (SysError & e) { + // swallow EPERM + if (e.errNo == EPERM) return false; + throw; + } +} + + +Path readLink(const Path & path) +{ + checkInterrupt(); + std::vector buf; + for (ssize_t bufSize = PATH_MAX/4; true; bufSize += bufSize/2) { + buf.resize(bufSize); + ssize_t rlSize = readlink(path.c_str(), buf.data(), bufSize); + if (rlSize == -1) + if (errno == EINVAL) + throw Error("'%1%' is not a symlink", path); + else + throw SysError("reading symbolic link '%1%'", path); + else if (rlSize < bufSize) + return std::string(buf.data(), rlSize); + } +} + + +bool isLink(const Path & path) +{ + struct stat st = lstat(path); + return S_ISLNK(st.st_mode); +} + + +DirEntries readDirectory(DIR *dir, const Path & path) +{ + DirEntries entries; + entries.reserve(64); + + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir)) { /* sic */ + checkInterrupt(); + std::string name = dirent->d_name; + if (name == "." || name == "..") continue; + entries.emplace_back(name, dirent->d_ino, +#ifdef HAVE_STRUCT_DIRENT_D_TYPE + dirent->d_type +#else + DT_UNKNOWN +#endif + ); + } + if (errno) throw SysError("reading directory '%1%'", path); + + return entries; +} + +DirEntries readDirectory(const Path & path) +{ + AutoCloseDir dir(opendir(path.c_str())); + if (!dir) throw SysError("opening directory '%1%'", path); + + return readDirectory(dir.get(), path); +} + + +unsigned char getFileType(const Path & path) +{ + struct stat st = lstat(path); + if (S_ISDIR(st.st_mode)) return DT_DIR; + if (S_ISLNK(st.st_mode)) return DT_LNK; + if (S_ISREG(st.st_mode)) return DT_REG; + return DT_UNKNOWN; +} + + +std::string readFile(const Path & path) +{ + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) + throw SysError("opening file '%1%'", path); + return readFile(fd.get()); +} + + +void readFile(const Path & path, Sink & sink) +{ + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) + throw SysError("opening file '%s'", path); + drainFD(fd.get(), sink); +} + + +void writeFile(const Path & path, std::string_view s, mode_t mode, bool sync) +{ + AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); + if (!fd) + throw SysError("opening file '%1%'", path); + try { + writeFull(fd.get(), s); + } catch (Error & e) { + e.addTrace({}, "writing file '%1%'", path); + throw; + } + if (sync) + fd.fsync(); + // Explicitly close to make sure exceptions are propagated. + fd.close(); + if (sync) + syncParent(path); +} + + +void writeFile(const Path & path, Source & source, mode_t mode, bool sync) +{ + AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); + if (!fd) + throw SysError("opening file '%1%'", path); + + std::vector buf(64 * 1024); + + try { + while (true) { + try { + auto n = source.read(buf.data(), buf.size()); + writeFull(fd.get(), {buf.data(), n}); + } catch (EndOfFile &) { break; } + } + } catch (Error & e) { + e.addTrace({}, "writing file '%1%'", path); + throw; + } + if (sync) + fd.fsync(); + // Explicitly close to make sure exceptions are propagated. + fd.close(); + if (sync) + syncParent(path); +} + +void syncParent(const Path & path) +{ + AutoCloseFD fd = open(dirOf(path).c_str(), O_RDONLY, 0); + if (!fd) + throw SysError("opening file '%1%'", path); + fd.fsync(); +} + + +static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed) +{ + checkInterrupt(); + + std::string name(baseNameOf(path)); + + struct stat st; + if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) { + if (errno == ENOENT) return; + throw SysError("getting status of '%1%'", path); + } + + if (!S_ISDIR(st.st_mode)) { + /* We are about to delete a file. Will it likely free space? */ + + switch (st.st_nlink) { + /* Yes: last link. */ + case 1: + bytesFreed += st.st_size; + break; + /* Maybe: yes, if 'auto-optimise-store' or manual optimisation + was performed. Instead of checking for real let's assume + it's an optimised file and space will be freed. + + In worst case we will double count on freed space for files + with exactly two hardlinks for unoptimised packages. + */ + case 2: + bytesFreed += st.st_size; + break; + /* No: 3+ links. */ + default: + break; + } + } + + if (S_ISDIR(st.st_mode)) { + /* Make the directory accessible. */ + const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR; + if ((st.st_mode & PERM_MASK) != PERM_MASK) { + if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1) + throw SysError("chmod '%1%'", path); + } + + int fd = openat(parentfd, path.c_str(), O_RDONLY); + if (fd == -1) + throw SysError("opening directory '%1%'", path); + AutoCloseDir dir(fdopendir(fd)); + if (!dir) + throw SysError("opening directory '%1%'", path); + for (auto & i : readDirectory(dir.get(), path)) + _deletePath(dirfd(dir.get()), path + "/" + i.name, bytesFreed); + } + + int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0; + if (unlinkat(parentfd, name.c_str(), flags) == -1) { + if (errno == ENOENT) return; + throw SysError("cannot unlink '%1%'", path); + } +} + +static void _deletePath(const Path & path, uint64_t & bytesFreed) +{ + Path dir = dirOf(path); + if (dir == "") + dir = "/"; + + AutoCloseFD dirfd{open(dir.c_str(), O_RDONLY)}; + if (!dirfd) { + if (errno == ENOENT) return; + throw SysError("opening directory '%1%'", path); + } + + _deletePath(dirfd.get(), path, bytesFreed); +} + + +void deletePath(const Path & path) +{ + uint64_t dummy; + deletePath(path, dummy); +} + + +Paths createDirs(const Path & path) +{ + Paths created; + if (path == "/") return created; + + struct stat st; + if (lstat(path.c_str(), &st) == -1) { + created = createDirs(dirOf(path)); + if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST) + throw SysError("creating directory '%1%'", path); + st = lstat(path); + created.push_back(path); + } + + if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1) + throw SysError("statting symlink '%1%'", path); + + if (!S_ISDIR(st.st_mode)) throw Error("'%1%' is not a directory", path); + + return created; +} + + +void deletePath(const Path & path, uint64_t & bytesFreed) +{ + //Activity act(*logger, lvlDebug, "recursively deleting path '%1%'", path); + bytesFreed = 0; + _deletePath(path, bytesFreed); +} + + +////////////////////////////////////////////////////////////////////// + +AutoDelete::AutoDelete() : del{false} {} + +AutoDelete::AutoDelete(const std::string & p, bool recursive) : path(p) +{ + del = true; + this->recursive = recursive; +} + +AutoDelete::~AutoDelete() +{ + try { + if (del) { + if (recursive) + deletePath(path); + else { + if (remove(path.c_str()) == -1) + throw SysError("cannot unlink '%1%'", path); + } + } + } catch (...) { + ignoreException(); + } +} + +void AutoDelete::cancel() +{ + del = false; +} + +void AutoDelete::reset(const Path & p, bool recursive) { + path = p; + this->recursive = recursive; + del = true; +} + +////////////////////////////////////////////////////////////////////// + +////////////////////////////////////////////////////////////////////// + +static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, + std::atomic & counter) +{ + tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); + if (includePid) + return fmt("%1%/%2%-%3%-%4%", tmpRoot, prefix, getpid(), counter++); + else + return fmt("%1%/%2%-%3%", tmpRoot, prefix, counter++); +} + +Path createTempDir(const Path & tmpRoot, const Path & prefix, + bool includePid, bool useGlobalCounter, mode_t mode) +{ + static std::atomic globalCounter = 0; + std::atomic localCounter = 0; + auto & counter(useGlobalCounter ? globalCounter : localCounter); + + while (1) { + checkInterrupt(); + Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); + if (mkdir(tmpDir.c_str(), mode) == 0) { +#if __FreeBSD__ + /* Explicitly set the group of the directory. This is to + work around around problems caused by BSD's group + ownership semantics (directories inherit the group of + the parent). For instance, the group of /tmp on + FreeBSD is "wheel", so all directories created in /tmp + will be owned by "wheel"; but if the user is not in + "wheel", then "tar" will fail to unpack archives that + have the setgid bit set on directories. */ + if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) + throw SysError("setting group of directory '%1%'", tmpDir); +#endif + return tmpDir; + } + if (errno != EEXIST) + throw SysError("creating directory '%1%'", tmpDir); + } +} + + +std::pair createTempFile(const Path & prefix) +{ + Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); + // Strictly speaking, this is UB, but who cares... + // FIXME: use O_TMPFILE. + AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); + if (!fd) + throw SysError("creating temporary file '%s'", tmpl); + closeOnExec(fd.get()); + return {std::move(fd), tmpl}; +} + +void createSymlink(const Path & target, const Path & link) +{ + if (symlink(target.c_str(), link.c_str())) + throw SysError("creating symlink from '%1%' to '%2%'", link, target); +} + +void replaceSymlink(const Path & target, const Path & link) +{ + for (unsigned int n = 0; true; n++) { + Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); + + try { + createSymlink(target, tmp); + } catch (SysError & e) { + if (e.errNo == EEXIST) continue; + throw; + } + + renameFile(tmp, link); + + break; + } +} + +void setWriteTime(const fs::path & p, const struct stat & st) +{ + struct timeval times[2]; + times[0] = { + .tv_sec = st.st_atime, + .tv_usec = 0, + }; + times[1] = { + .tv_sec = st.st_mtime, + .tv_usec = 0, + }; + if (lutimes(p.c_str(), times) != 0) + throw SysError("changing modification time of '%s'", p); +} + +void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete) +{ + // TODO: Rewrite the `is_*` to use `symlink_status()` + auto statOfFrom = lstat(from.path().c_str()); + auto fromStatus = from.symlink_status(); + + // Mark the directory as writable so that we can delete its children + if (andDelete && fs::is_directory(fromStatus)) { + fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); + } + + + if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) { + fs::copy(from.path(), to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing); + } else if (fs::is_directory(fromStatus)) { + fs::create_directory(to); + for (auto & entry : fs::directory_iterator(from.path())) { + copy(entry, to / entry.path().filename(), andDelete); + } + } else { + throw Error("file '%s' has an unsupported type", from.path()); + } + + setWriteTime(to, statOfFrom); + if (andDelete) { + if (!fs::is_symlink(fromStatus)) + fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); + fs::remove(from.path()); + } +} + +void renameFile(const Path & oldName, const Path & newName) +{ + fs::rename(oldName, newName); +} + +void moveFile(const Path & oldName, const Path & newName) +{ + try { + renameFile(oldName, newName); + } catch (fs::filesystem_error & e) { + auto oldPath = fs::path(oldName); + auto newPath = fs::path(newName); + // For the move to be as atomic as possible, copy to a temporary + // directory + fs::path temp = createTempDir(newPath.parent_path(), "rename-tmp"); + Finally removeTemp = [&]() { fs::remove(temp); }; + auto tempCopyTarget = temp / "copy-target"; + if (e.code().value() == EXDEV) { + fs::remove(newPath); + warn("Can’t rename %s as %s, copying instead", oldName, newName); + copy(fs::directory_entry(oldPath), tempCopyTarget, true); + renameFile(tempCopyTarget, newPath); + } + } +} + +////////////////////////////////////////////////////////////////////// + +} diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh new file mode 100644 index 000000000000..4637507b35b8 --- /dev/null +++ b/src/libutil/file-system.hh @@ -0,0 +1,238 @@ +#pragma once +/** + * @file + * + * Utiltities for working with the file sytem and file paths. + */ + +#include "types.hh" +#include "error.hh" +#include "logging.hh" +#include "file-descriptor.hh" + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +#ifndef HAVE_STRUCT_DIRENT_D_TYPE +#define DT_UNKNOWN 0 +#define DT_REG 1 +#define DT_LNK 2 +#define DT_DIR 3 +#endif + +namespace nix { + +struct Sink; +struct Source; + +/** + * @return An absolutized path, resolving paths relative to the + * specified directory, or the current directory otherwise. The path + * is also canonicalised. + */ +Path absPath(Path path, + std::optional dir = {}, + bool resolveSymlinks = false); + +/** + * Canonicalise a path by removing all `.` or `..` components and + * double or trailing slashes. Optionally resolves all symlink + * components such that each component of the resulting path is *not* + * a symbolic link. + */ +Path canonPath(PathView path, bool resolveSymlinks = false); + +/** + * @return The directory part of the given canonical path, i.e., + * everything before the final `/`. If the path is the root or an + * immediate child thereof (e.g., `/foo`), this means `/` + * is returned. + */ +Path dirOf(const PathView path); + +/** + * @return the base name of the given canonical path, i.e., everything + * following the final `/` (trailing slashes are removed). + */ +std::string_view baseNameOf(std::string_view path); + +/** + * Check whether 'path' is a descendant of 'dir'. Both paths must be + * canonicalized. + */ +bool isInDir(std::string_view path, std::string_view dir); + +/** + * Check whether 'path' is equal to 'dir' or a descendant of + * 'dir'. Both paths must be canonicalized. + */ +bool isDirOrInDir(std::string_view path, std::string_view dir); + +/** + * Get status of `path`. + */ +struct stat stat(const Path & path); +struct stat lstat(const Path & path); + +/** + * @return true iff the given path exists. + */ +bool pathExists(const Path & path); + +/** + * A version of pathExists that returns false on a permission error. + * Useful for inferring default paths across directories that might not + * be readable. + * @return true iff the given path can be accessed and exists + */ +bool pathAccessible(const Path & path); + +/** + * Read the contents (target) of a symbolic link. The result is not + * in any way canonicalised. + */ +Path readLink(const Path & path); + +bool isLink(const Path & path); + +/** + * Read the contents of a directory. The entries `.` and `..` are + * removed. + */ +struct DirEntry +{ + std::string name; + ino_t ino; + /** + * one of DT_* + */ + unsigned char type; + DirEntry(std::string name, ino_t ino, unsigned char type) + : name(std::move(name)), ino(ino), type(type) { } +}; + +typedef std::vector DirEntries; + +DirEntries readDirectory(const Path & path); + +unsigned char getFileType(const Path & path); + +/** + * Read the contents of a file into a string. + */ +std::string readFile(const Path & path); +void readFile(const Path & path, Sink & sink); + +/** + * Write a string to a file. + */ +void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false); + +void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false); + +/** + * Flush a file's parent directory to disk + */ +void syncParent(const Path & path); + +/** + * Delete a path; i.e., in the case of a directory, it is deleted + * recursively. It's not an error if the path does not exist. The + * second variant returns the number of bytes and blocks freed. + */ +void deletePath(const Path & path); + +void deletePath(const Path & path, uint64_t & bytesFreed); + +/** + * Create a directory and all its parents, if necessary. Returns the + * list of created directories, in order of creation. + */ +Paths createDirs(const Path & path); +inline Paths createDirs(PathView path) +{ + return createDirs(Path(path)); +} + +/** + * Create a symlink. + */ +void createSymlink(const Path & target, const Path & link); + +/** + * Atomically create or replace a symlink. + */ +void replaceSymlink(const Path & target, const Path & link); + +void renameFile(const Path & src, const Path & dst); + +/** + * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst` + * are on a different filesystem. + * + * Beware that this might not be atomic because of the copy that happens behind + * the scenes + */ +void moveFile(const Path & src, const Path & dst); + + +/** + * Automatic cleanup of resources. + */ +class AutoDelete +{ + Path path; + bool del; + bool recursive; +public: + AutoDelete(); + AutoDelete(const Path & p, bool recursive = true); + ~AutoDelete(); + void cancel(); + void reset(const Path & p, bool recursive = true); + operator Path() const { return path; } + operator PathView() const { return path; } +}; + + +struct DIRDeleter +{ + void operator()(DIR * dir) const { + closedir(dir); + } +}; + +typedef std::unique_ptr AutoCloseDir; + + +/** + * Create a temporary directory. + */ +Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", + bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); + +/** + * Create a temporary file, returning a file handle and its path. + */ +std::pair createTempFile(const Path & prefix = "nix"); + + +/** + * Used in various places. + */ +typedef std::function PathFilter; + +extern PathFilter defaultPathFilter; + +} diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc deleted file mode 100644 index 11cc0c0e7c81..000000000000 --- a/src/libutil/filesystem.cc +++ /dev/null @@ -1,162 +0,0 @@ -#include -#include -#include - -#include "finally.hh" -#include "util.hh" -#include "types.hh" - -namespace fs = std::filesystem; - -namespace nix { - -static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, - std::atomic & counter) -{ - tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); - if (includePid) - return fmt("%1%/%2%-%3%-%4%", tmpRoot, prefix, getpid(), counter++); - else - return fmt("%1%/%2%-%3%", tmpRoot, prefix, counter++); -} - -Path createTempDir(const Path & tmpRoot, const Path & prefix, - bool includePid, bool useGlobalCounter, mode_t mode) -{ - static std::atomic globalCounter = 0; - std::atomic localCounter = 0; - auto & counter(useGlobalCounter ? globalCounter : localCounter); - - while (1) { - checkInterrupt(); - Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); - if (mkdir(tmpDir.c_str(), mode) == 0) { -#if __FreeBSD__ - /* Explicitly set the group of the directory. This is to - work around around problems caused by BSD's group - ownership semantics (directories inherit the group of - the parent). For instance, the group of /tmp on - FreeBSD is "wheel", so all directories created in /tmp - will be owned by "wheel"; but if the user is not in - "wheel", then "tar" will fail to unpack archives that - have the setgid bit set on directories. */ - if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) - throw SysError("setting group of directory '%1%'", tmpDir); -#endif - return tmpDir; - } - if (errno != EEXIST) - throw SysError("creating directory '%1%'", tmpDir); - } -} - - -std::pair createTempFile(const Path & prefix) -{ - Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); - // Strictly speaking, this is UB, but who cares... - // FIXME: use O_TMPFILE. - AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); - if (!fd) - throw SysError("creating temporary file '%s'", tmpl); - closeOnExec(fd.get()); - return {std::move(fd), tmpl}; -} - -void createSymlink(const Path & target, const Path & link) -{ - if (symlink(target.c_str(), link.c_str())) - throw SysError("creating symlink from '%1%' to '%2%'", link, target); -} - -void replaceSymlink(const Path & target, const Path & link) -{ - for (unsigned int n = 0; true; n++) { - Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); - - try { - createSymlink(target, tmp); - } catch (SysError & e) { - if (e.errNo == EEXIST) continue; - throw; - } - - renameFile(tmp, link); - - break; - } -} - -void setWriteTime(const fs::path & p, const struct stat & st) -{ - struct timeval times[2]; - times[0] = { - .tv_sec = st.st_atime, - .tv_usec = 0, - }; - times[1] = { - .tv_sec = st.st_mtime, - .tv_usec = 0, - }; - if (lutimes(p.c_str(), times) != 0) - throw SysError("changing modification time of '%s'", p); -} - -void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete) -{ - // TODO: Rewrite the `is_*` to use `symlink_status()` - auto statOfFrom = lstat(from.path().c_str()); - auto fromStatus = from.symlink_status(); - - // Mark the directory as writable so that we can delete its children - if (andDelete && fs::is_directory(fromStatus)) { - fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); - } - - - if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) { - fs::copy(from.path(), to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing); - } else if (fs::is_directory(fromStatus)) { - fs::create_directory(to); - for (auto & entry : fs::directory_iterator(from.path())) { - copy(entry, to / entry.path().filename(), andDelete); - } - } else { - throw Error("file '%s' has an unsupported type", from.path()); - } - - setWriteTime(to, statOfFrom); - if (andDelete) { - if (!fs::is_symlink(fromStatus)) - fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); - fs::remove(from.path()); - } -} - -void renameFile(const Path & oldName, const Path & newName) -{ - fs::rename(oldName, newName); -} - -void moveFile(const Path & oldName, const Path & newName) -{ - try { - renameFile(oldName, newName); - } catch (fs::filesystem_error & e) { - auto oldPath = fs::path(oldName); - auto newPath = fs::path(newName); - // For the move to be as atomic as possible, copy to a temporary - // directory - fs::path temp = createTempDir(newPath.parent_path(), "rename-tmp"); - Finally removeTemp = [&]() { fs::remove(temp); }; - auto tempCopyTarget = temp / "copy-target"; - if (e.code().value() == EXDEV) { - fs::remove(newPath); - warn("Can’t rename %s as %s, copying instead", oldName, newName); - copy(fs::directory_entry(oldPath), tempCopyTarget, true); - renameFile(tempCopyTarget, newPath); - } - } -} - -} diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index 727255b4581e..ac72e47fbb3e 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -44,6 +44,11 @@ inline std::string fmt(const std::string & s) return s; } +inline std::string fmt(std::string_view s) +{ + return std::string(s); +} + inline std::string fmt(const char * s) { return s; diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc new file mode 100644 index 000000000000..d1a77644852d --- /dev/null +++ b/src/libutil/fs-sink.cc @@ -0,0 +1,127 @@ +#include + +#include "config.hh" +#include "fs-sink.hh" + +namespace nix { + +void copyRecursive( + SourceAccessor & accessor, const CanonPath & from, + ParseSink & sink, const Path & to) +{ + auto stat = accessor.lstat(from); + + switch (stat.type) { + case SourceAccessor::tSymlink: + { + sink.createSymlink(to, accessor.readLink(from)); + } + + case SourceAccessor::tRegular: + { + sink.createRegularFile(to); + if (stat.isExecutable) + sink.isExecutable(); + LambdaSink sink2 { + [&](auto d) { + sink.receiveContents(d); + } + }; + accessor.readFile(from, sink2, [&](uint64_t size) { + sink.preallocateContents(size); + }); + break; + } + + case SourceAccessor::tDirectory: + { + sink.createDirectory(to); + for (auto & [name, _] : accessor.readDirectory(from)) { + copyRecursive( + accessor, from + name, + sink, to + "/" + name); + break; + } + } + + case SourceAccessor::tMisc: + throw Error("file '%1%' has an unsupported type", from); + + default: + abort(); + } +} + + +struct RestoreSinkSettings : Config +{ + Setting preallocateContents{this, false, "preallocate-contents", + "Whether to preallocate files when writing objects with known size."}; +}; + +static RestoreSinkSettings restoreSinkSettings; + +static GlobalConfig::Register r1(&restoreSinkSettings); + + +void RestoreSink::createDirectory(const Path & path) +{ + Path p = dstPath + path; + auto mode = (protect && (path == "" || path == "/")) ? 0770 : 0777; + if (mkdir(p.c_str(), mode) == -1) + throw SysError("creating directory '%1%'", p); +}; + +void RestoreSink::createRegularFile(const Path & path) +{ + Path p = dstPath + path; + auto mode = (protect && (path == "" || path == "/")) ? 0660 : 0666; + fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, mode); + if (!fd) throw SysError("creating file '%1%'", p); +} + +void RestoreSink::closeRegularFile() +{ + /* Call close explicitly to make sure the error is checked */ + fd.close(); +} + +void RestoreSink::isExecutable() +{ + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError("fstat"); + if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) + throw SysError("fchmod"); +} + +void RestoreSink::preallocateContents(uint64_t len) +{ + if (!restoreSinkSettings.preallocateContents) + return; + +#if HAVE_POSIX_FALLOCATE + if (len) { + errno = posix_fallocate(fd.get(), 0, len); + /* Note that EINVAL may indicate that the underlying + filesystem doesn't support preallocation (e.g. on + OpenSolaris). Since preallocation is just an + optimisation, ignore it. */ + if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) + throw SysError("preallocating file of %1% bytes", len); + } +#endif +} + +void RestoreSink::receiveContents(std::string_view data) +{ + writeFull(fd.get(), data); +} + +void RestoreSink::createSymlink(const Path & path, const std::string & target) +{ + Path p = dstPath + path; + nix::createSymlink(target, p); +} + +} diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh new file mode 100644 index 000000000000..9f337c84a5fc --- /dev/null +++ b/src/libutil/fs-sink.hh @@ -0,0 +1,107 @@ +#pragma once +///@file + +#include "types.hh" +#include "serialise.hh" +#include "source-accessor.hh" +#include "file-system.hh" + +namespace nix { + +/** + * \todo Fix this API, it sucks. + */ +struct ParseSink +{ + virtual void createDirectory(const Path & path) = 0; + + virtual void createRegularFile(const Path & path) = 0; + virtual void receiveContents(std::string_view data) = 0; + virtual void isExecutable() = 0; + virtual void closeRegularFile() = 0; + + virtual void createSymlink(const Path & path, const std::string & target) = 0; + + /** + * An optimization. By default, do nothing. + */ + virtual void preallocateContents(uint64_t size) { }; +}; + +/** + * Recusively copy file system objects from the source into the sink. + */ +void copyRecursive( + SourceAccessor & accessor, const CanonPath & sourcePath, + ParseSink & sink, const Path & destPath); + +/** + * Ignore everything and do nothing + */ +struct NullParseSink : ParseSink +{ + void createDirectory(const Path & path) override { } + void receiveContents(std::string_view data) override { } + void createSymlink(const Path & path, const std::string & target) override { } + void createRegularFile(const Path & path) override { } + void closeRegularFile() override { } + void isExecutable() override { } +}; + +/** + * Write files at the given path + */ +struct RestoreSink : ParseSink +{ + Path dstPath; + + bool protect; + + void createDirectory(const Path & path) override; + + void createRegularFile(const Path & path) override; + void receiveContents(std::string_view data) override; + void isExecutable() override; + void closeRegularFile() override; + + void createSymlink(const Path & path, const std::string & target) override; + + void preallocateContents(uint64_t size) override; + +private: + AutoCloseFD fd; +}; + +/** + * Restore a single file at the top level, passing along + * `receiveContents` to the underlying `Sink`. For anything but a single + * file, set `regular = true` so the caller can fail accordingly. + */ +struct RegularFileSink : ParseSink +{ + bool regular = true; + Sink & sink; + + RegularFileSink(Sink & sink) : sink(sink) { } + + void createDirectory(const Path & path) override + { + regular = false; + } + + void receiveContents(std::string_view data) override + { + sink(data); + } + + void createSymlink(const Path & path, const std::string & target) override + { + regular = false; + } + + void createRegularFile(const Path & path) override { } + void closeRegularFile() override { } + void isExecutable() override { } +}; + +} diff --git a/src/libutil/git.cc b/src/libutil/git.cc index f35c2fdb75cf..a4bd60096467 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -1,9 +1,263 @@ +#include +#include +#include +#include +#include +#include // for strcasecmp + +#include "signals.hh" +#include "config.hh" +#include "hash.hh" +#include "posix-source-accessor.hh" + #include "git.hh" +#include "serialise.hh" -#include +namespace nix::git { + +using namespace nix; +using namespace std::string_literals; + +std::optional decodeMode(RawMode m) { + switch (m) { + case (RawMode) Mode::Directory: + case (RawMode) Mode::Executable: + case (RawMode) Mode::Regular: + case (RawMode) Mode::Symlink: + return (Mode) m; + default: + return std::nullopt; + } +} + + +static std::string getStringUntil(Source & source, char byte) +{ + std::string s; + char n[1]; + source(std::string_view { n, 1 }); + while (*n != byte) { + s += *n; + source(std::string_view { n, 1 }); + } + return s; +} + + +static std::string getString(Source & source, int n) +{ + std::string v; + v.resize(n); + source(v); + return v; +} + + +void parse( + ParseSink & sink, + const Path & sinkPath, + Source & source, + std::function hook, + const ExperimentalFeatureSettings & xpSettings) +{ + xpSettings.require(Xp::GitHashing); + + auto type = getString(source, 5); + + if (type == "blob ") { + sink.createRegularFile(sinkPath); + + unsigned long long size = std::stoi(getStringUntil(source, 0)); + + sink.preallocateContents(size); + + unsigned long long left = size; + std::string buf; + buf.reserve(65536); + + while (left) { + checkInterrupt(); + buf.resize(std::min((unsigned long long)buf.capacity(), left)); + source(buf); + sink.receiveContents(buf); + left -= buf.size(); + } + } else if (type == "tree ") { + unsigned long long size = std::stoi(getStringUntil(source, 0)); + unsigned long long left = size; + + sink.createDirectory(sinkPath); + + while (left) { + std::string perms = getStringUntil(source, ' '); + left -= perms.size(); + left -= 1; + + RawMode rawMode = std::stoi(perms, 0, 8); + auto modeOpt = decodeMode(rawMode); + if (!modeOpt) + throw Error("Unknown Git permission: %o", perms); + auto mode = std::move(*modeOpt); + + std::string name = getStringUntil(source, '\0'); + left -= name.size(); + left -= 1; + + std::string hashs = getString(source, 20); + left -= 20; + + Hash hash(htSHA1); + std::copy(hashs.begin(), hashs.end(), hash.hash); + + hook(name, TreeEntry { + .mode = mode, + .hash = hash, + }); + + if (mode == Mode::Executable) + sink.isExecutable(); + } + } else throw Error("input doesn't look like a Git object"); +} + + +std::optional convertMode(SourceAccessor::Type type) +{ + switch (type) { + case SourceAccessor::tSymlink: return Mode::Symlink; + case SourceAccessor::tRegular: return Mode::Regular; + case SourceAccessor::tDirectory: return Mode::Directory; + case SourceAccessor::tMisc: return std::nullopt; + default: abort(); + } +} + + +void restore(ParseSink & sink, Source & source, std::function hook) +{ + parse(sink, "", source, [&](Path name, TreeEntry entry) { + auto [accessor, from] = hook(entry.hash); + auto stat = accessor->lstat(from); + auto gotOpt = convertMode(stat.type); + if (!gotOpt) + throw Error("file '%s' (git hash %s) has an unsupported type", + from, + entry.hash.to_string(HashFormat::Base16, false)); + auto & got = *gotOpt; + if (got != entry.mode) + throw Error("git mode of file '%s' (git hash %s) is %o but expected %o", + from, + entry.hash.to_string(HashFormat::Base16, false), + (RawMode) got, + (RawMode) entry.mode); + copyRecursive( + *accessor, from, + sink, name); + }); +} + + +void dumpBlobPrefix( + uint64_t size, Sink & sink, + const ExperimentalFeatureSettings & xpSettings) +{ + xpSettings.require(Xp::GitHashing); + auto s = fmt("blob %d\0"s, std::to_string(size)); + sink(s); +} + + +void dumpTree(const Tree & entries, Sink & sink, + const ExperimentalFeatureSettings & xpSettings) +{ + xpSettings.require(Xp::GitHashing); + + std::string v1; + + for (auto & [name, entry] : entries) { + auto name2 = name; + if (entry.mode == Mode::Directory) { + assert(name2.back() == '/'); + name2.pop_back(); + } + v1 += fmt("%o %s\0"s, static_cast(entry.mode), name2); + std::copy(entry.hash.hash, entry.hash.hash + entry.hash.hashSize, std::back_inserter(v1)); + } + + { + auto s = fmt("tree %d\0"s, v1.size()); + sink(s); + } + + sink(v1); +} + + +Mode dump( + SourceAccessor & accessor, const CanonPath & path, + Sink & sink, + std::function hook, + PathFilter & filter, + const ExperimentalFeatureSettings & xpSettings) +{ + auto st = accessor.lstat(path); + + switch (st.type) { + case SourceAccessor::tRegular: + { + accessor.readFile(path, sink, [&](uint64_t size) { + dumpBlobPrefix(size, sink, xpSettings); + }); + return st.isExecutable + ? Mode::Executable + : Mode::Regular; + } + + case SourceAccessor::tDirectory: + { + Tree entries; + for (auto & [name, _] : accessor.readDirectory(path)) { + auto child = path + name; + if (!filter(child.abs())) continue; + + auto entry = hook(child); + + auto name2 = name; + if (entry.mode == Mode::Directory) + name2 += "/"; + + entries.insert_or_assign(std::move(name2), std::move(entry)); + } + dumpTree(entries, sink, xpSettings); + return Mode::Directory; + } + + case SourceAccessor::tSymlink: + case SourceAccessor::tMisc: + default: + throw Error("file '%1%' has an unsupported type", path); + } +} + + +TreeEntry dumpHash( + HashType ht, + SourceAccessor & accessor, const CanonPath & path, PathFilter & filter) +{ + std::function hook; + hook = [&](const CanonPath & path) -> TreeEntry { + auto hashSink = HashSink(ht); + auto mode = dump(accessor, path, hashSink, hook, filter); + auto hash = hashSink.finish().first; + return { + .mode = mode, + .hash = hash, + }; + }; + + return hook(path); +} -namespace nix { -namespace git { std::optional parseLsRemoteLine(std::string_view line) { @@ -22,4 +276,3 @@ std::optional parseLsRemoteLine(std::string_view line) } } -} diff --git a/src/libutil/git.hh b/src/libutil/git.hh index bf2b9a2869ab..30346007280d 100644 --- a/src/libutil/git.hh +++ b/src/libutil/git.hh @@ -5,9 +5,127 @@ #include #include -namespace nix { +#include "types.hh" +#include "serialise.hh" +#include "hash.hh" +#include "source-accessor.hh" +#include "fs-sink.hh" -namespace git { +namespace nix::git { + +using RawMode = uint32_t; + +enum struct Mode : RawMode { + Directory = 0040000, + Executable = 0100755, + Regular = 0100644, + Symlink = 0120000, +}; + +std::optional decodeMode(RawMode m); + +/** + * An anonymous Git tree object entry (no name part). + */ +struct TreeEntry +{ + Mode mode; + Hash hash; + + GENERATE_CMP(TreeEntry, me->mode, me->hash); +}; + +/** + * A Git tree object, fully decoded and stored in memory. + * + * Directory names must end in a `/` for sake of sorting. See + * https://github.com/mirage/irmin/issues/352 + */ +using Tree = std::map; + +/** + * Callback for processing a child hash with `parse` + * + * The function should + * + * 1. Obtain the file system objects denoted by `gitHash` + * + * 2. Ensure they match `mode` + * + * 3. Feed them into the same sink `parse` was called with + * + * Implementations may seek to memoize resources (bandwidth, storage, + * etc.) for the same Git hash. + */ +using SinkHook = void(const Path & name, TreeEntry entry); + +void parse( + ParseSink & sink, const Path & sinkPath, + Source & source, + std::function hook, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + +/** + * Assists with writing a `SinkHook` step (2). + */ +std::optional convertMode(SourceAccessor::Type type); + +/** + * Simplified version of `SinkHook` for `restore`. + * + * Given a `Hash`, return a `SourceAccessor` and `CanonPath` pointing to + * the file system object with that path. + */ +using RestoreHook = std::pair(Hash); + +/** + * Wrapper around `parse` and `RestoreSink` + */ +void restore(ParseSink & sink, Source & source, std::function hook); + +/** + * Dumps a single file to a sink + * + * @param xpSettings for testing purposes + */ +void dumpBlobPrefix( + uint64_t size, Sink & sink, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + +/** + * Dumps a representation of a git tree to a sink + */ +void dumpTree( + const Tree & entries, Sink & sink, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + +/** + * Callback for processing a child with `dump` + * + * The function should return the Git hash and mode of the file at the + * given path in the accessor passed to `dump`. + * + * Note that if the child is a directory, its child in must also be so + * processed in order to compute this information. + */ +using DumpHook = TreeEntry(const CanonPath & path); + +Mode dump( + SourceAccessor & accessor, const CanonPath & path, + Sink & sink, + std::function hook, + PathFilter & filter = defaultPathFilter, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + +/** + * Recursively dumps path, hashing as we go. + * + * A smaller wrapper around `dump`. + */ +TreeEntry dumpHash( + HashType ht, + SourceAccessor & accessor, const CanonPath & path, + PathFilter & filter = defaultPathFilter); /** * A line from the output of `git ls-remote --symref`. @@ -16,15 +134,17 @@ namespace git { * * - Symbolic references of the form * - * ref: {target} {reference} - * - * where {target} is itself a reference and {reference} is optional + * ``` + * ref: {target} {reference} + * ``` + * where {target} is itself a reference and {reference} is optional * * - Object references of the form * - * {target} {reference} - * - * where {target} is a commit id and {reference} is mandatory + * ``` + * {target} {reference} + * ``` + * where {target} is a commit id and {reference} is mandatory */ struct LsRemoteRefLine { enum struct Kind { @@ -36,8 +156,9 @@ struct LsRemoteRefLine { std::optional reference; }; +/** + * Parse an `LsRemoteRefLine` + */ std::optional parseLsRemoteLine(std::string_view line); } - -} diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 2c36d9d9498e..144f7ae7ed4e 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -9,7 +9,6 @@ #include "hash.hh" #include "archive.hh" #include "split.hh" -#include "util.hh" #include #include @@ -111,26 +110,26 @@ static std::string printHash32(const Hash & hash) std::string printHash16or32(const Hash & hash) { assert(hash.type); - return hash.to_string(hash.type == htMD5 ? Base16 : Base32, false); + return hash.to_string(hash.type == htMD5 ? HashFormat::Base16 : HashFormat::Base32, false); } -std::string Hash::to_string(Base base, bool includeType) const +std::string Hash::to_string(HashFormat hashFormat, bool includeType) const { std::string s; - if (base == SRI || includeType) { + if (hashFormat == HashFormat::SRI || includeType) { s += printHashType(type); - s += base == SRI ? '-' : ':'; + s += hashFormat == HashFormat::SRI ? '-' : ':'; } - switch (base) { - case Base16: + switch (hashFormat) { + case HashFormat::Base16: s += printHash16(*this); break; - case Base32: + case HashFormat::Base32: s += printHash32(*this); break; - case Base64: - case SRI: + case HashFormat::Base64: + case HashFormat::SRI: s += base64Encode(std::string_view((const char *) hash, hashSize)); break; } @@ -267,7 +266,7 @@ Hash newHashAllowEmpty(std::string_view hashStr, std::optional ht) if (!ht) throw BadHash("empty hash requires explicit hash type"); Hash h(*ht); - warn("found empty hash, assuming '%s'", h.to_string(SRI, true)); + warn("found empty hash, assuming '%s'", h.to_string(HashFormat::SRI, true)); return h; } else return Hash::parseAny(hashStr, ht); @@ -386,13 +385,48 @@ Hash compressHash(const Hash & hash, unsigned int newSize) } +std::optional parseHashFormatOpt(std::string_view hashFormatName) +{ + if (hashFormatName == "base16") return HashFormat::Base16; + if (hashFormatName == "base32") return HashFormat::Base32; + if (hashFormatName == "base64") return HashFormat::Base64; + if (hashFormatName == "sri") return HashFormat::SRI; + return std::nullopt; +} + +HashFormat parseHashFormat(std::string_view hashFormatName) +{ + auto opt_f = parseHashFormatOpt(hashFormatName); + if (opt_f) + return *opt_f; + throw UsageError("unknown hash format '%1%', expect 'base16', 'base32', 'base64', or 'sri'", hashFormatName); +} + +std::string_view printHashFormat(HashFormat HashFormat) +{ + switch (HashFormat) { + case HashFormat::Base64: + return "base64"; + case HashFormat::Base32: + return "base32"; + case HashFormat::Base16: + return "base16"; + case HashFormat::SRI: + return "sri"; + default: + // illegal hash base enum value internally, as opposed to external input + // which should be validated with nice error message. + assert(false); + } +} + std::optional parseHashTypeOpt(std::string_view s) { if (s == "md5") return htMD5; - else if (s == "sha1") return htSHA1; - else if (s == "sha256") return htSHA256; - else if (s == "sha512") return htSHA512; - else return std::optional {}; + if (s == "sha1") return htSHA1; + if (s == "sha256") return htSHA256; + if (s == "sha512") return htSHA512; + return std::nullopt; } HashType parseHashType(std::string_view s) @@ -401,7 +435,7 @@ HashType parseHashType(std::string_view s) if (opt_h) return *opt_h; else - throw UsageError("unknown hash algorithm '%1%'", s); + throw UsageError("unknown hash algorithm '%1%', expect 'md5', 'sha1', 'sha256', or 'sha512'", s); } std::string_view printHashType(HashType ht) diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index ae3ee40f4e34..6ade6555c89a 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -3,6 +3,7 @@ #include "types.hh" #include "serialise.hh" +#include "file-system.hh" namespace nix { @@ -23,7 +24,21 @@ extern std::set hashTypes; extern const std::string base32Chars; -enum Base : int { Base64, Base32, Base16, SRI }; +/** + * @brief Enumeration representing the hash formats. + */ +enum struct HashFormat : int { + /// @brief Base 64 encoding. + /// @see [IETF RFC 4648, section 4](https://datatracker.ietf.org/doc/html/rfc4648#section-4). + Base64, + /// @brief Nix-specific base-32 encoding. @see base32Chars + Base32, + /// @brief Lowercase hexadecimal encoding. @see base16Chars + Base16, + /// @brief ":", format of the SRI integrity attribute. + /// @see W3C recommendation [Subresource Intergrity](https://www.w3.org/TR/SRI/). + SRI +}; struct Hash @@ -114,16 +129,16 @@ public: * or base-64. By default, this is prefixed by the hash type * (e.g. "sha256:"). */ - std::string to_string(Base base, bool includeType) const; + std::string to_string(HashFormat hashFormat, bool includeType) const; std::string gitRev() const { - return to_string(Base16, false); + return to_string(HashFormat::Base16, false); } std::string gitShortRev() const { - return std::string(to_string(Base16, false), 0, 7); + return std::string(to_string(HashFormat::Base16, false), 0, 7); } static Hash dummy; @@ -145,13 +160,17 @@ std::string printHash16or32(const Hash & hash); Hash hashString(HashType ht, std::string_view s); /** - * Compute the hash of the given file. + * Compute the hash of the given file, hashing its contents directly. + * + * (Metadata, such as the executable permission bit, is ignored.) */ Hash hashFile(HashType ht, const Path & path); /** - * Compute the hash of the given path. The hash is defined as - * (essentially) hashString(ht, dumpPath(path)). + * Compute the hash of the given path, serializing as a Nix Archive and + * then hashing that. + * + * The hash is defined as (essentially) hashString(ht, dumpPath(path)). */ typedef std::pair HashResult; HashResult hashPath(HashType ht, const Path & path, @@ -163,6 +182,21 @@ HashResult hashPath(HashType ht, const Path & path, */ Hash compressHash(const Hash & hash, unsigned int newSize); +/** + * Parse a string representing a hash format. + */ +HashFormat parseHashFormat(std::string_view hashFormatName); + +/** + * std::optional version of parseHashFormat that doesn't throw error. + */ +std::optional parseHashFormatOpt(std::string_view hashFormatName); + +/** + * The reverse of parseHashFormat. + */ +std::string_view printHashFormat(HashFormat hashFormat); + /** * Parse a string representing a hash type. */ diff --git a/src/libutil/json-utils.cc b/src/libutil/json-utils.cc index d7220e71d488..61cef743d00e 100644 --- a/src/libutil/json-utils.cc +++ b/src/libutil/json-utils.cc @@ -1,4 +1,5 @@ #include "json-utils.hh" +#include "error.hh" namespace nix { @@ -16,4 +17,27 @@ nlohmann::json * get(nlohmann::json & map, const std::string & key) return &*i; } +const nlohmann::json & valueAt( + const nlohmann::json & map, + const std::string & key) +{ + if (!map.contains(key)) + throw Error("Expected JSON object to contain key '%s' but it doesn't", key); + + return map[key]; +} + +const nlohmann::json & ensureType( + const nlohmann::json & value, + nlohmann::json::value_type expectedType + ) +{ + if (value.type() != expectedType) + throw Error( + "Expected JSON value to be of type '%s' but it is of type '%s'", + nlohmann::json(expectedType).type_name(), + value.type_name()); + + return value; +} } diff --git a/src/libutil/json-utils.hh b/src/libutil/json-utils.hh index 5e63c1af4b7b..06dd80cf7d02 100644 --- a/src/libutil/json-utils.hh +++ b/src/libutil/json-utils.hh @@ -10,6 +10,28 @@ const nlohmann::json * get(const nlohmann::json & map, const std::string & key); nlohmann::json * get(nlohmann::json & map, const std::string & key); +/** + * Get the value of a json object at a key safely, failing + * with a Nix Error if the key does not exist. + * + * Use instead of nlohmann::json::at() to avoid ugly exceptions. + * + * _Does not check whether `map` is an object_, use `ensureType` for that. + */ +const nlohmann::json & valueAt( + const nlohmann::json & map, + const std::string & key); + +/** + * Ensure the type of a json object is what you expect, failing + * with a Nix Error if it isn't. + * + * Use before type conversions and element access to avoid ugly exceptions. + */ +const nlohmann::json & ensureType( + const nlohmann::json & value, + nlohmann::json::value_type expectedType); + /** * For `adl_serializer>` below, we need to track what * types are not already using `null`. Only for them can we use `null` @@ -56,20 +78,29 @@ namespace nlohmann { */ template struct adl_serializer> { - static std::optional from_json(const json & json) { + /** + * @brief Convert a JSON type to an `optional` treating + * `null` as `std::nullopt`. + */ + static void from_json(const json & json, std::optional & t) { static_assert( nix::json_avoids_null::value, "null is already in use for underlying type's JSON"); - return json.is_null() + t = json.is_null() ? std::nullopt - : std::optional { adl_serializer::from_json(json) }; + : std::make_optional(json.template get()); } - static void to_json(json & json, std::optional t) { + + /** + * @brief Convert an optional type to a JSON type treating `std::nullopt` + * as `null`. + */ + static void to_json(json & json, const std::optional & t) { static_assert( nix::json_avoids_null::value, "null is already in use for underlying type's JSON"); if (t) - adl_serializer::to_json(json, *t); + json = *t; else json = nullptr; } diff --git a/src/libutil/local.mk b/src/libutil/local.mk index 69cfcdfe96b5..de44e59ea5e7 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -6,7 +6,7 @@ libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc) -libutil_CXXFLAGS := -g +libutil_CXXFLAGS += -I src/libutil libutil_LDFLAGS += -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context @@ -14,6 +14,9 @@ ifdef HOST_LINUX libutil_LDFLAGS += -lacl endif +$(foreach i, $(wildcard $(d)/args/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/args, 0644))) + ifeq ($(HAVE_LIBCPUID), 1) libutil_LDFLAGS += -lcpuid endif diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 5a2dd99afca7..60b0865bf2c1 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -1,4 +1,7 @@ #include "logging.hh" +#include "file-descriptor.hh" +#include "environment-variables.hh" +#include "terminal.hh" #include "util.hh" #include "config.hh" @@ -67,7 +70,7 @@ class SimpleLogger : public Logger case lvlWarn: c = '4'; break; case lvlNotice: case lvlInfo: c = '5'; break; case lvlTalkative: case lvlChatty: c = '6'; break; - case lvlDebug: case lvlVomit: c = '7'; + case lvlDebug: case lvlVomit: c = '7'; break; default: c = '7'; break; // should not happen, and missing enum case is reported by -Werror=switch-enum } prefix = std::string("<") + c + ">"; @@ -220,8 +223,8 @@ struct JSONLogger : Logger { json["level"] = lvl; json["type"] = type; json["text"] = s; + json["parent"] = parent; addFields(json, fields); - // FIXME: handle parent write(json); } diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 5aa6bee956ac..183f2d8e13e9 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -23,6 +23,7 @@ typedef enum { actQueryPathInfo = 109, actPostBuildHook = 110, actBuildWaiting = 111, + actFetchTree = 112, } ActivityType; typedef enum { @@ -34,6 +35,7 @@ typedef enum { resProgress = 105, resSetExpected = 106, resPostBuildLogLine = 107, + resFetchStatus = 108, } ResultType; typedef uint64_t ActivityId; diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc new file mode 100644 index 000000000000..78a4dd29815f --- /dev/null +++ b/src/libutil/memory-source-accessor.cc @@ -0,0 +1,180 @@ +#include "memory-source-accessor.hh" + +namespace nix { + +MemorySourceAccessor::File * +MemorySourceAccessor::open(const CanonPath & path, std::optional create) +{ + File * cur = &root; + + bool newF = false; + + for (std::string_view name : path) + { + auto * curDirP = std::get_if(&cur->raw); + if (!curDirP) + return nullptr; + auto & curDir = *curDirP; + + auto i = curDir.contents.find(name); + if (i == curDir.contents.end()) { + if (!create) + return nullptr; + else { + newF = true; + i = curDir.contents.insert(i, { + std::string { name }, + File::Directory {}, + }); + } + } + cur = &i->second; + } + + if (newF && create) *cur = std::move(*create); + + return cur; +} + +std::string MemorySourceAccessor::readFile(const CanonPath & path) +{ + auto * f = open(path, std::nullopt); + if (!f) + throw Error("file '%s' does not exist", path); + if (auto * r = std::get_if(&f->raw)) + return r->contents; + else + throw Error("file '%s' is not a regular file", path); +} + +bool MemorySourceAccessor::pathExists(const CanonPath & path) +{ + return open(path, std::nullopt); +} + +MemorySourceAccessor::Stat MemorySourceAccessor::File::lstat() const +{ + return std::visit(overloaded { + [](const Regular & r) { + return Stat { + .type = tRegular, + .fileSize = r.contents.size(), + .isExecutable = r.executable, + }; + }, + [](const Directory &) { + return Stat { + .type = tDirectory, + }; + }, + [](const Symlink &) { + return Stat { + .type = tSymlink, + }; + }, + }, this->raw); +} + +std::optional +MemorySourceAccessor::maybeLstat(const CanonPath & path) +{ + const auto * f = open(path, std::nullopt); + return f ? std::optional { f->lstat() } : std::nullopt; +} + +MemorySourceAccessor::DirEntries MemorySourceAccessor::readDirectory(const CanonPath & path) +{ + auto * f = open(path, std::nullopt); + if (!f) + throw Error("file '%s' does not exist", path); + if (auto * d = std::get_if(&f->raw)) { + DirEntries res; + for (auto & [name, file] : d->contents) + res.insert_or_assign(name, file.lstat().type); + return res; + } else + throw Error("file '%s' is not a directory", path); + return {}; +} + +std::string MemorySourceAccessor::readLink(const CanonPath & path) +{ + auto * f = open(path, std::nullopt); + if (!f) + throw Error("file '%s' does not exist", path); + if (auto * s = std::get_if(&f->raw)) + return s->target; + else + throw Error("file '%s' is not a symbolic link", path); +} + +CanonPath MemorySourceAccessor::addFile(CanonPath path, std::string && contents) +{ + auto * f = open(path, File { File::Regular {} }); + if (!f) + throw Error("file '%s' cannot be made because some parent file is not a directory", path); + if (auto * r = std::get_if(&f->raw)) + r->contents = std::move(contents); + else + throw Error("file '%s' is not a regular file", path); + + return path; +} + + +using File = MemorySourceAccessor::File; + +void MemorySink::createDirectory(const Path & path) +{ + auto * f = dst.open(CanonPath{path}, File { File::Directory { } }); + if (!f) + throw Error("file '%s' cannot be made because some parent file is not a directory", path); + + if (!std::holds_alternative(f->raw)) + throw Error("file '%s' is not a directory", path); +}; + +void MemorySink::createRegularFile(const Path & path) +{ + auto * f = dst.open(CanonPath{path}, File { File::Regular {} }); + if (!f) + throw Error("file '%s' cannot be made because some parent file is not a directory", path); + if (!(r = std::get_if(&f->raw))) + throw Error("file '%s' is not a regular file", path); +} + +void MemorySink::closeRegularFile() +{ + r = nullptr; +} + +void MemorySink::isExecutable() +{ + assert(r); + r->executable = true; +} + +void MemorySink::preallocateContents(uint64_t len) +{ + assert(r); + r->contents.reserve(len); +} + +void MemorySink::receiveContents(std::string_view data) +{ + assert(r); + r->contents += data; +} + +void MemorySink::createSymlink(const Path & path, const std::string & target) +{ + auto * f = dst.open(CanonPath{path}, File { File::Symlink { } }); + if (!f) + throw Error("file '%s' cannot be made because some parent file is not a directory", path); + if (auto * s = std::get_if(&f->raw)) + s->target = target; + else + throw Error("file '%s' is not a symbolic link", path); +} + +} diff --git a/src/libutil/memory-source-accessor.hh b/src/libutil/memory-source-accessor.hh new file mode 100644 index 000000000000..b908f3713c0d --- /dev/null +++ b/src/libutil/memory-source-accessor.hh @@ -0,0 +1,99 @@ +#include "source-accessor.hh" +#include "fs-sink.hh" +#include "variant-wrapper.hh" + +namespace nix { + +/** + * An source accessor for an in-memory file system. + */ +struct MemorySourceAccessor : virtual SourceAccessor +{ + /** + * In addition to being part of the implementation of + * `MemorySourceAccessor`, this has a side benefit of nicely + * defining what a "file system object" is in Nix. + */ + struct File { + struct Regular { + bool executable = false; + std::string contents; + + GENERATE_CMP(Regular, me->executable, me->contents); + }; + + struct Directory { + using Name = std::string; + + std::map> contents; + + GENERATE_CMP(Directory, me->contents); + }; + + struct Symlink { + std::string target; + + GENERATE_CMP(Symlink, me->target); + }; + + using Raw = std::variant; + Raw raw; + + MAKE_WRAPPER_CONSTRUCTOR(File); + + GENERATE_CMP(File, me->raw); + + Stat lstat() const; + }; + + File root { File::Directory {} }; + + GENERATE_CMP(MemorySourceAccessor, me->root); + + std::string readFile(const CanonPath & path) override; + bool pathExists(const CanonPath & path) override; + std::optional maybeLstat(const CanonPath & path) override; + DirEntries readDirectory(const CanonPath & path) override; + std::string readLink(const CanonPath & path) override; + + /** + * @param create If present, create this file and any parent directories + * that are needed. + * + * Return null if + * + * - `create = false`: File does not exist. + * + * - `create = true`: some parent file was not a dir, so couldn't + * look/create inside. + */ + File * open(const CanonPath & path, std::optional create); + + CanonPath addFile(CanonPath path, std::string && contents); +}; + +/** + * Write to a `MemorySourceAccessor` at the given path + */ +struct MemorySink : ParseSink +{ + MemorySourceAccessor & dst; + + MemorySink(MemorySourceAccessor & dst) : dst(dst) { } + + void createDirectory(const Path & path) override; + + void createRegularFile(const Path & path) override; + void receiveContents(std::string_view data) override; + void isExecutable() override; + void closeRegularFile() override; + + void createSymlink(const Path & path, const std::string & target) override; + + void preallocateContents(uint64_t size) override; + +private: + MemorySourceAccessor::File::Regular * r; +}; + +} diff --git a/src/libutil/monitor-fd.hh b/src/libutil/monitor-fd.hh index 86d0115fc3c7..228fb13f853c 100644 --- a/src/libutil/monitor-fd.hh +++ b/src/libutil/monitor-fd.hh @@ -10,6 +10,8 @@ #include #include +#include "signals.hh" + namespace nix { diff --git a/src/libutil/namespaces.cc b/src/libutil/namespaces.cc index f66accb10ada..a789b321e174 100644 --- a/src/libutil/namespaces.cc +++ b/src/libutil/namespaces.cc @@ -1,13 +1,22 @@ -#if __linux__ - -#include "namespaces.hh" +#include "current-process.hh" #include "util.hh" #include "finally.hh" +#include "file-system.hh" +#include "processes.hh" +#include "signals.hh" + +#if __linux__ +# include +# include +# include "cgroup.hh" +#endif #include namespace nix { +#if __linux__ + bool userNamespacesSupported() { static auto res = [&]() -> bool @@ -92,6 +101,60 @@ bool mountAndPidNamespacesSupported() return res; } +#endif + + +////////////////////////////////////////////////////////////////////// + +#if __linux__ +static AutoCloseFD fdSavedMountNamespace; +static AutoCloseFD fdSavedRoot; +#endif + +void saveMountNamespace() +{ +#if __linux__ + static std::once_flag done; + std::call_once(done, []() { + fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY); + if (!fdSavedMountNamespace) + throw SysError("saving parent mount namespace"); + + fdSavedRoot = open("/proc/self/root", O_RDONLY); + }); +#endif +} + +void restoreMountNamespace() +{ +#if __linux__ + try { + auto savedCwd = absPath("."); + + if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) + throw SysError("restoring parent mount namespace"); + + if (fdSavedRoot) { + if (fchdir(fdSavedRoot.get())) + throw SysError("chdir into saved root"); + if (chroot(".")) + throw SysError("chroot into saved root"); + } + + if (chdir(savedCwd.c_str()) == -1) + throw SysError("restoring cwd"); + } catch (Error & e) { + debug(e.msg()); + } +#endif } +void unshareFilesystem() +{ +#ifdef __linux__ + if (unshare(CLONE_FS) != 0 && errno != EPERM) + throw SysError("unsharing filesystem state in download thread"); #endif +} + +} diff --git a/src/libutil/namespaces.hh b/src/libutil/namespaces.hh index 0b7eeb66cc4c..7e4e921a80af 100644 --- a/src/libutil/namespaces.hh +++ b/src/libutil/namespaces.hh @@ -1,8 +1,31 @@ #pragma once ///@file +#include + +#include "types.hh" + namespace nix { +/** + * Save the current mount namespace. Ignored if called more than + * once. + */ +void saveMountNamespace(); + +/** + * Restore the mount namespace saved by saveMountNamespace(). Ignored + * if saveMountNamespace() was never called. + */ +void restoreMountNamespace(); + +/** + * Cause this thread to not share any FS attributes with the main + * thread, because this causes setns() in restoreMountNamespace() to + * fail. + */ +void unshareFilesystem(); + #if __linux__ bool userNamespacesSupported(); diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc new file mode 100644 index 000000000000..dc96f84e5132 --- /dev/null +++ b/src/libutil/posix-source-accessor.cc @@ -0,0 +1,92 @@ +#include "posix-source-accessor.hh" +#include "signals.hh" + +namespace nix { + +void PosixSourceAccessor::readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) +{ + // FIXME: add O_NOFOLLOW since symlinks should be resolved by the + // caller? + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) + throw SysError("opening file '%1%'", path); + + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError("statting file"); + + sizeCallback(st.st_size); + + off_t left = st.st_size; + + std::vector buf(64 * 1024); + while (left) { + checkInterrupt(); + ssize_t rd = read(fd.get(), buf.data(), (size_t) std::min(left, (off_t) buf.size())); + if (rd == -1) { + if (errno != EINTR) + throw SysError("reading from file '%s'", showPath(path)); + } + else if (rd == 0) + throw SysError("unexpected end-of-file reading '%s'", showPath(path)); + else { + assert(rd <= left); + sink({(char *) buf.data(), (size_t) rd}); + left -= rd; + } + } +} + +bool PosixSourceAccessor::pathExists(const CanonPath & path) +{ + return nix::pathExists(path.abs()); +} + +std::optional PosixSourceAccessor::maybeLstat(const CanonPath & path) +{ + struct stat st; + if (::lstat(path.c_str(), &st)) { + if (errno == ENOENT) return std::nullopt; + throw SysError("getting status of '%s'", showPath(path)); + } + mtime = std::max(mtime, st.st_mtime); + return Stat { + .type = + S_ISREG(st.st_mode) ? tRegular : + S_ISDIR(st.st_mode) ? tDirectory : + S_ISLNK(st.st_mode) ? tSymlink : + tMisc, + .fileSize = S_ISREG(st.st_mode) ? std::optional(st.st_size) : std::nullopt, + .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR, + }; +} + +SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & path) +{ + DirEntries res; + for (auto & entry : nix::readDirectory(path.abs())) { + std::optional type; + switch (entry.type) { + case DT_REG: type = Type::tRegular; break; + case DT_LNK: type = Type::tSymlink; break; + case DT_DIR: type = Type::tDirectory; break; + } + res.emplace(entry.name, type); + } + return res; +} + +std::string PosixSourceAccessor::readLink(const CanonPath & path) +{ + return nix::readLink(path.abs()); +} + +std::optional PosixSourceAccessor::getPhysicalPath(const CanonPath & path) +{ + return path; +} + +} diff --git a/src/libutil/posix-source-accessor.hh b/src/libutil/posix-source-accessor.hh new file mode 100644 index 000000000000..a45d96bf8a1a --- /dev/null +++ b/src/libutil/posix-source-accessor.hh @@ -0,0 +1,34 @@ +#pragma once + +#include "source-accessor.hh" + +namespace nix { + +/** + * A source accessor that uses the Unix filesystem. + */ +struct PosixSourceAccessor : virtual SourceAccessor +{ + /** + * The most recent mtime seen by lstat(). This is a hack to + * support dumpPathAndGetMtime(). Should remove this eventually. + */ + time_t mtime = 0; + + void readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) override; + + bool pathExists(const CanonPath & path) override; + + std::optional maybeLstat(const CanonPath & path) override; + + DirEntries readDirectory(const CanonPath & path) override; + + std::string readLink(const CanonPath & path) override; + + std::optional getPhysicalPath(const CanonPath & path) override; +}; + +} diff --git a/src/libutil/processes.cc b/src/libutil/processes.cc new file mode 100644 index 000000000000..91a0ea66fda2 --- /dev/null +++ b/src/libutil/processes.cc @@ -0,0 +1,421 @@ +#include "current-process.hh" +#include "environment-variables.hh" +#include "signals.hh" +#include "processes.hh" +#include "finally.hh" +#include "serialise.hh" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#ifdef __APPLE__ +# include +#endif + +#ifdef __linux__ +# include +# include +#endif + + +namespace nix { + +Pid::Pid() +{ +} + + +Pid::Pid(pid_t pid) + : pid(pid) +{ +} + + +Pid::~Pid() +{ + if (pid != -1) kill(); +} + + +void Pid::operator =(pid_t pid) +{ + if (this->pid != -1 && this->pid != pid) kill(); + this->pid = pid; + killSignal = SIGKILL; // reset signal to default +} + + +Pid::operator pid_t() +{ + return pid; +} + + +int Pid::kill() +{ + assert(pid != -1); + + debug("killing process %1%", pid); + + /* Send the requested signal to the child. If it has its own + process group, send the signal to every process in the child + process group (which hopefully includes *all* its children). */ + if (::kill(separatePG ? -pid : pid, killSignal) != 0) { + /* On BSDs, killing a process group will return EPERM if all + processes in the group are zombies (or something like + that). So try to detect and ignore that situation. */ +#if __FreeBSD__ || __APPLE__ + if (errno != EPERM || ::kill(pid, 0) != 0) +#endif + logError(SysError("killing process %d", pid).info()); + } + + return wait(); +} + + +int Pid::wait() +{ + assert(pid != -1); + while (1) { + int status; + int res = waitpid(pid, &status, 0); + if (res == pid) { + pid = -1; + return status; + } + if (errno != EINTR) + throw SysError("cannot get exit status of PID %d", pid); + checkInterrupt(); + } +} + + +void Pid::setSeparatePG(bool separatePG) +{ + this->separatePG = separatePG; +} + + +void Pid::setKillSignal(int signal) +{ + this->killSignal = signal; +} + + +pid_t Pid::release() +{ + pid_t p = pid; + pid = -1; + return p; +} + + +void killUser(uid_t uid) +{ + debug("killing all processes running under uid '%1%'", uid); + + assert(uid != 0); /* just to be safe... */ + + /* The system call kill(-1, sig) sends the signal `sig' to all + users to which the current process can send signals. So we + fork a process, switch to uid, and send a mass kill. */ + + Pid pid = startProcess([&]() { + + if (setuid(uid) == -1) + throw SysError("setting uid"); + + while (true) { +#ifdef __APPLE__ + /* OSX's kill syscall takes a third parameter that, among + other things, determines if kill(-1, signo) affects the + calling process. In the OSX libc, it's set to true, + which means "follow POSIX", which we don't want here + */ + if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break; +#else + if (kill(-1, SIGKILL) == 0) break; +#endif + if (errno == ESRCH || errno == EPERM) break; /* no more processes */ + if (errno != EINTR) + throw SysError("cannot kill processes for uid '%1%'", uid); + } + + _exit(0); + }); + + int status = pid.wait(); + if (status != 0) + throw Error("cannot kill processes for uid '%1%': %2%", uid, statusToString(status)); + + /* !!! We should really do some check to make sure that there are + no processes left running under `uid', but there is no portable + way to do so (I think). The most reliable way may be `ps -eo + uid | grep -q $uid'. */ +} + + +////////////////////////////////////////////////////////////////////// + + +/* Wrapper around vfork to prevent the child process from clobbering + the caller's stack frame in the parent. */ +static pid_t doFork(bool allowVfork, std::function fun) __attribute__((noinline)); +static pid_t doFork(bool allowVfork, std::function fun) +{ +#ifdef __linux__ + pid_t pid = allowVfork ? vfork() : fork(); +#else + pid_t pid = fork(); +#endif + if (pid != 0) return pid; + fun(); + abort(); +} + + +#if __linux__ +static int childEntry(void * arg) +{ + auto main = (std::function *) arg; + (*main)(); + return 1; +} +#endif + + +pid_t startProcess(std::function fun, const ProcessOptions & options) +{ + std::function wrapper = [&]() { + if (!options.allowVfork) + logger = makeSimpleLogger(); + try { +#if __linux__ + if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) + throw SysError("setting death signal"); +#endif + fun(); + } catch (std::exception & e) { + try { + std::cerr << options.errorPrefix << e.what() << "\n"; + } catch (...) { } + } catch (...) { } + if (options.runExitHandlers) + exit(1); + else + _exit(1); + }; + + pid_t pid = -1; + + if (options.cloneFlags) { + #ifdef __linux__ + // Not supported, since then we don't know when to free the stack. + assert(!(options.cloneFlags & CLONE_VM)); + + size_t stackSize = 1 * 1024 * 1024; + auto stack = (char *) mmap(0, stackSize, + PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + if (stack == MAP_FAILED) throw SysError("allocating stack"); + + Finally freeStack([&]() { munmap(stack, stackSize); }); + + pid = clone(childEntry, stack + stackSize, options.cloneFlags | SIGCHLD, &wrapper); + #else + throw Error("clone flags are only supported on Linux"); + #endif + } else + pid = doFork(options.allowVfork, wrapper); + + if (pid == -1) throw SysError("unable to fork"); + + return pid; +} + + +std::string runProgram(Path program, bool searchPath, const Strings & args, + const std::optional & input, bool isInteractive) +{ + auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input, .isInteractive = isInteractive}); + + if (!statusOk(res.first)) + throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first)); + + return res.second; +} + +// Output = error code + "standard out" output stream +std::pair runProgram(RunOptions && options) +{ + StringSink sink; + options.standardOut = &sink; + + int status = 0; + + try { + runProgram2(options); + } catch (ExecError & e) { + status = e.status; + } + + return {status, std::move(sink.s)}; +} + +void runProgram2(const RunOptions & options) +{ + checkInterrupt(); + + assert(!(options.standardIn && options.input)); + + std::unique_ptr source_; + Source * source = options.standardIn; + + if (options.input) { + source_ = std::make_unique(*options.input); + source = source_.get(); + } + + /* Create a pipe. */ + Pipe out, in; + if (options.standardOut) out.create(); + if (source) in.create(); + + ProcessOptions processOptions; + // vfork implies that the environment of the main process and the fork will + // be shared (technically this is undefined, but in practice that's the + // case), so we can't use it if we alter the environment + processOptions.allowVfork = !options.environment; + + std::optional>> resumeLoggerDefer; + if (options.isInteractive) { + logger->pause(); + resumeLoggerDefer.emplace( + []() { + logger->resume(); + } + ); + } + + /* Fork. */ + Pid pid = startProcess([&]() { + if (options.environment) + replaceEnv(*options.environment); + if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); + if (options.mergeStderrToStdout) + if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) + throw SysError("cannot dup stdout into stderr"); + if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) + throw SysError("dupping stdin"); + + if (options.chdir && chdir((*options.chdir).c_str()) == -1) + throw SysError("chdir failed"); + if (options.gid && setgid(*options.gid) == -1) + throw SysError("setgid failed"); + /* Drop all other groups if we're setgid. */ + if (options.gid && setgroups(0, 0) == -1) + throw SysError("setgroups failed"); + if (options.uid && setuid(*options.uid) == -1) + throw SysError("setuid failed"); + + Strings args_(options.args); + args_.push_front(options.program); + + restoreProcessContext(); + + if (options.searchPath) + execvp(options.program.c_str(), stringsToCharPtrs(args_).data()); + // This allows you to refer to a program with a pathname relative + // to the PATH variable. + else + execv(options.program.c_str(), stringsToCharPtrs(args_).data()); + + throw SysError("executing '%1%'", options.program); + }, processOptions); + + out.writeSide.close(); + + std::thread writerThread; + + std::promise promise; + + Finally doJoin([&]() { + if (writerThread.joinable()) + writerThread.join(); + }); + + + if (source) { + in.readSide.close(); + writerThread = std::thread([&]() { + try { + std::vector buf(8 * 1024); + while (true) { + size_t n; + try { + n = source->read(buf.data(), buf.size()); + } catch (EndOfFile &) { + break; + } + writeFull(in.writeSide.get(), {buf.data(), n}); + } + promise.set_value(); + } catch (...) { + promise.set_exception(std::current_exception()); + } + in.writeSide.close(); + }); + } + + if (options.standardOut) + drainFD(out.readSide.get(), *options.standardOut); + + /* Wait for the child to finish. */ + int status = pid.wait(); + + /* Wait for the writer thread to finish. */ + if (source) promise.get_future().get(); + + if (status) + throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status)); +} + +////////////////////////////////////////////////////////////////////// + +std::string statusToString(int status) +{ + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + if (WIFEXITED(status)) + return fmt("failed with exit code %1%", WEXITSTATUS(status)); + else if (WIFSIGNALED(status)) { + int sig = WTERMSIG(status); +#if HAVE_STRSIGNAL + const char * description = strsignal(sig); + return fmt("failed due to signal %1% (%2%)", sig, description); +#else + return fmt("failed due to signal %1%", sig); +#endif + } + else + return "died abnormally"; + } else return "succeeded"; +} + + +bool statusOk(int status) +{ + return WIFEXITED(status) && WEXITSTATUS(status) == 0; +} + +} diff --git a/src/libutil/processes.hh b/src/libutil/processes.hh new file mode 100644 index 000000000000..978c37105c67 --- /dev/null +++ b/src/libutil/processes.hh @@ -0,0 +1,123 @@ +#pragma once +///@file + +#include "types.hh" +#include "error.hh" +#include "logging.hh" +#include "ansicolor.hh" + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +namespace nix { + +struct Sink; +struct Source; + +class Pid +{ + pid_t pid = -1; + bool separatePG = false; + int killSignal = SIGKILL; +public: + Pid(); + Pid(pid_t pid); + ~Pid(); + void operator =(pid_t pid); + operator pid_t(); + int kill(); + int wait(); + + void setSeparatePG(bool separatePG); + void setKillSignal(int signal); + pid_t release(); +}; + + +/** + * Kill all processes running under the specified uid by sending them + * a SIGKILL. + */ +void killUser(uid_t uid); + + +/** + * Fork a process that runs the given function, and return the child + * pid to the caller. + */ +struct ProcessOptions +{ + std::string errorPrefix = ""; + bool dieWithParent = true; + bool runExitHandlers = false; + bool allowVfork = false; + /** + * use clone() with the specified flags (Linux only) + */ + int cloneFlags = 0; +}; + +pid_t startProcess(std::function fun, const ProcessOptions & options = ProcessOptions()); + + +/** + * Run a program and return its stdout in a string (i.e., like the + * shell backtick operator). + */ +std::string runProgram(Path program, bool searchPath = false, + const Strings & args = Strings(), + const std::optional & input = {}, bool isInteractive = false); + +struct RunOptions +{ + Path program; + bool searchPath = true; + Strings args; + std::optional uid; + std::optional gid; + std::optional chdir; + std::optional> environment; + std::optional input; + Source * standardIn = nullptr; + Sink * standardOut = nullptr; + bool mergeStderrToStdout = false; + bool isInteractive = false; +}; + +std::pair runProgram(RunOptions && options); + +void runProgram2(const RunOptions & options); + + +class ExecError : public Error +{ +public: + int status; + + template + ExecError(int status, const Args & ... args) + : Error(args...), status(status) + { } +}; + + +/** + * Convert the exit status of a child as returned by wait() into an + * error string. + */ +std::string statusToString(int status); + +bool statusOk(int status); + +} diff --git a/src/libutil/references.cc b/src/libutil/references.cc index 7f59b4c09d0a..9d75606ef69d 100644 --- a/src/libutil/references.cc +++ b/src/libutil/references.cc @@ -1,6 +1,5 @@ #include "references.hh" #include "hash.hh" -#include "util.hh" #include "archive.hh" #include diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 3d5121a19fa6..f465bd0defd3 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -1,5 +1,5 @@ #include "serialise.hh" -#include "util.hh" +#include "signals.hh" #include #include @@ -74,6 +74,10 @@ void Source::operator () (char * data, size_t len) } } +void Source::operator () (std::string_view data) +{ + (*this)((char *)data.data(), data.size()); +} void Source::drainInto(Sink & sink) { @@ -444,7 +448,7 @@ Error readError(Source & source) auto msg = readString(source); ErrorInfo info { .level = level, - .msg = hintformat(fmt("%s", msg)), + .msg = hintfmt(msg), }; auto havePos = readNum(source); assert(havePos == 0); @@ -453,7 +457,7 @@ Error readError(Source & source) havePos = readNum(source); assert(havePos == 0); info.traces.push_back(Trace { - .hint = hintformat(fmt("%s", readString(source))) + .hint = hintfmt(readString(source)) }); } return Error(std::move(info)); diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 333c254ea8e3..3f57ce88ba73 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -5,6 +5,7 @@ #include "types.hh" #include "util.hh" +#include "file-descriptor.hh" namespace boost::context { struct stack_context; } @@ -72,6 +73,7 @@ struct Source * an error if it is not going to be available. */ void operator () (char * data, size_t len); + void operator () (std::string_view data); /** * Store up to ‘len’ in the buffer pointed to by ‘data’, and diff --git a/src/libutil/signals.cc b/src/libutil/signals.cc new file mode 100644 index 000000000000..4632aa319d80 --- /dev/null +++ b/src/libutil/signals.cc @@ -0,0 +1,188 @@ +#include "signals.hh" +#include "util.hh" +#include "error.hh" +#include "sync.hh" +#include "terminal.hh" + +#include + +namespace nix { + +std::atomic _isInterrupted = false; + +static thread_local bool interruptThrown = false; +thread_local std::function interruptCheck; + +void setInterruptThrown() +{ + interruptThrown = true; +} + +void _interrupted() +{ + /* Block user interrupts while an exception is being handled. + Throwing an exception while another exception is being handled + kills the program! */ + if (!interruptThrown && !std::uncaught_exceptions()) { + interruptThrown = true; + throw Interrupted("interrupted by the user"); + } +} + + +////////////////////////////////////////////////////////////////////// + + +/* We keep track of interrupt callbacks using integer tokens, so we can iterate + safely without having to lock the data structure while executing arbitrary + functions. + */ +struct InterruptCallbacks { + typedef int64_t Token; + + /* We use unique tokens so that we can't accidentally delete the wrong + handler because of an erroneous double delete. */ + Token nextToken = 0; + + /* Used as a list, see InterruptCallbacks comment. */ + std::map> callbacks; +}; + +static Sync _interruptCallbacks; + +static void signalHandlerThread(sigset_t set) +{ + while (true) { + int signal = 0; + sigwait(&set, &signal); + + if (signal == SIGINT || signal == SIGTERM || signal == SIGHUP) + triggerInterrupt(); + + else if (signal == SIGWINCH) { + updateWindowSize(); + } + } +} + +void triggerInterrupt() +{ + _isInterrupted = true; + + { + InterruptCallbacks::Token i = 0; + while (true) { + std::function callback; + { + auto interruptCallbacks(_interruptCallbacks.lock()); + auto lb = interruptCallbacks->callbacks.lower_bound(i); + if (lb == interruptCallbacks->callbacks.end()) + break; + + callback = lb->second; + i = lb->first + 1; + } + + try { + callback(); + } catch (...) { + ignoreException(); + } + } + } +} + + +static sigset_t savedSignalMask; +static bool savedSignalMaskIsSet = false; + +void setChildSignalMask(sigset_t * sigs) +{ + assert(sigs); // C style function, but think of sigs as a reference + +#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE + sigemptyset(&savedSignalMask); + // There's no "assign" or "copy" function, so we rely on (math) idempotence + // of the or operator: a or a = a. + sigorset(&savedSignalMask, sigs, sigs); +#else + // Without sigorset, our best bet is to assume that sigset_t is a type that + // can be assigned directly, such as is the case for a sigset_t defined as + // an integer type. + savedSignalMask = *sigs; +#endif + + savedSignalMaskIsSet = true; +} + +void saveSignalMask() { + if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) + throw SysError("querying signal mask"); + + savedSignalMaskIsSet = true; +} + +void startSignalHandlerThread() +{ + updateWindowSize(); + + saveSignalMask(); + + sigset_t set; + sigemptyset(&set); + sigaddset(&set, SIGINT); + sigaddset(&set, SIGTERM); + sigaddset(&set, SIGHUP); + sigaddset(&set, SIGPIPE); + sigaddset(&set, SIGWINCH); + if (pthread_sigmask(SIG_BLOCK, &set, nullptr)) + throw SysError("blocking signals"); + + std::thread(signalHandlerThread, set).detach(); +} + +void restoreSignals() +{ + // If startSignalHandlerThread wasn't called, that means we're not running + // in a proper libmain process, but a process that presumably manages its + // own signal handlers. Such a process should call either + // - initNix(), to be a proper libmain process + // - startSignalHandlerThread(), to resemble libmain regarding signal + // handling only + // - saveSignalMask(), for processes that define their own signal handling + // thread + // TODO: Warn about this? Have a default signal mask? The latter depends on + // whether we should generally inherit signal masks from the caller. + // I don't know what the larger unix ecosystem expects from us here. + if (!savedSignalMaskIsSet) + return; + + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) + throw SysError("restoring signals"); +} + + +/* RAII helper to automatically deregister a callback. */ +struct InterruptCallbackImpl : InterruptCallback +{ + InterruptCallbacks::Token token; + ~InterruptCallbackImpl() override + { + auto interruptCallbacks(_interruptCallbacks.lock()); + interruptCallbacks->callbacks.erase(token); + } +}; + +std::unique_ptr createInterruptCallback(std::function callback) +{ + auto interruptCallbacks(_interruptCallbacks.lock()); + auto token = interruptCallbacks->nextToken++; + interruptCallbacks->callbacks.emplace(token, callback); + + auto res = std::make_unique(); + res->token = token; + + return std::unique_ptr(res.release()); +} + +} diff --git a/src/libutil/signals.hh b/src/libutil/signals.hh new file mode 100644 index 000000000000..7e8beff33266 --- /dev/null +++ b/src/libutil/signals.hh @@ -0,0 +1,104 @@ +#pragma once +///@file + +#include "types.hh" +#include "error.hh" +#include "logging.hh" +#include "ansicolor.hh" + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +namespace nix { + +/* User interruption. */ + +extern std::atomic _isInterrupted; + +extern thread_local std::function interruptCheck; + +void setInterruptThrown(); + +void _interrupted(); + +void inline checkInterrupt() +{ + if (_isInterrupted || (interruptCheck && interruptCheck())) + _interrupted(); +} + +MakeError(Interrupted, BaseError); + + +/** + * Start a thread that handles various signals. Also block those signals + * on the current thread (and thus any threads created by it). + * Saves the signal mask before changing the mask to block those signals. + * See saveSignalMask(). + */ +void startSignalHandlerThread(); + +/** + * Saves the signal mask, which is the signal mask that nix will restore + * before creating child processes. + * See setChildSignalMask() to set an arbitrary signal mask instead of the + * current mask. + */ +void saveSignalMask(); + +/** + * To use in a process that already called `startSignalHandlerThread()` + * or `saveSignalMask()` first. + */ +void restoreSignals(); + +/** + * Sets the signal mask. Like saveSignalMask() but for a signal set that doesn't + * necessarily match the current thread's mask. + * See saveSignalMask() to set the saved mask to the current mask. + */ +void setChildSignalMask(sigset_t *sigs); + +struct InterruptCallback +{ + virtual ~InterruptCallback() { }; +}; + +/** + * Register a function that gets called on SIGINT (in a non-signal + * context). + */ +std::unique_ptr createInterruptCallback( + std::function callback); + +void triggerInterrupt(); + +/** + * A RAII class that causes the current thread to receive SIGUSR1 when + * the signal handler thread receives SIGINT. That is, this allows + * SIGINT to be multiplexed to multiple threads. + */ +struct ReceiveInterrupts +{ + pthread_t target; + std::unique_ptr callback; + + ReceiveInterrupts() + : target(pthread_self()) + , callback(createInterruptCallback([&]() { pthread_kill(target, SIGUSR1); })) + { } +}; + + +} diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc new file mode 100644 index 000000000000..7813433a746b --- /dev/null +++ b/src/libutil/source-accessor.cc @@ -0,0 +1,70 @@ +#include "source-accessor.hh" +#include "archive.hh" + +namespace nix { + +static std::atomic nextNumber{0}; + +SourceAccessor::SourceAccessor() + : number(++nextNumber) + , displayPrefix{"«unknown»"} +{ +} + +bool SourceAccessor::pathExists(const CanonPath & path) +{ + return maybeLstat(path).has_value(); +} + +std::string SourceAccessor::readFile(const CanonPath & path) +{ + StringSink sink; + std::optional size; + readFile(path, sink, [&](uint64_t _size) + { + size = _size; + }); + assert(size && *size == sink.s.size()); + return std::move(sink.s); +} + +void SourceAccessor::readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) +{ + auto s = readFile(path); + sizeCallback(s.size()); + sink(s); +} + +Hash SourceAccessor::hashPath( + const CanonPath & path, + PathFilter & filter, + HashType ht) +{ + HashSink sink(ht); + dumpPath(path, sink, filter); + return sink.finish().first; +} + +SourceAccessor::Stat SourceAccessor::lstat(const CanonPath & path) +{ + if (auto st = maybeLstat(path)) + return *st; + else + throw Error("path '%s' does not exist", showPath(path)); +} + +void SourceAccessor::setPathDisplay(std::string displayPrefix, std::string displaySuffix) +{ + this->displayPrefix = std::move(displayPrefix); + this->displaySuffix = std::move(displaySuffix); +} + +std::string SourceAccessor::showPath(const CanonPath & path) +{ + return displayPrefix + path.abs() + displaySuffix; +} + +} diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh new file mode 100644 index 000000000000..264caab16d5d --- /dev/null +++ b/src/libutil/source-accessor.hh @@ -0,0 +1,127 @@ +#pragma once + +#include "canon-path.hh" +#include "hash.hh" + +namespace nix { + +struct Sink; + +/** + * A read-only filesystem abstraction. This is used by the Nix + * evaluator and elsewhere for accessing sources in various + * filesystem-like entities (such as the real filesystem, tarballs or + * Git repositories). + */ +struct SourceAccessor +{ + const size_t number; + + std::string displayPrefix, displaySuffix; + + SourceAccessor(); + + virtual ~SourceAccessor() + { } + + /** + * Return the contents of a file as a string. + */ + virtual std::string readFile(const CanonPath & path); + + /** + * Write the contents of a file as a sink. `sizeCallback` must be + * called with the size of the file before any data is written to + * the sink. + * + * Note: subclasses of `SourceAccessor` need to implement at least + * one of the `readFile()` variants. + */ + virtual void readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback = [](uint64_t size){}); + + virtual bool pathExists(const CanonPath & path); + + enum Type { + tRegular, tSymlink, tDirectory, + /** + Any other node types that may be encountered on the file system, such as device nodes, sockets, named pipe, and possibly even more exotic things. + + Responsible for `"unknown"` from `builtins.readFileType "/dev/null"`. + + Unlike `DT_UNKNOWN`, this must not be used for deferring the lookup of types. + */ + tMisc + }; + + struct Stat + { + Type type = tMisc; + + /** + * For regular files only: the size of the file. Not all + * accessors return this since it may be too expensive to + * compute. + */ + std::optional fileSize; + + /** + * For regular files only: whether this is an executable. + */ + bool isExecutable = false; + + /** + * For regular files only: the position of the contents of this + * file in the NAR. Only returned by NAR accessors. + */ + std::optional narOffset; + }; + + Stat lstat(const CanonPath & path); + + virtual std::optional maybeLstat(const CanonPath & path) = 0; + + typedef std::optional DirEntry; + + typedef std::map DirEntries; + + virtual DirEntries readDirectory(const CanonPath & path) = 0; + + virtual std::string readLink(const CanonPath & path) = 0; + + virtual void dumpPath( + const CanonPath & path, + Sink & sink, + PathFilter & filter = defaultPathFilter); + + Hash hashPath( + const CanonPath & path, + PathFilter & filter = defaultPathFilter, + HashType ht = htSHA256); + + /** + * Return a corresponding path in the root filesystem, if + * possible. This is only possible for filesystems that are + * materialized in the root filesystem. + */ + virtual std::optional getPhysicalPath(const CanonPath & path) + { return std::nullopt; } + + bool operator == (const SourceAccessor & x) const + { + return number == x.number; + } + + bool operator < (const SourceAccessor & x) const + { + return number < x.number; + } + + void setPathDisplay(std::string displayPrefix, std::string displaySuffix = ""); + + virtual std::string showPath(const CanonPath & path); +}; + +} diff --git a/src/libutil/suggestions.cc b/src/libutil/suggestions.cc index 9510a5f0c415..e67e986fb597 100644 --- a/src/libutil/suggestions.cc +++ b/src/libutil/suggestions.cc @@ -1,7 +1,9 @@ #include "suggestions.hh" #include "ansicolor.hh" -#include "util.hh" +#include "terminal.hh" + #include +#include namespace nix { diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index 5060a8f24a06..1733c791c34d 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -3,6 +3,7 @@ #include "serialise.hh" #include "tarfile.hh" +#include "file-system.hh" namespace nix { diff --git a/src/libutil/terminal.cc b/src/libutil/terminal.cc new file mode 100644 index 000000000000..8febc8771e36 --- /dev/null +++ b/src/libutil/terminal.cc @@ -0,0 +1,108 @@ +#include "terminal.hh" +#include "environment-variables.hh" +#include "sync.hh" + +#include +#include + +namespace nix { + +bool shouldANSI() +{ + return isatty(STDERR_FILENO) + && getEnv("TERM").value_or("dumb") != "dumb" + && !(getEnv("NO_COLOR").has_value() || getEnv("NOCOLOR").has_value()); +} + +std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int width) +{ + std::string t, e; + size_t w = 0; + auto i = s.begin(); + + while (w < (size_t) width && i != s.end()) { + + if (*i == '\e') { + std::string e; + e += *i++; + char last = 0; + + if (i != s.end() && *i == '[') { + e += *i++; + // eat parameter bytes + while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++; + // eat intermediate bytes + while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++; + // eat final byte + if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++; + } else { + if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++; + } + + if (!filterAll && last == 'm') + t += e; + } + + else if (*i == '\t') { + i++; t += ' '; w++; + while (w < (size_t) width && w % 8) { + t += ' '; w++; + } + } + + else if (*i == '\r' || *i == '\a') + // do nothing for now + i++; + + else { + w++; + // Copy one UTF-8 character. + if ((*i & 0xe0) == 0xc0) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; + } else if ((*i & 0xf0) == 0xe0) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; + } + } else if ((*i & 0xf8) == 0xf0) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; + } + } + } else + t += *i++; + } + } + + return t; +} + + +////////////////////////////////////////////////////////////////////// + +static Sync> windowSize{{0, 0}}; + + +void updateWindowSize() +{ + struct winsize ws; + if (ioctl(2, TIOCGWINSZ, &ws) == 0) { + auto windowSize_(windowSize.lock()); + windowSize_->first = ws.ws_row; + windowSize_->second = ws.ws_col; + } +} + + +std::pair getWindowSize() +{ + return *windowSize.lock(); +} + +} diff --git a/src/libutil/terminal.hh b/src/libutil/terminal.hh new file mode 100644 index 000000000000..9cb191308da8 --- /dev/null +++ b/src/libutil/terminal.hh @@ -0,0 +1,38 @@ +#pragma once +///@file + +#include "types.hh" + +namespace nix { +/** + * Determine whether ANSI escape sequences are appropriate for the + * present output. + */ +bool shouldANSI(); + +/** + * Truncate a string to 'width' printable characters. If 'filterAll' + * is true, all ANSI escape sequences are filtered out. Otherwise, + * some escape sequences (such as colour setting) are copied but not + * included in the character count. Also, tabs are expanded to + * spaces. + */ +std::string filterANSIEscapes(std::string_view s, + bool filterAll = false, + unsigned int width = std::numeric_limits::max()); + +/** + * Recalculate the window size, updating a global variable. Used in the + * `SIGWINCH` signal handler. + */ +void updateWindowSize(); + +/** + * @return the number of rows and columns of the terminal. + * + * The value is cached so this is quick. The cached result is computed + * by `updateWindowSize()`. + */ +std::pair getWindowSize(); + +} diff --git a/src/libutil/tests/args.cc b/src/libutil/tests/args.cc new file mode 100644 index 000000000000..bea74a8c8393 --- /dev/null +++ b/src/libutil/tests/args.cc @@ -0,0 +1,168 @@ +#include "../args.hh" +#include "libutil/fs-sink.hh" +#include + +#include +#include + +namespace nix { + + TEST(parseShebangContent, basic) { + std::list r = parseShebangContent("hi there"); + ASSERT_EQ(r.size(), 2); + auto i = r.begin(); + ASSERT_EQ(*i++, "hi"); + ASSERT_EQ(*i++, "there"); + } + + TEST(parseShebangContent, empty) { + std::list r = parseShebangContent(""); + ASSERT_EQ(r.size(), 0); + } + + TEST(parseShebangContent, doubleBacktick) { + std::list r = parseShebangContent("``\"ain't that nice\"``"); + ASSERT_EQ(r.size(), 1); + auto i = r.begin(); + ASSERT_EQ(*i++, "\"ain't that nice\""); + } + + TEST(parseShebangContent, doubleBacktickEmpty) { + std::list r = parseShebangContent("````"); + ASSERT_EQ(r.size(), 1); + auto i = r.begin(); + ASSERT_EQ(*i++, ""); + } + + TEST(parseShebangContent, doubleBacktickMarkdownInlineCode) { + std::list r = parseShebangContent("``# I'm markdown section about `coolFunction` ``"); + ASSERT_EQ(r.size(), 1); + auto i = r.begin(); + ASSERT_EQ(*i++, "# I'm markdown section about `coolFunction`"); + } + + TEST(parseShebangContent, doubleBacktickMarkdownCodeBlockNaive) { + std::list r = parseShebangContent("``Example 1\n```nix\na: a\n``` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "Example 1\n``nix\na: a\n``"); + } + + TEST(parseShebangContent, doubleBacktickMarkdownCodeBlockCorrect) { + std::list r = parseShebangContent("``Example 1\n````nix\na: a\n```` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "Example 1\n```nix\na: a\n```"); + } + + TEST(parseShebangContent, doubleBacktickMarkdownCodeBlock2) { + std::list r = parseShebangContent("``Example 1\n````nix\na: a\n````\nExample 2\n````nix\na: a\n```` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "Example 1\n```nix\na: a\n```\nExample 2\n```nix\na: a\n```"); + } + + TEST(parseShebangContent, singleBacktickInDoubleBacktickQuotes) { + std::list r = parseShebangContent("``` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "`"); + } + + TEST(parseShebangContent, singleBacktickAndSpaceInDoubleBacktickQuotes) { + std::list r = parseShebangContent("``` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "` "); + } + + TEST(parseShebangContent, doubleBacktickInDoubleBacktickQuotes) { + std::list r = parseShebangContent("````` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "``"); + } + + TEST(parseShebangContent, increasingQuotes) { + std::list r = parseShebangContent("```` ``` `` ````` `` `````` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 4); + ASSERT_EQ(*i++, ""); + ASSERT_EQ(*i++, "`"); + ASSERT_EQ(*i++, "``"); + ASSERT_EQ(*i++, "```"); + } + + +#ifndef COVERAGE + +// quick and dirty +static inline std::string escape(std::string_view s_) { + + std::string_view s = s_; + std::string r = "``"; + + // make a guess to allocate ahead of time + r.reserve( + // plain chars + s.size() + // quotes + + 5 + // some "escape" backticks + + s.size() / 8); + + while (!s.empty()) { + if (s[0] == '`' && s.size() >= 2 && s[1] == '`') { + // escape it + r += "`"; + while (!s.empty() && s[0] == '`') { + r += "`"; + s = s.substr(1); + } + } else { + r += s[0]; + s = s.substr(1); + } + } + + if (!r.empty() + && ( + r[r.size() - 1] == '`' + || r[r.size() - 1] == ' ' + )) { + r += " "; + } + + r += "``"; + + return r; +}; + +RC_GTEST_PROP( + parseShebangContent, + prop_round_trip_single, + (const std::string & orig)) +{ + auto escaped = escape(orig); + // RC_LOG() << "escaped: <[[" << escaped << "]]>" << std::endl; + auto ss = parseShebangContent(escaped); + RC_ASSERT(ss.size() == 1); + RC_ASSERT(*ss.begin() == orig); +} + +RC_GTEST_PROP( + parseShebangContent, + prop_round_trip_two, + (const std::string & one, const std::string & two)) +{ + auto ss = parseShebangContent(escape(one) + " " + escape(two)); + RC_ASSERT(ss.size() == 2); + auto i = ss.begin(); + RC_ASSERT(*i++ == one); + RC_ASSERT(*i++ == two); +} + + +#endif + +} \ No newline at end of file diff --git a/src/libutil/tests/characterization.hh b/src/libutil/tests/characterization.hh new file mode 100644 index 000000000000..6eb513d68746 --- /dev/null +++ b/src/libutil/tests/characterization.hh @@ -0,0 +1,108 @@ +#pragma once +///@file + +#include + +#include "types.hh" +#include "environment-variables.hh" + +namespace nix { + +/** + * The path to the `unit-test-data` directory. See the contributing + * guide in the manual for further details. + */ +static Path getUnitTestData() { + return getEnv("_NIX_TEST_UNIT_DATA").value(); +} + +/** + * Whether we should update "golden masters" instead of running tests + * against them. See the contributing guide in the manual for further + * details. + */ +static bool testAccept() { + return getEnv("_NIX_TEST_ACCEPT") == "1"; +} + +/** + * Mixin class for writing characterization tests + */ +class CharacterizationTest : public virtual ::testing::Test +{ +protected: + /** + * While the "golden master" for this characterization test is + * located. It should not be shared with any other test. + */ + virtual Path goldenMaster(PathView testStem) const = 0; + +public: + /** + * Golden test for reading + * + * @param test hook that takes the contents of the file and does the + * actual work + */ + void readTest(PathView testStem, auto && test) + { + auto file = goldenMaster(testStem); + + if (testAccept()) + { + GTEST_SKIP() + << "Cannot read golden master " + << file + << "because another test is also updating it"; + } + else + { + test(readFile(file)); + } + } + + /** + * Golden test for writing + * + * @param test hook that produces contents of the file and does the + * actual work + */ + void writeTest( + PathView testStem, auto && test, auto && readFile2, auto && writeFile2) + { + auto file = goldenMaster(testStem); + + auto got = test(); + + if (testAccept()) + { + createDirs(dirOf(file)); + writeFile2(file, got); + GTEST_SKIP() + << "Updating golden master " + << file; + } + else + { + decltype(got) expected = readFile2(file); + ASSERT_EQ(got, expected); + } + } + + /** + * Specialize to `std::string` + */ + void writeTest(PathView testStem, auto && test) + { + writeTest( + testStem, test, + [](const Path & f) -> std::string { + return readFile(f); + }, + [](const Path & f, const std::string & c) { + return writeFile(f, c); + }); + } +}; + +} diff --git a/src/libutil/tests/git.cc b/src/libutil/tests/git.cc index 5b5715fc26bb..2842ea4d03f5 100644 --- a/src/libutil/tests/git.cc +++ b/src/libutil/tests/git.cc @@ -1,33 +1,236 @@ -#include "git.hh" #include +#include "git.hh" +#include "memory-source-accessor.hh" + +#include "tests/characterization.hh" + namespace nix { - TEST(GitLsRemote, parseSymrefLineWithReference) { - auto line = "ref: refs/head/main HEAD"; - auto res = git::parseLsRemoteLine(line); - ASSERT_TRUE(res.has_value()); - ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Symbolic); - ASSERT_EQ(res->target, "refs/head/main"); - ASSERT_EQ(res->reference, "HEAD"); - } +using namespace git; + +class GitTest : public CharacterizationTest +{ + Path unitTestData = getUnitTestData() + "/libutil/git"; - TEST(GitLsRemote, parseSymrefLineWithNoReference) { - auto line = "ref: refs/head/main"; - auto res = git::parseLsRemoteLine(line); - ASSERT_TRUE(res.has_value()); - ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Symbolic); - ASSERT_EQ(res->target, "refs/head/main"); - ASSERT_EQ(res->reference, std::nullopt); +public: + + Path goldenMaster(std::string_view testStem) const override { + return unitTestData + "/" + testStem; } - TEST(GitLsRemote, parseObjectRefLine) { - auto line = "abc123 refs/head/main"; - auto res = git::parseLsRemoteLine(line); - ASSERT_TRUE(res.has_value()); - ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Object); - ASSERT_EQ(res->target, "abc123"); - ASSERT_EQ(res->reference, "refs/head/main"); + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + +private: + + void SetUp() override + { + mockXpSettings.set("experimental-features", "git-hashing"); } +}; + +TEST(GitMode, gitMode_directory) { + Mode m = Mode::Directory; + RawMode r = 0040000; + ASSERT_EQ(static_cast(m), r); + ASSERT_EQ(decodeMode(r), std::optional { m }); +}; + +TEST(GitMode, gitMode_executable) { + Mode m = Mode::Executable; + RawMode r = 0100755; + ASSERT_EQ(static_cast(m), r); + ASSERT_EQ(decodeMode(r), std::optional { m }); +}; + +TEST(GitMode, gitMode_regular) { + Mode m = Mode::Regular; + RawMode r = 0100644; + ASSERT_EQ(static_cast(m), r); + ASSERT_EQ(decodeMode(r), std::optional { m }); +}; + +TEST(GitMode, gitMode_symlink) { + Mode m = Mode::Symlink; + RawMode r = 0120000; + ASSERT_EQ(static_cast(m), r); + ASSERT_EQ(decodeMode(r), std::optional { m }); +}; + +TEST_F(GitTest, blob_read) { + readTest("hello-world-blob.bin", [&](const auto & encoded) { + StringSource in { encoded }; + StringSink out; + RegularFileSink out2 { out }; + parse(out2, "", in, [](auto &, auto) {}, mockXpSettings); + + auto expected = readFile(goldenMaster("hello-world.bin")); + + ASSERT_EQ(out.s, expected); + }); +} + +TEST_F(GitTest, blob_write) { + writeTest("hello-world-blob.bin", [&]() { + auto decoded = readFile(goldenMaster("hello-world.bin")); + StringSink s; + dumpBlobPrefix(decoded.size(), s, mockXpSettings); + s(decoded); + return s.s; + }); +} + +/** + * This data is for "shallow" tree tests. However, we use "real" hashes + * so that we can check our test data in the corresponding functional + * test (`git-hashing/unit-test-data`). + */ +const static Tree tree = { + { + "Foo", + { + .mode = Mode::Regular, + // hello world with special chars from above + .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", htSHA1), + }, + }, + { + "bAr", + { + .mode = Mode::Executable, + // ditto + .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", htSHA1), + }, + }, + { + "baZ/", + { + .mode = Mode::Directory, + // Empty directory hash + .hash = Hash::parseAny("4b825dc642cb6eb9a060e54bf8d69288fbee4904", htSHA1), + }, + }, +}; + +TEST_F(GitTest, tree_read) { + readTest("tree.bin", [&](const auto & encoded) { + StringSource in { encoded }; + NullParseSink out; + Tree got; + parse(out, "", in, [&](auto & name, auto entry) { + auto name2 = name; + if (entry.mode == Mode::Directory) + name2 += '/'; + got.insert_or_assign(name2, std::move(entry)); + }, mockXpSettings); + + ASSERT_EQ(got, tree); + }); +} + +TEST_F(GitTest, tree_write) { + writeTest("tree.bin", [&]() { + StringSink s; + dumpTree(tree, s, mockXpSettings); + return s.s; + }); } +TEST_F(GitTest, both_roundrip) { + using File = MemorySourceAccessor::File; + + MemorySourceAccessor files; + files.root = File::Directory { + .contents { + { + "foo", + File::Regular { + .contents = "hello\n\0\n\tworld!", + }, + }, + { + "bar", + File::Directory { + .contents = { + { + "baz", + File::Regular { + .executable = true, + .contents = "good day,\n\0\n\tworld!", + }, + }, + }, + }, + }, + }, + }; + + std::map cas; + + std::function dumpHook; + dumpHook = [&](const CanonPath & path) { + StringSink s; + HashSink hashSink { htSHA1 }; + TeeSink s2 { s, hashSink }; + auto mode = dump( + files, path, s2, dumpHook, + defaultPathFilter, mockXpSettings); + auto hash = hashSink.finish().first; + cas.insert_or_assign(hash, std::move(s.s)); + return TreeEntry { + .mode = mode, + .hash = hash, + }; + }; + + auto root = dumpHook(CanonPath::root); + + MemorySourceAccessor files2; + + MemorySink sinkFiles2 { files2 }; + + std::function mkSinkHook; + mkSinkHook = [&](const Path prefix, const Hash & hash) { + StringSource in { cas[hash] }; + parse(sinkFiles2, prefix, in, [&](const Path & name, const auto & entry) { + mkSinkHook(prefix + "/" + name, entry.hash); + }, mockXpSettings); + }; + + mkSinkHook("", root.hash); + + ASSERT_EQ(files, files2); +} + +TEST(GitLsRemote, parseSymrefLineWithReference) { + auto line = "ref: refs/head/main HEAD"; + auto res = parseLsRemoteLine(line); + ASSERT_TRUE(res.has_value()); + ASSERT_EQ(res->kind, LsRemoteRefLine::Kind::Symbolic); + ASSERT_EQ(res->target, "refs/head/main"); + ASSERT_EQ(res->reference, "HEAD"); +} + +TEST(GitLsRemote, parseSymrefLineWithNoReference) { + auto line = "ref: refs/head/main"; + auto res = parseLsRemoteLine(line); + ASSERT_TRUE(res.has_value()); + ASSERT_EQ(res->kind, LsRemoteRefLine::Kind::Symbolic); + ASSERT_EQ(res->target, "refs/head/main"); + ASSERT_EQ(res->reference, std::nullopt); +} + +TEST(GitLsRemote, parseObjectRefLine) { + auto line = "abc123 refs/head/main"; + auto res = parseLsRemoteLine(line); + ASSERT_TRUE(res.has_value()); + ASSERT_EQ(res->kind, LsRemoteRefLine::Kind::Object); + ASSERT_EQ(res->target, "abc123"); + ASSERT_EQ(res->reference, "refs/head/main"); +} + +} diff --git a/src/libutil/tests/hash.cc b/src/libutil/tests/hash.cc index e4e928b3b40d..9a5ebbb30fdc 100644 --- a/src/libutil/tests/hash.cc +++ b/src/libutil/tests/hash.cc @@ -18,28 +18,28 @@ namespace nix { // values taken from: https://tools.ietf.org/html/rfc1321 auto s1 = ""; auto hash = hashString(HashType::htMD5, s1); - ASSERT_EQ(hash.to_string(Base::Base16, true), "md5:d41d8cd98f00b204e9800998ecf8427e"); + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "md5:d41d8cd98f00b204e9800998ecf8427e"); } TEST(hashString, testKnownMD5Hashes2) { // values taken from: https://tools.ietf.org/html/rfc1321 auto s2 = "abc"; auto hash = hashString(HashType::htMD5, s2); - ASSERT_EQ(hash.to_string(Base::Base16, true), "md5:900150983cd24fb0d6963f7d28e17f72"); + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "md5:900150983cd24fb0d6963f7d28e17f72"); } TEST(hashString, testKnownSHA1Hashes1) { // values taken from: https://tools.ietf.org/html/rfc3174 auto s = "abc"; auto hash = hashString(HashType::htSHA1, s); - ASSERT_EQ(hash.to_string(Base::Base16, true),"sha1:a9993e364706816aba3e25717850c26c9cd0d89d"); + ASSERT_EQ(hash.to_string(HashFormat::Base16, true),"sha1:a9993e364706816aba3e25717850c26c9cd0d89d"); } TEST(hashString, testKnownSHA1Hashes2) { // values taken from: https://tools.ietf.org/html/rfc3174 auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; auto hash = hashString(HashType::htSHA1, s); - ASSERT_EQ(hash.to_string(Base::Base16, true),"sha1:84983e441c3bd26ebaae4aa1f95129e5e54670f1"); + ASSERT_EQ(hash.to_string(HashFormat::Base16, true),"sha1:84983e441c3bd26ebaae4aa1f95129e5e54670f1"); } TEST(hashString, testKnownSHA256Hashes1) { @@ -47,7 +47,7 @@ namespace nix { auto s = "abc"; auto hash = hashString(HashType::htSHA256, s); - ASSERT_EQ(hash.to_string(Base::Base16, true), + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); } @@ -55,7 +55,7 @@ namespace nix { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; auto hash = hashString(HashType::htSHA256, s); - ASSERT_EQ(hash.to_string(Base::Base16, true), + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha256:248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1"); } @@ -63,7 +63,7 @@ namespace nix { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abc"; auto hash = hashString(HashType::htSHA512, s); - ASSERT_EQ(hash.to_string(Base::Base16, true), + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha512:ddaf35a193617abacc417349ae20413112e6fa4e89a9" "7ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd" "454d4423643ce80e2a9ac94fa54ca49f"); @@ -74,11 +74,26 @@ namespace nix { auto s = "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"; auto hash = hashString(HashType::htSHA512, s); - ASSERT_EQ(hash.to_string(Base::Base16, true), + ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha512:8e959b75dae313da8cf4f72814fc143f8f7779c6eb9f7fa1" "7299aeadb6889018501d289e4900f7e4331b99dec4b5433a" "c7d329eeb6dd26545e96e55b874be909"); } + + /* ---------------------------------------------------------------------------- + * parseHashFormat, parseHashFormatOpt, printHashFormat + * --------------------------------------------------------------------------*/ + + TEST(hashFormat, testRoundTripPrintParse) { + for (const HashFormat hashFormat: { HashFormat::Base64, HashFormat::Base32, HashFormat::Base16, HashFormat::SRI}) { + ASSERT_EQ(parseHashFormat(printHashFormat(hashFormat)), hashFormat); + ASSERT_EQ(*parseHashFormatOpt(printHashFormat(hashFormat)), hashFormat); + } + } + + TEST(hashFormat, testParseHashFormatOptException) { + ASSERT_EQ(parseHashFormatOpt("sha0042"), std::nullopt); + } } namespace rc { diff --git a/src/libutil/tests/json-utils.cc b/src/libutil/tests/json-utils.cc new file mode 100644 index 000000000000..f0ce15c93b5c --- /dev/null +++ b/src/libutil/tests/json-utils.cc @@ -0,0 +1,58 @@ +#include +#include + +#include + +#include "json-utils.hh" + +namespace nix { + +/* Test `to_json` and `from_json` with `std::optional` types. + * We are specifically interested in whether we can _nest_ optionals in STL + * containers so we that we can leverage existing adl_serializer templates. */ + +TEST(to_json, optionalInt) { + std::optional val = std::make_optional(420); + ASSERT_EQ(nlohmann::json(val), nlohmann::json(420)); + val = std::nullopt; + ASSERT_EQ(nlohmann::json(val), nlohmann::json(nullptr)); +} + +TEST(to_json, vectorOfOptionalInts) { + std::vector> vals = { + std::make_optional(420), + std::nullopt, + }; + ASSERT_EQ(nlohmann::json(vals), nlohmann::json::parse("[420,null]")); +} + +TEST(to_json, optionalVectorOfInts) { + std::optional> val = std::make_optional(std::vector { + -420, + 420, + }); + ASSERT_EQ(nlohmann::json(val), nlohmann::json::parse("[-420,420]")); + val = std::nullopt; + ASSERT_EQ(nlohmann::json(val), nlohmann::json(nullptr)); +} + +TEST(from_json, optionalInt) { + nlohmann::json json = 420; + std::optional val = json; + ASSERT_TRUE(val.has_value()); + ASSERT_EQ(*val, 420); + json = nullptr; + json.get_to(val); + ASSERT_FALSE(val.has_value()); +} + +TEST(from_json, vectorOfOptionalInts) { + nlohmann::json json = { 420, nullptr }; + std::vector> vals = json; + ASSERT_EQ(vals.size(), 2); + ASSERT_TRUE(vals.at(0).has_value()); + ASSERT_EQ(*vals.at(0), 420); + ASSERT_FALSE(vals.at(1).has_value()); +} + +} /* namespace nix */ diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk index 167915439fd0..66886c45fc2d 100644 --- a/src/libutil/tests/local.mk +++ b/src/libutil/tests/local.mk @@ -1,12 +1,16 @@ -check: libutil-tests_RUN +check: libutil-tests-exe_RUN -programs += libutil-tests +programs += libutil-tests-exe libutil-tests-exe_NAME = libnixutil-tests libutil-tests-exe_DIR := $(d) -libutil-tests-exe_INSTALL_DIR := +ifeq ($(INSTALL_UNIT_TESTS), yes) + libutil-tests-exe_INSTALL_DIR := $(checkbindir) +else + libutil-tests-exe_INSTALL_DIR := +endif libutil-tests-exe_LIBS = libutil-tests @@ -18,7 +22,11 @@ libutil-tests_NAME = libnixutil-tests libutil-tests_DIR := $(d) -libutil-tests_INSTALL_DIR := +ifeq ($(INSTALL_UNIT_TESTS), yes) + libutil-tests_INSTALL_DIR := $(checklibdir) +else + libutil-tests_INSTALL_DIR := +endif libutil-tests_SOURCES := $(wildcard $(d)/*.cc) @@ -27,3 +35,7 @@ libutil-tests_CXXFLAGS += -I src/libutil libutil-tests_LIBS = libutil libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) + +check: unit-test-data/libutil/git/check-data.sh.test + +$(eval $(call run-test,unit-test-data/libutil/git/check-data.sh)) diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc index 2ffdc2e9b889..c6dfe63d39b3 100644 --- a/src/libutil/tests/logging.cc +++ b/src/libutil/tests/logging.cc @@ -2,7 +2,6 @@ #include "logging.hh" #include "nixexpr.hh" -#include "util.hh" #include #include diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc index f3c1e8248a03..568f03f702d4 100644 --- a/src/libutil/tests/tests.cc +++ b/src/libutil/tests/tests.cc @@ -1,5 +1,8 @@ #include "util.hh" #include "types.hh" +#include "file-system.hh" +#include "processes.hh" +#include "terminal.hh" #include #include diff --git a/src/libutil/tests/url.cc b/src/libutil/tests/url.cc index a908631e6418..a678dad20414 100644 --- a/src/libutil/tests/url.cc +++ b/src/libutil/tests/url.cc @@ -335,4 +335,13 @@ namespace nix { ASSERT_EQ(d, s); } + TEST(percentEncode, yen) { + // https://en.wikipedia.org/wiki/Percent-encoding#Character_data + std::string s = reinterpret_cast(u8"円"); + std::string e = "%E5%86%86"; + + ASSERT_EQ(percentEncode(s), e); + ASSERT_EQ(percentDecode(e), s); + } + } diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc index dc4067f1b3a1..c5e735617391 100644 --- a/src/libutil/thread-pool.cc +++ b/src/libutil/thread-pool.cc @@ -1,4 +1,6 @@ #include "thread-pool.hh" +#include "signals.hh" +#include "util.hh" namespace nix { diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh index 0e09fae97628..02765badc822 100644 --- a/src/libutil/thread-pool.hh +++ b/src/libutil/thread-pool.hh @@ -1,8 +1,8 @@ #pragma once ///@file +#include "error.hh" #include "sync.hh" -#include "util.hh" #include #include diff --git a/src/libutil/unix-domain-socket.cc b/src/libutil/unix-domain-socket.cc new file mode 100644 index 000000000000..8949461d247e --- /dev/null +++ b/src/libutil/unix-domain-socket.cc @@ -0,0 +1,100 @@ +#include "file-system.hh" +#include "processes.hh" +#include "unix-domain-socket.hh" + +#include +#include +#include + +namespace nix { + +AutoCloseFD createUnixDomainSocket() +{ + AutoCloseFD fdSocket = socket(PF_UNIX, SOCK_STREAM + #ifdef SOCK_CLOEXEC + | SOCK_CLOEXEC + #endif + , 0); + if (!fdSocket) + throw SysError("cannot create Unix domain socket"); + closeOnExec(fdSocket.get()); + return fdSocket; +} + + +AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) +{ + auto fdSocket = nix::createUnixDomainSocket(); + + bind(fdSocket.get(), path); + + if (chmod(path.c_str(), mode) == -1) + throw SysError("changing permissions on '%1%'", path); + + if (listen(fdSocket.get(), 100) == -1) + throw SysError("cannot listen on socket '%1%'", path); + + return fdSocket; +} + + +void bind(int fd, const std::string & path) +{ + unlink(path.c_str()); + + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + + if (path.size() + 1 >= sizeof(addr.sun_path)) { + Pid pid = startProcess([&]() { + Path dir = dirOf(path); + if (chdir(dir.c_str()) == -1) + throw SysError("chdir to '%s' failed", dir); + std::string base(baseNameOf(path)); + if (base.size() + 1 >= sizeof(addr.sun_path)) + throw Error("socket path '%s' is too long", base); + memcpy(addr.sun_path, base.c_str(), base.size() + 1); + if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot bind to socket '%s'", path); + _exit(0); + }); + int status = pid.wait(); + if (status != 0) + throw Error("cannot bind to socket '%s'", path); + } else { + memcpy(addr.sun_path, path.c_str(), path.size() + 1); + if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot bind to socket '%s'", path); + } +} + + +void connect(int fd, const std::string & path) +{ + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + + if (path.size() + 1 >= sizeof(addr.sun_path)) { + Pid pid = startProcess([&]() { + Path dir = dirOf(path); + if (chdir(dir.c_str()) == -1) + throw SysError("chdir to '%s' failed", dir); + std::string base(baseNameOf(path)); + if (base.size() + 1 >= sizeof(addr.sun_path)) + throw Error("socket path '%s' is too long", base); + memcpy(addr.sun_path, base.c_str(), base.size() + 1); + if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot connect to socket at '%s'", path); + _exit(0); + }); + int status = pid.wait(); + if (status != 0) + throw Error("cannot connect to socket at '%s'", path); + } else { + memcpy(addr.sun_path, path.c_str(), path.size() + 1); + if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot connect to socket at '%s'", path); + } +} + +} diff --git a/src/libutil/unix-domain-socket.hh b/src/libutil/unix-domain-socket.hh new file mode 100644 index 000000000000..b78feb454b18 --- /dev/null +++ b/src/libutil/unix-domain-socket.hh @@ -0,0 +1,31 @@ +#pragma once +///@file + +#include "types.hh" +#include "file-descriptor.hh" + +#include + +namespace nix { + +/** + * Create a Unix domain socket. + */ +AutoCloseFD createUnixDomainSocket(); + +/** + * Create a Unix domain socket in listen mode. + */ +AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); + +/** + * Bind a Unix domain socket to a path. + */ +void bind(int fd, const std::string & path); + +/** + * Connect to a Unix domain socket. + */ +void connect(int fd, const std::string & path); + +} diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh index 98162b0f78a2..5c5a30dc2109 100644 --- a/src/libutil/url-parts.hh +++ b/src/libutil/url-parts.hh @@ -30,7 +30,7 @@ extern std::regex refRegex; /// Instead of defining what a good Git Ref is, we define what a bad Git Ref is /// This is because of the definition of a ref in refs.c in https://github.com/git/git -/// See tests/fetchGitRefs.sh for the full definition +/// See tests/functional/fetchGitRefs.sh for the full definition const static std::string badGitRefRegexS = "//|^[./]|/\\.|\\.\\.|[[:cntrl:][:space:]:?^~\[]|\\\\|\\*|\\.lock$|\\.lock/|@\\{|[/.]$|^@$|^$"; extern std::regex badGitRefRegex; @@ -41,7 +41,4 @@ extern std::regex revRegex; /// A ref or revision, or a ref followed by a revision. const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegexS + ")(?:/(" + revRegexS + "))?))"; -const static std::string flakeIdRegexS = "[a-zA-Z][a-zA-Z0-9_-]*"; -extern std::regex flakeIdRegex; - } diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 9e44241ac256..57b64d6074c1 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -2,13 +2,13 @@ #include "url-parts.hh" #include "util.hh" #include "split.hh" +#include "canon-path.hh" namespace nix { std::regex refRegex(refRegexS, std::regex::ECMAScript); std::regex badGitRefRegex(badGitRefRegexS, std::regex::ECMAScript); std::regex revRegex(revRegexS, std::regex::ECMAScript); -std::regex flakeIdRegex(flakeIdRegexS, std::regex::ECMAScript); ParsedURL parseURL(const std::string & url) { @@ -44,7 +44,7 @@ ParsedURL parseURL(const std::string & url) .base = base, .scheme = scheme, .authority = authority, - .path = path, + .path = percentDecode(path), .query = decodeQuery(query), .fragment = percentDecode(std::string(fragment)) }; @@ -103,7 +103,7 @@ std::string percentEncode(std::string_view s, std::string_view keep) || keep.find(c) != std::string::npos) res += c; else - res += fmt("%%%02X", (unsigned int) c); + res += fmt("%%%02X", c & 0xFF); return res; } @@ -142,6 +142,13 @@ bool ParsedURL::operator ==(const ParsedURL & other) const && fragment == other.fragment; } +ParsedURL ParsedURL::canonicalise() +{ + ParsedURL res(*this); + res.path = CanonPath(res.path).abs(); + return res; +} + /** * Parse a URL scheme of the form '(applicationScheme\+)?transportScheme' * into a tuple '(applicationScheme, transportScheme)' @@ -159,4 +166,21 @@ ParsedUrlScheme parseUrlScheme(std::string_view scheme) }; } +std::string fixGitURL(const std::string & url) +{ + std::regex scpRegex("([^/]*)@(.*):(.*)"); + if (!hasPrefix(url, "/") && std::regex_match(url, scpRegex)) + return std::regex_replace(url, scpRegex, "ssh://$1@$2/$3"); + else { + if (url.find("://") == std::string::npos) { + return (ParsedURL { + .scheme = "file", + .authority = "", + .path = url + }).to_string(); + } else + return url; + } +} + } diff --git a/src/libutil/url.hh b/src/libutil/url.hh index d2413ec0efc5..833f546787b3 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -19,6 +19,11 @@ struct ParsedURL std::string to_string() const; bool operator ==(const ParsedURL & other) const; + + /** + * Remove `.` and `..` path elements. + */ + ParsedURL canonicalise(); }; MakeError(BadURL, Error); @@ -45,4 +50,9 @@ struct ParsedUrlScheme { ParsedUrlScheme parseUrlScheme(std::string_view scheme); +/* Detects scp-style uris (e.g. git@github.com:NixOS/nix) and fixes + them by removing the `:` and assuming a scheme of `ssh://`. Also + changes absolute paths into file:// URLs. */ +std::string fixGitURL(const std::string & url); + } diff --git a/src/libutil/users.cc b/src/libutil/users.cc new file mode 100644 index 000000000000..ad62030251da --- /dev/null +++ b/src/libutil/users.cc @@ -0,0 +1,148 @@ +#include "util.hh" +#include "users.hh" +#include "environment-variables.hh" +#include "file-system.hh" + +#include +#include +#include +#include + +namespace nix { + +std::string getUserName(uid_t uid) +{ + auto pw = getpwuid(uid); + std::string name = pw ? pw->pw_name : getEnv("USER").value_or(""); + if (name.empty()) + throw Error("cannot figure out user name"); + return name; +} + +std::string getUserName() +{ + return getUserName(getuid()); +} + +std::vector getUserGroups(uid_t uid) { + struct passwd * pw = getpwuid(uid); + int ngroups = 0; + getgrouplist(pw->pw_name, pw->pw_gid, NULL, &ngroups); + gid_t _groups[ngroups]; +// Apple takes ints instead of gids for the second and third arguments +#if __APPLE__ + getgrouplist(pw->pw_name, (int) pw->pw_gid, (int *) _groups, &ngroups); +#else + getgrouplist(pw->pw_name, pw->pw_gid, _groups, &ngroups); +#endif + std::vector groups; + for (auto group : _groups) groups.push_back(group); + return groups; +} + +std::vector getUserGroupNames(uid_t uid) { + auto groups = getUserGroups(uid); + std::vector groupsWithNames; + for (auto group : groups) { + struct group * g = getgrgid(group); + groupsWithNames.push_back(g->gr_name); + } + return groupsWithNames; +} + +Path getHomeOf(uid_t userId) +{ + std::vector buf(16384); + struct passwd pwbuf; + struct passwd * pw; + if (getpwuid_r(userId, &pwbuf, buf.data(), buf.size(), &pw) != 0 + || !pw || !pw->pw_dir || !pw->pw_dir[0]) + throw Error("cannot determine user's home directory"); + return pw->pw_dir; +} + +Path getHome() +{ + static Path homeDir = []() + { + std::optional unownedUserHomeDir = {}; + auto homeDir = getEnv("HOME"); + if (homeDir) { + // Only use $HOME if doesn't exist or is owned by the current user. + struct stat st; + int result = stat(homeDir->c_str(), &st); + if (result != 0) { + if (errno != ENOENT) { + warn("couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno); + homeDir.reset(); + } + } else if (st.st_uid != geteuid()) { + unownedUserHomeDir.swap(homeDir); + } + } + if (!homeDir) { + homeDir = getHomeOf(geteuid()); + if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) { + warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir); + } + } + return *homeDir; + }(); + return homeDir; +} + + +Path getCacheDir() +{ + auto cacheDir = getEnv("XDG_CACHE_HOME"); + return cacheDir ? *cacheDir : getHome() + "/.cache"; +} + + +Path getConfigDir() +{ + auto configDir = getEnv("XDG_CONFIG_HOME"); + return configDir ? *configDir : getHome() + "/.config"; +} + +std::vector getConfigDirs() +{ + Path configHome = getConfigDir(); + auto configDirs = getEnv("XDG_CONFIG_DIRS").value_or("/etc/xdg"); + std::vector result = tokenizeString>(configDirs, ":"); + result.insert(result.begin(), configHome); + return result; +} + + +Path getDataDir() +{ + auto dataDir = getEnv("XDG_DATA_HOME"); + return dataDir ? *dataDir : getHome() + "/.local/share"; +} + +Path getStateDir() +{ + auto stateDir = getEnv("XDG_STATE_HOME"); + return stateDir ? *stateDir : getHome() + "/.local/state"; +} + +Path createNixStateDir() +{ + Path dir = getStateDir() + "/nix"; + createDirs(dir); + return dir; +} + + +std::string expandTilde(std::string_view path) +{ + // TODO: expand ~user ? + auto tilde = path.substr(0, 2); + if (tilde == "~/" || tilde == "~") + return getHome() + std::string(path.substr(1)); + else + return std::string(path); +} + +} diff --git a/src/libutil/users.hh b/src/libutil/users.hh new file mode 100644 index 000000000000..cecbb8bfb9ed --- /dev/null +++ b/src/libutil/users.hh @@ -0,0 +1,58 @@ +#pragma once +///@file + +#include "types.hh" + +#include + +namespace nix { + +std::string getUserName(); + +/** + * @return the given user's home directory from /etc/passwd. + */ +Path getHomeOf(uid_t userId); + +/** + * @return $HOME or the user's home directory from /etc/passwd. + */ +Path getHome(); + +/** + * @return $XDG_CACHE_HOME or $HOME/.cache. + */ +Path getCacheDir(); + +/** + * @return $XDG_CONFIG_HOME or $HOME/.config. + */ +Path getConfigDir(); + +/** + * @return the directories to search for user configuration files + */ +std::vector getConfigDirs(); + +/** + * @return $XDG_DATA_HOME or $HOME/.local/share. + */ +Path getDataDir(); + +/** + * @return $XDG_STATE_HOME or $HOME/.local/state. + */ +Path getStateDir(); + +/** + * Create the Nix state directory and return the path to it. + */ +Path createNixStateDir(); + +/** + * Perform tilde expansion on a path, replacing tilde with the user's + * home directory. + */ +std::string expandTilde(std::string_view path); + +} diff --git a/src/libutil/util.cc b/src/libutil/util.cc index bcbcd5ed1d08..9f5c66deea65 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1,1181 +1,38 @@ #include "util.hh" -#include "sync.hh" -#include "finally.hh" -#include "serialise.hh" -#include "cgroup.hh" +#include "fmt.hh" #include #include -#include -#include -#include -#include -#include -#include #include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef __APPLE__ -#include -#include -#endif - -#ifdef __linux__ -#include -#include -#include - -#include -#endif - -#include "execinfo.h" - - -extern char * * environ __attribute__((weak)); - - -namespace nix { - -void initLibUtil() { -} - -std::optional getEnv(const std::string & key) -{ - char * value = getenv(key.c_str()); - if (!value) return {}; - return std::string(value); -} - -std::optional getEnvNonEmpty(const std::string & key) { - auto value = getEnv(key); - if (value == "") return {}; - return value; -} - -std::map getEnv() -{ - std::map env; - for (size_t i = 0; environ[i]; ++i) { - auto s = environ[i]; - auto eq = strchr(s, '='); - if (!eq) - // invalid env, just keep going - continue; - env.emplace(std::string(s, eq), std::string(eq + 1)); - } - return env; -} - - -void clearEnv() -{ - for (auto & name : getEnv()) - unsetenv(name.first.c_str()); -} - -void replaceEnv(const std::map & newEnv) -{ - clearEnv(); - for (auto & newEnvVar : newEnv) - setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); -} - - -Path absPath(Path path, std::optional dir, bool resolveSymlinks) -{ - if (path[0] != '/') { - if (!dir) { -#ifdef __GNU__ - /* GNU (aka. GNU/Hurd) doesn't have any limitation on path - lengths and doesn't define `PATH_MAX'. */ - char *buf = getcwd(NULL, 0); - if (buf == NULL) -#else - char buf[PATH_MAX]; - if (!getcwd(buf, sizeof(buf))) -#endif - throw SysError("cannot get cwd"); - path = concatStrings(buf, "/", path); -#ifdef __GNU__ - free(buf); -#endif - } else - path = concatStrings(*dir, "/", path); - } - return canonPath(path, resolveSymlinks); -} - - -Path canonPath(PathView path, bool resolveSymlinks) -{ - assert(path != ""); - - std::string s; - s.reserve(256); - - if (path[0] != '/') - throw Error("not an absolute path: '%1%'", path); - - std::string temp; - - /* Count the number of times we follow a symlink and stop at some - arbitrary (but high) limit to prevent infinite loops. */ - unsigned int followCount = 0, maxFollow = 1024; - - while (1) { - - /* Skip slashes. */ - while (!path.empty() && path[0] == '/') path.remove_prefix(1); - if (path.empty()) break; - - /* Ignore `.'. */ - if (path == "." || path.substr(0, 2) == "./") - path.remove_prefix(1); - - /* If `..', delete the last component. */ - else if (path == ".." || path.substr(0, 3) == "../") - { - if (!s.empty()) s.erase(s.rfind('/')); - path.remove_prefix(2); - } - - /* Normal component; copy it. */ - else { - s += '/'; - if (const auto slash = path.find('/'); slash == std::string::npos) { - s += path; - path = {}; - } else { - s += path.substr(0, slash); - path = path.substr(slash); - } - - /* If s points to a symlink, resolve it and continue from there */ - if (resolveSymlinks && isLink(s)) { - if (++followCount >= maxFollow) - throw Error("infinite symlink recursion in path '%1%'", path); - temp = concatStrings(readLink(s), path); - path = temp; - if (!temp.empty() && temp[0] == '/') { - s.clear(); /* restart for symlinks pointing to absolute path */ - } else { - s = dirOf(s); - if (s == "/") { // we don’t want trailing slashes here, which dirOf only produces if s = / - s.clear(); - } - } - } - } - } - - return s.empty() ? "/" : std::move(s); -} - - -Path dirOf(const PathView path) -{ - Path::size_type pos = path.rfind('/'); - if (pos == std::string::npos) - return "."; - return pos == 0 ? "/" : Path(path, 0, pos); -} - - -std::string_view baseNameOf(std::string_view path) -{ - if (path.empty()) - return ""; - - auto last = path.size() - 1; - if (path[last] == '/' && last > 0) - last -= 1; - - auto pos = path.rfind('/', last); - if (pos == std::string::npos) - pos = 0; - else - pos += 1; - - return path.substr(pos, last - pos + 1); -} - - -std::string expandTilde(std::string_view path) -{ - // TODO: expand ~user ? - auto tilde = path.substr(0, 2); - if (tilde == "~/" || tilde == "~") - return getHome() + std::string(path.substr(1)); - else - return std::string(path); -} - - -bool isInDir(std::string_view path, std::string_view dir) -{ - return path.substr(0, 1) == "/" - && path.substr(0, dir.size()) == dir - && path.size() >= dir.size() + 2 - && path[dir.size()] == '/'; -} - - -bool isDirOrInDir(std::string_view path, std::string_view dir) -{ - return path == dir || isInDir(path, dir); -} - - -struct stat stat(const Path & path) -{ - struct stat st; - if (stat(path.c_str(), &st)) - throw SysError("getting status of '%1%'", path); - return st; -} - - -struct stat lstat(const Path & path) -{ - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError("getting status of '%1%'", path); - return st; -} - - -bool pathExists(const Path & path) -{ - int res; - struct stat st; - res = lstat(path.c_str(), &st); - if (!res) return true; - if (errno != ENOENT && errno != ENOTDIR) - throw SysError("getting status of %1%", path); - return false; -} - -bool pathAccessible(const Path & path) -{ - try { - return pathExists(path); - } catch (SysError & e) { - // swallow EPERM - if (e.errNo == EPERM) return false; - throw; - } -} - - -Path readLink(const Path & path) -{ - checkInterrupt(); - std::vector buf; - for (ssize_t bufSize = PATH_MAX/4; true; bufSize += bufSize/2) { - buf.resize(bufSize); - ssize_t rlSize = readlink(path.c_str(), buf.data(), bufSize); - if (rlSize == -1) - if (errno == EINVAL) - throw Error("'%1%' is not a symlink", path); - else - throw SysError("reading symbolic link '%1%'", path); - else if (rlSize < bufSize) - return std::string(buf.data(), rlSize); - } -} - - -bool isLink(const Path & path) -{ - struct stat st = lstat(path); - return S_ISLNK(st.st_mode); -} - - -DirEntries readDirectory(DIR *dir, const Path & path) -{ - DirEntries entries; - entries.reserve(64); - - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir)) { /* sic */ - checkInterrupt(); - std::string name = dirent->d_name; - if (name == "." || name == "..") continue; - entries.emplace_back(name, dirent->d_ino, -#ifdef HAVE_STRUCT_DIRENT_D_TYPE - dirent->d_type -#else - DT_UNKNOWN -#endif - ); - } - if (errno) throw SysError("reading directory '%1%'", path); - - return entries; -} - -DirEntries readDirectory(const Path & path) -{ - AutoCloseDir dir(opendir(path.c_str())); - if (!dir) throw SysError("opening directory '%1%'", path); - - return readDirectory(dir.get(), path); -} - - -unsigned char getFileType(const Path & path) -{ - struct stat st = lstat(path); - if (S_ISDIR(st.st_mode)) return DT_DIR; - if (S_ISLNK(st.st_mode)) return DT_LNK; - if (S_ISREG(st.st_mode)) return DT_REG; - return DT_UNKNOWN; -} - - -std::string readFile(int fd) -{ - struct stat st; - if (fstat(fd, &st) == -1) - throw SysError("statting file"); - - return drainFD(fd, true, st.st_size); -} - - -std::string readFile(const Path & path) -{ - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) - throw SysError("opening file '%1%'", path); - return readFile(fd.get()); -} - - -void readFile(const Path & path, Sink & sink) -{ - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) - throw SysError("opening file '%s'", path); - drainFD(fd.get(), sink); -} - - -void writeFile(const Path & path, std::string_view s, mode_t mode, bool sync) -{ - AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); - if (!fd) - throw SysError("opening file '%1%'", path); - try { - writeFull(fd.get(), s); - } catch (Error & e) { - e.addTrace({}, "writing file '%1%'", path); - throw; - } - if (sync) - fd.fsync(); - // Explicitly close to make sure exceptions are propagated. - fd.close(); - if (sync) - syncParent(path); -} - - -void writeFile(const Path & path, Source & source, mode_t mode, bool sync) -{ - AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); - if (!fd) - throw SysError("opening file '%1%'", path); - - std::vector buf(64 * 1024); - - try { - while (true) { - try { - auto n = source.read(buf.data(), buf.size()); - writeFull(fd.get(), {buf.data(), n}); - } catch (EndOfFile &) { break; } - } - } catch (Error & e) { - e.addTrace({}, "writing file '%1%'", path); - throw; - } - if (sync) - fd.fsync(); - // Explicitly close to make sure exceptions are propagated. - fd.close(); - if (sync) - syncParent(path); -} - -void syncParent(const Path & path) -{ - AutoCloseFD fd = open(dirOf(path).c_str(), O_RDONLY, 0); - if (!fd) - throw SysError("opening file '%1%'", path); - fd.fsync(); -} - -std::string readLine(int fd) -{ - std::string s; - while (1) { - checkInterrupt(); - char ch; - // FIXME: inefficient - ssize_t rd = read(fd, &ch, 1); - if (rd == -1) { - if (errno != EINTR) - throw SysError("reading a line"); - } else if (rd == 0) - throw EndOfFile("unexpected EOF reading a line"); - else { - if (ch == '\n') return s; - s += ch; - } - } -} - - -void writeLine(int fd, std::string s) -{ - s += '\n'; - writeFull(fd, s); -} - - -static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed) -{ - checkInterrupt(); - - std::string name(baseNameOf(path)); - - struct stat st; - if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) { - if (errno == ENOENT) return; - throw SysError("getting status of '%1%'", path); - } - - if (!S_ISDIR(st.st_mode)) { - /* We are about to delete a file. Will it likely free space? */ - - switch (st.st_nlink) { - /* Yes: last link. */ - case 1: - bytesFreed += st.st_size; - break; - /* Maybe: yes, if 'auto-optimise-store' or manual optimisation - was performed. Instead of checking for real let's assume - it's an optimised file and space will be freed. - - In worst case we will double count on freed space for files - with exactly two hardlinks for unoptimised packages. - */ - case 2: - bytesFreed += st.st_size; - break; - /* No: 3+ links. */ - default: - break; - } - } - - if (S_ISDIR(st.st_mode)) { - /* Make the directory accessible. */ - const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR; - if ((st.st_mode & PERM_MASK) != PERM_MASK) { - if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1) - throw SysError("chmod '%1%'", path); - } - - int fd = openat(parentfd, path.c_str(), O_RDONLY); - if (fd == -1) - throw SysError("opening directory '%1%'", path); - AutoCloseDir dir(fdopendir(fd)); - if (!dir) - throw SysError("opening directory '%1%'", path); - for (auto & i : readDirectory(dir.get(), path)) - _deletePath(dirfd(dir.get()), path + "/" + i.name, bytesFreed); - } - - int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0; - if (unlinkat(parentfd, name.c_str(), flags) == -1) { - if (errno == ENOENT) return; - throw SysError("cannot unlink '%1%'", path); - } -} - -static void _deletePath(const Path & path, uint64_t & bytesFreed) -{ - Path dir = dirOf(path); - if (dir == "") - dir = "/"; - - AutoCloseFD dirfd{open(dir.c_str(), O_RDONLY)}; - if (!dirfd) { - if (errno == ENOENT) return; - throw SysError("opening directory '%1%'", path); - } - - _deletePath(dirfd.get(), path, bytesFreed); -} - - -void deletePath(const Path & path) -{ - uint64_t dummy; - deletePath(path, dummy); -} - - -void deletePath(const Path & path, uint64_t & bytesFreed) -{ - //Activity act(*logger, lvlDebug, "recursively deleting path '%1%'", path); - bytesFreed = 0; - _deletePath(path, bytesFreed); -} - - -std::string getUserName(uid_t uid) -{ - auto pw = getpwuid(uid); - std::string name = pw ? pw->pw_name : getEnv("USER").value_or(""); - if (name.empty()) - throw Error("cannot figure out user name"); - return name; -} - -std::string getUserName() -{ - return getUserName(getuid()); -} - -std::vector getUserGroups(uid_t uid) { - struct passwd * pw = getpwuid(uid); - int ngroups = 0; - getgrouplist(pw->pw_name, pw->pw_gid, NULL, &ngroups); - gid_t _groups[ngroups]; -// Apple takes ints instead of gids for the second and third arguments -#if __APPLE__ - getgrouplist(pw->pw_name, (int) pw->pw_gid, (int *) _groups, &ngroups); -#else - getgrouplist(pw->pw_name, pw->pw_gid, _groups, &ngroups); -#endif - std::vector groups; - for (auto group : _groups) groups.push_back(group); - return groups; -} - -std::vector getUserGroupNames(uid_t uid) { - auto groups = getUserGroups(uid); - std::vector groupsWithNames; - for (auto group : groups) { - struct group * g = getgrgid(group); - groupsWithNames.push_back(g->gr_name); - } - return groupsWithNames; -} - -Path getHomeOf(uid_t userId) -{ - std::vector buf(16384); - struct passwd pwbuf; - struct passwd * pw; - if (getpwuid_r(userId, &pwbuf, buf.data(), buf.size(), &pw) != 0 - || !pw || !pw->pw_dir || !pw->pw_dir[0]) - throw Error("cannot determine user's home directory"); - return pw->pw_dir; -} - -Path getHome() -{ - static Path homeDir = []() - { - std::optional unownedUserHomeDir = {}; - auto homeDir = getEnv("HOME"); - if (homeDir) { - // Only use $HOME if doesn't exist or is owned by the current user. - struct stat st; - int result = stat(homeDir->c_str(), &st); - if (result != 0) { - if (errno != ENOENT) { - warn("couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno); - homeDir.reset(); - } - } else if (st.st_uid != geteuid()) { - unownedUserHomeDir.swap(homeDir); - } - } - if (!homeDir) { - homeDir = getHomeOf(geteuid()); - if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) { - warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir); - } - } - return *homeDir; - }(); - return homeDir; -} - - -Path getCacheDir() -{ - auto cacheDir = getEnv("XDG_CACHE_HOME"); - return cacheDir ? *cacheDir : getHome() + "/.cache"; -} - - -Path getConfigDir() -{ - auto configDir = getEnv("XDG_CONFIG_HOME"); - return configDir ? *configDir : getHome() + "/.config"; -} - -std::vector getConfigDirs() -{ - Path configHome = getConfigDir(); - auto configDirs = getEnv("XDG_CONFIG_DIRS").value_or("/etc/xdg"); - std::vector result = tokenizeString>(configDirs, ":"); - result.insert(result.begin(), configHome); - return result; -} - - -Path getDataDir() -{ - auto dataDir = getEnv("XDG_DATA_HOME"); - return dataDir ? *dataDir : getHome() + "/.local/share"; -} - -Path getStateDir() -{ - auto stateDir = getEnv("XDG_STATE_HOME"); - return stateDir ? *stateDir : getHome() + "/.local/state"; -} - -Path createNixStateDir() -{ - Path dir = getStateDir() + "/nix"; - createDirs(dir); - return dir; -} - - -std::optional getSelfExe() -{ - static auto cached = []() -> std::optional - { - #if __linux__ - return readLink("/proc/self/exe"); - #elif __APPLE__ - char buf[1024]; - uint32_t size = sizeof(buf); - if (_NSGetExecutablePath(buf, &size) == 0) - return buf; - else - return std::nullopt; - #else - return std::nullopt; - #endif - }(); - return cached; -} - - -Paths createDirs(const Path & path) -{ - Paths created; - if (path == "/") return created; - - struct stat st; - if (lstat(path.c_str(), &st) == -1) { - created = createDirs(dirOf(path)); - if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST) - throw SysError("creating directory '%1%'", path); - st = lstat(path); - created.push_back(path); - } - - if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1) - throw SysError("statting symlink '%1%'", path); - - if (!S_ISDIR(st.st_mode)) throw Error("'%1%' is not a directory", path); - - return created; -} - - -void readFull(int fd, char * buf, size_t count) -{ - while (count) { - checkInterrupt(); - ssize_t res = read(fd, buf, count); - if (res == -1) { - if (errno == EINTR) continue; - throw SysError("reading from file"); - } - if (res == 0) throw EndOfFile("unexpected end-of-file"); - count -= res; - buf += res; - } -} - - -void writeFull(int fd, std::string_view s, bool allowInterrupts) -{ - while (!s.empty()) { - if (allowInterrupts) checkInterrupt(); - ssize_t res = write(fd, s.data(), s.size()); - if (res == -1 && errno != EINTR) - throw SysError("writing to file"); - if (res > 0) - s.remove_prefix(res); - } -} - - -std::string drainFD(int fd, bool block, const size_t reserveSize) -{ - // the parser needs two extra bytes to append terminating characters, other users will - // not care very much about the extra memory. - StringSink sink(reserveSize + 2); - drainFD(fd, sink, block); - return std::move(sink.s); -} - - -void drainFD(int fd, Sink & sink, bool block) -{ - // silence GCC maybe-uninitialized warning in finally - int saved = 0; - - if (!block) { - saved = fcntl(fd, F_GETFL); - if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) - throw SysError("making file descriptor non-blocking"); - } - - Finally finally([&]() { - if (!block) { - if (fcntl(fd, F_SETFL, saved) == -1) - throw SysError("making file descriptor blocking"); - } - }); - - std::vector buf(64 * 1024); - while (1) { - checkInterrupt(); - ssize_t rd = read(fd, buf.data(), buf.size()); - if (rd == -1) { - if (!block && (errno == EAGAIN || errno == EWOULDBLOCK)) - break; - if (errno != EINTR) - throw SysError("reading from file"); - } - else if (rd == 0) break; - else sink({(char *) buf.data(), (size_t) rd}); - } -} - -////////////////////////////////////////////////////////////////////// - -unsigned int getMaxCPU() -{ - #if __linux__ - try { - auto cgroupFS = getCgroupFS(); - if (!cgroupFS) return 0; - - auto cgroups = getCgroups("/proc/self/cgroup"); - auto cgroup = cgroups[""]; - if (cgroup == "") return 0; - - auto cpuFile = *cgroupFS + "/" + cgroup + "/cpu.max"; - - auto cpuMax = readFile(cpuFile); - auto cpuMaxParts = tokenizeString>(cpuMax, " \n"); - auto quota = cpuMaxParts[0]; - auto period = cpuMaxParts[1]; - if (quota != "max") - return std::ceil(std::stoi(quota) / std::stof(period)); - } catch (Error &) { ignoreException(lvlDebug); } - #endif - - return 0; -} - -////////////////////////////////////////////////////////////////////// - - -AutoDelete::AutoDelete() : del{false} {} - -AutoDelete::AutoDelete(const std::string & p, bool recursive) : path(p) -{ - del = true; - this->recursive = recursive; -} - -AutoDelete::~AutoDelete() -{ - try { - if (del) { - if (recursive) - deletePath(path); - else { - if (remove(path.c_str()) == -1) - throw SysError("cannot unlink '%1%'", path); - } - } - } catch (...) { - ignoreException(); - } -} - -void AutoDelete::cancel() -{ - del = false; -} - -void AutoDelete::reset(const Path & p, bool recursive) { - path = p; - this->recursive = recursive; - del = true; -} - - - -////////////////////////////////////////////////////////////////////// - - -AutoCloseFD::AutoCloseFD() : fd{-1} {} - - -AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {} - - -AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd} -{ - that.fd = -1; -} - - -AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that) -{ - close(); - fd = that.fd; - that.fd = -1; - return *this; -} - - -AutoCloseFD::~AutoCloseFD() -{ - try { - close(); - } catch (...) { - ignoreException(); - } -} - - -int AutoCloseFD::get() const -{ - return fd; -} - - -void AutoCloseFD::close() -{ - if (fd != -1) { - if (::close(fd) == -1) - /* This should never happen. */ - throw SysError("closing file descriptor %1%", fd); - fd = -1; - } -} - -void AutoCloseFD::fsync() -{ - if (fd != -1) { - int result; -#if __APPLE__ - result = ::fcntl(fd, F_FULLFSYNC); -#else - result = ::fsync(fd); -#endif - if (result == -1) - throw SysError("fsync file descriptor %1%", fd); - } -} - - -AutoCloseFD::operator bool() const -{ - return fd != -1; -} - - -int AutoCloseFD::release() -{ - int oldFD = fd; - fd = -1; - return oldFD; -} - - -void Pipe::create() -{ - int fds[2]; -#if HAVE_PIPE2 - if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe"); -#else - if (pipe(fds) != 0) throw SysError("creating pipe"); - closeOnExec(fds[0]); - closeOnExec(fds[1]); -#endif - readSide = fds[0]; - writeSide = fds[1]; -} - - -void Pipe::close() -{ - readSide.close(); - writeSide.close(); -} - - -////////////////////////////////////////////////////////////////////// - - -Pid::Pid() -{ -} - - -Pid::Pid(pid_t pid) - : pid(pid) -{ -} - - -Pid::~Pid() -{ - if (pid != -1) kill(); -} - - -void Pid::operator =(pid_t pid) -{ - if (this->pid != -1 && this->pid != pid) kill(); - this->pid = pid; - killSignal = SIGKILL; // reset signal to default -} - - -Pid::operator pid_t() -{ - return pid; -} - - -int Pid::kill() -{ - assert(pid != -1); - - debug("killing process %1%", pid); - - /* Send the requested signal to the child. If it has its own - process group, send the signal to every process in the child - process group (which hopefully includes *all* its children). */ - if (::kill(separatePG ? -pid : pid, killSignal) != 0) { - /* On BSDs, killing a process group will return EPERM if all - processes in the group are zombies (or something like - that). So try to detect and ignore that situation. */ -#if __FreeBSD__ || __APPLE__ - if (errno != EPERM || ::kill(pid, 0) != 0) -#endif - logError(SysError("killing process %d", pid).info()); - } - - return wait(); -} - - -int Pid::wait() -{ - assert(pid != -1); - while (1) { - int status; - int res = waitpid(pid, &status, 0); - if (res == pid) { - pid = -1; - return status; - } - if (errno != EINTR) - throw SysError("cannot get exit status of PID %d", pid); - checkInterrupt(); - } -} - - -void Pid::setSeparatePG(bool separatePG) -{ - this->separatePG = separatePG; -} - - -void Pid::setKillSignal(int signal) -{ - this->killSignal = signal; -} - - -pid_t Pid::release() -{ - pid_t p = pid; - pid = -1; - return p; -} - - -void killUser(uid_t uid) -{ - debug("killing all processes running under uid '%1%'", uid); - - assert(uid != 0); /* just to be safe... */ - - /* The system call kill(-1, sig) sends the signal `sig' to all - users to which the current process can send signals. So we - fork a process, switch to uid, and send a mass kill. */ - - Pid pid = startProcess([&]() { - - if (setuid(uid) == -1) - throw SysError("setting uid"); - - while (true) { -#ifdef __APPLE__ - /* OSX's kill syscall takes a third parameter that, among - other things, determines if kill(-1, signo) affects the - calling process. In the OSX libc, it's set to true, - which means "follow POSIX", which we don't want here - */ - if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break; -#else - if (kill(-1, SIGKILL) == 0) break; -#endif - if (errno == ESRCH || errno == EPERM) break; /* no more processes */ - if (errno != EINTR) - throw SysError("cannot kill processes for uid '%1%'", uid); - } +#include +#include +#include - _exit(0); - }); - int status = pid.wait(); - if (status != 0) - throw Error("cannot kill processes for uid '%1%': %2%", uid, statusToString(status)); +namespace nix { - /* !!! We should really do some check to make sure that there are - no processes left running under `uid', but there is no portable - way to do so (I think). The most reliable way may be `ps -eo - uid | grep -q $uid'. */ +void initLibUtil() { + // Check that exception handling works. Exception handling has been observed + // not to work on darwin when the linker flags aren't quite right. + // In this case we don't want to expose the user to some unrelated uncaught + // exception, but rather tell them exactly that exception handling is + // broken. + // When exception handling fails, the message tends to be printed by the + // C++ runtime, followed by an abort. + // For example on macOS we might see an error such as + // libc++abi: terminating with uncaught exception of type nix::SysError: error: C++ exception handling is broken. This would appear to be a problem with the way Nix was compiled and/or linked and/or loaded. + bool caught = false; + try { + throwExceptionSelfCheck(); + } catch (const nix::Error & _e) { + caught = true; + } + // This is not actually the main point of this check, but let's make sure anyway: + assert(caught); } - ////////////////////////////////////////////////////////////////////// - -/* Wrapper around vfork to prevent the child process from clobbering - the caller's stack frame in the parent. */ -static pid_t doFork(bool allowVfork, std::function fun) __attribute__((noinline)); -static pid_t doFork(bool allowVfork, std::function fun) -{ -#ifdef __linux__ - pid_t pid = allowVfork ? vfork() : fork(); -#else - pid_t pid = fork(); -#endif - if (pid != 0) return pid; - fun(); - abort(); -} - - -#if __linux__ -static int childEntry(void * arg) -{ - auto main = (std::function *) arg; - (*main)(); - return 1; -} -#endif - - -pid_t startProcess(std::function fun, const ProcessOptions & options) -{ - std::function wrapper = [&]() { - if (!options.allowVfork) - logger = makeSimpleLogger(); - try { -#if __linux__ - if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) - throw SysError("setting death signal"); -#endif - fun(); - } catch (std::exception & e) { - try { - std::cerr << options.errorPrefix << e.what() << "\n"; - } catch (...) { } - } catch (...) { } - if (options.runExitHandlers) - exit(1); - else - _exit(1); - }; - - pid_t pid = -1; - - if (options.cloneFlags) { - #ifdef __linux__ - // Not supported, since then we don't know when to free the stack. - assert(!(options.cloneFlags & CLONE_VM)); - - size_t stackSize = 1 * 1024 * 1024; - auto stack = (char *) mmap(0, stackSize, - PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); - if (stack == MAP_FAILED) throw SysError("allocating stack"); - - Finally freeStack([&]() { munmap(stack, stackSize); }); - - pid = clone(childEntry, stack + stackSize, options.cloneFlags | SIGCHLD, &wrapper); - #else - throw Error("clone flags are only supported on Linux"); - #endif - } else - pid = doFork(options.allowVfork, wrapper); - - if (pid == -1) throw SysError("unable to fork"); - - return pid; -} - - std::vector stringsToCharPtrs(const Strings & ss) { std::vector res; @@ -1184,211 +41,6 @@ std::vector stringsToCharPtrs(const Strings & ss) return res; } -std::string runProgram(Path program, bool searchPath, const Strings & args, - const std::optional & input, bool isInteractive) -{ - auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input, .isInteractive = isInteractive}); - - if (!statusOk(res.first)) - throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first)); - - return res.second; -} - -// Output = error code + "standard out" output stream -std::pair runProgram(RunOptions && options) -{ - StringSink sink; - options.standardOut = &sink; - - int status = 0; - - try { - runProgram2(options); - } catch (ExecError & e) { - status = e.status; - } - - return {status, std::move(sink.s)}; -} - -void runProgram2(const RunOptions & options) -{ - checkInterrupt(); - - assert(!(options.standardIn && options.input)); - - std::unique_ptr source_; - Source * source = options.standardIn; - - if (options.input) { - source_ = std::make_unique(*options.input); - source = source_.get(); - } - - /* Create a pipe. */ - Pipe out, in; - if (options.standardOut) out.create(); - if (source) in.create(); - - ProcessOptions processOptions; - // vfork implies that the environment of the main process and the fork will - // be shared (technically this is undefined, but in practice that's the - // case), so we can't use it if we alter the environment - processOptions.allowVfork = !options.environment; - - std::optional>> resumeLoggerDefer; - if (options.isInteractive) { - logger->pause(); - resumeLoggerDefer.emplace( - []() { - logger->resume(); - } - ); - } - - /* Fork. */ - Pid pid = startProcess([&]() { - if (options.environment) - replaceEnv(*options.environment); - if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); - if (options.mergeStderrToStdout) - if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) - throw SysError("cannot dup stdout into stderr"); - if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) - throw SysError("dupping stdin"); - - if (options.chdir && chdir((*options.chdir).c_str()) == -1) - throw SysError("chdir failed"); - if (options.gid && setgid(*options.gid) == -1) - throw SysError("setgid failed"); - /* Drop all other groups if we're setgid. */ - if (options.gid && setgroups(0, 0) == -1) - throw SysError("setgroups failed"); - if (options.uid && setuid(*options.uid) == -1) - throw SysError("setuid failed"); - - Strings args_(options.args); - args_.push_front(options.program); - - restoreProcessContext(); - - if (options.searchPath) - execvp(options.program.c_str(), stringsToCharPtrs(args_).data()); - // This allows you to refer to a program with a pathname relative - // to the PATH variable. - else - execv(options.program.c_str(), stringsToCharPtrs(args_).data()); - - throw SysError("executing '%1%'", options.program); - }, processOptions); - - out.writeSide.close(); - - std::thread writerThread; - - std::promise promise; - - Finally doJoin([&]() { - if (writerThread.joinable()) - writerThread.join(); - }); - - - if (source) { - in.readSide.close(); - writerThread = std::thread([&]() { - try { - std::vector buf(8 * 1024); - while (true) { - size_t n; - try { - n = source->read(buf.data(), buf.size()); - } catch (EndOfFile &) { - break; - } - writeFull(in.writeSide.get(), {buf.data(), n}); - } - promise.set_value(); - } catch (...) { - promise.set_exception(std::current_exception()); - } - in.writeSide.close(); - }); - } - - if (options.standardOut) - drainFD(out.readSide.get(), *options.standardOut); - - /* Wait for the child to finish. */ - int status = pid.wait(); - - /* Wait for the writer thread to finish. */ - if (source) promise.get_future().get(); - - if (status) - throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status)); -} - - -void closeMostFDs(const std::set & exceptions) -{ -#if __linux__ - try { - for (auto & s : readDirectory("/proc/self/fd")) { - auto fd = std::stoi(s.name); - if (!exceptions.count(fd)) { - debug("closing leaked FD %d", fd); - close(fd); - } - } - return; - } catch (SysError &) { - } -#endif - - int maxFD = 0; - maxFD = sysconf(_SC_OPEN_MAX); - for (int fd = 0; fd < maxFD; ++fd) - if (!exceptions.count(fd)) - close(fd); /* ignore result */ -} - - -void closeOnExec(int fd) -{ - int prev; - if ((prev = fcntl(fd, F_GETFD, 0)) == -1 || - fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1) - throw SysError("setting close-on-exec flag"); -} - - -////////////////////////////////////////////////////////////////////// - - -std::atomic _isInterrupted = false; - -static thread_local bool interruptThrown = false; -thread_local std::function interruptCheck; - -void setInterruptThrown() -{ - interruptThrown = true; -} - -void _interrupted() -{ - /* Block user interrupts while an exception is being handled. - Throwing an exception while another exception is being handled - kills the program! */ - if (!interruptThrown && !std::uncaught_exceptions()) { - interruptThrown = true; - throw Interrupted("interrupted by the user"); - } -} - ////////////////////////////////////////////////////////////////////// @@ -1454,32 +106,6 @@ std::string rewriteStrings(std::string s, const StringMap & rewrites) } -std::string statusToString(int status) -{ - if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { - if (WIFEXITED(status)) - return fmt("failed with exit code %1%", WEXITSTATUS(status)); - else if (WIFSIGNALED(status)) { - int sig = WTERMSIG(status); -#if HAVE_STRSIGNAL - const char * description = strsignal(sig); - return fmt("failed due to signal %1% (%2%)", sig, description); -#else - return fmt("failed due to signal %1%", sig); -#endif - } - else - return "died abnormally"; - } else return "succeeded"; -} - - -bool statusOk(int status) -{ - return WIFEXITED(status) && WEXITSTATUS(status) == 0; -} - - bool hasPrefix(std::string_view s, std::string_view prefix) { return s.compare(0, prefix.size(), prefix) == 0; @@ -1527,82 +153,6 @@ void ignoreException(Verbosity lvl) } catch (...) { } } -bool shouldANSI() -{ - return isatty(STDERR_FILENO) - && getEnv("TERM").value_or("dumb") != "dumb" - && !getEnv("NO_COLOR").has_value(); -} - -std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int width) -{ - std::string t, e; - size_t w = 0; - auto i = s.begin(); - - while (w < (size_t) width && i != s.end()) { - - if (*i == '\e') { - std::string e; - e += *i++; - char last = 0; - - if (i != s.end() && *i == '[') { - e += *i++; - // eat parameter bytes - while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++; - // eat intermediate bytes - while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++; - // eat final byte - if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++; - } else { - if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++; - } - - if (!filterAll && last == 'm') - t += e; - } - - else if (*i == '\t') { - i++; t += ' '; w++; - while (w < (size_t) width && w % 8) { - t += ' '; w++; - } - } - - else if (*i == '\r' || *i == '\a') - // do nothing for now - i++; - - else { - w++; - // Copy one UTF-8 character. - if ((*i & 0xe0) == 0xc0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } else if ((*i & 0xf0) == 0xe0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } - } else if ((*i & 0xf8) == 0xf0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } - } - } else - t += *i++; - } - } - - return t; -} - constexpr char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; @@ -1719,386 +269,9 @@ std::pair getLine(std::string_view s) } -////////////////////////////////////////////////////////////////////// - -static Sync> windowSize{{0, 0}}; - - -static void updateWindowSize() -{ - struct winsize ws; - if (ioctl(2, TIOCGWINSZ, &ws) == 0) { - auto windowSize_(windowSize.lock()); - windowSize_->first = ws.ws_row; - windowSize_->second = ws.ws_col; - } -} - - -std::pair getWindowSize() -{ - return *windowSize.lock(); -} - - -/* We keep track of interrupt callbacks using integer tokens, so we can iterate - safely without having to lock the data structure while executing arbitrary - functions. - */ -struct InterruptCallbacks { - typedef int64_t Token; - - /* We use unique tokens so that we can't accidentally delete the wrong - handler because of an erroneous double delete. */ - Token nextToken = 0; - - /* Used as a list, see InterruptCallbacks comment. */ - std::map> callbacks; -}; - -static Sync _interruptCallbacks; - -static void signalHandlerThread(sigset_t set) -{ - while (true) { - int signal = 0; - sigwait(&set, &signal); - - if (signal == SIGINT || signal == SIGTERM || signal == SIGHUP) - triggerInterrupt(); - - else if (signal == SIGWINCH) { - updateWindowSize(); - } - } -} - -void triggerInterrupt() -{ - _isInterrupted = true; - - { - InterruptCallbacks::Token i = 0; - while (true) { - std::function callback; - { - auto interruptCallbacks(_interruptCallbacks.lock()); - auto lb = interruptCallbacks->callbacks.lower_bound(i); - if (lb == interruptCallbacks->callbacks.end()) - break; - - callback = lb->second; - i = lb->first + 1; - } - - try { - callback(); - } catch (...) { - ignoreException(); - } - } - } -} - -static sigset_t savedSignalMask; -static bool savedSignalMaskIsSet = false; - -void setChildSignalMask(sigset_t * sigs) -{ - assert(sigs); // C style function, but think of sigs as a reference - -#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE - sigemptyset(&savedSignalMask); - // There's no "assign" or "copy" function, so we rely on (math) idempotence - // of the or operator: a or a = a. - sigorset(&savedSignalMask, sigs, sigs); -#else - // Without sigorset, our best bet is to assume that sigset_t is a type that - // can be assigned directly, such as is the case for a sigset_t defined as - // an integer type. - savedSignalMask = *sigs; -#endif - - savedSignalMaskIsSet = true; -} - -void saveSignalMask() { - if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) - throw SysError("querying signal mask"); - - savedSignalMaskIsSet = true; -} - -void startSignalHandlerThread() -{ - updateWindowSize(); - - saveSignalMask(); - - sigset_t set; - sigemptyset(&set); - sigaddset(&set, SIGINT); - sigaddset(&set, SIGTERM); - sigaddset(&set, SIGHUP); - sigaddset(&set, SIGPIPE); - sigaddset(&set, SIGWINCH); - if (pthread_sigmask(SIG_BLOCK, &set, nullptr)) - throw SysError("blocking signals"); - - std::thread(signalHandlerThread, set).detach(); -} - -static void restoreSignals() -{ - // If startSignalHandlerThread wasn't called, that means we're not running - // in a proper libmain process, but a process that presumably manages its - // own signal handlers. Such a process should call either - // - initNix(), to be a proper libmain process - // - startSignalHandlerThread(), to resemble libmain regarding signal - // handling only - // - saveSignalMask(), for processes that define their own signal handling - // thread - // TODO: Warn about this? Have a default signal mask? The latter depends on - // whether we should generally inherit signal masks from the caller. - // I don't know what the larger unix ecosystem expects from us here. - if (!savedSignalMaskIsSet) - return; - - if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) - throw SysError("restoring signals"); -} - -#if __linux__ -rlim_t savedStackSize = 0; -#endif - -void setStackSize(size_t stackSize) -{ - #if __linux__ - struct rlimit limit; - if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) { - savedStackSize = limit.rlim_cur; - limit.rlim_cur = stackSize; - setrlimit(RLIMIT_STACK, &limit); - } - #endif -} - -#if __linux__ -static AutoCloseFD fdSavedMountNamespace; -static AutoCloseFD fdSavedRoot; -#endif - -void saveMountNamespace() -{ -#if __linux__ - static std::once_flag done; - std::call_once(done, []() { - fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY); - if (!fdSavedMountNamespace) - throw SysError("saving parent mount namespace"); - - fdSavedRoot = open("/proc/self/root", O_RDONLY); - }); -#endif -} - -void restoreMountNamespace() -{ -#if __linux__ - try { - auto savedCwd = absPath("."); - - if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) - throw SysError("restoring parent mount namespace"); - - if (fdSavedRoot) { - if (fchdir(fdSavedRoot.get())) - throw SysError("chdir into saved root"); - if (chroot(".")) - throw SysError("chroot into saved root"); - } - - if (chdir(savedCwd.c_str()) == -1) - throw SysError("restoring cwd"); - } catch (Error & e) { - debug(e.msg()); - } -#endif -} - -void unshareFilesystem() -{ -#ifdef __linux__ - if (unshare(CLONE_FS) != 0 && errno != EPERM) - throw SysError("unsharing filesystem state in download thread"); -#endif -} - -void restoreProcessContext(bool restoreMounts) -{ - restoreSignals(); - if (restoreMounts) { - restoreMountNamespace(); - } - - #if __linux__ - if (savedStackSize) { - struct rlimit limit; - if (getrlimit(RLIMIT_STACK, &limit) == 0) { - limit.rlim_cur = savedStackSize; - setrlimit(RLIMIT_STACK, &limit); - } - } - #endif -} - -/* RAII helper to automatically deregister a callback. */ -struct InterruptCallbackImpl : InterruptCallback -{ - InterruptCallbacks::Token token; - ~InterruptCallbackImpl() override - { - auto interruptCallbacks(_interruptCallbacks.lock()); - interruptCallbacks->callbacks.erase(token); - } -}; - -std::unique_ptr createInterruptCallback(std::function callback) -{ - auto interruptCallbacks(_interruptCallbacks.lock()); - auto token = interruptCallbacks->nextToken++; - interruptCallbacks->callbacks.emplace(token, callback); - - auto res = std::make_unique(); - res->token = token; - - return std::unique_ptr(res.release()); -} - - -AutoCloseFD createUnixDomainSocket() -{ - AutoCloseFD fdSocket = socket(PF_UNIX, SOCK_STREAM - #ifdef SOCK_CLOEXEC - | SOCK_CLOEXEC - #endif - , 0); - if (!fdSocket) - throw SysError("cannot create Unix domain socket"); - closeOnExec(fdSocket.get()); - return fdSocket; -} - - -AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) -{ - auto fdSocket = nix::createUnixDomainSocket(); - - bind(fdSocket.get(), path); - - if (chmod(path.c_str(), mode) == -1) - throw SysError("changing permissions on '%1%'", path); - - if (listen(fdSocket.get(), 100) == -1) - throw SysError("cannot listen on socket '%1%'", path); - - return fdSocket; -} - - -void bind(int fd, const std::string & path) -{ - unlink(path.c_str()); - - struct sockaddr_un addr; - addr.sun_family = AF_UNIX; - - if (path.size() + 1 >= sizeof(addr.sun_path)) { - Pid pid = startProcess([&]() { - Path dir = dirOf(path); - if (chdir(dir.c_str()) == -1) - throw SysError("chdir to '%s' failed", dir); - std::string base(baseNameOf(path)); - if (base.size() + 1 >= sizeof(addr.sun_path)) - throw Error("socket path '%s' is too long", base); - memcpy(addr.sun_path, base.c_str(), base.size() + 1); - if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot bind to socket '%s'", path); - _exit(0); - }); - int status = pid.wait(); - if (status != 0) - throw Error("cannot bind to socket '%s'", path); - } else { - memcpy(addr.sun_path, path.c_str(), path.size() + 1); - if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot bind to socket '%s'", path); - } -} - - -void connect(int fd, const std::string & path) -{ - struct sockaddr_un addr; - addr.sun_family = AF_UNIX; - - if (path.size() + 1 >= sizeof(addr.sun_path)) { - Pid pid = startProcess([&]() { - Path dir = dirOf(path); - if (chdir(dir.c_str()) == -1) - throw SysError("chdir to '%s' failed", dir); - std::string base(baseNameOf(path)); - if (base.size() + 1 >= sizeof(addr.sun_path)) - throw Error("socket path '%s' is too long", base); - memcpy(addr.sun_path, base.c_str(), base.size() + 1); - if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot connect to socket at '%s'", path); - _exit(0); - }); - int status = pid.wait(); - if (status != 0) - throw Error("cannot connect to socket at '%s'", path); - } else { - memcpy(addr.sun_path, path.c_str(), path.size() + 1); - if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot connect to socket at '%s'", path); - } -} - - std::string showBytes(uint64_t bytes) { return fmt("%.2f MiB", bytes / (1024.0 * 1024.0)); } - -// FIXME: move to libstore/build -void commonChildInit() -{ - logger = makeSimpleLogger(); - - const static std::string pathNullDevice = "/dev/null"; - restoreProcessContext(false); - - /* Put the child in a separate session (and thus a separate - process group) so that it has no controlling terminal (meaning - that e.g. ssh cannot open /dev/tty) and it doesn't receive - terminal signals. */ - if (setsid() == -1) - throw SysError("creating a new session"); - - /* Dup stderr to stdout. */ - if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) - throw SysError("cannot dup stderr into stdout"); - - /* Reroute stdin to /dev/null. */ - int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); - if (fdDevNull == -1) - throw SysError("cannot open '%1%'", pathNullDevice); - if (dup2(fdDevNull, STDIN_FILENO) == -1) - throw SysError("cannot dup null device into stdin"); - close(fdDevNull); -} - } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index c71cf0864b31..19153cf1515f 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -4,197 +4,24 @@ #include "types.hh" #include "error.hh" #include "logging.hh" -#include "ansicolor.hh" - -#include -#include -#include -#include -#include #include -#include #include #include #include #include -#ifndef HAVE_STRUCT_DIRENT_D_TYPE -#define DT_UNKNOWN 0 -#define DT_REG 1 -#define DT_LNK 2 -#define DT_DIR 3 -#endif - namespace nix { -struct Sink; -struct Source; - void initLibUtil(); /** - * The system for which Nix is compiled. - */ -extern const std::string nativeSystem; - - -/** - * @return an environment variable. - */ -std::optional getEnv(const std::string & key); - -/** - * @return a non empty environment variable. Returns nullopt if the env - * variable is set to "" - */ -std::optional getEnvNonEmpty(const std::string & key); - -/** - * Get the entire environment. - */ -std::map getEnv(); - -/** - * Clear the environment. - */ -void clearEnv(); - -/** - * @return An absolutized path, resolving paths relative to the - * specified directory, or the current directory otherwise. The path - * is also canonicalised. - */ -Path absPath(Path path, - std::optional dir = {}, - bool resolveSymlinks = false); - -/** - * Canonicalise a path by removing all `.` or `..` components and - * double or trailing slashes. Optionally resolves all symlink - * components such that each component of the resulting path is *not* - * a symbolic link. - */ -Path canonPath(PathView path, bool resolveSymlinks = false); - -/** - * @return The directory part of the given canonical path, i.e., - * everything before the final `/`. If the path is the root or an - * immediate child thereof (e.g., `/foo`), this means `/` - * is returned. - */ -Path dirOf(const PathView path); - -/** - * @return the base name of the given canonical path, i.e., everything - * following the final `/` (trailing slashes are removed). - */ -std::string_view baseNameOf(std::string_view path); - -/** - * Perform tilde expansion on a path. - */ -std::string expandTilde(std::string_view path); - -/** - * Check whether 'path' is a descendant of 'dir'. Both paths must be - * canonicalized. - */ -bool isInDir(std::string_view path, std::string_view dir); - -/** - * Check whether 'path' is equal to 'dir' or a descendant of - * 'dir'. Both paths must be canonicalized. - */ -bool isDirOrInDir(std::string_view path, std::string_view dir); - -/** - * Get status of `path`. - */ -struct stat stat(const Path & path); -struct stat lstat(const Path & path); - -/** - * @return true iff the given path exists. - */ -bool pathExists(const Path & path); - -/** - * A version of pathExists that returns false on a permission error. - * Useful for inferring default paths across directories that might not - * be readable. - * @return true iff the given path can be accessed and exists - */ -bool pathAccessible(const Path & path); - -/** - * Read the contents (target) of a symbolic link. The result is not - * in any way canonicalised. - */ -Path readLink(const Path & path); - -bool isLink(const Path & path); - -/** - * Read the contents of a directory. The entries `.` and `..` are - * removed. - */ -struct DirEntry -{ - std::string name; - ino_t ino; - /** - * one of DT_* - */ - unsigned char type; - DirEntry(std::string name, ino_t ino, unsigned char type) - : name(std::move(name)), ino(ino), type(type) { } -}; - -typedef std::vector DirEntries; - -DirEntries readDirectory(const Path & path); - -unsigned char getFileType(const Path & path); - -/** - * Read the contents of a file into a string. - */ -std::string readFile(int fd); -std::string readFile(const Path & path); -void readFile(const Path & path, Sink & sink); - -/** - * Write a string to a file. - */ -void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false); - -void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false); - -/** - * Flush a file's parent directory to disk - */ -void syncParent(const Path & path); - -/** - * Read a line from a file descriptor. - */ -std::string readLine(int fd); - -/** - * Write a line to a file descriptor. - */ -void writeLine(int fd, std::string s); - -/** - * Delete a path; i.e., in the case of a directory, it is deleted - * recursively. It's not an error if the path does not exist. The - * second variant returns the number of bytes and blocks freed. + * Convert a list of strings to a null-terminated vector of `char + * *`s. The result must not be accessed beyond the lifetime of the + * list of strings. */ -void deletePath(const Path & path); - -void deletePath(const Path & path, uint64_t & bytesFreed); +std::vector stringsToCharPtrs(const Strings & ss); std::string getUserName(); std::string getUserName(uid_t uid); @@ -209,335 +36,6 @@ std::vector getUserGroups(uid_t uid); */ std::vector getUserGroupNames(uid_t uid); -/** - * @return the given user's home directory from /etc/passwd. - */ -Path getHomeOf(uid_t userId); - -/** - * @return $HOME or the user's home directory from /etc/passwd. - */ -Path getHome(); - -/** - * @return $XDG_CACHE_HOME or $HOME/.cache. - */ -Path getCacheDir(); - -/** - * @return $XDG_CONFIG_HOME or $HOME/.config. - */ -Path getConfigDir(); - -/** - * @return the directories to search for user configuration files - */ -std::vector getConfigDirs(); - -/** - * @return $XDG_DATA_HOME or $HOME/.local/share. - */ -Path getDataDir(); - -/** - * @return the path of the current executable. - */ -std::optional getSelfExe(); - -/** - * @return $XDG_STATE_HOME or $HOME/.local/state. - */ -Path getStateDir(); - -/** - * Create the Nix state directory and return the path to it. - */ -Path createNixStateDir(); - -/** - * Create a directory and all its parents, if necessary. Returns the - * list of created directories, in order of creation. - */ -Paths createDirs(const Path & path); -inline Paths createDirs(PathView path) -{ - return createDirs(Path(path)); -} - -/** - * Create a symlink. - */ -void createSymlink(const Path & target, const Path & link); - -/** - * Atomically create or replace a symlink. - */ -void replaceSymlink(const Path & target, const Path & link); - -void renameFile(const Path & src, const Path & dst); - -/** - * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst` - * are on a different filesystem. - * - * Beware that this might not be atomic because of the copy that happens behind - * the scenes - */ -void moveFile(const Path & src, const Path & dst); - - -/** - * Wrappers arount read()/write() that read/write exactly the - * requested number of bytes. - */ -void readFull(int fd, char * buf, size_t count); -void writeFull(int fd, std::string_view s, bool allowInterrupts = true); - -MakeError(EndOfFile, Error); - - -/** - * Read a file descriptor until EOF occurs. - */ -std::string drainFD(int fd, bool block = true, const size_t reserveSize=0); - -void drainFD(int fd, Sink & sink, bool block = true); - -/** - * If cgroups are active, attempt to calculate the number of CPUs available. - * If cgroups are unavailable or if cpu.max is set to "max", return 0. - */ -unsigned int getMaxCPU(); - -/** - * Automatic cleanup of resources. - */ - - -class AutoDelete -{ - Path path; - bool del; - bool recursive; -public: - AutoDelete(); - AutoDelete(const Path & p, bool recursive = true); - ~AutoDelete(); - void cancel(); - void reset(const Path & p, bool recursive = true); - operator Path() const { return path; } - operator PathView() const { return path; } -}; - - -class AutoCloseFD -{ - int fd; -public: - AutoCloseFD(); - AutoCloseFD(int fd); - AutoCloseFD(const AutoCloseFD & fd) = delete; - AutoCloseFD(AutoCloseFD&& fd); - ~AutoCloseFD(); - AutoCloseFD& operator =(const AutoCloseFD & fd) = delete; - AutoCloseFD& operator =(AutoCloseFD&& fd); - int get() const; - explicit operator bool() const; - int release(); - void close(); - void fsync(); -}; - - -/** - * Create a temporary directory. - */ -Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", - bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); - -/** - * Create a temporary file, returning a file handle and its path. - */ -std::pair createTempFile(const Path & prefix = "nix"); - - -class Pipe -{ -public: - AutoCloseFD readSide, writeSide; - void create(); - void close(); -}; - - -struct DIRDeleter -{ - void operator()(DIR * dir) const { - closedir(dir); - } -}; - -typedef std::unique_ptr AutoCloseDir; - - -class Pid -{ - pid_t pid = -1; - bool separatePG = false; - int killSignal = SIGKILL; -public: - Pid(); - Pid(pid_t pid); - ~Pid(); - void operator =(pid_t pid); - operator pid_t(); - int kill(); - int wait(); - - void setSeparatePG(bool separatePG); - void setKillSignal(int signal); - pid_t release(); -}; - - -/** - * Kill all processes running under the specified uid by sending them - * a SIGKILL. - */ -void killUser(uid_t uid); - - -/** - * Fork a process that runs the given function, and return the child - * pid to the caller. - */ -struct ProcessOptions -{ - std::string errorPrefix = ""; - bool dieWithParent = true; - bool runExitHandlers = false; - bool allowVfork = false; - /** - * use clone() with the specified flags (Linux only) - */ - int cloneFlags = 0; -}; - -pid_t startProcess(std::function fun, const ProcessOptions & options = ProcessOptions()); - - -/** - * Run a program and return its stdout in a string (i.e., like the - * shell backtick operator). - */ -std::string runProgram(Path program, bool searchPath = false, - const Strings & args = Strings(), - const std::optional & input = {}, bool isInteractive = false); - -struct RunOptions -{ - Path program; - bool searchPath = true; - Strings args; - std::optional uid; - std::optional gid; - std::optional chdir; - std::optional> environment; - std::optional input; - Source * standardIn = nullptr; - Sink * standardOut = nullptr; - bool mergeStderrToStdout = false; - bool isInteractive = false; -}; - -std::pair runProgram(RunOptions && options); - -void runProgram2(const RunOptions & options); - - -/** - * Change the stack size. - */ -void setStackSize(size_t stackSize); - - -/** - * Restore the original inherited Unix process context (such as signal - * masks, stack size). - - * See startSignalHandlerThread(), saveSignalMask(). - */ -void restoreProcessContext(bool restoreMounts = true); - -/** - * Save the current mount namespace. Ignored if called more than - * once. - */ -void saveMountNamespace(); - -/** - * Restore the mount namespace saved by saveMountNamespace(). Ignored - * if saveMountNamespace() was never called. - */ -void restoreMountNamespace(); - -/** - * Cause this thread to not share any FS attributes with the main - * thread, because this causes setns() in restoreMountNamespace() to - * fail. - */ -void unshareFilesystem(); - - -class ExecError : public Error -{ -public: - int status; - - template - ExecError(int status, const Args & ... args) - : Error(args...), status(status) - { } -}; - -/** - * Convert a list of strings to a null-terminated vector of `char - * *`s. The result must not be accessed beyond the lifetime of the - * list of strings. - */ -std::vector stringsToCharPtrs(const Strings & ss); - -/** - * Close all file descriptors except those listed in the given set. - * Good practice in child processes. - */ -void closeMostFDs(const std::set & exceptions); - -/** - * Set the close-on-exec flag for the given file descriptor. - */ -void closeOnExec(int fd); - - -/* User interruption. */ - -extern std::atomic _isInterrupted; - -extern thread_local std::function interruptCheck; - -void setInterruptThrown(); - -void _interrupted(); - -void inline checkInterrupt() -{ - if (_isInterrupted || (interruptCheck && interruptCheck())) - _interrupted(); -} - -MakeError(Interrupted, BaseError); - - MakeError(FormatError, Error); @@ -612,15 +110,6 @@ std::string replaceStrings( std::string rewriteStrings(std::string s, const StringMap & rewrites); -/** - * Convert the exit status of a child as returned by wait() into an - * error string. - */ -std::string statusToString(int status); - -bool statusOk(int status); - - /** * Parse a string into an integer. */ @@ -712,10 +201,8 @@ std::string toLower(const std::string & s); std::string shellEscape(const std::string_view s); -/** - * Exception handling in destructors: print an error message, then - * ignore the exception. - */ +/* Exception handling in destructors: print an error message, then + ignore the exception. */ void ignoreException(Verbosity lvl = lvlError); @@ -728,23 +215,6 @@ constexpr char treeLast[] = "└───"; constexpr char treeLine[] = "│ "; constexpr char treeNull[] = " "; -/** - * Determine whether ANSI escape sequences are appropriate for the - * present output. - */ -bool shouldANSI(); - -/** - * Truncate a string to 'width' printable characters. If 'filterAll' - * is true, all ANSI escape sequences are filtered out. Otherwise, - * some escape sequences (such as colour setting) are copied but not - * included in the character count. Also, tabs are expanded to - * spaces. - */ -std::string filterANSIEscapes(std::string_view s, - bool filterAll = false, - unsigned int width = std::numeric_limits::max()); - /** * Base64 encoding/decoding. @@ -832,61 +302,6 @@ template class Callback; -/** - * Start a thread that handles various signals. Also block those signals - * on the current thread (and thus any threads created by it). - * Saves the signal mask before changing the mask to block those signals. - * See saveSignalMask(). - */ -void startSignalHandlerThread(); - -/** - * Saves the signal mask, which is the signal mask that nix will restore - * before creating child processes. - * See setChildSignalMask() to set an arbitrary signal mask instead of the - * current mask. - */ -void saveSignalMask(); - -/** - * Sets the signal mask. Like saveSignalMask() but for a signal set that doesn't - * necessarily match the current thread's mask. - * See saveSignalMask() to set the saved mask to the current mask. - */ -void setChildSignalMask(sigset_t *sigs); - -struct InterruptCallback -{ - virtual ~InterruptCallback() { }; -}; - -/** - * Register a function that gets called on SIGINT (in a non-signal - * context). - */ -std::unique_ptr createInterruptCallback( - std::function callback); - -void triggerInterrupt(); - -/** - * A RAII class that causes the current thread to receive SIGUSR1 when - * the signal handler thread receives SIGINT. That is, this allows - * SIGINT to be multiplexed to multiple threads. - */ -struct ReceiveInterrupts -{ - pthread_t target; - std::unique_ptr callback; - - ReceiveInterrupts() - : target(pthread_self()) - , callback(createInterruptCallback([&]() { pthread_kill(target, SIGUSR1); })) - { } -}; - - - /** * A RAII helper that increments a counter on construction and * decrements it on destruction. @@ -901,45 +316,6 @@ struct MaintainCount }; -/** - * @return the number of rows and columns of the terminal. - */ -std::pair getWindowSize(); - - -/** - * Used in various places. - */ -typedef std::function PathFilter; - -extern PathFilter defaultPathFilter; - -/** - * Common initialisation performed in child processes. - */ -void commonChildInit(); - -/** - * Create a Unix domain socket. - */ -AutoCloseFD createUnixDomainSocket(); - -/** - * Create a Unix domain socket in listen mode. - */ -AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); - -/** - * Bind a Unix domain socket to a path. - */ -void bind(int fd, const std::string & path); - -/** - * Connect to a Unix domain socket. - */ -void connect(int fd, const std::string & path); - - /** * A Rust/Python-like enumerate() iterator adapter. * diff --git a/src/libutil/variant-wrapper.hh b/src/libutil/variant-wrapper.hh new file mode 100644 index 000000000000..cedcb999c03b --- /dev/null +++ b/src/libutil/variant-wrapper.hh @@ -0,0 +1,30 @@ +#pragma once +///@file + +// not used, but will be used by callers +#include + +/** + * Force the default versions of all constructors (copy, move, copy + * assignment). + */ +#define FORCE_DEFAULT_CONSTRUCTORS(CLASS_NAME) \ + CLASS_NAME(const CLASS_NAME &) = default; \ + CLASS_NAME(CLASS_NAME &) = default; \ + CLASS_NAME(CLASS_NAME &&) = default; \ + \ + CLASS_NAME & operator =(const CLASS_NAME &) = default; \ + CLASS_NAME & operator =(CLASS_NAME &) = default; + +/** + * Make a wrapper constructor. All args are forwarded to the + * construction of the "raw" field. (Which we assume is the only one.) + * + * The moral equivalent of `using Raw::Raw;` + */ +#define MAKE_WRAPPER_CONSTRUCTOR(CLASS_NAME) \ + FORCE_DEFAULT_CONSTRUCTORS(CLASS_NAME) \ + \ + CLASS_NAME(auto &&... arg) \ + : raw(std::forward(arg)...) \ + { } diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 6510df8f041b..75ce12a8c3e5 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -9,12 +9,12 @@ #include +#include "current-process.hh" #include "parsed-derivations.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "globals.hh" #include "derivations.hh" -#include "util.hh" #include "shared.hh" #include "path-with-outputs.hh" #include "eval.hh" @@ -34,13 +34,14 @@ extern char * * environ __attribute__((weak)); */ static std::vector shellwords(const std::string & s) { - std::regex whitespace("^(\\s+).*"); + std::regex whitespace("^\\s+"); auto begin = s.cbegin(); std::vector res; std::string cur; enum state { sBegin, - sQuote + sSingleQuote, + sDoubleQuote }; state st = sBegin; auto it = begin; @@ -50,26 +51,39 @@ static std::vector shellwords(const std::string & s) if (regex_search(it, s.cend(), match, whitespace)) { cur.append(begin, it); res.push_back(cur); - cur.clear(); - it = match[1].second; + it = match[0].second; + if (it == s.cend()) return res; begin = it; + cur.clear(); } } switch (*it) { + case '\'': + if (st != sDoubleQuote) { + cur.append(begin, it); + begin = it + 1; + st = st == sBegin ? sSingleQuote : sBegin; + } + break; case '"': - cur.append(begin, it); - begin = it + 1; - st = st == sBegin ? sQuote : sBegin; + if (st != sSingleQuote) { + cur.append(begin, it); + begin = it + 1; + st = st == sBegin ? sDoubleQuote : sBegin; + } break; case '\\': - /* perl shellwords mostly just treats the next char as part of the string with no special processing */ - cur.append(begin, it); - begin = ++it; + if (st != sSingleQuote) { + /* perl shellwords mostly just treats the next char as part of the string with no special processing */ + cur.append(begin, it); + begin = ++it; + } break; } } + if (st != sBegin) throw Error("unterminated quote in shebang line"); cur.append(begin, it); - if (!cur.empty()) res.push_back(cur); + res.push_back(cur); return res; } @@ -128,7 +142,7 @@ static void main_nix_build(int argc, char * * argv) for (auto line : lines) { line = chomp(line); std::smatch match; - if (std::regex_match(line, match, std::regex("^#!\\s*nix-shell (.*)$"))) + if (std::regex_match(line, match, std::regex("^#!\\s*nix-shell\\s+(.*)$"))) for (const auto & word : shellwords(match[1].str())) args.push_back(word); } @@ -344,7 +358,7 @@ static void main_nix_build(int argc, char * * argv) } } - state->printStats(); + state->maybePrintStats(); auto buildPaths = [&](const std::vector & paths) { /* Note: we do this even when !printMissing to efficiently @@ -393,7 +407,7 @@ static void main_nix_build(int argc, char * * argv) auto bashDrv = drv->requireDrvPath(); pathsToBuild.push_back(DerivedPath::Built { - .drvPath = bashDrv, + .drvPath = makeConstantStorePathRef(bashDrv), .outputs = OutputsSpec::Names {"out"}, }); pathsToCopy.insert(bashDrv); @@ -406,8 +420,22 @@ static void main_nix_build(int argc, char * * argv) } } + std::function, const DerivedPathMap::ChildNode &)> accumDerivedPath; + + accumDerivedPath = [&](ref inputDrv, const DerivedPathMap::ChildNode & inputNode) { + if (!inputNode.value.empty()) + pathsToBuild.push_back(DerivedPath::Built { + .drvPath = inputDrv, + .outputs = OutputsSpec::Names { inputNode.value }, + }); + for (const auto & [outputName, childNode] : inputNode.childMap) + accumDerivedPath( + make_ref(SingleDerivedPath::Built { inputDrv, outputName }), + childNode); + }; + // Build or fetch all dependencies of the derivation. - for (const auto & [inputDrv0, inputOutputs] : drv.inputDrvs) { + for (const auto & [inputDrv0, inputNode] : drv.inputDrvs.map) { // To get around lambda capturing restrictions in the // standard. const auto & inputDrv = inputDrv0; @@ -416,15 +444,12 @@ static void main_nix_build(int argc, char * * argv) return !std::regex_search(store->printStorePath(inputDrv), std::regex(exclude)); })) { - pathsToBuild.push_back(DerivedPath::Built { - .drvPath = inputDrv, - .outputs = OutputsSpec::Names { inputOutputs }, - }); + accumDerivedPath(makeConstantStorePathRef(inputDrv), inputNode); pathsToCopy.insert(inputDrv); } } for (const auto & src : drv.inputSrcs) { - pathsToBuild.push_back(DerivedPath::Opaque{src}); + pathsToBuild.emplace_back(DerivedPath::Opaque{src}); pathsToCopy.insert(src); } @@ -482,13 +507,21 @@ static void main_nix_build(int argc, char * * argv) if (env.count("__json")) { StorePathSet inputs; - for (auto & [depDrvPath, wantedDepOutputs] : drv.inputDrvs) { - auto outputs = evalStore->queryPartialDerivationOutputMap(depDrvPath); - for (auto & i : wantedDepOutputs) { + + std::function::ChildNode &)> accumInputClosure; + + accumInputClosure = [&](const StorePath & inputDrv, const DerivedPathMap::ChildNode & inputNode) { + auto outputs = evalStore->queryPartialDerivationOutputMap(inputDrv); + for (auto & i : inputNode.value) { auto o = outputs.at(i); store->computeFSClosure(*o, inputs); } - } + for (const auto & [outputName, childNode] : inputNode.childMap) + accumInputClosure(*outputs.at(outputName), childNode); + }; + + for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) + accumInputClosure(inputDrv, inputNode); ParsedDerivation parsedDrv(drvInfo.requireDrvPath(), drv); @@ -590,7 +623,10 @@ static void main_nix_build(int argc, char * * argv) if (outputName == "") throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath)); - pathsToBuild.push_back(DerivedPath::Built{drvPath, OutputsSpec::Names{outputName}}); + pathsToBuild.push_back(DerivedPath::Built{ + .drvPath = makeConstantStorePathRef(drvPath), + .outputs = OutputsSpec::Names{outputName}, + }); pathsToBuildOrdered.push_back({drvPath, {outputName}}); drvsToCopy.insert(drvPath); diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc old mode 100755 new mode 100644 index c1c8edd1d35c..79db7823696d --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -4,8 +4,9 @@ #include "filetransfer.hh" #include "store-api.hh" #include "legacy.hh" -#include "fetchers.hh" -#include "util.hh" +#include "eval-settings.hh" // for defexpr +#include "users.hh" +#include "tarball.hh" #include #include @@ -165,7 +166,7 @@ static int main_nix_channel(int argc, char ** argv) // Figure out the name of the `.nix-channels' file to use auto home = getHome(); channelsList = settings.useXDGBaseDirectories ? createNixStateDir() + "/channels" : home + "/.nix-channels"; - nixDefExpr = settings.useXDGBaseDirectories ? createNixStateDir() + "/defexpr" : home + "/.nix-defexpr"; + nixDefExpr = getNixDefExpr(); // Figure out the name of the channels profile. profile = profilesDir() + "/channels"; diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index 70af53b286ba..bb3f1bc6add8 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -1,3 +1,5 @@ +#include "file-system.hh" +#include "signals.hh" #include "store-api.hh" #include "store-cast.hh" #include "gc-store.hh" diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc old mode 100755 new mode 100644 diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 91b073b49503..86126c7ad2d5 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1,3 +1,4 @@ +#include "users.hh" #include "attr-path.hh" #include "common-eval-args.hh" #include "derivations.hh" @@ -11,10 +12,10 @@ #include "store-api.hh" #include "local-fs-store.hh" #include "user-env.hh" -#include "util.hh" #include "value-to-json.hh" #include "xml-writer.hh" #include "legacy.hh" +#include "eval-settings.hh" // for defexpr #include #include @@ -171,7 +172,7 @@ static void loadSourceExpr(EvalState & state, const SourcePath & path, Value & v directory). */ else if (st.type == InputAccessor::tDirectory) { auto attrs = state.buildBindings(maxAttrs); - attrs.alloc("_combineChannels").mkList(0); + state.mkList(attrs.alloc("_combineChannels"), 0); StringSet seen; getAllExprs(state, path, seen, attrs); v.mkAttrs(attrs); @@ -480,12 +481,12 @@ static void printMissing(EvalState & state, DrvInfos & elems) std::vector targets; for (auto & i : elems) if (auto drvPath = i.queryDrvPath()) - targets.push_back(DerivedPath::Built{ - .drvPath = *drvPath, + targets.emplace_back(DerivedPath::Built{ + .drvPath = makeConstantStorePathRef(*drvPath), .outputs = OutputsSpec::All { }, }); else - targets.push_back(DerivedPath::Opaque{ + targets.emplace_back(DerivedPath::Opaque{ .path = i.queryOutPath(), }); @@ -759,7 +760,7 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) std::vector paths { drvPath ? (DerivedPath) (DerivedPath::Built { - .drvPath = *drvPath, + .drvPath = makeConstantStorePathRef(*drvPath), .outputs = OutputsSpec::All { }, }) : (DerivedPath) (DerivedPath::Opaque { @@ -921,7 +922,7 @@ static VersionDiff compareVersionAgainstSet( } -static void queryJSON(Globals & globals, std::vector & elems, bool printOutPath, bool printMeta) +static void queryJSON(Globals & globals, std::vector & elems, bool printOutPath, bool printDrvPath, bool printMeta) { using nlohmann::json; json topObj = json::object(); @@ -952,6 +953,11 @@ static void queryJSON(Globals & globals, std::vector & elems, bool prin } } + if (printDrvPath) { + auto drvPath = i.queryDrvPath(); + if (drvPath) pkgObj["drvPath"] = globals.state->store->printStorePath(*drvPath); + } + if (printMeta) { json &metaObj = pkgObj["meta"]; metaObj = json::object(); @@ -1078,7 +1084,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) /* Print the desired columns, or XML output. */ if (jsonOutput) { - queryJSON(globals, elems, printOutPath, printMeta); + queryJSON(globals, elems, printOutPath, printDrvPath, printMeta); cout << '\n'; return; } @@ -1227,7 +1233,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) else { if (v->type() == nString) { attrs2["type"] = "string"; - attrs2["value"] = v->string.s; + attrs2["value"] = v->c_str(); xml.writeEmptyElement("meta", attrs2); } else if (v->type() == nInt) { attrs2["type"] = "int"; @@ -1247,7 +1253,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) for (auto elem : v->listItems()) { if (elem->type() != nString) continue; XMLAttrs attrs3; - attrs3["value"] = elem->string.s; + attrs3["value"] = elem->c_str(); xml.writeEmptyElement("string", attrs3); } } else if (v->type() == nAttrs) { @@ -1259,7 +1265,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) if(a.value->type() != nString) continue; XMLAttrs attrs3; attrs3["type"] = globals.state->symbols[i.name]; - attrs3["value"] = a.value->string.s; + attrs3["value"] = a.value->c_str(); xml.writeEmptyElement("string", attrs3); } } @@ -1399,7 +1405,7 @@ static int main_nix_env(int argc, char * * argv) globals.instSource.type = srcUnknown; globals.instSource.systemFilter = "*"; - Path nixExprPath = settings.useXDGBaseDirectories ? createNixStateDir() + "/defexpr" : getHome() + "/.nix-defexpr"; + Path nixExprPath = getNixDefExpr(); if (!pathExists(nixExprPath)) { try { @@ -1530,7 +1536,7 @@ static int main_nix_env(int argc, char * * argv) op(globals, std::move(opFlags), std::move(opArgs)); - globals.state->printStats(); + globals.state->maybePrintStats(); return 0; } diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index d12d70f332ea..250224e7da0a 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -1,5 +1,4 @@ #include "user-env.hh" -#include "util.hh" #include "derivations.hh" #include "store-api.hh" #include "path-with-outputs.hh" diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 446b27e667e8..c67409e89e1c 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -6,7 +6,6 @@ #include "attr-path.hh" #include "value-to-xml.hh" #include "value-to-json.hh" -#include "util.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "common-eval-args.hh" @@ -189,7 +188,7 @@ static int main_nix_instantiate(int argc, char * * argv) evalOnly, outputKind, xmlOutputSourceLocation, e); } - state->printStats(); + state->maybePrintStats(); return 0; } diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc index 577cadceb313..2c530999b551 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/nix-store/dotgraph.cc @@ -1,5 +1,4 @@ #include "dotgraph.hh" -#include "util.hh" #include "store-api.hh" #include diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc index 4395576589c3..3e789a2d8b37 100644 --- a/src/nix-store/graphml.cc +++ b/src/nix-store/graphml.cc @@ -1,5 +1,4 @@ #include "graphml.hh" -#include "util.hh" #include "store-api.hh" #include "derivations.hh" diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index caa0248f1aa9..25f0107bc4fb 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -9,13 +9,12 @@ #include "local-store.hh" #include "monitor-fd.hh" #include "serve-protocol.hh" +#include "serve-protocol-impl.hh" #include "shared.hh" -#include "util.hh" -#include "worker-protocol.hh" -#include "worker-protocol-impl.hh" #include "graphml.hh" #include "legacy.hh" #include "path-with-outputs.hh" +#include "posix-fs-canonicalise.hh" #include #include @@ -220,10 +219,8 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) std::string name = *i++; cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo { - .hash = { - .method = method, - .hash = Hash::parseAny(hash, hashAlgo), - }, + .method = method, + .hash = Hash::parseAny(hash, hashAlgo), .references = {}, }))); } @@ -285,7 +282,7 @@ static void opQuery(Strings opFlags, Strings opArgs) { enum QueryType { qOutputs, qRequisites, qReferences, qReferrers - , qReferrersClosure, qDeriver, qBinding, qHash, qSize + , qReferrersClosure, qDeriver, qValidDerivers, qBinding, qHash, qSize , qTree, qGraph, qGraphML, qResolve, qRoots }; std::optional query; bool useOutput = false; @@ -301,6 +298,7 @@ static void opQuery(Strings opFlags, Strings opArgs) else if (i == "--referrers" || i == "--referers") query = qReferrers; else if (i == "--referrers-closure" || i == "--referers-closure") query = qReferrersClosure; else if (i == "--deriver" || i == "-d") query = qDeriver; + else if (i == "--valid-derivers") query = qValidDerivers; else if (i == "--binding" || i == "-b") { if (opArgs.size() == 0) throw UsageError("expected binding name"); @@ -374,6 +372,21 @@ static void opQuery(Strings opFlags, Strings opArgs) } break; + case qValidDerivers: { + StorePathSet result; + for (auto & i : opArgs) { + auto derivers = store->queryValidDerivers(store->followLinksToStorePath(i)); + for (const auto &i: derivers) { + result.insert(i); + } + } + auto sorted = store->topoSortPaths(result); + for (StorePaths::reverse_iterator i = sorted.rbegin(); + i != sorted.rend(); ++i) + cout << fmt("%s\n", store->printStorePath(*i)); + break; + } + case qBinding: for (auto & i : opArgs) { auto path = useDeriver(store->followLinksToStorePath(i)); @@ -393,7 +406,7 @@ static void opQuery(Strings opFlags, Strings opArgs) auto info = store->queryPathInfo(j); if (query == qHash) { assert(info->narHash.type == htSHA256); - cout << fmt("%s\n", info->narHash.to_string(Base32, true)); + cout << fmt("%s\n", info->narHash.to_string(HashFormat::Base32, true)); } else if (query == qSize) cout << fmt("%d\n", info->narSize); } @@ -756,8 +769,8 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) if (current.first != info->narHash) { printError("path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(path), - info->narHash.to_string(Base32, true), - current.first.to_string(Base32, true)); + info->narHash.to_string(HashFormat::Base32, true), + current.first.to_string(HashFormat::Base32, true)); status = 1; } } @@ -805,10 +818,16 @@ static void opServe(Strings opFlags, Strings opArgs) if (magic != SERVE_MAGIC_1) throw Error("protocol mismatch"); out << SERVE_MAGIC_2 << SERVE_PROTOCOL_VERSION; out.flush(); - unsigned int clientVersion = readInt(in); + ServeProto::Version clientVersion = readInt(in); - WorkerProto::ReadConn rconn { .from = in }; - WorkerProto::WriteConn wconn { .to = out }; + ServeProto::ReadConn rconn { + .from = in, + .version = clientVersion, + }; + ServeProto::WriteConn wconn { + .to = out, + .version = clientVersion, + }; auto getBuildSettings = [&]() { // FIXME: changing options here doesn't work if we're @@ -853,7 +872,7 @@ static void opServe(Strings opFlags, Strings opArgs) case ServeProto::Command::QueryValidPaths: { bool lock = readInt(in); bool substitute = readInt(in); - auto paths = WorkerProto::Serialise::read(*store, rconn); + auto paths = ServeProto::Serialise::read(*store, rconn); if (lock && writeAllowed) for (auto & path : paths) store->addTempRoot(path); @@ -862,24 +881,24 @@ static void opServe(Strings opFlags, Strings opArgs) store->substitutePaths(paths); } - WorkerProto::write(*store, wconn, store->queryValidPaths(paths)); + ServeProto::write(*store, wconn, store->queryValidPaths(paths)); break; } case ServeProto::Command::QueryPathInfos: { - auto paths = WorkerProto::Serialise::read(*store, rconn); + auto paths = ServeProto::Serialise::read(*store, rconn); // !!! Maybe we want a queryPathInfos? for (auto & i : paths) { try { auto info = store->queryPathInfo(i); out << store->printStorePath(info->path) << (info->deriver ? store->printStorePath(*info->deriver) : ""); - WorkerProto::write(*store, wconn, info->references); + ServeProto::write(*store, wconn, info->references); // !!! Maybe we want compression? out << info->narSize // downloadSize << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << info->narHash.to_string(Base32, true) + out << info->narHash.to_string(HashFormat::Base32, true) << renderContentAddress(info->ca) << info->sigs; } catch (InvalidPath &) { @@ -902,7 +921,7 @@ static void opServe(Strings opFlags, Strings opArgs) case ServeProto::Command::ExportPaths: { readInt(in); // obsolete - store->exportPaths(WorkerProto::Serialise::read(*store, rconn), out); + store->exportPaths(ServeProto::Serialise::read(*store, rconn), out); break; } @@ -940,26 +959,16 @@ static void opServe(Strings opFlags, Strings opArgs) MonitorFdHup monitor(in.fd); auto status = store->buildDerivation(drvPath, drv); - out << status.status << status.errorMsg; - - if (GET_PROTOCOL_MINOR(clientVersion) >= 3) - out << status.timesBuilt << status.isNonDeterministic << status.startTime << status.stopTime; - if (GET_PROTOCOL_MINOR(clientVersion) >= 6) { - DrvOutputs builtOutputs; - for (auto & [output, realisation] : status.builtOutputs) - builtOutputs.insert_or_assign(realisation.id, realisation); - WorkerProto::write(*store, wconn, builtOutputs); - } - + ServeProto::write(*store, wconn, status); break; } case ServeProto::Command::QueryClosure: { bool includeOutputs = readInt(in); StorePathSet closure; - store->computeFSClosure(WorkerProto::Serialise::read(*store, rconn), + store->computeFSClosure(ServeProto::Serialise::read(*store, rconn), closure, false, includeOutputs); - WorkerProto::write(*store, wconn, closure); + ServeProto::write(*store, wconn, closure); break; } @@ -974,7 +983,7 @@ static void opServe(Strings opFlags, Strings opArgs) }; if (deriver != "") info.deriver = store->parseStorePath(deriver); - info.references = WorkerProto::Serialise::read(*store, rconn); + info.references = ServeProto::Serialise::read(*store, rconn); in >> info.registrationTime >> info.narSize >> info.ultimate; info.sigs = readStrings(in); info.ca = ContentAddress::parseOpt(readString(in)); diff --git a/src/nix/add-file.md b/src/nix/add-file.md deleted file mode 100644 index ed237a035004..000000000000 --- a/src/nix/add-file.md +++ /dev/null @@ -1,28 +0,0 @@ -R""( - -# Description - -Copy the regular file *path* to the Nix store, and print the resulting -store path on standard output. - -> **Warning** -> -> The resulting store path is not registered as a garbage -> collector root, so it could be deleted before you have a -> chance to register it. - -# Examples - -Add a regular file to the store: - -```console -# echo foo > bar - -# nix store add-file ./bar -/nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar - -# cat /nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar -foo -``` - -)"" diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 2aa146f82f59..4de5c787cba5 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -7,11 +7,22 @@ using namespace nix; +static FileIngestionMethod parseIngestionMethod(std::string_view input) +{ + if (input == "flat") { + return FileIngestionMethod::Flat; + } else if (input == "nar") { + return FileIngestionMethod::Recursive; + } else { + throw UsageError("Unknown hash mode '%s', expect `flat` or `nar`"); + } +} + struct CmdAddToStore : MixDryRun, MixProtect, StoreCommand { Path path; std::optional namePart; - FileIngestionMethod ingestionMethod; + FileIngestionMethod ingestionMethod = FileIngestionMethod::Recursive; CmdAddToStore() { @@ -25,6 +36,23 @@ struct CmdAddToStore : MixDryRun, MixProtect, StoreCommand .labels = {"name"}, .handler = {&namePart}, }); + + addFlag({ + .longName = "mode", + .shortName = 'n', + .description = R"( + How to compute the hash of the input. + One of: + + - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function. + + - `flat`: Assumes that the input is a single file and directly passes it to the hash function; + )", + .labels = {"hash-mode"}, + .handler = {[this](std::string s) { + this->ingestionMethod = parseIngestionMethod(s); + }}, + }); } void run(ref store) override @@ -47,10 +75,8 @@ struct CmdAddToStore : MixDryRun, MixProtect, StoreCommand *store, std::move(*namePart), FixedOutputInfo { - .hash = { - .method = std::move(ingestionMethod), - .hash = std::move(hash), - }, + .method = std::move(ingestionMethod), + .hash = std::move(hash), .references = {}, }, narHash, @@ -73,45 +99,43 @@ struct CmdAddToStore : MixDryRun, MixProtect, StoreCommand } }; -struct CmdAddFile : CmdAddToStore +struct CmdAdd : CmdAddToStore { - CmdAddFile() - { - ingestionMethod = FileIngestionMethod::Flat; - } std::string description() override { - return "add a regular file to the Nix store"; + return "Add a file or directory to the Nix store"; } std::string doc() override { return - #include "add-file.md" + #include "add.md" ; } }; -struct CmdAddPath : CmdAddToStore +struct CmdAddFile : CmdAddToStore { - CmdAddPath() + CmdAddFile() { - ingestionMethod = FileIngestionMethod::Recursive; + ingestionMethod = FileIngestionMethod::Flat; } std::string description() override { - return "add a path to the Nix store"; + return "Deprecated. Use [`nix store add --mode flat`](@docroot@/command-ref/new-cli/nix3-store-add.md) instead."; } +}; - std::string doc() override +struct CmdAddPath : CmdAddToStore +{ + std::string description() override { - return - #include "add-path.md" - ; + return "Deprecated alias to [`nix store add`](@docroot@/command-ref/new-cli/nix3-store-add.md)."; } }; static auto rCmdAddFile = registerCommand2({"store", "add-file"}); static auto rCmdAddPath = registerCommand2({"store", "add-path"}); +static auto rCmdAdd = registerCommand2({"store", "add"}); diff --git a/src/nix/add-path.md b/src/nix/add.md similarity index 94% rename from src/nix/add-path.md rename to src/nix/add.md index 87473611df4d..d38cd21d87f6 100644 --- a/src/nix/add-path.md +++ b/src/nix/add.md @@ -19,7 +19,7 @@ Add a directory to the store: # mkdir dir # echo foo > dir/bar -# nix store add-path ./dir +# nix store add ./dir /nix/store/6pmjx56pm94n66n4qw1nff0y1crm8nqg-dir # cat /nix/store/6pmjx56pm94n66n4qw1nff0y1crm8nqg-dir/bar diff --git a/src/nix/app.cc b/src/nix/app.cc index e678b54f0f76..935ed18ecbaa 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -20,13 +20,26 @@ StringPairs resolveRewrites( const std::vector & dependencies) { StringPairs res; - for (auto & dep : dependencies) - if (auto drvDep = std::get_if(&dep.path)) - for (auto & [ outputName, outputPath ] : drvDep->outputs) - res.emplace( - DownstreamPlaceholder::unknownCaOutput(drvDep->drvPath, outputName).render(), - store.printStorePath(outputPath) - ); + if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + return res; + } + for (auto &dep: dependencies) { + auto drvDep = std::get_if(&dep.path); + if (!drvDep) { + continue; + } + + for (const auto & [ outputName, outputPath ] : drvDep->outputs) { + res.emplace( + DownstreamPlaceholder::fromSingleDerivedPathBuilt( + SingleDerivedPath::Built { + .drvPath = make_ref(drvDep->drvPath->discardOutputPath()), + .output = outputName, + }).render(), + store.printStorePath(outputPath) + ); + } + } return res; } @@ -49,11 +62,11 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) auto type = cursor->getAttr("type")->getString(); - std::string expected = !attrPath.empty() && + std::string expectedType = !attrPath.empty() && (state.symbols[attrPath[0]] == "apps" || state.symbols[attrPath[0]] == "defaultApp") ? "app" : "derivation"; - if (type != expected) - throw Error("attribute '%s' should have type '%s'", cursor->getAttrPathStr(), expected); + if (type != expectedType) + throw Error("attribute '%s' should have type '%s'", cursor->getAttrPathStr(), expectedType); if (type == "app") { auto [program, context] = cursor->getAttr("program")->getStringWithContext(); @@ -64,7 +77,7 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) [&](const NixStringContextElem::DrvDeep & d) -> DerivedPath { /* We want all outputs of the drv */ return DerivedPath::Built { - .drvPath = d.drvPath, + .drvPath = makeConstantStorePathRef(d.drvPath), .outputs = OutputsSpec::All {}, }; }, @@ -79,10 +92,10 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) .path = o.path, }; }, - }, c.raw())); + }, c.raw)); } - return UnresolvedApp{App { + return UnresolvedApp { App { .context = std::move(context2), .program = program, }}; @@ -105,7 +118,7 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) auto program = outPath + "/bin/" + mainProgram; return UnresolvedApp { App { .context = { DerivedPath::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::Names { outputName }, } }, .program = program, diff --git a/src/nix/build.cc b/src/nix/build.cc index 4a5b29c0fe90..1245ad0c7791 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -9,18 +9,18 @@ using namespace nix; -nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, ref store) +static nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, Store & store) { auto res = nlohmann::json::array(); for (auto & t : paths) { - std::visit([&res, store](const auto & t) { + std::visit([&](const auto & t) { res.push_back(t.toJSON(store)); }, t.raw()); } return res; } -nlohmann::json builtPathsWithResultToJSON(const std::vector & buildables, ref store) +static nlohmann::json builtPathsWithResultToJSON(const std::vector & buildables, const Store & store) { auto res = nlohmann::json::array(); for (auto & b : buildables) { @@ -125,7 +125,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile, MixProtec printMissing(store, pathsToBuild, lvlError); if (json) - logger->cout("%s", derivedPathsToJSON(pathsToBuild, store).dump()); + logger->cout("%s", derivedPathsToJSON(pathsToBuild, *store).dump()); return; } @@ -137,7 +137,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile, MixProtec repair ? bmRepair : buildMode, protect); - if (json) logger->cout("%s", builtPathsWithResultToJSON(buildables, store).dump()); + if (json) logger->cout("%s", builtPathsWithResultToJSON(buildables, *store).dump()); if (outLink != "") if (auto store2 = store.dynamic_pointer_cast()) diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index bcc00d490dd4..54cc6a17f364 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -4,7 +4,6 @@ #include "shared.hh" #include "store-api.hh" #include "local-fs-store.hh" -#include "fs-accessor.hh" #include "eval-inline.hh" using namespace nix; @@ -21,8 +20,8 @@ struct CmdBundle : InstallableValueCommand .description = fmt("Use a custom bundler instead of the default (`%s`).", bundler), .labels = {"flake-url"}, .handler = {&bundler}, - .completer = {[&](size_t, std::string_view prefix) { - completeFlakeRef(getStore(), prefix); + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getStore(), prefix); }} }); @@ -80,7 +79,7 @@ struct CmdBundle : InstallableValueCommand auto [bundlerFlakeRef, bundlerName, extendedOutputsSpec] = parseFlakeRefWithFragmentAndExtendedOutputsSpec(bundler, absPath(".")); const flake::LockFlags lockFlags{ .writeLockFile = false }; InstallableFlake bundler{this, - evalState, std::move(bundlerFlakeRef), bundlerName, extendedOutputsSpec, + evalState, std::move(bundlerFlakeRef), bundlerName, std::move(extendedOutputsSpec), {"bundlers." + settings.thisSystem.get() + ".default", "defaultBundler." + settings.thisSystem.get() }, @@ -109,7 +108,7 @@ struct CmdBundle : InstallableValueCommand store->buildPaths({ DerivedPath::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All { }, }, }); diff --git a/src/nix/cat.cc b/src/nix/cat.cc index 60aa66ce0c66..4df086d4fbac 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -1,7 +1,7 @@ #include "command.hh" #include "store-api.hh" -#include "fs-accessor.hh" #include "nar-accessor.hh" +#include "progress-bar.hh" using namespace nix; @@ -9,15 +9,13 @@ struct MixCat : virtual Args { std::string path; - void cat(ref accessor) + void cat(ref accessor) { - auto st = accessor->stat(path); - if (st.type == FSAccessor::Type::tMissing) - throw Error("path '%1%' does not exist", path); - if (st.type != FSAccessor::Type::tRegular) + auto st = accessor->lstat(CanonPath(path)); + if (st.type != SourceAccessor::Type::tRegular) throw Error("path '%1%' is not a regular file", path); - - writeFull(STDOUT_FILENO, accessor->readFile(path)); + stopProgressBar(); + writeFull(STDOUT_FILENO, accessor->readFile(CanonPath(path))); } }; diff --git a/src/nix/show-config.cc b/src/nix/config.cc similarity index 67% rename from src/nix/show-config.cc rename to src/nix/config.cc index 3530584f9b78..5b280d11d25d 100644 --- a/src/nix/show-config.cc +++ b/src/nix/config.cc @@ -7,11 +7,31 @@ using namespace nix; -struct CmdShowConfig : Command, MixJSON +struct CmdConfig : virtual NixMultiCommand +{ + CmdConfig() : MultiCommand(RegisterCommand::getCommandsFor({"config"})) + { } + + std::string description() override + { + return "manipulate the Nix configuration"; + } + + Category category() override { return catUtility; } + + void run() override + { + if (!command) + throw UsageError("'nix config' requires a sub-command."); + command->second->run(); + } +}; + +struct CmdConfigShow : Command, MixJSON { std::optional name; - CmdShowConfig() { + CmdConfigShow() { expectArgs({ .label = {"name"}, .optional = true, @@ -56,4 +76,5 @@ struct CmdShowConfig : Command, MixJSON } }; -static auto rShowConfig = registerCommand("show-config"); +static auto rCmdConfig = registerCommand("config"); +static auto rShowConfig = registerCommand2({"config", "show"}); diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 06cc2519e7f7..e68a05c5519b 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -1,11 +1,12 @@ ///@file +#include "signals.hh" +#include "unix-domain-socket.hh" #include "command.hh" #include "shared.hh" #include "local-store.hh" #include "remote-store.hh" #include "remote-store-connection.hh" -#include "util.hh" #include "serialise.hh" #include "archive.hh" #include "globals.hh" @@ -419,16 +420,23 @@ static void processStdioConnection(ref store, AuthenticatedUser user) * * @param forceTrustClientOpt See `daemonLoop()` and the parameter with * the same name over there for details. + * + * @param procesOps Whether to force processing ops even if the next + * store also is a remote store and could process it directly. */ -static void runDaemon(bool stdio, std::optional forceTrustClientOpt) +static void runDaemon(bool stdio, std::optional forceTrustClientOpt, bool processOps) { if (stdio) { auto store = openUncachedStore(); + std::shared_ptr remoteStore; + // If --force-untrusted is passed, we cannot forward the connection and // must process it ourselves (before delegating to the next store) to // force untrusting the client. - if (auto remoteStore = store.dynamic_pointer_cast(); remoteStore && (!forceTrustClientOpt || *forceTrustClientOpt != NotTrusted)) + processOps |= !forceTrustClientOpt || *forceTrustClientOpt != NotTrusted; + + if (!processOps && (remoteStore = store.dynamic_pointer_cast())) forwardStdioConnection(*remoteStore); else { // `Trusted` is passed in the auto (no override case) because we @@ -446,6 +454,7 @@ static int main_nix_daemon(int argc, char * * argv) { auto stdio = false; std::optional isTrustedOpt = std::nullopt; + auto processOps = false; parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { if (*arg == "--daemon") @@ -465,11 +474,14 @@ static int main_nix_daemon(int argc, char * * argv) } else if (*arg == "--default-trust") { experimentalFeatureSettings.require(Xp::DaemonTrustOverride); isTrustedOpt = std::nullopt; + } else if (*arg == "--process-ops") { + experimentalFeatureSettings.require(Xp::MountedSSHStore); + processOps = true; } else return false; return true; }); - runDaemon(stdio, isTrustedOpt); + runDaemon(stdio, isTrustedOpt, processOps); return 0; } @@ -479,6 +491,59 @@ static RegisterLegacyCommand r_nix_daemon("nix-daemon", main_nix_daemon); struct CmdDaemon : StoreCommand { + bool stdio = false; + std::optional isTrustedOpt = std::nullopt; + bool processOps = false; + + CmdDaemon() + { + addFlag({ + .longName = "stdio", + .description = "Attach to standard I/O, instead of trying to bind to a UNIX socket.", + .handler = {&stdio, true}, + }); + + addFlag({ + .longName = "force-trusted", + .description = "Force the daemon to trust connecting clients.", + .handler = {[&]() { + isTrustedOpt = Trusted; + }}, + .experimentalFeature = Xp::DaemonTrustOverride, + }); + + addFlag({ + .longName = "force-untrusted", + .description = "Force the daemon to not trust connecting clients. The connection will be processed by the receiving daemon before forwarding commands.", + .handler = {[&]() { + isTrustedOpt = NotTrusted; + }}, + .experimentalFeature = Xp::DaemonTrustOverride, + }); + + addFlag({ + .longName = "default-trust", + .description = "Use Nix's default trust.", + .handler = {[&]() { + isTrustedOpt = std::nullopt; + }}, + .experimentalFeature = Xp::DaemonTrustOverride, + }); + + addFlag({ + .longName = "process-ops", + .description = R"( + Forces the daemon to process received commands itself rather than forwarding the commands straight to the remote store. + + This is useful for the `mounted-ssh://` store where some actions need to be performed on the remote end but as connected user, and not as the user of the underlying daemon on the remote end. + )", + .handler = {[&]() { + processOps = true; + }}, + .experimentalFeature = Xp::MountedSSHStore, + }); + } + std::string description() override { return "daemon to perform store operations on behalf of non-root clients"; @@ -495,7 +560,7 @@ struct CmdDaemon : StoreCommand void run(ref store) override { - runDaemon(false, std::nullopt); + runDaemon(stdio, isTrustedOpt, processOps); } }; diff --git a/src/nix/daemon.md b/src/nix/daemon.md index d5cdadf08e1f..b1ea850ede25 100644 --- a/src/nix/daemon.md +++ b/src/nix/daemon.md @@ -1,20 +1,44 @@ R""( -# Example +# Examples -* Run the daemon in the foreground: +* Run the daemon: ```console # nix daemon ``` +* Run the daemon and listen on standard I/O instead of binding to a UNIX socket: + + ```console + # nix daemon --stdio + ``` + +* Run the daemon and force all connections to be trusted: + + ```console + # nix daemon --force-trusted + ``` + +* Run the daemon and force all connections to be untrusted: + + ```console + # nix daemon --force-untrusted + ``` + +* Run the daemon, listen on standard I/O, and force all connections to use Nix's default trust: + + ```console + # nix daemon --stdio --default-trust + ``` + # Description This command runs the Nix daemon, which is a required component in multi-user Nix installations. It runs build tasks and other operations on the Nix store on behalf of non-root users. Usually you don't run the daemon directly; instead it's managed by a service -management framework such as `systemd`. +management framework such as `systemd` on Linux, or `launchctl` on Darwin. Note that this daemon does not fork into the background. diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 195eeaa21e9a..38482ed42a46 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -9,8 +9,10 @@ #include "progress-bar.hh" #include "run.hh" +#include #include #include +#include using namespace nix; @@ -51,6 +53,7 @@ struct BuildEnvironment std::map vars; std::map bashFunctions; + std::optional> structuredAttrs; static BuildEnvironment fromJSON(std::string_view in) { @@ -74,6 +77,10 @@ struct BuildEnvironment res.bashFunctions.insert({name, def}); } + if (json.contains("structuredAttrs")) { + res.structuredAttrs = {json["structuredAttrs"][".attrs.json"], json["structuredAttrs"][".attrs.sh"]}; + } + return res; } @@ -102,6 +109,13 @@ struct BuildEnvironment res["bashFunctions"] = bashFunctions; + if (providesStructuredAttrs()) { + auto contents = nlohmann::json::object(); + contents[".attrs.sh"] = getAttrsSH(); + contents[".attrs.json"] = getAttrsJSON(); + res["structuredAttrs"] = std::move(contents); + } + auto json = res.dump(); assert(BuildEnvironment::fromJSON(json) == *this); @@ -109,6 +123,23 @@ struct BuildEnvironment return json; } + bool providesStructuredAttrs() const + { + return structuredAttrs.has_value(); + } + + std::string getAttrsJSON() const + { + assert(providesStructuredAttrs()); + return structuredAttrs->first; + } + + std::string getAttrsSH() const + { + assert(providesStructuredAttrs()); + return structuredAttrs->second; + } + void toBash(std::ostream & out, const std::set & ignoreVars) const { for (auto & [name, value] : vars) { @@ -235,7 +266,7 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore /* Build the derivation. */ store->buildPaths( { DerivedPath::Built { - .drvPath = shellDrvPath, + .drvPath = makeConstantStorePathRef(shellDrvPath), .outputs = OutputsSpec::All { }, }}, bmNormal, evalStore); @@ -291,6 +322,7 @@ struct Common : InstallableCommand, MixProfile std::string makeRcScript( ref store, const BuildEnvironment & buildEnvironment, + const Path & tmpDir, const Path & outputsDir = absPath(".") + "/outputs") { // A list of colon-separated environment variables that should be @@ -353,9 +385,48 @@ struct Common : InstallableCommand, MixProfile } } + if (buildEnvironment.providesStructuredAttrs()) { + fixupStructuredAttrs( + "sh", + "NIX_ATTRS_SH_FILE", + buildEnvironment.getAttrsSH(), + rewrites, + buildEnvironment, + tmpDir + ); + fixupStructuredAttrs( + "json", + "NIX_ATTRS_JSON_FILE", + buildEnvironment.getAttrsJSON(), + rewrites, + buildEnvironment, + tmpDir + ); + } + return rewriteStrings(script, rewrites); } + /** + * Replace the value of NIX_ATTRS_*_FILE (`/build/.attrs.*`) with a tmp file + * that's accessible from the interactive shell session. + */ + void fixupStructuredAttrs( + const std::string & ext, + const std::string & envVar, + const std::string & content, + StringMap & rewrites, + const BuildEnvironment & buildEnvironment, + const Path & tmpDir) + { + auto targetFilePath = tmpDir + "/.attrs." + ext; + writeFile(targetFilePath, content); + + auto fileInBuilderEnv = buildEnvironment.vars.find(envVar); + assert(fileInBuilderEnv != buildEnvironment.vars.end()); + rewrites.insert({BuildEnvironment::getString(fileInBuilderEnv->second), targetFilePath}); + } + Strings getDefaultFlakeAttrPaths() override { Strings paths{ @@ -487,7 +558,9 @@ struct CmdDevelop : Common, MixEnvironment auto [rcFileFd, rcFilePath] = createTempFile("nix-shell"); - auto script = makeRcScript(store, buildEnvironment); + AutoDelete tmpDir(createTempDir("", "nix-develop"), true); + + auto script = makeRcScript(store, buildEnvironment, (Path) tmpDir); if (verbosity >= lvlDebug) script += "set -x\n"; @@ -547,7 +620,7 @@ struct CmdDevelop : Common, MixEnvironment state, std::move(nixpkgs), "bashInteractive", - DefaultOutputs(), + ExtendedOutputsSpec::Default(), Strings{}, Strings{"legacyPackages." + settings.thisSystem.get() + "."}, nixpkgsLockFlags); @@ -615,10 +688,12 @@ struct CmdPrintDevEnv : Common, MixJSON stopProgressBar(); - logger->writeToStdout( - json - ? buildEnvironment.toJSON() - : makeRcScript(store, buildEnvironment)); + if (json) { + logger->writeToStdout(buildEnvironment.toJSON()); + } else { + AutoDelete tmpDir(createTempDir("", "nix-dev-env"), true); + logger->writeToStdout(makeRcScript(store, buildEnvironment, tmpDir)); + } } }; diff --git a/src/nix/develop.md b/src/nix/develop.md index 1b5a8aeba0d5..c49b39669815 100644 --- a/src/nix/develop.md +++ b/src/nix/develop.md @@ -69,7 +69,7 @@ R""( * Run a series of script commands: ```console - # nix develop --command bash --command "mkdir build && cmake .. && make" + # nix develop --command bash -c "mkdir build && cmake .. && make" ``` # Description diff --git a/src/nix/doctor.cc b/src/nix/doctor.cc index 1aa6831d30f3..59f9e3e5d7a9 100644 --- a/src/nix/doctor.cc +++ b/src/nix/doctor.cc @@ -6,7 +6,6 @@ #include "shared.hh" #include "store-api.hh" #include "local-fs-store.hh" -#include "util.hh" #include "worker-protocol.hh" using namespace nix; diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index c4edc894b896..0850d4c1cdb5 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -61,4 +61,12 @@ struct CmdDumpPath2 : Command } }; -static auto rDumpPath2 = registerCommand2({"nar", "dump-path"}); +struct CmdNarDumpPath : CmdDumpPath2 { + void run() override { + warn("'nix nar dump-path' is a deprecated alias for 'nix nar pack'"); + CmdDumpPath2::run(); + } +}; + +static auto rCmdNarPack = registerCommand2({"nar", "pack"}); +static auto rCmdNarDumpPath = registerCommand2({"nar", "dump-path"}); diff --git a/src/nix/edit.cc b/src/nix/edit.cc index 66629fab03ad..9cbab230b0ff 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -1,3 +1,4 @@ +#include "current-process.hh" #include "command-installable-value.hh" #include "shared.hh" #include "eval.hh" diff --git a/src/nix/eval.cc b/src/nix/eval.cc index d880bef0a7b6..b34af34e0eb1 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -85,7 +85,7 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption state->forceValue(v, pos); if (v.type() == nString) // FIXME: disallow strings with contexts? - writeFile(path, v.string.s); + writeFile(path, v.string_view()); else if (v.type() == nAttrs) { if (mkdir(path.c_str(), 0777) == -1) throw SysError("creating directory '%s'", path); diff --git a/src/nix/flake-init.md b/src/nix/flake-init.md index fc1f4f805041..ea274bf29b3c 100644 --- a/src/nix/flake-init.md +++ b/src/nix/flake-init.md @@ -30,7 +30,7 @@ using `-t`. # Template definitions A flake can declare templates through its `templates` output -attribute. A template has two attributes: +attribute. A template has the following attributes: * `description`: A one-line description of the template, in CommonMark syntax. diff --git a/src/nix/flake-lock.md b/src/nix/flake-lock.md index 2af0ad81e0e8..6d10258e338f 100644 --- a/src/nix/flake-lock.md +++ b/src/nix/flake-lock.md @@ -2,37 +2,39 @@ R""( # Examples -* Update the `nixpkgs` and `nix` inputs of the flake in the current - directory: +* Create the lock file for the flake in the current directory: ```console - # nix flake lock --update-input nixpkgs --update-input nix - * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' - * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' + # nix flake lock + warning: creating lock file '/home/myself/repos/testflake/flake.lock': + • Added input 'nix': + 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' (2023-06-28) + • Added input 'nixpkgs': + 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) ``` -# Description - -This command updates the lock file of a flake (`flake.lock`) so that -it contains a lock for every flake input specified in -`flake.nix`. Existing lock file entries are not updated unless -required by a flag such as `--update-input`. +* Add missing inputs to the lock file for a flake in a different directory: -Note that every command that operates on a flake will also update the -lock file if needed, and supports the same flags. Therefore, + ```console + # nix flake lock ~/repos/another + warning: updating lock file '/home/myself/repos/another/flake.lock': + • Added input 'nixpkgs': + 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) + ``` -```console -# nix flake lock --update-input nixpkgs -# nix build -``` + > **Note** + > + > When trying to refer to a flake in a subdirectory, write `./another` + > instead of `another`. + > Otherwise Nix will try to look up the flake in the registry. -is equivalent to: +# Description -```console -# nix build --update-input nixpkgs -``` +This command adds inputs to the lock file of a flake (`flake.lock`) +so that it contains a lock for every flake input specified in +`flake.nix`. Existing lock file entries are not updated. -Thus, this command is only useful if you want to update the lock file -separately from any other action such as building. +If you want to update existing lock entries, use +[`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) )"" diff --git a/src/nix/flake-update.md b/src/nix/flake-update.md index 8c6042d94a3c..63df3b12afee 100644 --- a/src/nix/flake-update.md +++ b/src/nix/flake-update.md @@ -2,33 +2,57 @@ R""( # Examples -* Recreate the lock file (i.e. update all inputs) and commit the new - lock file: +* Update all inputs (i.e. recreate the lock file from scratch): ```console - # nix flake update --commit-lock-file - * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' - * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' - … - warning: committed new revision '158bcbd9d6cc08ab859c0810186c1beebc982aad' + # nix flake update + warning: updating lock file '/home/myself/repos/testflake/flake.lock': + • Updated input 'nix': + 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' (2023-06-28) + → 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' (2023-07-11) + • Updated input 'nixpkgs': + 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) + → 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05) ``` -# Description +* Update only a single input: + + ```console + # nix flake update nixpkgs + warning: updating lock file '/home/myself/repos/testflake/flake.lock': + • Updated input 'nixpkgs': + 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) + → 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05) + ``` + +* Update only a single input of a flake in a different directory: -This command recreates the lock file of a flake (`flake.lock`), thus -updating the lock for every unlocked input (like `nixpkgs`) to its -current version. This is equivalent to passing `--recreate-lock-file` -to any command that operates on a flake. That is, + ```console + # nix flake update nixpkgs --flake ~/repos/another + warning: updating lock file '/home/myself/repos/another/flake.lock': + • Updated input 'nixpkgs': + 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) + → 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05) + ``` + + > **Note** + > + > When trying to refer to a flake in a subdirectory, write `./another` + > instead of `another`. + > Otherwise Nix will try to look up the flake in the registry. + +# Description -```console -# nix flake update -# nix build -``` +This command updates the inputs in a lock file (`flake.lock`). +**By default, all inputs are updated**. If the lock file doesn't exist +yet, it will be created. If inputs are not in the lock file yet, they will be added. -is equivalent to: +Unlike other `nix flake` commands, `nix flake update` takes a list of names of inputs +to update as its positional arguments and operates on the flake in the current directory. +You can pass a different flake-url with `--flake` to override that default. -```console -# nix build --recreate-lock-file -``` +The related command [`nix flake lock`](@docroot@/command-ref/new-cli/nix3-flake-lock.md) +also creates lock files and adds missing inputs, but is safer as it +will never update inputs already in the lock file. )"" diff --git a/src/nix/flake.cc b/src/nix/flake.cc index b5f5d0cac800..e0c67fdfa44c 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -4,6 +4,7 @@ #include "shared.hh" #include "eval.hh" #include "eval-inline.hh" +#include "eval-settings.hh" #include "flake/flake.hh" #include "get-drvs.hh" #include "store-api.hh" @@ -14,6 +15,7 @@ #include "registry.hh" #include "eval-cache.hh" #include "markdown.hh" +#include "users.hh" #include #include @@ -23,8 +25,10 @@ using namespace nix; using namespace nix::flake; using json = nlohmann::json; +struct CmdFlakeUpdate; class FlakeCommand : virtual Args, public MixFlakeOptions { +protected: std::string flakeUrl = "."; public: @@ -35,8 +39,8 @@ class FlakeCommand : virtual Args, public MixFlakeOptions .label = "flake-url", .optional = true, .handler = {&flakeUrl}, - .completer = {[&](size_t, std::string_view prefix) { - completeFlakeRef(getStore(), prefix); + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getStore(), prefix); }} }); } @@ -51,14 +55,19 @@ class FlakeCommand : virtual Args, public MixFlakeOptions return flake::lockFlake(*getEvalState(), getFlakeRef(), lockFlags); } - std::vector getFlakesForCompletion() override + std::vector getFlakeRefsForCompletion() override { - return {flakeUrl}; + return { + // Like getFlakeRef but with expandTilde calld first + parseFlakeRef(expandTilde(flakeUrl), absPath(".")) + }; } }; struct CmdFlakeUpdate : FlakeCommand { +public: + std::string description() override { return "update flake lock file"; @@ -66,9 +75,37 @@ struct CmdFlakeUpdate : FlakeCommand CmdFlakeUpdate() { + expectedArgs.clear(); + addFlag({ + .longName="flake", + .description="The flake to operate on. Default is the current directory.", + .labels={"flake-url"}, + .handler={&flakeUrl}, + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getStore(), prefix); + }} + }); + expectArgs({ + .label="inputs", + .optional=true, + .handler={[&](std::string inputToUpdate){ + InputPath inputPath; + try { + inputPath = flake::parseInputPath(inputToUpdate); + } catch (Error & e) { + warn("Invalid flake input '%s'. To update a specific flake, use 'nix flake update --flake %s' instead.", inputToUpdate, inputToUpdate); + throw e; + } + if (lockFlags.inputUpdates.contains(inputPath)) + warn("Input '%s' was specified multiple times. You may have done this by accident."); + lockFlags.inputUpdates.insert(inputPath); + }}, + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix); + }} + }); + /* Remove flags that don't make sense. */ - removeFlag("recreate-lock-file"); - removeFlag("update-input"); removeFlag("no-update-lock-file"); removeFlag("no-write-lock-file"); } @@ -83,8 +120,9 @@ struct CmdFlakeUpdate : FlakeCommand void run(nix::ref store) override { settings.tarballTtl = 0; + auto updateAll = lockFlags.inputUpdates.empty(); - lockFlags.recreateLockFile = true; + lockFlags.recreateLockFile = updateAll; lockFlags.writeLockFile = true; lockFlags.applyNixConfig = true; @@ -178,14 +216,14 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl j["locked"] = fetchers::attrsToJSON(flake.lockedRef.toAttrs()); if (auto rev = flake.lockedRef.input.getRev()) - j["revision"] = rev->to_string(Base16, false); + j["revision"] = rev->to_string(HashFormat::Base16, false); if (auto dirtyRev = fetchers::maybeGetStrAttr(flake.lockedRef.toAttrs(), "dirtyRev")) j["dirtyRevision"] = *dirtyRev; if (auto revCount = flake.lockedRef.input.getRevCount()) j["revCount"] = *revCount; if (auto lastModified = flake.lockedRef.input.getLastModified()) j["lastModified"] = *lastModified; - j["path"] = store->printStorePath(flake.sourceInfo->storePath); + j["path"] = store->printStorePath(flake.storePath); j["locks"] = lockedFlake.lockFile.toJSON(); logger->cout("%s", j.dump()); } else { @@ -201,11 +239,11 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON *flake.description); logger->cout( ANSI_BOLD "Path:" ANSI_NORMAL " %s", - store->printStorePath(flake.sourceInfo->storePath)); + store->printStorePath(flake.storePath)); if (auto rev = flake.lockedRef.input.getRev()) logger->cout( ANSI_BOLD "Revision:" ANSI_NORMAL " %s", - rev->to_string(Base16, false)); + rev->to_string(HashFormat::Base16, false)); if (auto dirtyRev = fetchers::maybeGetStrAttr(flake.lockedRef.toAttrs(), "dirtyRev")) logger->cout( ANSI_BOLD "Revision:" ANSI_NORMAL " %s", @@ -232,9 +270,13 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON bool last = i + 1 == node.inputs.size(); if (auto lockedNode = std::get_if<0>(&input.second)) { - logger->cout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s", + std::string lastModifiedStr = ""; + if (auto lastModified = (*lockedNode)->lockedRef.input.getLastModified()) + lastModifiedStr = fmt(" (%s)", std::put_time(std::gmtime(&*lastModified), "%F %T")); + logger->cout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s%s", prefix + (last ? treeLast : treeConn), input.first, - (*lockedNode)->lockedRef); + (*lockedNode)->lockedRef, + lastModifiedStr); bool firstVisit = visited.insert(*lockedNode).second; @@ -543,9 +585,9 @@ struct CmdFlakeCheck : FlakeCommand *attr2.value, attr2.pos); if (drvPath && attr_name == settings.thisSystem.get()) { drvPaths.push_back(DerivedPath::Built { - .drvPath = *drvPath, - .outputs = OutputsSpec::All { }, - }); + .drvPath = makeConstantStorePathRef(*drvPath), + .outputs = OutputsSpec::All { }, + }); } } } @@ -757,8 +799,9 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand .description = "The template to use.", .labels = {"template"}, .handler = {&templateUrl}, - .completer = {[&](size_t, std::string_view prefix) { + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { completeFlakeRefWithFragment( + completions, getEvalState(), lockFlags, defaultTemplateAttrPathsPrefixes, @@ -777,7 +820,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment(templateUrl, absPath(".")); auto installable = InstallableFlake(nullptr, - evalState, std::move(templateFlakeRef), templateName, DefaultOutputs(), + evalState, std::move(templateFlakeRef), templateName, ExtendedOutputsSpec::Default(), defaultTemplateAttrPaths, defaultTemplateAttrPathsPrefixes, lockFlags); @@ -971,7 +1014,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun StorePathSet sources; - sources.insert(flake.flake.sourceInfo->storePath); + sources.insert(flake.flake.storePath); // FIXME: use graph output, handle cycles. std::function traverse; @@ -983,7 +1026,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun auto storePath = dryRun ? (*inputNode)->lockedRef.input.computeStorePath(*store) - : (*inputNode)->lockedRef.input.fetch(store).first.storePath; + : (*inputNode)->lockedRef.input.fetch(store).first; if (json) { auto& jsonObj3 = jsonObj2[inputName]; jsonObj3["path"] = store->printStorePath(storePath); @@ -1000,7 +1043,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun if (json) { nlohmann::json jsonRoot = { - {"path", store->printStorePath(flake.flake.sourceInfo->storePath)}, + {"path", store->printStorePath(flake.flake.storePath)}, {"inputs", traverse(*flake.lockFile.root)}, }; logger->cout("%s", jsonRoot); @@ -1334,19 +1377,21 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON { auto originalRef = getFlakeRef(); auto resolvedRef = originalRef.resolve(store); - auto [tree, lockedRef] = resolvedRef.fetchTree(store); - auto hash = store->queryPathInfo(tree.storePath)->narHash; + auto [storePath, lockedRef] = resolvedRef.fetchTree(store); + auto hash = store->queryPathInfo(storePath)->narHash; if (json) { auto res = nlohmann::json::object(); - res["storePath"] = store->printStorePath(tree.storePath); - res["hash"] = hash.to_string(SRI, true); + res["storePath"] = store->printStorePath(storePath); + res["hash"] = hash.to_string(HashFormat::SRI, true); + res["original"] = fetchers::attrsToJSON(resolvedRef.toAttrs()); + res["locked"] = fetchers::attrsToJSON(lockedRef.toAttrs()); logger->cout(res.dump()); } else { notice("Downloaded '%s' to '%s' (hash '%s').", lockedRef.to_string(), - store->printStorePath(tree.storePath), - hash.to_string(SRI, true)); + store->printStorePath(storePath), + hash.to_string(HashFormat::SRI, true)); } } }; diff --git a/src/nix/flake.md b/src/nix/flake.md index 92f477917fdb..d8b5bf435f1f 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -67,6 +67,11 @@ inputs.nixpkgs = { }; ``` +Following [RFC 3986](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1), +characters outside of the allowed range (i.e. neither [reserved characters](https://datatracker.ietf.org/doc/html/rfc3986#section-2.2) +nor [unreserved characters](https://datatracker.ietf.org/doc/html/rfc3986#section-2.3)) +must be percent-encoded. + ### Examples Here are some examples of flake references in their URL-like representation: @@ -93,7 +98,15 @@ Here are some examples of flake references in their URL-like representation: ## Path-like syntax -Flakes corresponding to a local path can also be referred to by a direct path reference, either `/absolute/path/to/the/flake` or `./relative/path/to/the/flake` (note that the leading `./` is mandatory for relative paths to avoid any ambiguity). +Flakes corresponding to a local path can also be referred to by a direct +path reference, either `/absolute/path/to/the/flake` or`./relative/path/to/the/flake`. +Note that the leading `./` is mandatory for relative paths. If it is +omitted, the path will be interpreted as [URL-like syntax](#url-like-syntax), +which will cause error messages like this: + +```console +error: cannot find flake 'flake:relative/path/to/the/flake' in the flake registries +``` The semantic of such a path is as follows: @@ -103,10 +116,14 @@ The semantic of such a path is as follows: 2. The filesystem root (/), or 3. A folder on a different mount point. +Contrary to URL-like references, path-like flake references can contain arbitrary unicode characters (except `#` and `?`). + ### Examples * `.`: The flake to which the current directory belongs to. * `/home/alice/src/patchelf`: A flake in some other directory. +* `./../sub directory/with Ûñî©ôδ€`: A flake in another relative directory that + has Unicode characters in its name. ## Flake reference attributes @@ -144,18 +161,39 @@ can occur in *locked* flake references and are available to Nix code: Currently the `type` attribute can be one of the following: -* `path`: arbitrary local directories, or local Git trees. The - required attribute `path` specifies the path of the flake. The URL - form is +* `indirect`: *The default*. Indirection through the flake registry. + These have the form + + ``` + [flake:](/(/rev)?)? + ``` + + These perform a lookup of `` in the flake registry. For + example, `nixpkgs` and `nixpkgs/release-20.09` are indirect flake + references. The specified `rev` and/or `ref` are merged with the + entry in the registry; see [nix registry](./nix3-registry.md) for + details. + + For example, these are valid indirect flake references: + + * `nixpkgs` + * `nixpkgs/nixos-unstable` + * `nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293` + * `nixpkgs/nixos-unstable/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293` + * `sub/dir` (if a flake named `sub` is in the registry) + +* `path`: arbitrary local directories. The required attribute `path` + specifies the path of the flake. The URL form is ``` - [path:](\?(\?)? ``` - where *path* is an absolute path. + where *path* is an absolute path to a directory in the file system + containing a file named `flake.nix`. - *path* must be a directory in the file system containing a file - named `flake.nix`. + If the flake at *path* is not inside a git repository, the `path:` + prefix is implied and can be omitted. *path* generally must be an absolute path. However, on the command line, it can be a relative path (e.g. `.` or `./foo`) which is @@ -164,15 +202,25 @@ Currently the `type` attribute can be one of the following: (e.g. `nixpkgs` is a registry lookup; `./nixpkgs` is a relative path). + For example, these are valid path flake references: + + * `path:/home/user/sub/dir` + * `/home/user/sub/dir` (if `dir/flake.nix` is *not* in a git repository) + * `./sub/dir` (when used on the command line and `dir/flake.nix` is *not* in a git repository) + * `git`: Git repositories. The location of the repository is specified by the attribute `url`. They have the URL form ``` - git(+http|+https|+ssh|+git|+file|):(//)?(\?)? + git(+http|+https|+ssh|+git|+file):(//)?(\?)? ``` + If *path* starts with `/` (or `./` when used as an argument on the + command line) and is a local path to a git repository, the leading + `git:` or `+file` prefixes are implied and can be omitted. + The `ref` attribute defaults to resolving the `HEAD` reference. The `rev` attribute must denote a commit that exists in the branch @@ -188,6 +236,9 @@ Currently the `type` attribute can be one of the following: For example, the following are valid Git flake references: + * `git:/home/user/sub/dir` + * `/home/user/sub/dir` (if `dir/flake.nix` is in a git repository) + * `./sub/dir` (when used on the command line and `dir/flake.nix` is in a git repository) * `git+https://example.org/my/repo` * `git+https://example.org/my/repo?dir=flake1` * `git+ssh://git@github.com/NixOS/nix?ref=v1.2.3` @@ -309,19 +360,6 @@ Currently the `type` attribute can be one of the following: * `sourcehut:~misterio/nix-colors/182b4b8709b8ffe4e9774a4c5d6877bf6bb9a21c` * `sourcehut:~misterio/nix-colors/21c1a380a6915d890d408e9f22203436a35bb2de?host=hg.sr.ht` -* `indirect`: Indirections through the flake registry. These have the - form - - ``` - [flake:](/(/rev)?)? - ``` - - These perform a lookup of `` in the flake registry. For - example, `nixpkgs` and `nixpkgs/release-20.09` are indirect flake - references. The specified `rev` and/or `ref` are merged with the - entry in the registry; see [nix registry](./nix3-registry.md) for - details. - # Flake format As an example, here is a simple `flake.nix` that depends on the diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh index a7a8a01b9945..832cc2f1134f 100644 --- a/src/nix/get-env.sh +++ b/src/nix/get-env.sh @@ -1,5 +1,5 @@ set -e -if [ -e .attrs.sh ]; then source .attrs.sh; fi +if [ -e "$NIX_ATTRS_SH_FILE" ]; then source "$NIX_ATTRS_SH_FILE"; fi export IN_NIX_SHELL=impure export dontAddDisableDepTrack=1 @@ -101,7 +101,21 @@ __dumpEnv() { printf "}" done < <(printf "%s\n" "$__vars") - printf '\n }\n}' + printf '\n }' + + if [ -e "$NIX_ATTRS_SH_FILE" ]; then + printf ',\n "structuredAttrs": {\n ' + __escapeString ".attrs.sh" + printf ': ' + __escapeString "$(<"$NIX_ATTRS_SH_FILE")" + printf ',\n ' + __escapeString ".attrs.json" + printf ': ' + __escapeString "$(<"$NIX_ATTRS_JSON_FILE")" + printf '\n }' + fi + + printf '\n}' } __escapeString() { @@ -117,7 +131,7 @@ __escapeString() { # In case of `__structuredAttrs = true;` the list of outputs is an associative # array with a format like `outname => /nix/store/hash-drvname-outname`, so `__olist` # must contain the array's keys (hence `${!...[@]}`) in this case. -if [ -e .attrs.sh ]; then +if [ -e "$NIX_ATTRS_SH_FILE" ]; then __olist="${!outputs[@]}" else __olist=$outputs diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 9feca934557c..d6595dcca059 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -11,7 +11,7 @@ using namespace nix; struct CmdHashBase : Command { FileIngestionMethod mode; - Base base = SRI; + HashFormat hashFormat = HashFormat::SRI; bool truncate = false; HashType ht = htSHA256; std::vector paths; @@ -22,25 +22,25 @@ struct CmdHashBase : Command addFlag({ .longName = "sri", .description = "Print the hash in SRI format.", - .handler = {&base, SRI}, + .handler = {&hashFormat, HashFormat::SRI}, }); addFlag({ .longName = "base64", .description = "Print the hash in base-64 format.", - .handler = {&base, Base64}, + .handler = {&hashFormat, HashFormat::Base64}, }); addFlag({ .longName = "base32", .description = "Print the hash in base-32 (Nix-specific) format.", - .handler = {&base, Base32}, + .handler = {&hashFormat, HashFormat::Base32}, }); addFlag({ .longName = "base16", .description = "Print the hash in base-16 format.", - .handler = {&base, Base16}, + .handler = {&hashFormat, HashFormat::Base16}, }); addFlag(Flag::mkHashTypeFlag("type", &ht)); @@ -94,18 +94,18 @@ struct CmdHashBase : Command Hash h = hashSink->finish().first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); - logger->cout(h.to_string(base, base == SRI)); + logger->cout(h.to_string(hashFormat, hashFormat == HashFormat::SRI)); } } }; struct CmdToBase : Command { - Base base; + HashFormat hashFormat; std::optional ht; std::vector args; - CmdToBase(Base base) : base(base) + CmdToBase(HashFormat hashFormat) : hashFormat(hashFormat) { addFlag(Flag::mkHashTypeOptFlag("type", &ht)); expectArgs("strings", &args); @@ -114,16 +114,16 @@ struct CmdToBase : Command std::string description() override { return fmt("convert a hash to %s representation", - base == Base16 ? "base-16" : - base == Base32 ? "base-32" : - base == Base64 ? "base-64" : + hashFormat == HashFormat::Base16 ? "base-16" : + hashFormat == HashFormat::Base32 ? "base-32" : + hashFormat == HashFormat::Base64 ? "base-64" : "SRI"); } void run() override { for (auto s : args) - logger->cout(Hash::parseAny(s, ht).to_string(base, base == SRI)); + logger->cout(Hash::parseAny(s, ht).to_string(hashFormat, hashFormat == HashFormat::SRI)); } }; @@ -133,10 +133,10 @@ struct CmdHash : NixMultiCommand : MultiCommand({ {"file", []() { return make_ref(FileIngestionMethod::Flat);; }}, {"path", []() { return make_ref(FileIngestionMethod::Recursive); }}, - {"to-base16", []() { return make_ref(Base16); }}, - {"to-base32", []() { return make_ref(Base32); }}, - {"to-base64", []() { return make_ref(Base64); }}, - {"to-sri", []() { return make_ref(SRI); }}, + {"to-base16", []() { return make_ref(HashFormat::Base16); }}, + {"to-base32", []() { return make_ref(HashFormat::Base32); }}, + {"to-base64", []() { return make_ref(HashFormat::Base64); }}, + {"to-sri", []() { return make_ref(HashFormat::SRI); }}, }) { } @@ -162,7 +162,7 @@ static int compatNixHash(int argc, char * * argv) { std::optional ht; bool flat = false; - Base base = Base16; + HashFormat hashFormat = HashFormat::Base16; bool truncate = false; enum { opHash, opTo } op = opHash; std::vector ss; @@ -173,10 +173,10 @@ static int compatNixHash(int argc, char * * argv) else if (*arg == "--version") printVersion("nix-hash"); else if (*arg == "--flat") flat = true; - else if (*arg == "--base16") base = Base16; - else if (*arg == "--base32") base = Base32; - else if (*arg == "--base64") base = Base64; - else if (*arg == "--sri") base = SRI; + else if (*arg == "--base16") hashFormat = HashFormat::Base16; + else if (*arg == "--base32") hashFormat = HashFormat::Base32; + else if (*arg == "--base64") hashFormat = HashFormat::Base64; + else if (*arg == "--sri") hashFormat = HashFormat::SRI; else if (*arg == "--truncate") truncate = true; else if (*arg == "--type") { std::string s = getArg(*arg, arg, end); @@ -184,19 +184,19 @@ static int compatNixHash(int argc, char * * argv) } else if (*arg == "--to-base16") { op = opTo; - base = Base16; + hashFormat = HashFormat::Base16; } else if (*arg == "--to-base32") { op = opTo; - base = Base32; + hashFormat = HashFormat::Base32; } else if (*arg == "--to-base64") { op = opTo; - base = Base64; + hashFormat = HashFormat::Base64; } else if (*arg == "--to-sri") { op = opTo; - base = SRI; + hashFormat = HashFormat::SRI; } else if (*arg != "" && arg->at(0) == '-') return false; @@ -209,14 +209,14 @@ static int compatNixHash(int argc, char * * argv) CmdHashBase cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive); if (!ht.has_value()) ht = htMD5; cmd.ht = ht.value(); - cmd.base = base; + cmd.hashFormat = hashFormat; cmd.truncate = truncate; cmd.paths = ss; cmd.run(); } else { - CmdToBase cmd(base); + CmdToBase cmd(hashFormat); cmd.args = ss; if (ht.has_value()) cmd.ht = ht; cmd.run(); diff --git a/src/nix/local.mk b/src/nix/local.mk index 20ea29d10fac..a21aa705ffb1 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -31,10 +31,23 @@ src/nix/develop.cc: src/nix/get-env.sh.gen.hh src/nix-channel/nix-channel.cc: src/nix-channel/unpack-channel.nix.gen.hh -src/nix/main.cc: doc/manual/generate-manpage.nix.gen.hh doc/manual/utils.nix.gen.hh +src/nix/main.cc: \ + doc/manual/generate-manpage.nix.gen.hh \ + doc/manual/utils.nix.gen.hh doc/manual/generate-settings.nix.gen.hh \ + doc/manual/generate-store-info.nix.gen.hh \ + src/nix/generated-doc/help-stores.md -src/nix/doc/files/%.md: doc/manual/src/command-ref/files/%.md +src/nix/generated-doc/files/%.md: doc/manual/src/command-ref/files/%.md @mkdir -p $$(dirname $@) @cp $< $@ -src/nix/profile.cc: src/nix/profile.md src/nix/doc/files/profiles.md.gen.hh +src/nix/profile.cc: src/nix/profile.md src/nix/generated-doc/files/profiles.md.gen.hh + +src/nix/generated-doc/help-stores.md: doc/manual/src/store/types/index.md.in + @mkdir -p $$(dirname $@) + @echo 'R"(' >> $@.tmp + @echo >> $@.tmp + @cat $^ >> $@.tmp + @echo >> $@.tmp + @echo ')"' >> $@.tmp + @mv $@.tmp $@ diff --git a/src/nix/log.cc b/src/nix/log.cc index aaf82976404f..9a9bd30f9619 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -33,6 +33,17 @@ struct CmdLog : InstallableCommand auto b = installable->toDerivedPath(); + // For compat with CLI today, TODO revisit + auto oneUp = std::visit(overloaded { + [&](const DerivedPath::Opaque & bo) { + return make_ref(bo); + }, + [&](const DerivedPath::Built & bfd) { + return bfd.drvPath; + }, + }, b.path.raw()); + auto path = resolveDerivedPath(*store, *oneUp); + RunPager pager; for (auto & sub : subs) { auto * logSubP = dynamic_cast(&*sub); @@ -42,14 +53,7 @@ struct CmdLog : InstallableCommand } auto & logSub = *logSubP; - auto log = std::visit(overloaded { - [&](const DerivedPath::Opaque & bo) { - return logSub.getBuildLog(bo.path); - }, - [&](const DerivedPath::Built & bfd) { - return logSub.getBuildLog(bfd.drvPath); - }, - }, b.path.raw()); + auto log = logSub.getBuildLog(path); if (!log) continue; stopProgressBar(); printInfo("got build log for '%s' from '%s'", installable->what(), logSub.getUri()); diff --git a/src/nix/ls.cc b/src/nix/ls.cc index c990a303c12f..231456c9c6c2 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -1,6 +1,5 @@ #include "command.hh" #include "store-api.hh" -#include "fs-accessor.hh" #include "nar-accessor.hh" #include "common-args.hh" #include @@ -39,61 +38,58 @@ struct MixLs : virtual Args, MixJSON }); } - void listText(ref accessor) + void listText(ref accessor) { - std::function doPath; + std::function doPath; - auto showFile = [&](const Path & curPath, const std::string & relPath) { + auto showFile = [&](const CanonPath & curPath, std::string_view relPath) { if (verbose) { - auto st = accessor->stat(curPath); + auto st = accessor->lstat(curPath); std::string tp = - st.type == FSAccessor::Type::tRegular ? + st.type == SourceAccessor::Type::tRegular ? (st.isExecutable ? "-r-xr-xr-x" : "-r--r--r--") : - st.type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" : + st.type == SourceAccessor::Type::tSymlink ? "lrwxrwxrwx" : "dr-xr-xr-x"; - auto line = fmt("%s %20d %s", tp, st.fileSize, relPath); - if (st.type == FSAccessor::Type::tSymlink) + auto line = fmt("%s %20d %s", tp, st.fileSize.value_or(0), relPath); + if (st.type == SourceAccessor::Type::tSymlink) line += " -> " + accessor->readLink(curPath); logger->cout(line); - if (recursive && st.type == FSAccessor::Type::tDirectory) + if (recursive && st.type == SourceAccessor::Type::tDirectory) doPath(st, curPath, relPath, false); } else { logger->cout(relPath); if (recursive) { - auto st = accessor->stat(curPath); - if (st.type == FSAccessor::Type::tDirectory) + auto st = accessor->lstat(curPath); + if (st.type == SourceAccessor::Type::tDirectory) doPath(st, curPath, relPath, false); } } }; - doPath = [&](const FSAccessor::Stat & st, const Path & curPath, - const std::string & relPath, bool showDirectory) + doPath = [&](const SourceAccessor::Stat & st, const CanonPath & curPath, + std::string_view relPath, bool showDirectory) { - if (st.type == FSAccessor::Type::tDirectory && !showDirectory) { + if (st.type == SourceAccessor::Type::tDirectory && !showDirectory) { auto names = accessor->readDirectory(curPath); - for (auto & name : names) - showFile(curPath + "/" + name, relPath + "/" + name); + for (auto & [name, type] : names) + showFile(curPath + name, relPath + "/" + name); } else showFile(curPath, relPath); }; - auto st = accessor->stat(path); - if (st.type == FSAccessor::Type::tMissing) - throw Error("path '%1%' does not exist", path); - doPath(st, path, - st.type == FSAccessor::Type::tDirectory ? "." : std::string(baseNameOf(path)), + auto path2 = CanonPath(path); + auto st = accessor->lstat(path2); + doPath(st, path2, + st.type == SourceAccessor::Type::tDirectory ? "." : path2.baseName().value_or(""), showDirectory); } - void list(ref accessor) + void list(ref accessor) { - if (path == "/") path = ""; - if (json) { if (showDirectory) throw UsageError("'--directory' is useless with '--json'"); - logger->cout("%s", listNar(accessor, path, recursive)); + logger->cout("%s", listNar(accessor, CanonPath(path), recursive)); } else listText(accessor); } diff --git a/src/nix/main.cc b/src/nix/main.cc index 650c79d1483c..49e637fb0c40 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -1,8 +1,12 @@ #include +#include "args/root.hh" +#include "current-process.hh" +#include "namespaces.hh" #include "command.hh" #include "common-args.hh" #include "eval.hh" +#include "eval-settings.hh" #include "globals.hh" #include "legacy.hh" #include "shared.hh" @@ -11,12 +15,14 @@ #include "finally.hh" #include "loggers.hh" #include "markdown.hh" +#include "memory-input-accessor.hh" #include #include #include #include #include +#include #include @@ -54,7 +60,7 @@ static bool haveInternet() std::string programPath; -struct NixArgs : virtual MultiCommand, virtual MixCommonArgs +struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs { bool useNet = true; bool refresh = false; @@ -128,6 +134,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs {"ping-store", {"store", "ping"}}, {"sign-paths", {"store", "sign"}}, {"show-derivation", {"derivation", "show"}}, + {"show-config", {"config", "show"}}, {"to-base16", {"hash", "to-base16"}}, {"to-base32", {"hash", "to-base32"}}, {"to-base64", {"hash", "to-base64"}}, @@ -179,10 +186,13 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs for (auto & implem : *Implementations::registered) { auto storeConfig = implem.getConfig(); auto storeName = storeConfig->name(); - stores[storeName]["doc"] = storeConfig->doc(); - stores[storeName]["settings"] = storeConfig->toJSON(); + auto & j = stores[storeName]; + j["doc"] = storeConfig->doc(); + j["settings"] = storeConfig->toJSON(); + j["experimentalFeature"] = storeConfig->experimentalFeature(); } res["stores"] = std::move(stores); + res["fetchers"] = fetchers::dumpRegisterInputSchemeInfo(); return res.dump(); } @@ -201,21 +211,29 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) auto vGenerateManpage = state.allocValue(); state.eval(state.parseExprFromString( #include "generate-manpage.nix.gen.hh" - , CanonPath::root), *vGenerateManpage); + , state.rootPath(CanonPath::root)), *vGenerateManpage); - auto vUtils = state.allocValue(); - state.cacheFile( - CanonPath("/utils.nix"), CanonPath("/utils.nix"), - state.parseExprFromString( - #include "utils.nix.gen.hh" - , CanonPath::root), - *vUtils); + state.corepkgsFS->addFile( + CanonPath("utils.nix"), + #include "utils.nix.gen.hh" + ); + + state.corepkgsFS->addFile( + CanonPath("/generate-settings.nix"), + #include "generate-settings.nix.gen.hh" + ); + + state.corepkgsFS->addFile( + CanonPath("/generate-store-info.nix"), + #include "generate-store-info.nix.gen.hh" + ); auto vDump = state.allocValue(); vDump->mkString(toplevel.dumpCli()); auto vRes = state.allocValue(); - state.callFunction(*vGenerateManpage, *vDump, *vRes, noPos); + state.callFunction(*vGenerateManpage, state.getBuiltin("false"), *vRes, noPos); + state.callFunction(*vRes, *vDump, *vRes, noPos); auto attr = vRes->attrs->get(state.symbols.create(mdName + ".md")); if (!attr) @@ -229,10 +247,7 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) static NixArgs & getNixArgs(Command & cmd) { - assert(cmd.parent); - MultiCommand * toplevel = cmd.parent; - while (toplevel->parent) toplevel = toplevel->parent; - return dynamic_cast(*toplevel); + return dynamic_cast(cmd.getRoot()); } struct CmdHelp : Command @@ -282,7 +297,7 @@ struct CmdHelpStores : Command std::string doc() override { return - #include "help-stores.md" + #include "generated-doc/help-stores.md" ; } @@ -356,6 +371,7 @@ void mainWrapped(int argc, char * * argv) experimentalFeatureSettings.experimentalFeatures = { Xp::Flakes, Xp::FetchClosure, + Xp::DynamicDerivations, }; evalSettings.pureEval = false; EvalState state({}, openStore("dummy://")); @@ -399,24 +415,26 @@ void mainWrapped(int argc, char * * argv) Finally printCompletions([&]() { - if (completions) { - switch (completionType) { - case ctNormal: + if (args.completions) { + switch (args.completions->type) { + case Completions::Type::Normal: logger->cout("normal"); break; - case ctFilenames: + case Completions::Type::Filenames: logger->cout("filenames"); break; - case ctAttrs: + case Completions::Type::Attrs: logger->cout("attrs"); break; } - for (auto & s : *completions) + for (auto & s : args.completions->completions) logger->cout(s.completion + "\t" + trim(s.description)); } }); try { - args.parseCmdline(argvToStrings(argc, argv)); + auto isNixCommand = std::regex_search(programName, std::regex("nix$")); + auto allowShebang = isNixCommand && argc > 1; + args.parseCmdline(argvToStrings(argc, argv),allowShebang); } catch (UsageError &) { - if (!args.helpRequested && !completions) throw; + if (!args.helpRequested && !args.completions) throw; } if (args.helpRequested) { @@ -433,10 +451,7 @@ void mainWrapped(int argc, char * * argv) return; } - if (completions) { - args.completionHook(); - return; - } + if (args.completions) return; if (args.showVersion) { printVersion(programName); diff --git a/src/nix/nar-dump-path.md b/src/nix/nar-dump-path.md index 26191ad250f0..de82202decd0 100644 --- a/src/nix/nar-dump-path.md +++ b/src/nix/nar-dump-path.md @@ -5,7 +5,7 @@ R""( * To serialise directory `foo` as a NAR: ```console - # nix nar dump-path ./foo > foo.nar + # nix nar pack ./foo > foo.nar ``` # Description diff --git a/src/nix/nix.md b/src/nix/nix.md index 6d9e40dbc272..749456014b2e 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -63,7 +63,7 @@ The following types of installable are supported by most commands: - [Nix file](#nix-file), optionally qualified by an attribute path - [Nix expression](#nix-expression), optionally qualified by an attribute path -For most commands, if no installable is specified, `.` as assumed. +For most commands, if no installable is specified, `.` is assumed. That is, Nix will operate on the default flake output attribute of the flake in the current directory. ### Flake output attribute @@ -132,6 +132,8 @@ subcommands, these are `packages.`*system*, attributes `packages.x86_64-linux.hello`, `legacyPackages.x86_64-linux.hello` and `hello`. +If *attrpath* begins with `.` then no prefixes or defaults are attempted. This allows the form *flakeref*[`#.`*attrpath*], such as `github:NixOS/nixpkgs#.lib.fakeSha256` to avoid a search of `packages.*system*.lib.fakeSha256` + ### Store path Example: `/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10` @@ -233,7 +235,74 @@ operate are determined as follows: # Nix stores -Most `nix` subcommands operate on a *Nix store*. These are documented -in [`nix help-stores`](./nix3-help-stores.md). +Most `nix` subcommands operate on a *Nix store*. +The various store types are documented in the +[Store Types](@docroot@/store/types/index.md) +section of the manual. + +The same information is also available from the [`nix help-stores`](./nix3-help-stores.md) command. + +# Shebang interpreter + +The `nix` command can be used as a `#!` interpreter. +Arguments to Nix can be passed on subsequent lines in the script. + +Verbatim strings may be passed in double backtick (```` `` ````) quotes. +Sequences of _n_ backticks of 3 or longer are parsed as _n-1_ literal backticks. +A single space before the closing ```` `` ```` is ignored if present. + +`--file` and `--expr` resolve relative paths based on the script location. + +Examples: + +``` +#!/usr/bin/env nix +#! nix shell --file ```` hello cowsay --command bash + +hello | cowsay +``` + +or with **flakes**: + +``` +#!/usr/bin/env nix +#! nix shell nixpkgs#bash nixpkgs#hello nixpkgs#cowsay --command bash + +hello | cowsay +``` + +or with an **expression**: + +```bash +#! /usr/bin/env nix +#! nix shell --impure --expr `` +#! nix with (import (builtins.getFlake "nixpkgs") {}); +#! nix terraform.withPlugins (plugins: [ plugins.openstack ]) +#! nix `` +#! nix --command bash + +terraform "$@" +``` + +or with cascading interpreters. Note that the `#! nix` lines don't need to follow after the first line, to accomodate other interpreters. + +``` +#!/usr/bin/env nix +//! ```cargo +//! [dependencies] +//! time = "0.1.25" +//! ``` +/* +#!nix shell nixpkgs#rustc nixpkgs#rust-script nixpkgs#cargo --command rust-script +*/ +fn main() { + for argument in std::env::args().skip(1) { + println!("{}", argument); + }; + println!("{}", std::env::var("HOME").expect("")); + println!("{}", time::now().rfc822z()); +} +// vim: ft=rust +``` )"" diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 613c5b1918dc..080d6bbf169f 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -9,6 +9,75 @@ #include using namespace nix; +using nlohmann::json; + +/** + * @return the total size of a set of store objects (specified by path), + * that is, the sum of the size of the NAR serialisation of each object + * in the set. + */ +static uint64_t getStoreObjectsTotalSize(Store & store, const StorePathSet & closure) +{ + uint64_t totalNarSize = 0; + for (auto & p : closure) { + totalNarSize += store.queryPathInfo(p)->narSize; + } + return totalNarSize; +} + + +/** + * Write a JSON representation of store object metadata, such as the + * hash and the references. + * + * @param showClosureSize If true, the closure size of each path is + * included. + */ +static json pathInfoToJSON( + Store & store, + const StorePathSet & storePaths, + bool showClosureSize) +{ + json::object_t jsonAllObjects = json::object(); + + for (auto & storePath : storePaths) { + json jsonObject; + + try { + auto info = store.queryPathInfo(storePath); + + jsonObject = info->toJSON(store, true, HashFormat::SRI); + + if (showClosureSize) { + StorePathSet closure; + store.computeFSClosure(storePath, closure, false, false); + + jsonObject["closureSize"] = getStoreObjectsTotalSize(store, closure); + + if (dynamic_cast(&*info)) { + uint64_t totalDownloadSize = 0; + for (auto & p : closure) { + auto depInfo = store.queryPathInfo(p); + if (auto * depNarInfo = dynamic_cast(&*depInfo)) + totalDownloadSize += depNarInfo->fileSize; + else + throw Error("Missing .narinfo for dep %s of %s", + store.printStorePath(p), + store.printStorePath(storePath)); + } + jsonObject["closureDownloadSize"] = totalDownloadSize; + } + } + + } catch (InvalidPath &) { + jsonObject = nullptr; + } + + jsonAllObjects[store.printStorePath(storePath)] = std::move(jsonObject); + } + return jsonAllObjects; +} + struct CmdPathInfo : StorePathsCommand, MixJSON { @@ -87,10 +156,11 @@ struct CmdPathInfo : StorePathsCommand, MixJSON pathLen = std::max(pathLen, store->printStorePath(storePath).size()); if (json) { - std::cout << store->pathInfoToJSON( + std::cout << pathInfoToJSON( + *store, // FIXME: preserve order? StorePathSet(storePaths.begin(), storePaths.end()), - true, showClosureSize, SRI, AllowInvalid).dump(); + showClosureSize).dump(); } else { @@ -107,8 +177,11 @@ struct CmdPathInfo : StorePathsCommand, MixJSON if (showSize) printSize(info->narSize); - if (showClosureSize) - printSize(store->getClosureSize(info->path).first); + if (showClosureSize) { + StorePathSet closure; + store->computeFSClosure(storePath, closure, false, false); + printSize(getStoreObjectsTotalSize(*store, closure)); + } if (showSigs) { std::cout << '\t'; diff --git a/src/nix/path-info.md b/src/nix/path-info.md index 2dda866d05d7..4594854eba2d 100644 --- a/src/nix/path-info.md +++ b/src/nix/path-info.md @@ -43,7 +43,7 @@ R""( command): ```console - # nix path-info --json --all | jq -r 'sort_by(.registrationTime)[-11:-1][].path' + # nix path-info --json --all | jq -r 'to_entries | sort_by(.value.registrationTime) | .[-11:-1][] | .key' ``` * Show the size of the entire Nix store: @@ -58,13 +58,13 @@ R""( ```console # nix path-info --json --all --closure-size \ - | jq 'map(select(.closureSize > 1e9)) | sort_by(.closureSize) | map([.path, .closureSize])' + | jq 'map_values(.closureSize | select(. < 1e9)) | to_entries | sort_by(.value)' [ …, - [ - "/nix/store/zqamz3cz4dbzfihki2mk7a63mbkxz9xq-nixos-system-machine-20.09.20201112.3090c65", - 5887562256 - ] + { + .key = "/nix/store/zqamz3cz4dbzfihki2mk7a63mbkxz9xq-nixos-system-machine-20.09.20201112.3090c65", + .value = 5887562256, + } ] ``` diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 3b2e225f6efa..3ed7946a8b84 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -71,10 +71,8 @@ std::tuple prefetchFile( if (expectedHash) { hashType = expectedHash->type; storePath = store->makeFixedOutputPath(*name, FixedOutputInfo { - .hash = { - .method = ingestionMethod, - .hash = *expectedHash, - }, + .method = ingestionMethod, + .hash = *expectedHash, .references = {}, }); if (store->isValidPath(*storePath)) @@ -127,7 +125,7 @@ std::tuple prefetchFile( auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash); storePath = info.path; assert(info.ca); - hash = info.ca->getHash(); + hash = info.ca->hash; } return {storePath.value(), hash.value()}; @@ -312,13 +310,13 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON if (json) { auto res = nlohmann::json::object(); res["storePath"] = store->printStorePath(storePath); - res["hash"] = hash.to_string(SRI, true); + res["hash"] = hash.to_string(HashFormat::SRI, true); logger->cout(res.dump()); } else { notice("Downloaded '%s' to '%s' (hash '%s').", url, store->printStorePath(storePath), - hash.to_string(SRI, true)); + hash.to_string(HashFormat::SRI, true)); } } }; diff --git a/src/nix/profile-list.md b/src/nix/profile-list.md index fa786162f3e1..5d7fcc0ecc70 100644 --- a/src/nix/profile-list.md +++ b/src/nix/profile-list.md @@ -6,26 +6,48 @@ R""( ```console # nix profile list - 0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.spotify /nix/store/akpdsid105phbbvknjsdh7hl4v3fhjkr-spotify-1.1.46.916.g416cacf1 - 1 flake:nixpkgs#legacyPackages.x86_64-linux.zoom-us github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.zoom-us /nix/store/89pmjmbih5qpi7accgacd17ybpgp4xfm-zoom-us-5.4.53350.1027 - 2 flake:blender-bin#packages.x86_64-linux.default github:edolstra/nix-warez/d09d7eea893dcb162e89bc67f6dc1ced14abfc27?dir=blender#packages.x86_64-linux.default /nix/store/zfgralhqjnam662kqsgq6isjw8lhrflz-blender-bin-2.91.0 + Index: 0 + Flake attribute: legacyPackages.x86_64-linux.gdb + Original flake URL: flake:nixpkgs + Locked flake URL: github:NixOS/nixpkgs/7b38b03d76ab71bdc8dc325e3f6338d984cc35ca + Store paths: /nix/store/indzcw5wvlhx6vwk7k4iq29q15chvr3d-gdb-11.1 + + Index: 1 + Flake attribute: packages.x86_64-linux.default + Original flake URL: flake:blender-bin + Locked flake URL: github:edolstra/nix-warez/91f2ffee657bf834e4475865ae336e2379282d34?dir=blender + Store paths: /nix/store/i798sxl3j40wpdi1rgf391id1b5klw7g-blender-bin-3.1.2 ``` + Note that you can unambiguously rebuild a package from a profile + through its locked flake URL and flake attribute, e.g. + + ```console + # nix build github:edolstra/nix-warez/91f2ffee657bf834e4475865ae336e2379282d34?dir=blender#packages.x86_64-linux.default + ``` + + will build the package with index 1 shown above. + # Description This command shows what packages are currently installed in a -profile. The output consists of one line per package, with the -following fields: +profile. For each installed package, it shows the following +information: + +* `Index`: An integer that can be used to unambiguously identify the + package in invocations of `nix profile remove` and `nix profile + upgrade`. -* An integer that can be used to unambiguously identify the package in - invocations of `nix profile remove` and `nix profile upgrade`. +* `Flake attribute`: The flake output attribute path that provides the + package (e.g. `packages.x86_64-linux.hello`). -* The original ("unlocked") flake reference and output attribute path - used at installation time. +* `Original flake URL`: The original ("unlocked") flake reference + specified by the user when the package was first installed via `nix + profile install`. -* The locked flake reference to which the unlocked flake reference was - resolved. +* `Locked flake URL`: The locked flake reference to which the original + flake reference was resolved. -* The store path(s) of the package. +* `Store paths`: The store path(s) of the package. )"" diff --git a/src/nix/profile.cc b/src/nix/profile.cc index f3b73f10d789..476ddcd609b5 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -21,7 +21,7 @@ struct ProfileElementSource { FlakeRef originalRef; // FIXME: record original attrpath. - FlakeRef resolvedRef; + FlakeRef lockedRef; std::string attrPath; ExtendedOutputsSpec outputs; @@ -168,7 +168,7 @@ struct ProfileManifest } } - std::string toJSON(Store & store) const + nlohmann::json toJSON(Store & store) const { auto array = nlohmann::json::array(); for (auto & element : elements) { @@ -181,7 +181,7 @@ struct ProfileManifest obj["priority"] = element.priority; if (element.source) { obj["originalUrl"] = element.source->originalRef.to_string(); - obj["url"] = element.source->resolvedRef.to_string(); + obj["url"] = element.source->lockedRef.to_string(); obj["attrPath"] = element.source->attrPath; obj["outputs"] = element.source->outputs; } @@ -190,7 +190,7 @@ struct ProfileManifest nlohmann::json json; json["version"] = 2; json["elements"] = array; - return json.dump(); + return json; } StorePath build(ref store) @@ -210,7 +210,7 @@ struct ProfileManifest buildProfile(tempDir, std::move(pkgs)); - writeFile(tempDir + "/manifest.json", toJSON(*store)); + writeFile(tempDir + "/manifest.json", toJSON(*store).dump()); /* Add the symlink tree to the store. */ StringSink sink; @@ -222,10 +222,8 @@ struct ProfileManifest *store, "profile", FixedOutputInfo { - .hash = { - .method = FileIngestionMethod::Recursive, - .hash = narHash, - }, + .method = FileIngestionMethod::Recursive, + .hash = narHash, .references = { .others = std::move(references), // profiles never refer to themselves @@ -349,7 +347,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile if (auto * info2 = dynamic_cast(&*info)) { element.source = ProfileElementSource { .originalRef = info2->flake.originalRef, - .resolvedRef = info2->flake.resolvedRef, + .lockedRef = info2->flake.lockedRef, .attrPath = info2->value.attrPath, .outputs = info2->value.extendedOutputsSpec, }; @@ -588,14 +586,14 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf assert(infop); auto & info = *infop; - if (element.source->resolvedRef == info.flake.resolvedRef) continue; + if (element.source->lockedRef == info.flake.lockedRef) continue; printInfo("upgrading '%s' from flake '%s' to '%s'", - element.source->attrPath, element.source->resolvedRef, info.flake.resolvedRef); + element.source->attrPath, element.source->lockedRef, info.flake.lockedRef); element.source = ProfileElementSource { .originalRef = installable->flakeRef, - .resolvedRef = info.flake.resolvedRef, + .lockedRef = info.flake.lockedRef, .attrPath = info.value.attrPath, .outputs = installable->extendedOutputsSpec, }; @@ -635,7 +633,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf } }; -struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile +struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile, MixJSON { std::string description() override { @@ -653,12 +651,22 @@ struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultPro { ProfileManifest manifest(*getEvalState(), *profile); - for (size_t i = 0; i < manifest.elements.size(); ++i) { - auto & element(manifest.elements[i]); - logger->cout("%d %s %s %s", i, - element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath + element.source->outputs.to_string() : "-", - element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath + element.source->outputs.to_string() : "-", - concatStringsSep(" ", store->printStorePathSet(element.storePaths))); + if (json) { + std::cout << manifest.toJSON(*store).dump() << "\n"; + } else { + for (size_t i = 0; i < manifest.elements.size(); ++i) { + auto & element(manifest.elements[i]); + if (i) logger->cout(""); + logger->cout("Index: " ANSI_BOLD "%s" ANSI_NORMAL "%s", + i, + element.active ? "" : " " ANSI_RED "(inactive)" ANSI_NORMAL); + if (element.source) { + logger->cout("Flake attribute: %s%s", element.source->attrPath, element.source->outputs.to_string()); + logger->cout("Original flake URL: %s", element.source->originalRef.to_string()); + logger->cout("Locked flake URL: %s", element.source->lockedRef.to_string()); + } + logger->cout("Store paths: %s", concatStringsSep(" ", store->printStorePathSet(element.storePaths))); + } } } }; diff --git a/src/nix/profile.md b/src/nix/profile.md index bd13f906f1cf..9b2f86f4ac63 100644 --- a/src/nix/profile.md +++ b/src/nix/profile.md @@ -11,7 +11,7 @@ them to be rolled back easily. )"" -#include "doc/files/profiles.md.gen.hh" +#include "generated-doc/files/profiles.md.gen.hh" R""( diff --git a/src/nix/registry.cc b/src/nix/registry.cc index cb94bbd317fa..f509ccae8403 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -175,8 +175,8 @@ struct CmdRegistryPin : RegistryCommand, EvalCommand .label = "locked", .optional = true, .handler = {&locked}, - .completer = {[&](size_t, std::string_view prefix) { - completeFlakeRef(getStore(), prefix); + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getStore(), prefix); }} }); } diff --git a/src/nix/repl.cc b/src/nix/repl.cc index bb14f3f99c21..63fe3044bb23 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -1,4 +1,5 @@ #include "eval.hh" +#include "eval-settings.hh" #include "globals.hh" #include "command.hh" #include "installable-value.hh" @@ -46,7 +47,7 @@ struct CmdRepl : RawInstallablesCommand void applyDefaultInstallables(std::vector & rawInstallables) override { - if (!experimentalFeatureSettings.isEnabled(Xp::ReplFlake) && !(file) && rawInstallables.size() >= 1) { + if (!experimentalFeatureSettings.isEnabled(Xp::Flakes) && !(file) && rawInstallables.size() >= 1) { warn("future versions of Nix will require using `--file` to load a file"); if (rawInstallables.size() > 1) warn("more than one input file is not currently supported"); diff --git a/src/nix/repl.md b/src/nix/repl.md index c5113be61d27..32c08e24b240 100644 --- a/src/nix/repl.md +++ b/src/nix/repl.md @@ -36,16 +36,13 @@ R""( Loading Installable ''... Added 1 variables. - # nix repl --extra-experimental-features 'flakes repl-flake' nixpkgs + # nix repl --extra-experimental-features 'flakes' nixpkgs Loading Installable 'flake:nixpkgs#'... Added 5 variables. nix-repl> legacyPackages.x86_64-linux.emacs.name "emacs-27.1" - nix-repl> legacyPackages.x86_64-linux.emacs.name - "emacs-27.1" - nix-repl> :q # nix repl --expr 'import {}' diff --git a/src/nix/run.cc b/src/nix/run.cc index 1baf299ab9fb..ea0a1789711a 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -1,3 +1,4 @@ +#include "current-process.hh" #include "run.hh" #include "command-installable-value.hh" #include "common-args.hh" @@ -6,7 +7,7 @@ #include "derivations.hh" #include "local-store.hh" #include "finally.hh" -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "progress-bar.hh" #include "eval.hh" #include "build/personality.hh" @@ -119,9 +120,9 @@ struct CmdShell : InstallablesCommand, MixEnvironment if (true) unixPath.push_front(store->printStorePath(path) + "/bin"); - auto propPath = store->printStorePath(path) + "/nix-support/propagated-user-env-packages"; - if (accessor->stat(propPath).type == FSAccessor::tRegular) { - for (auto & p : tokenizeString(readFile(propPath))) + auto propPath = CanonPath(store->printStorePath(path)) + "nix-support" + "propagated-user-env-packages"; + if (auto st = accessor->maybeLstat(propPath); st && st->type == SourceAccessor::tRegular) { + for (auto & p : tokenizeString(accessor->readFile(propPath))) todo.push(store->parseStorePath(p)); } } diff --git a/src/nix/search.cc b/src/nix/search.cc index c92ed1663edd..ef0139e093f8 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -2,6 +2,7 @@ #include "globals.hh" #include "eval.hh" #include "eval-inline.hh" +#include "eval-settings.hh" #include "names.hh" #include "get-drvs.hh" #include "common-args.hh" diff --git a/src/nix/shell.md b/src/nix/shell.md index 1668104b128c..7c315fb3f029 100644 --- a/src/nix/shell.md +++ b/src/nix/shell.md @@ -26,7 +26,7 @@ R""( * Run multiple commands in a shell environment: ```console - # nix shell nixpkgs#gnumake --command sh --command "cd src && make" + # nix shell nixpkgs#gnumake --command sh -c "cd src && make" ``` * Run GNU Hello in a chroot store: @@ -51,4 +51,120 @@ R""( provides the specified [*installables*](./nix.md#installable). If no command is specified, it starts the default shell of your user account specified by `$SHELL`. +# Use as a `#!`-interpreter + +You can use `nix` as a script interpreter to allow scripts written +in arbitrary languages to obtain their own dependencies via Nix. This is +done by starting the script with the following lines: + +```bash +#! /usr/bin/env nix +#! nix shell installables --command real-interpreter +``` + +where *real-interpreter* is the “real” script interpreter that will be +invoked by `nix shell` after it has obtained the dependencies and +initialised the environment, and *installables* are the attribute names of +the dependencies in Nixpkgs. + +The lines starting with `#! nix` specify options (see above). Note that you +cannot write `#! /usr/bin/env nix shell -i ...` because many operating systems +only allow one argument in `#!` lines. + +For example, here is a Python script that depends on Python and the +`prettytable` package: + +```python +#! /usr/bin/env nix +#! nix shell github:tomberek/-#python3With.prettytable --command python + +import prettytable + +# Print a simple table. +t = prettytable.PrettyTable(["N", "N^2"]) +for n in range(1, 10): t.add_row([n, n * n]) +print t +``` + +Similarly, the following is a Perl script that specifies that it +requires Perl and the `HTML::TokeParser::Simple` and `LWP` packages: + +```perl +#! /usr/bin/env nix +#! nix shell github:tomberek/-#perlWith.HTMLTokeParserSimple.LWP --command perl -x + +use HTML::TokeParser::Simple; + +# Fetch nixos.org and print all hrefs. +my $p = HTML::TokeParser::Simple->new(url => 'http://nixos.org/'); + +while (my $token = $p->get_tag("a")) { + my $href = $token->get_attr("href"); + print "$href\n" if $href; +} +``` + +Sometimes you need to pass a simple Nix expression to customize a +package like Terraform: + +```bash +#! /usr/bin/env nix +#! nix shell --impure --expr `` +#! nix with (import (builtins.getFlake ''nixpkgs'') {}); +#! nix terraform.withPlugins (plugins: [ plugins.openstack ]) +#! nix `` +#! nix --command bash + +terraform "$@" +``` + +> **Note** +> +> You must use double backticks (```` `` ````) when passing a simple Nix expression +> in a nix shell shebang. + +Finally, using the merging of multiple nix shell shebangs the following +Haskell script uses a specific branch of Nixpkgs/NixOS (the 21.11 stable +branch): + +```haskell +#!/usr/bin/env nix +#!nix shell --override-input nixpkgs github:NixOS/nixpkgs/nixos-21.11 +#!nix github:tomberek/-#haskellWith.download-curl.tagsoup --command runghc + +import Network.Curl.Download +import Text.HTML.TagSoup +import Data.Either +import Data.ByteString.Char8 (unpack) + +-- Fetch nixos.org and print all hrefs. +main = do + resp <- openURI "https://nixos.org/" + let tags = filter (isTagOpenName "a") $ parseTags $ unpack $ fromRight undefined resp + let tags' = map (fromAttrib "href") tags + mapM_ putStrLn $ filter (/= "") tags' +``` + +If you want to be even more precise, you can specify a specific revision +of Nixpkgs: + + #!nix shell --override-input nixpkgs github:NixOS/nixpkgs/eabc38219184cc3e04a974fe31857d8e0eac098d + +You can also use a Nix expression to build your own dependencies. For example, +the Python example could have been written as: + +```python +#! /usr/bin/env nix +#! nix shell --impure --file deps.nix -i python +``` + +where the file `deps.nix` in the same directory as the `#!`-script +contains: + +```nix +with import {}; +python3.withPackages (ps: with ps; [ prettytable ]) +``` + + )"" diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 45cd2e1a6c20..39555c9eae30 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -1,7 +1,9 @@ +#include "signals.hh" #include "command.hh" #include "shared.hh" #include "store-api.hh" #include "thread-pool.hh" +#include "progress-bar.hh" #include @@ -173,6 +175,7 @@ struct CmdKeyGenerateSecret : Command if (!keyName) throw UsageError("required argument '--key-name' is missing"); + stopProgressBar(); writeFull(STDOUT_FILENO, SecretKey::generate(*keyName).to_string()); } }; @@ -194,6 +197,7 @@ struct CmdKeyConvertSecretToPublic : Command void run() override { SecretKey secretKey(drainFD(STDIN_FILENO)); + stopProgressBar(); writeFull(STDOUT_FILENO, secretKey.toPublicKey().to_string()); } }; diff --git a/src/nix/ping-store.cc b/src/nix/store-info.cc similarity index 78% rename from src/nix/ping-store.cc rename to src/nix/store-info.cc index ec450e8e0a97..a7c59576146d 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/store-info.cc @@ -17,7 +17,7 @@ struct CmdPingStore : StoreCommand, MixJSON std::string doc() override { return - #include "ping-store.md" + #include "store-info.md" ; } @@ -46,4 +46,15 @@ struct CmdPingStore : StoreCommand, MixJSON } }; -static auto rCmdPingStore = registerCommand2({"store", "ping"}); +struct CmdInfoStore : CmdPingStore +{ + void run(nix::ref store) override + { + warn("'nix store ping' is a deprecated alias for 'nix store info'"); + CmdPingStore::run(store); + } +}; + + +static auto rCmdPingStore = registerCommand2({"store", "info"}); +static auto rCmdInfoStore = registerCommand2({"store", "ping"}); diff --git a/src/nix/ping-store.md b/src/nix/store-info.md similarity index 82% rename from src/nix/ping-store.md rename to src/nix/store-info.md index 8c846791b9fd..f86efd722492 100644 --- a/src/nix/ping-store.md +++ b/src/nix/store-info.md @@ -5,19 +5,19 @@ R""( * Test whether connecting to a remote Nix store via SSH works: ```console - # nix store ping --store ssh://mac1 + # nix store info --store ssh://mac1 ``` * Test whether a URL is a valid binary cache: ```console - # nix store ping --store https://cache.nixos.org + # nix store info --store https://cache.nixos.org ``` * Test whether the Nix daemon is up and running: ```console - # nix store ping --store daemon + # nix store info --store daemon ``` # Description diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 3997c98bfc7f..4c7a74e16f19 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -1,8 +1,10 @@ +#include "processes.hh" #include "command.hh" #include "common-args.hh" #include "store-api.hh" #include "filetransfer.hh" #include "eval.hh" +#include "eval-settings.hh" #include "attr-path.hh" #include "names.hh" #include "progress-bar.hh" @@ -12,7 +14,6 @@ using namespace nix; struct CmdUpgradeNix : MixDryRun, StoreCommand { Path profileDir; - std::string storePathsUrl = "https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix"; CmdUpgradeNix() { @@ -28,7 +29,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand .longName = "nix-store-paths-url", .description = "The URL of the file that contains the store paths of the latest Nix release.", .labels = {"url"}, - .handler = {&storePathsUrl} + .handler = {&(std::string&) settings.upgradeNixStorePathUrl} }); } @@ -42,7 +43,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand std::string description() override { - return "upgrade Nix to the stable version declared in Nixpkgs"; + return "upgrade Nix to the latest stable version"; } std::string doc() override @@ -143,10 +144,10 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand Activity act(*logger, lvlInfo, actUnknown, "querying latest Nix version"); // FIXME: use nixos.org? - auto req = FileTransferRequest(storePathsUrl); + auto req = FileTransferRequest((std::string&) settings.upgradeNixStorePathUrl); auto res = getFileTransfer()->download(req); - auto state = std::make_unique(Strings(), store); + auto state = std::make_unique(SearchPath{}, store); auto v = state->allocValue(); state->eval(state->parseExprFromString(res.data, state->rootPath(CanonPath("/no-such-path"))), *v); Bindings & bindings(*state->allocBindings(0)); diff --git a/src/nix/upgrade-nix.md b/src/nix/upgrade-nix.md index cce88c3970db..3a3bf61b9b05 100644 --- a/src/nix/upgrade-nix.md +++ b/src/nix/upgrade-nix.md @@ -16,8 +16,10 @@ R""( # Description -This command upgrades Nix to the stable version declared in Nixpkgs. -This stable version is defined in [nix-fallback-paths.nix](https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix) +This command upgrades Nix to the stable version. + +By default, the latest stable version is defined by Nixpkgs, in +[nix-fallback-paths.nix](https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix) and updated manually. It may not always be the latest tagged release. By default, it locates the directory containing the `nix` binary in the `$PATH` diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 0b306cc11597..78cb765ce172 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -4,6 +4,7 @@ #include "sync.hh" #include "thread-pool.hh" #include "references.hh" +#include "signals.hh" #include @@ -108,8 +109,8 @@ struct CmdVerify : StorePathsCommand act2.result(resCorruptedPath, store->printStorePath(info->path)); printError("path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(info->path), - info->narHash.to_string(Base32, true), - hash.first.to_string(Base32, true)); + info->narHash.to_string(HashFormat::Base32, true), + hash.first.to_string(HashFormat::Base32, true)); } } diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index a3a9dc698892..aecf65922220 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -1,7 +1,7 @@ #include "command.hh" #include "store-api.hh" #include "progress-bar.hh" -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "shared.hh" #include @@ -38,17 +38,13 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions expectArgs({ .label = "package", .handler = {&_package}, - .completer = {[&](size_t, std::string_view prefix) { - completeInstallable(prefix); - }} + .completer = getCompleteInstallable(), }); expectArgs({ .label = "dependency", .handler = {&_dependency}, - .completer = {[&](size_t, std::string_view prefix) { - completeInstallable(prefix); - }} + .completer = getCompleteInstallable(), }); addFlag({ @@ -179,7 +175,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions struct BailOut { }; printNode = [&](Node & node, const std::string & firstPad, const std::string & tailPad) { - auto pathS = store->printStorePath(node.path); + CanonPath pathS(store->printStorePath(node.path)); assert(node.dist != inf); if (precise) { @@ -187,7 +183,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions firstPad, node.visited ? "\e[38;5;244m" : "", firstPad != "" ? "→ " : "", - pathS); + pathS.abs()); } if (node.path == dependencyPath && !all @@ -214,24 +210,25 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions contain the reference. */ std::map hits; - std::function visitPath; + std::function visitPath; - visitPath = [&](const Path & p) { - auto st = accessor->stat(p); + visitPath = [&](const CanonPath & p) { + auto st = accessor->maybeLstat(p); + assert(st); - auto p2 = p == pathS ? "/" : std::string(p, pathS.size() + 1); + auto p2 = p == pathS ? "/" : p.abs().substr(pathS.abs().size() + 1); auto getColour = [&](const std::string & hash) { return hash == dependencyPathHash ? ANSI_GREEN : ANSI_BLUE; }; - if (st.type == FSAccessor::Type::tDirectory) { + if (st->type == SourceAccessor::Type::tDirectory) { auto names = accessor->readDirectory(p); - for (auto & name : names) - visitPath(p + "/" + name); + for (auto & [name, type] : names) + visitPath(p + name); } - else if (st.type == FSAccessor::Type::tRegular) { + else if (st->type == SourceAccessor::Type::tRegular) { auto contents = accessor->readFile(p); for (auto & hash : hashes) { @@ -239,7 +236,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions if (pos != std::string::npos) { size_t margin = 32; auto pos2 = pos >= margin ? pos - margin : 0; - hits[hash].emplace_back(fmt("%s: …%s…\n", + hits[hash].emplace_back(fmt("%s: …%s…", p2, hilite(filterPrintable( std::string(contents, pos2, pos - pos2 + hash.size() + margin)), @@ -249,13 +246,13 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions } } - else if (st.type == FSAccessor::Type::tSymlink) { + else if (st->type == SourceAccessor::Type::tSymlink) { auto target = accessor->readLink(p); for (auto & hash : hashes) { auto pos = target.find(hash); if (pos != std::string::npos) - hits[hash].emplace_back(fmt("%s -> %s\n", p2, + hits[hash].emplace_back(fmt("%s -> %s", p2, hilite(target, pos, StorePath::HashLen, getColour(hash)))); } } @@ -272,9 +269,9 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions for (auto & hit : hits[hash]) { bool first = hit == *hits[hash].begin(); - std::cout << tailPad - << (first ? (last ? treeLast : treeConn) : (last ? treeNull : treeLine)) - << hit; + logger->cout("%s%s%s", tailPad, + (first ? (last ? treeLast : treeConn) : (last ? treeNull : treeLine)), + hit); if (!all) break; } diff --git a/tests/acls.sh b/tests/acls.sh deleted file mode 100755 index f92843701ba0..000000000000 --- a/tests/acls.sh +++ /dev/null @@ -1,59 +0,0 @@ -source common.sh - -USER=$(whoami) - -setfacl -m "u:$USER:r" example || skipTest "ACLs not supported" - -# Adds the "dummy" file to the nix store and check that we can access it -EXAMPLE_PATH=$(nix store add-path dummy) -nix store access info "$EXAMPLE_PATH" --json | grep '"protected":false' -cat "$EXAMPLE_PATH" -getfacl "$EXAMPLE_PATH" - -# Protect a file and check that we cannot access it anymore -nix store access protect "$EXAMPLE_PATH" -! cat "$EXAMPLE_PATH" -nix store access info "$EXAMPLE_PATH" --json | grep '"protected":true' -nix store access info "$EXAMPLE_PATH" --json | grep '"users":\[\]' - -# Grant permission and check that we can access the file -nix store access grant "$EXAMPLE_PATH" --user "$USER" -cat "$EXAMPLE_PATH" -nix store access info "$EXAMPLE_PATH" --json | grep '"users":\["'$USER'"\]' - -# Revoke permission and check that we cannot access the file anymore -nix store access revoke "$EXAMPLE_PATH" --user "$USER" -nix store access info "$EXAMPLE_PATH" --json | grep '"users":\[\]' - -# Check setting permissions from a nix file -cp dummy "$TEST_ROOT" -cp config.nix "$TEST_ROOT" -cat > "$TEST_ROOT/test-acls.nix"< "test.nix"< "test.nix"< $flake2Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/default.nix < $nonFlakeDir/README.md < $flake1Dir/foo -git -C $flake1Dir add $flake1Dir/foo -[[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "$hash1-dirty" ]] - -echo -n '# foo' >> $flake1Dir/flake.nix -flake1OriginalCommit=$(git -C $flake1Dir rev-parse HEAD) -git -C $flake1Dir commit -a -m 'Foo' -flake1NewCommit=$(git -C $flake1Dir rev-parse HEAD) -hash2=$(nix flake metadata flake1 --json --refresh | jq -r .revision) -[[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "null" ]] -[[ $hash1 != $hash2 ]] - -# Test 'nix build' on a flake. -nix build -o $TEST_ROOT/result flake1#foo -[[ -e $TEST_ROOT/result/hello ]] - -# Test packages.default. -nix build -o $TEST_ROOT/result flake1 -[[ -e $TEST_ROOT/result/hello ]] - -nix build -o $TEST_ROOT/result $flake1Dir -nix build -o $TEST_ROOT/result git+file://$flake1Dir - -# Check that store symlinks inside a flake are not interpreted as flakes. -nix build -o $flake1Dir/result git+file://$flake1Dir -nix path-info $flake1Dir/result - -# 'getFlake' on an unlocked flakeref should fail in pure mode, but -# succeed in impure mode. -(! nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"$flake1Dir\").packages.$system.default") -nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"$flake1Dir\").packages.$system.default" --impure - -# 'getFlake' on a locked flakeref should succeed even in pure mode. -nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"git+file://$flake1Dir?rev=$hash2\").packages.$system.default" - -# Building a flake with an unlocked dependency should fail in pure mode. -(! nix build -o $TEST_ROOT/result flake2#bar --no-registries) -(! nix build -o $TEST_ROOT/result flake2#bar --no-use-registries) -(! nix eval --expr "builtins.getFlake \"$flake2Dir\"") - -# But should succeed in impure mode. -(! nix build -o $TEST_ROOT/result flake2#bar --impure) -nix build -o $TEST_ROOT/result flake2#bar --impure --no-write-lock-file -nix eval --expr "builtins.getFlake \"$flake2Dir\"" --impure - -# Building a local flake with an unlocked dependency should fail with --no-update-lock-file. -expect 1 nix build -o $TEST_ROOT/result $flake2Dir#bar --no-update-lock-file 2>&1 | grep 'requires lock file changes' - -# But it should succeed without that flag. -nix build -o $TEST_ROOT/result $flake2Dir#bar --no-write-lock-file -expect 1 nix build -o $TEST_ROOT/result $flake2Dir#bar --no-update-lock-file 2>&1 | grep 'requires lock file changes' -nix build -o $TEST_ROOT/result $flake2Dir#bar --commit-lock-file -[[ -e $flake2Dir/flake.lock ]] -[[ -z $(git -C $flake2Dir diff main || echo failed) ]] - -# Rerunning the build should not change the lockfile. -nix build -o $TEST_ROOT/result $flake2Dir#bar -[[ -z $(git -C $flake2Dir diff main || echo failed) ]] - -# Building with a lockfile should not require a fetch of the registry. -nix build -o $TEST_ROOT/result --flake-registry file:///no-registry.json $flake2Dir#bar --refresh -nix build -o $TEST_ROOT/result --no-registries $flake2Dir#bar --refresh -nix build -o $TEST_ROOT/result --no-use-registries $flake2Dir#bar --refresh - -# Updating the flake should not change the lockfile. -nix flake lock $flake2Dir -[[ -z $(git -C $flake2Dir diff main || echo failed) ]] - -# Now we should be able to build the flake in pure mode. -nix build -o $TEST_ROOT/result flake2#bar - -# Or without a registry. -nix build -o $TEST_ROOT/result --no-registries git+file://$flake2Dir#bar --refresh -nix build -o $TEST_ROOT/result --no-use-registries git+file://$flake2Dir#bar --refresh - -# Test whether indirect dependencies work. -nix build -o $TEST_ROOT/result $flake3Dir#xyzzy -git -C $flake3Dir add flake.lock - -# Add dependency to flake3. -rm $flake3Dir/flake.nix - -cat > $flake3Dir/flake.nix < $flake3Dir/flake.nix < \$out - [[ \$(cat \${inputs.nonFlake}/README.md) = \$(cat \${inputs.nonFlakeFile}) ]] - [[ \${inputs.nonFlakeFile} = \${inputs.nonFlakeFile2} ]] - ''; - }; - }; -} -EOF - -cp ../config.nix $flake3Dir - -git -C $flake3Dir add flake.nix config.nix -git -C $flake3Dir commit -m 'Add nonFlakeInputs' - -# Check whether `nix build` works with a lockfile which is missing a -# nonFlakeInputs. -nix build -o $TEST_ROOT/result $flake3Dir#sth --commit-lock-file - -nix build -o $TEST_ROOT/result flake3#fnord -[[ $(cat $TEST_ROOT/result) = FNORD ]] - -# Check whether flake input fetching is lazy: flake3#sth does not -# depend on flake2, so this shouldn't fail. -rm -rf $TEST_HOME/.cache -clearStore -mv $flake2Dir $flake2Dir.tmp -mv $nonFlakeDir $nonFlakeDir.tmp -nix build -o $TEST_ROOT/result flake3#sth -(! nix build -o $TEST_ROOT/result flake3#xyzzy) -(! nix build -o $TEST_ROOT/result flake3#fnord) -mv $flake2Dir.tmp $flake2Dir -mv $nonFlakeDir.tmp $nonFlakeDir -nix build -o $TEST_ROOT/result flake3#xyzzy flake3#fnord - -# Test doing multiple `lookupFlake`s -nix build -o $TEST_ROOT/result flake4#xyzzy - -# Test 'nix flake update' and --override-flake. -nix flake lock $flake3Dir -[[ -z $(git -C $flake3Dir diff master || echo failed) ]] - -nix flake update $flake3Dir --override-flake flake2 nixpkgs -[[ ! -z $(git -C $flake3Dir diff master || echo failed) ]] - -# Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore -git -C $flake3Dir checkout -b removeXyzzy -rm $flake3Dir/flake.nix - -cat > $flake3Dir/flake.nix < \$out - ''; - }; - }; -} -EOF -nix flake lock $flake3Dir -git -C $flake3Dir add flake.nix flake.lock -git -C $flake3Dir commit -m 'Remove packages.xyzzy' -git -C $flake3Dir checkout master - -# Test whether fuzzy-matching works for registry entries. -(! nix build -o $TEST_ROOT/result flake4/removeXyzzy#xyzzy) -nix build -o $TEST_ROOT/result flake4/removeXyzzy#sth - -# Testing the nix CLI -nix registry add flake1 flake3 -[[ $(nix registry list | wc -l) == 6 ]] -nix registry pin flake1 -[[ $(nix registry list | wc -l) == 6 ]] -nix registry pin flake1 flake3 -[[ $(nix registry list | wc -l) == 6 ]] -nix registry remove flake1 -[[ $(nix registry list | wc -l) == 5 ]] - -# Test 'nix registry list' with a disabled global registry. -nix registry add user-flake1 git+file://$flake1Dir -nix registry add user-flake2 git+file://$flake2Dir -[[ $(nix --flake-registry "" registry list | wc -l) == 2 ]] -nix --flake-registry "" registry list | grepQuietInverse '^global' # nothing in global registry -nix --flake-registry "" registry list | grepQuiet '^user' -nix registry remove user-flake1 -nix registry remove user-flake2 -[[ $(nix registry list | wc -l) == 5 ]] - -# Test 'nix flake clone'. -rm -rf $TEST_ROOT/flake1-v2 -nix flake clone flake1 --dest $TEST_ROOT/flake1-v2 -[ -e $TEST_ROOT/flake1-v2/flake.nix ] - -# Test 'follows' inputs. -cat > $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $flake3Dir/flake.nix < $badFlakeDir/flake.nix -nix store delete $(nix store add-path $badFlakeDir) - -[[ $(nix path-info $(nix store add-path $flake1Dir)) =~ flake1 ]] -[[ $(nix path-info path:$(nix store add-path $flake1Dir)) =~ simple ]] - -# Test fetching flakerefs in the legacy CLI. -[[ $(nix-instantiate --eval flake:flake3 -A x) = 123 ]] -[[ $(nix-instantiate --eval flake:git+file://$flake3Dir -A x) = 123 ]] -[[ $(nix-instantiate -I flake3=flake:flake3 --eval '' -A x) = 123 ]] -[[ $(NIX_PATH=flake3=flake:flake3 nix-instantiate --eval '' -A x) = 123 ]] - -# Test alternate lockfile paths. -nix flake lock $flake2Dir --output-lock-file $TEST_ROOT/flake2.lock -cmp $flake2Dir/flake.lock $TEST_ROOT/flake2.lock >/dev/null # lockfiles should be identical, since we're referencing flake2's original one - -nix flake lock $flake2Dir --output-lock-file $TEST_ROOT/flake2-overridden.lock --override-input flake1 git+file://$flake1Dir?rev=$flake1OriginalCommit -expectStderr 1 cmp $flake2Dir/flake.lock $TEST_ROOT/flake2-overridden.lock -nix flake metadata $flake2Dir --reference-lock-file $TEST_ROOT/flake2-overridden.lock | grepQuiet $flake1OriginalCommit - -# reference-lock-file can only be used if allow-dirty is set. -expectStderr 1 nix flake metadata $flake2Dir --no-allow-dirty --reference-lock-file $TEST_ROOT/flake2-overridden.lock diff --git a/tests/flakes/follow-paths.sh b/tests/flakes/follow-paths.sh deleted file mode 100644 index fe9b51c652ef..000000000000 --- a/tests/flakes/follow-paths.sh +++ /dev/null @@ -1,150 +0,0 @@ -source ./common.sh - -requireGit - -flakeFollowsA=$TEST_ROOT/follows/flakeA -flakeFollowsB=$TEST_ROOT/follows/flakeA/flakeB -flakeFollowsC=$TEST_ROOT/follows/flakeA/flakeB/flakeC -flakeFollowsD=$TEST_ROOT/follows/flakeA/flakeD -flakeFollowsE=$TEST_ROOT/follows/flakeA/flakeE - -# Test following path flakerefs. -createGitRepo $flakeFollowsA -mkdir -p $flakeFollowsB -mkdir -p $flakeFollowsC -mkdir -p $flakeFollowsD -mkdir -p $flakeFollowsE - -cat > $flakeFollowsA/flake.nix < $flakeFollowsB/flake.nix < $flakeFollowsC/flake.nix < $flakeFollowsD/flake.nix < $flakeFollowsE/flake.nix < $flakeFollowsA/flake.nix < $flakeFollowsA/flake.nix <&1 | grep 'points outside' - -# Non-existant follows should print a warning. -cat >$flakeFollowsA/flake.nix <&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid'" -nix flake lock $flakeFollowsA 2>&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid2'" diff --git a/tests/add.sh b/tests/functional/add.sh similarity index 62% rename from tests/add.sh rename to tests/functional/add.sh index 5c3eed7931a0..d0fedcb251cf 100644 --- a/tests/add.sh +++ b/tests/functional/add.sh @@ -26,3 +26,20 @@ hash2=$(nix-hash --type sha256 --base32 ./dummy) echo $hash2 test "$hash1" = "sha256:$hash2" + +#### New style commands + +clearStore + +( + path1=$(nix store add ./dummy) + path2=$(nix store add --mode nar ./dummy) + path3=$(nix store add-path ./dummy) + [[ "$path1" == "$path2" ]] + [[ "$path1" == "$path3" ]] +) +( + path1=$(nix store add --mode flat ./dummy) + path2=$(nix store add-file ./dummy) + [[ "$path1" == "$path2" ]] +) diff --git a/tests/bad.tar.xz b/tests/functional/bad.tar.xz similarity index 100% rename from tests/bad.tar.xz rename to tests/functional/bad.tar.xz diff --git a/tests/bash-profile.sh b/tests/functional/bash-profile.sh similarity index 78% rename from tests/bash-profile.sh rename to tests/functional/bash-profile.sh index e2e0d1090804..3faeaaba1dcb 100644 --- a/tests/bash-profile.sh +++ b/tests/functional/bash-profile.sh @@ -1,6 +1,6 @@ source common.sh -sed -e "s|@localstatedir@|$TEST_ROOT/profile-var|g" -e "s|@coreutils@|$coreutils|g" < ../scripts/nix-profile.sh.in > $TEST_ROOT/nix-profile.sh +sed -e "s|@localstatedir@|$TEST_ROOT/profile-var|g" -e "s|@coreutils@|$coreutils|g" < ../../scripts/nix-profile.sh.in > $TEST_ROOT/nix-profile.sh user=$(whoami) rm -rf $TEST_HOME $TEST_ROOT/profile-var diff --git a/tests/big-derivation-attr.nix b/tests/functional/big-derivation-attr.nix similarity index 100% rename from tests/big-derivation-attr.nix rename to tests/functional/big-derivation-attr.nix diff --git a/tests/binary-cache-build-remote.sh b/tests/functional/binary-cache-build-remote.sh similarity index 100% rename from tests/binary-cache-build-remote.sh rename to tests/functional/binary-cache-build-remote.sh diff --git a/tests/binary-cache.sh b/tests/functional/binary-cache.sh similarity index 100% rename from tests/binary-cache.sh rename to tests/functional/binary-cache.sh diff --git a/tests/brotli.sh b/tests/functional/brotli.sh similarity index 100% rename from tests/brotli.sh rename to tests/functional/brotli.sh diff --git a/tests/build-delete.sh b/tests/functional/build-delete.sh similarity index 100% rename from tests/build-delete.sh rename to tests/functional/build-delete.sh diff --git a/tests/build-dry.sh b/tests/functional/build-dry.sh similarity index 100% rename from tests/build-dry.sh rename to tests/functional/build-dry.sh diff --git a/tests/build-hook-ca-fixed.nix b/tests/functional/build-hook-ca-fixed.nix similarity index 91% rename from tests/build-hook-ca-fixed.nix rename to tests/functional/build-hook-ca-fixed.nix index 4cb9e85d1296..0ce6d9b128b8 100644 --- a/tests/build-hook-ca-fixed.nix +++ b/tests/functional/build-hook-ca-fixed.nix @@ -8,7 +8,10 @@ let derivation ({ inherit system; builder = busybox; - args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" '' + if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi; + eval "$buildCommand" + '')]; outputHashMode = "recursive"; outputHashAlgo = "sha256"; } // removeAttrs args ["builder" "meta" "passthru"]) diff --git a/tests/build-hook-ca-floating.nix b/tests/functional/build-hook-ca-floating.nix similarity index 100% rename from tests/build-hook-ca-floating.nix rename to tests/functional/build-hook-ca-floating.nix diff --git a/tests/build-hook.nix b/tests/functional/build-hook.nix similarity index 90% rename from tests/build-hook.nix rename to tests/functional/build-hook.nix index 7effd79037f6..99a13aee483d 100644 --- a/tests/build-hook.nix +++ b/tests/functional/build-hook.nix @@ -14,7 +14,10 @@ let derivation ({ inherit system; builder = busybox; - args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" '' + if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi; + eval "$buildCommand" + '')]; } // removeAttrs args ["builder" "meta" "passthru"] // caArgs) // { meta = args.meta or {}; passthru = args.passthru or {}; }; diff --git a/tests/build-remote-content-addressed-fixed.sh b/tests/functional/build-remote-content-addressed-fixed.sh similarity index 100% rename from tests/build-remote-content-addressed-fixed.sh rename to tests/functional/build-remote-content-addressed-fixed.sh diff --git a/tests/build-remote-content-addressed-floating.sh b/tests/functional/build-remote-content-addressed-floating.sh similarity index 100% rename from tests/build-remote-content-addressed-floating.sh rename to tests/functional/build-remote-content-addressed-floating.sh diff --git a/tests/build-remote-input-addressed.sh b/tests/functional/build-remote-input-addressed.sh similarity index 100% rename from tests/build-remote-input-addressed.sh rename to tests/functional/build-remote-input-addressed.sh diff --git a/tests/build-remote-trustless-after.sh b/tests/functional/build-remote-trustless-after.sh similarity index 100% rename from tests/build-remote-trustless-after.sh rename to tests/functional/build-remote-trustless-after.sh diff --git a/tests/build-remote-trustless-should-fail-0.sh b/tests/functional/build-remote-trustless-should-fail-0.sh similarity index 100% rename from tests/build-remote-trustless-should-fail-0.sh rename to tests/functional/build-remote-trustless-should-fail-0.sh diff --git a/tests/build-remote-trustless-should-pass-0.sh b/tests/functional/build-remote-trustless-should-pass-0.sh similarity index 100% rename from tests/build-remote-trustless-should-pass-0.sh rename to tests/functional/build-remote-trustless-should-pass-0.sh diff --git a/tests/build-remote-trustless-should-pass-1.sh b/tests/functional/build-remote-trustless-should-pass-1.sh similarity index 100% rename from tests/build-remote-trustless-should-pass-1.sh rename to tests/functional/build-remote-trustless-should-pass-1.sh diff --git a/tests/build-remote-trustless-should-pass-2.sh b/tests/functional/build-remote-trustless-should-pass-2.sh similarity index 100% rename from tests/build-remote-trustless-should-pass-2.sh rename to tests/functional/build-remote-trustless-should-pass-2.sh diff --git a/tests/build-remote-trustless-should-pass-3.sh b/tests/functional/build-remote-trustless-should-pass-3.sh similarity index 100% rename from tests/build-remote-trustless-should-pass-3.sh rename to tests/functional/build-remote-trustless-should-pass-3.sh diff --git a/tests/build-remote-trustless.sh b/tests/functional/build-remote-trustless.sh similarity index 82% rename from tests/build-remote-trustless.sh rename to tests/functional/build-remote-trustless.sh index 9df44e0c5ef7..81e5253bf298 100644 --- a/tests/build-remote-trustless.sh +++ b/tests/functional/build-remote-trustless.sh @@ -6,7 +6,7 @@ unset NIX_STATE_DIR remoteDir=$TEST_ROOT/remote -# Note: ssh{-ng}://localhost bypasses ssh. See tests/build-remote.sh for +# Note: ssh{-ng}://localhost bypasses ssh. See tests/functional/build-remote.sh for # more details. nix-build $file -o $TEST_ROOT/result --max-jobs 0 \ --arg busybox $busybox \ diff --git a/tests/functional/build-remote-with-mounted-ssh-ng.sh b/tests/functional/build-remote-with-mounted-ssh-ng.sh new file mode 100644 index 000000000000..443acb6ca788 --- /dev/null +++ b/tests/functional/build-remote-with-mounted-ssh-ng.sh @@ -0,0 +1,22 @@ +source common.sh + +requireSandboxSupport +[[ $busybox =~ busybox ]] || skipTest "no busybox" + +enableFeatures mounted-ssh-store + +nix build -Lvf simple.nix \ + --arg busybox $busybox \ + --out-link $TEST_ROOT/result-from-remote \ + --store mounted-ssh-ng://localhost + +nix build -Lvf simple.nix \ + --arg busybox $busybox \ + --out-link $TEST_ROOT/result-from-remote-new-cli \ + --store 'mounted-ssh-ng://localhost?remote-program=nix daemon' + +# This verifies that the out link was actually created and valid. The ability +# to create out links (permanent gc roots) is the distinguishing feature of +# the mounted-ssh-ng store. +cat $TEST_ROOT/result-from-remote/hello | grepQuiet 'Hello World!' +cat $TEST_ROOT/result-from-remote-new-cli/hello | grepQuiet 'Hello World!' diff --git a/tests/build-remote.sh b/tests/functional/build-remote.sh similarity index 98% rename from tests/build-remote.sh rename to tests/functional/build-remote.sh index 78e12b477cf6..d2a2132c1a41 100644 --- a/tests/build-remote.sh +++ b/tests/functional/build-remote.sh @@ -1,6 +1,7 @@ requireSandboxSupport [[ $busybox =~ busybox ]] || skipTest "no busybox" +# Avoid store dir being inside sandbox build-dir unset NIX_STORE_DIR unset NIX_STATE_DIR diff --git a/tests/build.sh b/tests/functional/build.sh similarity index 98% rename from tests/build.sh rename to tests/functional/build.sh index 8ae20f0dff5e..7fbdb0f07491 100644 --- a/tests/build.sh +++ b/tests/functional/build.sh @@ -78,7 +78,7 @@ expectStderr 1 nix build --impure --expr 'with (import ./multiple-outputs.nix).e | grepQuiet "has 2 entries in its context. It should only have exactly one entry" nix build --impure --json --expr 'builtins.unsafeDiscardOutputDependency (import ./multiple-outputs.nix).e.a_a.drvPath' --no-link | jq --exit-status ' - (.[0] | .path | match(".*multiple-outputs-e.drv")) + (.[0] | match(".*multiple-outputs-e.drv")) ' # Test building from raw store path to drv not expression. diff --git a/tests/ca-shell.nix b/tests/functional/ca-shell.nix similarity index 100% rename from tests/ca-shell.nix rename to tests/functional/ca-shell.nix diff --git a/tests/functional/ca/build-cache.sh b/tests/functional/ca/build-cache.sh new file mode 100644 index 000000000000..6a4080fecf5f --- /dev/null +++ b/tests/functional/ca/build-cache.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +source common.sh + +# The substituters didn't work prior to this time. +requireDaemonNewerThan "2.18.0pre20230808" + +drv=$(nix-instantiate ./content-addressed.nix -A rootCA --arg seed 1)^out +nix derivation show "$drv" --arg seed 1 + +buildAttr () { + local derivationPath=$1 + local seedValue=$2 + shift; shift + local args=("./content-addressed.nix" "-A" "$derivationPath" --arg seed "$seedValue" "--no-out-link") + args+=("$@") + nix-build "${args[@]}" +} + +copyAttr () { + local derivationPath=$1 + local seedValue=$2 + shift; shift + local args=("-f" "./content-addressed.nix" "$derivationPath" --arg seed "$seedValue") + args+=("$@") + # Note: to copy CA derivations, we need to copy the realisations, which + # currently requires naming the installables, not just the derivation output + # path. + nix copy --to file://$cacheDir "${args[@]}" +} + +testRemoteCacheFor () { + local derivationPath=$1 + clearCache + copyAttr "$derivationPath" 1 + clearStore + # Check nothing gets built. + buildAttr "$derivationPath" 1 --option substituters file://$cacheDir --no-require-sigs |& grepQuietInverse " will be built:" +} + +testRemoteCache () { + testRemoteCacheFor rootCA + testRemoteCacheFor dependentCA + testRemoteCacheFor dependentNonCA + testRemoteCacheFor dependentFixedOutput + testRemoteCacheFor dependentForBuildCA + testRemoteCacheFor dependentForBuildNonCA +} + +clearStore +testRemoteCache \ No newline at end of file diff --git a/tests/ca/build-dry.sh b/tests/functional/ca/build-dry.sh similarity index 100% rename from tests/ca/build-dry.sh rename to tests/functional/ca/build-dry.sh diff --git a/tests/ca/build-with-garbage-path.sh b/tests/functional/ca/build-with-garbage-path.sh similarity index 100% rename from tests/ca/build-with-garbage-path.sh rename to tests/functional/ca/build-with-garbage-path.sh diff --git a/tests/ca/build.sh b/tests/functional/ca/build.sh similarity index 82% rename from tests/ca/build.sh rename to tests/functional/ca/build.sh index 7754ad276576..e1a8a76252b1 100644 --- a/tests/ca/build.sh +++ b/tests/functional/ca/build.sh @@ -2,7 +2,7 @@ source common.sh -drv=$(nix-instantiate ./content-addressed.nix -A rootCA --arg seed 1) +drv=$(nix-instantiate ./content-addressed.nix -A rootCA --arg seed 1)^out nix derivation show "$drv" --arg seed 1 buildAttr () { @@ -14,14 +14,6 @@ buildAttr () { nix-build "${args[@]}" } -testRemoteCache () { - clearCache - local outPath=$(buildAttr dependentNonCA 1) - nix copy --to file://$cacheDir $outPath - clearStore - buildAttr dependentNonCA 1 --option substituters file://$cacheDir --no-require-sigs |& grepQuietInverse "building dependent-non-ca" -} - testDeterministicCA () { [[ $(buildAttr rootCA 1) = $(buildAttr rootCA 2) ]] } @@ -66,8 +58,6 @@ testNormalization () { test "$(stat -c %Y $outPath)" -eq 1 } -# Disabled until we have it properly working -# testRemoteCache clearStore testNormalization testDeterministicCA diff --git a/tests/ca/common.sh b/tests/functional/ca/common.sh similarity index 100% rename from tests/ca/common.sh rename to tests/functional/ca/common.sh diff --git a/tests/ca/concurrent-builds.sh b/tests/functional/ca/concurrent-builds.sh similarity index 100% rename from tests/ca/concurrent-builds.sh rename to tests/functional/ca/concurrent-builds.sh diff --git a/tests/ca/config.nix.in b/tests/functional/ca/config.nix.in similarity index 100% rename from tests/ca/config.nix.in rename to tests/functional/ca/config.nix.in diff --git a/tests/ca/content-addressed.nix b/tests/functional/ca/content-addressed.nix similarity index 83% rename from tests/ca/content-addressed.nix rename to tests/functional/ca/content-addressed.nix index 81bc4bf5c211..2559c562f926 100644 --- a/tests/ca/content-addressed.nix +++ b/tests/functional/ca/content-addressed.nix @@ -61,6 +61,24 @@ rec { echo ${rootCA}/non-ca-hello > $out/dep ''; }; + dependentForBuildCA = mkCADerivation { + name = "dependent-for-build-ca"; + buildCommand = '' + echo "Depends on rootCA for building only" + mkdir -p $out + echo ${rootCA} + touch $out + ''; + }; + dependentForBuildNonCA = mkDerivation { + name = "dependent-for-build-non-ca"; + buildCommand = '' + echo "Depends on rootCA for building only" + mkdir -p $out + echo ${rootCA} + touch $out + ''; + }; dependentFixedOutput = mkDerivation { name = "dependent-fixed-output"; outputHashMode = "recursive"; diff --git a/tests/ca/derivation-json.sh b/tests/functional/ca/derivation-json.sh similarity index 100% rename from tests/ca/derivation-json.sh rename to tests/functional/ca/derivation-json.sh diff --git a/tests/ca/duplicate-realisation-in-closure.sh b/tests/functional/ca/duplicate-realisation-in-closure.sh similarity index 100% rename from tests/ca/duplicate-realisation-in-closure.sh rename to tests/functional/ca/duplicate-realisation-in-closure.sh diff --git a/tests/ca/flake.nix b/tests/functional/ca/flake.nix similarity index 100% rename from tests/ca/flake.nix rename to tests/functional/ca/flake.nix diff --git a/tests/ca/gc.sh b/tests/functional/ca/gc.sh similarity index 100% rename from tests/ca/gc.sh rename to tests/functional/ca/gc.sh diff --git a/tests/ca/import-derivation.sh b/tests/functional/ca/import-derivation.sh similarity index 100% rename from tests/ca/import-derivation.sh rename to tests/functional/ca/import-derivation.sh diff --git a/tests/functional/ca/local.mk b/tests/functional/ca/local.mk new file mode 100644 index 000000000000..fd87b8d1f8c6 --- /dev/null +++ b/tests/functional/ca/local.mk @@ -0,0 +1,28 @@ +ca-tests := \ + $(d)/build-with-garbage-path.sh \ + $(d)/build.sh \ + $(d)/build-cache.sh \ + $(d)/concurrent-builds.sh \ + $(d)/derivation-json.sh \ + $(d)/duplicate-realisation-in-closure.sh \ + $(d)/gc.sh \ + $(d)/import-derivation.sh \ + $(d)/new-build-cmd.sh \ + $(d)/nix-copy.sh \ + $(d)/nix-run.sh \ + $(d)/nix-shell.sh \ + $(d)/post-hook.sh \ + $(d)/recursive.sh \ + $(d)/repl.sh \ + $(d)/selfref-gc.sh \ + $(d)/signatures.sh \ + $(d)/substitute.sh \ + $(d)/why-depends.sh + +install-tests-groups += ca + +clean-files += \ + $(d)/config.nix + +test-deps += \ + tests/functional/ca/config.nix diff --git a/tests/ca/new-build-cmd.sh b/tests/functional/ca/new-build-cmd.sh similarity index 100% rename from tests/ca/new-build-cmd.sh rename to tests/functional/ca/new-build-cmd.sh diff --git a/tests/ca/nix-copy.sh b/tests/functional/ca/nix-copy.sh similarity index 100% rename from tests/ca/nix-copy.sh rename to tests/functional/ca/nix-copy.sh diff --git a/tests/ca/nix-run.sh b/tests/functional/ca/nix-run.sh similarity index 100% rename from tests/ca/nix-run.sh rename to tests/functional/ca/nix-run.sh diff --git a/tests/ca/nix-shell.sh b/tests/functional/ca/nix-shell.sh similarity index 100% rename from tests/ca/nix-shell.sh rename to tests/functional/ca/nix-shell.sh diff --git a/tests/ca/nondeterministic.nix b/tests/functional/ca/nondeterministic.nix similarity index 100% rename from tests/ca/nondeterministic.nix rename to tests/functional/ca/nondeterministic.nix diff --git a/tests/ca/post-hook.sh b/tests/functional/ca/post-hook.sh similarity index 100% rename from tests/ca/post-hook.sh rename to tests/functional/ca/post-hook.sh diff --git a/tests/ca/racy.nix b/tests/functional/ca/racy.nix similarity index 100% rename from tests/ca/racy.nix rename to tests/functional/ca/racy.nix diff --git a/tests/ca/recursive.sh b/tests/functional/ca/recursive.sh similarity index 100% rename from tests/ca/recursive.sh rename to tests/functional/ca/recursive.sh diff --git a/tests/ca/repl.sh b/tests/functional/ca/repl.sh similarity index 100% rename from tests/ca/repl.sh rename to tests/functional/ca/repl.sh diff --git a/tests/ca/selfref-gc.sh b/tests/functional/ca/selfref-gc.sh similarity index 100% rename from tests/ca/selfref-gc.sh rename to tests/functional/ca/selfref-gc.sh diff --git a/tests/ca/signatures.sh b/tests/functional/ca/signatures.sh similarity index 100% rename from tests/ca/signatures.sh rename to tests/functional/ca/signatures.sh diff --git a/tests/ca/substitute.sh b/tests/functional/ca/substitute.sh similarity index 100% rename from tests/ca/substitute.sh rename to tests/functional/ca/substitute.sh diff --git a/tests/ca/why-depends.sh b/tests/functional/ca/why-depends.sh similarity index 100% rename from tests/ca/why-depends.sh rename to tests/functional/ca/why-depends.sh diff --git a/tests/case-hack.sh b/tests/functional/case-hack.sh similarity index 100% rename from tests/case-hack.sh rename to tests/functional/case-hack.sh diff --git a/tests/case.nar b/tests/functional/case.nar similarity index 100% rename from tests/case.nar rename to tests/functional/case.nar diff --git a/tests/check-refs.nix b/tests/functional/check-refs.nix similarity index 97% rename from tests/check-refs.nix rename to tests/functional/check-refs.nix index 99d69a22636a..89690e456c14 100644 --- a/tests/check-refs.nix +++ b/tests/functional/check-refs.nix @@ -2,7 +2,7 @@ with import ./config.nix; rec { - dep = import ./dependencies.nix; + dep = import ./dependencies.nix {}; makeTest = nr: args: mkDerivation ({ name = "check-refs-" + toString nr; diff --git a/tests/check-refs.sh b/tests/functional/check-refs.sh similarity index 93% rename from tests/check-refs.sh rename to tests/functional/check-refs.sh index 2778e491d871..3b587d1e56f6 100644 --- a/tests/check-refs.sh +++ b/tests/functional/check-refs.sh @@ -42,8 +42,10 @@ nix-build -o $RESULT check-refs.nix -A test7 nix-build -o $RESULT check-refs.nix -A test10 if isDaemonNewer 2.12pre20230103; then - enableFeatures discard-references - restartDaemon + if ! isDaemonNewer 2.16.0; then + enableFeatures discard-references + restartDaemon + fi # test11 should succeed. test11=$(nix-build -o $RESULT check-refs.nix -A test11) diff --git a/tests/check-reqs.nix b/tests/functional/check-reqs.nix similarity index 100% rename from tests/check-reqs.nix rename to tests/functional/check-reqs.nix diff --git a/tests/check-reqs.sh b/tests/functional/check-reqs.sh similarity index 100% rename from tests/check-reqs.sh rename to tests/functional/check-reqs.sh diff --git a/tests/check.nix b/tests/functional/check.nix similarity index 100% rename from tests/check.nix rename to tests/functional/check.nix diff --git a/tests/check.sh b/tests/functional/check.sh similarity index 100% rename from tests/check.sh rename to tests/functional/check.sh diff --git a/tests/common.sh b/tests/functional/common.sh similarity index 65% rename from tests/common.sh rename to tests/functional/common.sh index 8941671d67d9..7b0922c9f320 100644 --- a/tests/common.sh +++ b/tests/functional/common.sh @@ -4,7 +4,7 @@ if [[ -z "${COMMON_SH_SOURCED-}" ]]; then COMMON_SH_SOURCED=1 -source "$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")/common/vars-and-functions.sh" +source "$(readlink -f "$(dirname "${BASH_SOURCE[0]-$0}")")/common/vars-and-functions.sh" if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then startDaemon fi diff --git a/tests/common/vars-and-functions.sh.in b/tests/functional/common/vars-and-functions.sh.in similarity index 97% rename from tests/common/vars-and-functions.sh.in rename to tests/functional/common/vars-and-functions.sh.in index a9e6c802fe5d..848988af9966 100644 --- a/tests/common/vars-and-functions.sh.in +++ b/tests/functional/common/vars-and-functions.sh.in @@ -4,9 +4,9 @@ if [[ -z "${COMMON_VARS_AND_FUNCTIONS_SH_SOURCED-}" ]]; then COMMON_VARS_AND_FUNCTIONS_SH_SOURCED=1 -export PS4='+(${BASH_SOURCE[0]}:$LINENO) ' +set +x -export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default} +export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default/tests\/functional//} export NIX_STORE_DIR if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then # Maybe the build directory is symlinked. @@ -195,7 +195,7 @@ expect() { shift "$@" && res=0 || res="$?" if [[ $res -ne $expected ]]; then - echo "Expected '$expected' but got '$res' while running '${*@Q}'" >&2 + echo "Expected exit code '$expected' but got '$res' from command ${*@Q}" >&2 return 1 fi return 0 @@ -209,7 +209,7 @@ expectStderr() { shift "$@" 2>&1 && res=0 || res="$?" if [[ $res -ne $expected ]]; then - echo "Expected '$expected' but got '$res' while running '${*@Q}'" >&2 + echo "Expected exit code '$expected' but got '$res' from command ${*@Q}" >&2 return 1 fi return 0 diff --git a/tests/completions.sh b/tests/functional/completions.sh similarity index 81% rename from tests/completions.sh rename to tests/functional/completions.sh index 19dc610989e6..d3d5bbd48b1d 100644 --- a/tests/completions.sh +++ b/tests/functional/completions.sh @@ -44,13 +44,18 @@ EOF # Input override completion [[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '')" == $'normal\na\t' ]] [[ "$(NIX_GET_COMPLETIONS=5 nix flake show ./foo --override-input '')" == $'normal\na\t' ]] +cd ./foo +[[ "$(NIX_GET_COMPLETIONS=3 nix flake update '')" == $'normal\na\t' ]] +cd .. +[[ "$(NIX_GET_COMPLETIONS=5 nix flake update --flake './foo' '')" == $'normal\na\t' ]] ## With multiple input flakes [[ "$(NIX_GET_COMPLETIONS=5 nix build ./foo ./bar --override-input '')" == $'normal\na\t\nb\t' ]] ## With tilde expansion [[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix build '~/foo' --override-input '')" == $'normal\na\t' ]] +[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=5 nix flake update --flake '~/foo' '')" == $'normal\na\t' ]] ## Out of order -[[ "$(NIX_GET_COMPLETIONS=3 nix build --update-input '' ./foo)" == $'normal\na\t' ]] -[[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --update-input '' ./bar)" == $'normal\na\t\nb\t' ]] +[[ "$(NIX_GET_COMPLETIONS=3 nix build --override-input '' '' ./foo)" == $'normal\na\t' ]] +[[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '' '' ./bar)" == $'normal\na\t\nb\t' ]] # Cli flag completion NIX_GET_COMPLETIONS=2 nix build --log-form | grep -- "--log-format" diff --git a/tests/compression-levels.sh b/tests/functional/compression-levels.sh similarity index 100% rename from tests/compression-levels.sh rename to tests/functional/compression-levels.sh diff --git a/tests/compute-levels.sh b/tests/functional/compute-levels.sh similarity index 100% rename from tests/compute-levels.sh rename to tests/functional/compute-levels.sh diff --git a/tests/config.nix.in b/tests/functional/config.nix.in similarity index 82% rename from tests/config.nix.in rename to tests/functional/config.nix.in index 7facbdcbc98e..00dc007e12fc 100644 --- a/tests/config.nix.in +++ b/tests/functional/config.nix.in @@ -20,7 +20,10 @@ rec { derivation ({ inherit system; builder = shell; - args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + args = ["-e" args.builder or (builtins.toFile "builder-${args.name}.sh" '' + if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi; + eval "$buildCommand" + '')]; PATH = path; } // caArgs // removeAttrs args ["builder" "meta"]) // { meta = args.meta or {}; }; diff --git a/tests/config.sh b/tests/functional/config.sh similarity index 83% rename from tests/config.sh rename to tests/functional/config.sh index 723f575ed1d7..324fe95bd71d 100644 --- a/tests/config.sh +++ b/tests/functional/config.sh @@ -40,19 +40,20 @@ files=$(nix-build --verbose --version | grep "User config" | cut -d ':' -f2- | x # Test that it's possible to load the config from a custom location here=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")") export NIX_USER_CONF_FILES=$here/config/nix-with-substituters.conf -var=$(nix show-config | grep '^substituters =' | cut -d '=' -f 2 | xargs) +var=$(nix config show | grep '^substituters =' | cut -d '=' -f 2 | xargs) [[ $var == https://example.com ]] # Test that it's possible to load config from the environment -prev=$(nix show-config | grep '^cores' | cut -d '=' -f 2 | xargs) +prev=$(nix config show | grep '^cores' | cut -d '=' -f 2 | xargs) export NIX_CONFIG="cores = 4242"$'\n'"experimental-features = nix-command flakes" -exp_cores=$(nix show-config | grep '^cores' | cut -d '=' -f 2 | xargs) -exp_features=$(nix show-config | grep '^experimental-features' | cut -d '=' -f 2 | xargs) +exp_cores=$(nix config show | grep '^cores' | cut -d '=' -f 2 | xargs) +exp_features=$(nix config show | grep '^experimental-features' | cut -d '=' -f 2 | xargs) [[ $prev != $exp_cores ]] [[ $exp_cores == "4242" ]] -[[ $exp_features == "flakes nix-command" ]] +# flakes implies fetch-tree +[[ $exp_features == "fetch-tree flakes nix-command" ]] # Test that it's possible to retrieve a single setting's value -val=$(nix show-config | grep '^warn-dirty' | cut -d '=' -f 2 | xargs) -val2=$(nix show-config warn-dirty) +val=$(nix config show | grep '^warn-dirty' | cut -d '=' -f 2 | xargs) +val2=$(nix config show warn-dirty) [[ $val == $val2 ]] diff --git a/tests/config/nix-with-substituters.conf b/tests/functional/config/nix-with-substituters.conf similarity index 100% rename from tests/config/nix-with-substituters.conf rename to tests/functional/config/nix-with-substituters.conf diff --git a/tests/db-migration.sh b/tests/functional/db-migration.sh similarity index 100% rename from tests/db-migration.sh rename to tests/functional/db-migration.sh diff --git a/tests/dependencies.builder0.sh b/tests/functional/dependencies.builder0.sh similarity index 100% rename from tests/dependencies.builder0.sh rename to tests/functional/dependencies.builder0.sh diff --git a/tests/dependencies.nix b/tests/functional/dependencies.nix similarity index 69% rename from tests/dependencies.nix rename to tests/functional/dependencies.nix index 45aca179347b..be1a7ae9a6ec 100644 --- a/tests/dependencies.nix +++ b/tests/functional/dependencies.nix @@ -1,3 +1,4 @@ +{ hashInvalidator ? "" }: with import ./config.nix; let { @@ -21,6 +22,17 @@ let { ''; }; + fod_input = mkDerivation { + name = "fod-input"; + buildCommand = '' + echo ${hashInvalidator} + echo FOD > $out + ''; + outputHashMode = "flat"; + outputHashAlgo = "sha256"; + outputHash = "1dq9p0hnm1y75q2x40fws5887bq1r840hzdxak0a9djbwvx0b16d"; + }; + body = mkDerivation { name = "dependencies-top"; builder = ./dependencies.builder0.sh + "/FOOBAR/../."; @@ -29,6 +41,7 @@ let { input1_drv = input1; input2_drv = input2; input0_drv = input0; + fod_input_drv = fod_input; meta.description = "Random test package"; }; diff --git a/tests/dependencies.sh b/tests/functional/dependencies.sh similarity index 63% rename from tests/dependencies.sh rename to tests/functional/dependencies.sh index f9da0c6bcb9b..b93dacac0b98 100644 --- a/tests/dependencies.sh +++ b/tests/functional/dependencies.sh @@ -15,6 +15,9 @@ if test -n "$dot"; then $dot < $TEST_ROOT/graph fi +# Test GraphML graph generation +nix-store -q --graphml "$drvPath" > $TEST_ROOT/graphml + outPath=$(nix-store -rvv "$drvPath") || fail "build failed" # Test Graphviz graph generation. @@ -50,3 +53,20 @@ nix-store -q --referrers-closure "$input2OutPath" | grep "$outPath" # Check that the derivers are set properly. test $(nix-store -q --deriver "$outPath") = "$drvPath" nix-store -q --deriver "$input2OutPath" | grepQuiet -- "-input-2.drv" + +# --valid-derivers returns the currently single valid .drv file +test "$(nix-store -q --valid-derivers "$outPath")" = "$drvPath" + +# instantiate a different drv with the same output +drvPath2=$(nix-instantiate dependencies.nix --argstr hashInvalidator yay) + +# now --valid-derivers returns both +test "$(nix-store -q --valid-derivers "$outPath" | sort)" = "$(sort <<< "$drvPath"$'\n'"$drvPath2")" + +# check that nix-store --valid-derivers only returns existing drv +nix-store --delete "$drvPath" +test "$(nix-store -q --valid-derivers "$outPath")" = "$drvPath2" + +# check that --valid-derivers returns nothing when there are no valid derivers +nix-store --delete "$drvPath2" +test -z "$(nix-store -q --valid-derivers "$outPath")" diff --git a/tests/derivation-json.sh b/tests/functional/derivation-json.sh similarity index 100% rename from tests/derivation-json.sh rename to tests/functional/derivation-json.sh diff --git a/tests/dummy b/tests/functional/dummy similarity index 100% rename from tests/dummy rename to tests/functional/dummy diff --git a/tests/dump-db.sh b/tests/functional/dump-db.sh similarity index 100% rename from tests/dump-db.sh rename to tests/functional/dump-db.sh diff --git a/tests/functional/dyn-drv/build-built-drv.sh b/tests/functional/dyn-drv/build-built-drv.sh new file mode 100644 index 000000000000..647be9457167 --- /dev/null +++ b/tests/functional/dyn-drv/build-built-drv.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +source common.sh + +# In the corresponding nix file, we have two derivations: the first, named `hello`, +# is a normal recursive derivation, while the second, named dependent, has the +# new outputHashMode "text". Note that in "dependent", we don't refer to the +# build output of `hello`, but only to the path of the drv file. For this reason, +# we only need to: +# +# - instantiate `hello` +# - build `producingDrv` +# - check that the path of the output coincides with that of the original derivation + +out1=$(nix build -f ./text-hashed-output.nix hello --no-link) + +clearStore + +drvDep=$(nix-instantiate ./text-hashed-output.nix -A producingDrv) + +expectStderr 1 nix build "${drvDep}^out^out" --no-link | grepQuiet "Building dynamic derivations in one shot is not yet implemented" diff --git a/tests/dyn-drv/common.sh b/tests/functional/dyn-drv/common.sh similarity index 100% rename from tests/dyn-drv/common.sh rename to tests/functional/dyn-drv/common.sh diff --git a/tests/dyn-drv/config.nix.in b/tests/functional/dyn-drv/config.nix.in similarity index 100% rename from tests/dyn-drv/config.nix.in rename to tests/functional/dyn-drv/config.nix.in diff --git a/tests/functional/dyn-drv/dep-built-drv.sh b/tests/functional/dyn-drv/dep-built-drv.sh new file mode 100644 index 000000000000..4f6e9b080faa --- /dev/null +++ b/tests/functional/dyn-drv/dep-built-drv.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +source common.sh + +out1=$(nix-build ./text-hashed-output.nix -A hello --no-out-link) + +clearStore + +expectStderr 1 nix-build ./text-hashed-output.nix -A wrapper --no-out-link | grepQuiet "Building dynamic derivations in one shot is not yet implemented" + +# diff -r $out1 $out2 diff --git a/tests/functional/dyn-drv/eval-outputOf.sh b/tests/functional/dyn-drv/eval-outputOf.sh new file mode 100644 index 000000000000..9467feb8d23f --- /dev/null +++ b/tests/functional/dyn-drv/eval-outputOf.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +source ./common.sh + +# Without the dynamic-derivations XP feature, we don't have the builtin. +nix --experimental-features 'nix-command' eval --impure --expr \ + 'assert ! (builtins ? outputOf); ""' + +# Test that a string is required. +# +# We currently require a string to be passed, rather than a derivation +# object that could be coerced to a string. We might liberalise this in +# the future so it does work, but there are some design questions to +# resolve first. Adding a test so we don't liberalise it by accident. +expectStderr 1 nix --experimental-features 'nix-command dynamic-derivations' eval --impure --expr \ + 'builtins.outputOf (import ../dependencies.nix {}) "out"' \ + | grepQuiet "value is a set while a string was expected" + +# Test that "DrvDeep" string contexts are not supported at this time +# +# Like the above, this is a restriction we could relax later. +expectStderr 1 nix --experimental-features 'nix-command dynamic-derivations' eval --impure --expr \ + 'builtins.outputOf (import ../dependencies.nix {}).drvPath "out"' \ + | grepQuiet "has a context which refers to a complete source and binary closure. This is not supported at this time" + +# Test using `builtins.outputOf` with static derivations +testStaticHello () { + nix eval --impure --expr \ + 'with (import ./text-hashed-output.nix); let + a = hello.outPath; + b = builtins.outputOf (builtins.unsafeDiscardOutputDependency hello.drvPath) "out"; + in builtins.trace a + (builtins.trace b + (assert a == b; null))' +} + +# Test with a regular old input-addresed derivation +# +# `builtins.outputOf` works without ca-derivations and doesn't create a +# placeholder but just returns the output path. +testStaticHello + +# Test with content addressed derivation. +NIX_TESTS_CA_BY_DEFAULT=1 testStaticHello + +# Test with derivation-producing derivation +# +# This is hardly different from the preceding cases, except that we're +# only taking 1 outputOf out of 2 possible outputOfs. Note that +# `.outPath` could be defined as `outputOf drvPath`, which is what we're +# testing here. The other `outputOf` that we're not testing here is the +# use of _dynamic_ derivations. +nix eval --impure --expr \ + 'with (import ./text-hashed-output.nix); let + a = producingDrv.outPath; + b = builtins.outputOf (builtins.builtins.unsafeDiscardOutputDependency producingDrv.drvPath) "out"; + in builtins.trace a + (builtins.trace b + (assert a == b; null))' + +# Test with unbuilt output of derivation-producing derivation. +# +# This function similar to `testStaticHello` used above, but instead of +# checking the property on a constant derivation, we check it on a +# derivation that's from another derivation's output (outPath). +testDynamicHello () { + nix eval --impure --expr \ + 'with (import ./text-hashed-output.nix); let + a = builtins.outputOf producingDrv.outPath "out"; + b = builtins.outputOf (builtins.outputOf (builtins.unsafeDiscardOutputDependency producingDrv.drvPath) "out") "out"; + in builtins.trace a + (builtins.trace b + (assert a == b; null))' +} + +# inner dynamic derivation is input-addressed +testDynamicHello + +# inner dynamic derivation is content-addressed +NIX_TESTS_CA_BY_DEFAULT=1 testDynamicHello diff --git a/tests/functional/dyn-drv/local.mk b/tests/functional/dyn-drv/local.mk new file mode 100644 index 000000000000..c87534944b1f --- /dev/null +++ b/tests/functional/dyn-drv/local.mk @@ -0,0 +1,15 @@ +dyn-drv-tests := \ + $(d)/text-hashed-output.sh \ + $(d)/recursive-mod-json.sh \ + $(d)/build-built-drv.sh \ + $(d)/eval-outputOf.sh \ + $(d)/dep-built-drv.sh \ + $(d)/old-daemon-error-hack.sh + +install-tests-groups += dyn-drv + +clean-files += \ + $(d)/config.nix + +test-deps += \ + tests/functional/dyn-drv/config.nix diff --git a/tests/functional/dyn-drv/old-daemon-error-hack.nix b/tests/functional/dyn-drv/old-daemon-error-hack.nix new file mode 100644 index 000000000000..c9d4a62d4f4b --- /dev/null +++ b/tests/functional/dyn-drv/old-daemon-error-hack.nix @@ -0,0 +1,20 @@ +with import ./config.nix; + +# A simple content-addressed derivation. +# The derivation can be arbitrarily modified by passing a different `seed`, +# but the output will always be the same +rec { + stub = mkDerivation { + name = "stub"; + buildCommand = '' + echo stub > $out + ''; + }; + wrapper = mkDerivation { + name = "has-dynamic-drv-dep"; + buildCommand = '' + exit 1 # we're not building this derivation + ${builtins.outputOf stub.outPath "out"} + ''; + }; +} diff --git a/tests/functional/dyn-drv/old-daemon-error-hack.sh b/tests/functional/dyn-drv/old-daemon-error-hack.sh new file mode 100644 index 000000000000..43b049973964 --- /dev/null +++ b/tests/functional/dyn-drv/old-daemon-error-hack.sh @@ -0,0 +1,11 @@ +# Purposely bypassing our usual common for this subgroup +source ../common.sh + +# Need backend to support text-hashing too +isDaemonNewer "2.18.0pre20230906" && skipTest "Daemon is too new" + +enableFeatures "ca-derivations dynamic-derivations" + +restartDaemon + +expectStderr 1 nix-instantiate --read-write-mode ./old-daemon-error-hack.nix | grepQuiet "the daemon is too old to understand dependencies on dynamic derivations" diff --git a/tests/dyn-drv/recursive-mod-json.nix b/tests/functional/dyn-drv/recursive-mod-json.nix similarity index 100% rename from tests/dyn-drv/recursive-mod-json.nix rename to tests/functional/dyn-drv/recursive-mod-json.nix diff --git a/tests/dyn-drv/recursive-mod-json.sh b/tests/functional/dyn-drv/recursive-mod-json.sh similarity index 93% rename from tests/dyn-drv/recursive-mod-json.sh rename to tests/functional/dyn-drv/recursive-mod-json.sh index 070c5c2cb80c..0698b81bd112 100644 --- a/tests/dyn-drv/recursive-mod-json.sh +++ b/tests/functional/dyn-drv/recursive-mod-json.sh @@ -3,6 +3,8 @@ source common.sh # FIXME if [[ $(uname) != Linux ]]; then skipTest "Not running Linux"; fi +export NIX_TESTS_CA_BY_DEFAULT=1 + enableFeatures 'recursive-nix' restartDaemon diff --git a/tests/dyn-drv/text-hashed-output.nix b/tests/functional/dyn-drv/text-hashed-output.nix similarity index 74% rename from tests/dyn-drv/text-hashed-output.nix rename to tests/functional/dyn-drv/text-hashed-output.nix index a700fd102de0..99203b518496 100644 --- a/tests/dyn-drv/text-hashed-output.nix +++ b/tests/functional/dyn-drv/text-hashed-output.nix @@ -12,9 +12,6 @@ rec { mkdir -p $out echo "Hello World" > $out/hello ''; - __contentAddressed = true; - outputHashMode = "recursive"; - outputHashAlgo = "sha256"; }; producingDrv = mkDerivation { name = "hello.drv"; @@ -26,4 +23,11 @@ rec { outputHashMode = "text"; outputHashAlgo = "sha256"; }; + wrapper = mkDerivation { + name = "use-dynamic-drv-in-non-dynamic-drv"; + buildCommand = '' + echo "Copying the output of the dynamic derivation" + cp -r ${builtins.outputOf producingDrv.outPath "out"} $out + ''; + }; } diff --git a/tests/dyn-drv/text-hashed-output.sh b/tests/functional/dyn-drv/text-hashed-output.sh similarity index 100% rename from tests/dyn-drv/text-hashed-output.sh rename to tests/functional/dyn-drv/text-hashed-output.sh diff --git a/tests/eval-store.sh b/tests/functional/eval-store.sh similarity index 100% rename from tests/eval-store.sh rename to tests/functional/eval-store.sh diff --git a/tests/eval.nix b/tests/functional/eval.nix similarity index 100% rename from tests/eval.nix rename to tests/functional/eval.nix diff --git a/tests/eval.sh b/tests/functional/eval.sh similarity index 100% rename from tests/eval.sh rename to tests/functional/eval.sh diff --git a/tests/experimental-features.sh b/tests/functional/experimental-features.sh similarity index 93% rename from tests/experimental-features.sh rename to tests/functional/experimental-features.sh index 607bf0a8eff5..9ee4a53d46cc 100644 --- a/tests/experimental-features.sh +++ b/tests/functional/experimental-features.sh @@ -31,7 +31,7 @@ source common.sh NIX_CONFIG=' experimental-features = nix-command accept-flake-config = true -' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr grepQuiet "false" $TEST_ROOT/stdout grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr @@ -39,7 +39,7 @@ grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature ' NIX_CONFIG=' accept-flake-config = true experimental-features = nix-command -' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr grepQuiet "false" $TEST_ROOT/stdout grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr @@ -47,7 +47,7 @@ grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature ' NIX_CONFIG=' experimental-features = nix-command flakes accept-flake-config = true -' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr grepQuiet "true" $TEST_ROOT/stdout grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr @@ -55,7 +55,7 @@ grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr NIX_CONFIG=' accept-flake-config = true experimental-features = nix-command flakes -' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr grepQuiet "true" $TEST_ROOT/stdout grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr diff --git a/tests/export-graph.nix b/tests/functional/export-graph.nix similarity index 81% rename from tests/export-graph.nix rename to tests/functional/export-graph.nix index fdac9583db2c..64fe36bd1ef8 100644 --- a/tests/export-graph.nix +++ b/tests/functional/export-graph.nix @@ -17,13 +17,13 @@ rec { foo."bar.runtimeGraph" = mkDerivation { name = "dependencies"; builder = builtins.toFile "build-graph-builder" "${printRefs}"; - exportReferencesGraph = ["refs" (import ./dependencies.nix)]; + exportReferencesGraph = ["refs" (import ./dependencies.nix {})]; }; foo."bar.buildGraph" = mkDerivation { name = "dependencies"; builder = builtins.toFile "build-graph-builder" "${printRefs}"; - exportReferencesGraph = ["refs" (import ./dependencies.nix).drvPath]; + exportReferencesGraph = ["refs" (import ./dependencies.nix {}).drvPath]; }; } diff --git a/tests/export-graph.sh b/tests/functional/export-graph.sh similarity index 100% rename from tests/export-graph.sh rename to tests/functional/export-graph.sh diff --git a/tests/export.sh b/tests/functional/export.sh similarity index 100% rename from tests/export.sh rename to tests/functional/export.sh diff --git a/tests/failing.nix b/tests/functional/failing.nix similarity index 74% rename from tests/failing.nix rename to tests/functional/failing.nix index 2a0350d4d25e..d25e2d6b62b6 100644 --- a/tests/failing.nix +++ b/tests/functional/failing.nix @@ -6,7 +6,10 @@ let derivation ({ inherit system; builder = busybox; - args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + args = ["sh" "-e" args.builder or (builtins.toFile "builder-${args.name}.sh" '' + if [ -e "$NIX_ATTRS_SH_FILE" ]; then source $NIX_ATTRS_SH_FILE; fi; + eval "$buildCommand" + '')]; } // removeAttrs args ["builder" "meta"]) // { meta = args.meta or {}; }; in diff --git a/tests/functional/fetchClosure.sh b/tests/functional/fetchClosure.sh new file mode 100644 index 000000000000..a02d1ce7a285 --- /dev/null +++ b/tests/functional/fetchClosure.sh @@ -0,0 +1,150 @@ +source common.sh + +enableFeatures "fetch-closure" + +clearStore +clearCacheCache + +# Old daemons don't properly zero out the self-references when +# calculating the CA hashes, so this breaks `nix store +# make-content-addressed` which expects the client and the daemon to +# compute the same hash +requireDaemonNewerThan "2.16.0pre20230524" + +# Initialize binary cache. +nonCaPath=$(nix build --json --file ./dependencies.nix --no-link | jq -r .[].outputs.out) +caPath=$(nix store make-content-addressed --json $nonCaPath | jq -r '.rewrites | map(.) | .[]') +nix copy --to file://$cacheDir $nonCaPath + +# Test basic fetchClosure rewriting from non-CA to CA. +clearStore + +[ ! -e $nonCaPath ] +[ ! -e $caPath ] + +[[ $(nix eval -v --raw --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $nonCaPath; + toPath = $caPath; + } +") = $caPath ]] + +[ ! -e $nonCaPath ] +[ -e $caPath ] + +clearStore + +# The daemon will reject input addressed paths unless configured to trust the +# cache key or the user. This behavior should be covered by another test, so we +# skip this part when using the daemon. +if [[ "$NIX_REMOTE" != "daemon" ]]; then + + # If we want to return a non-CA path, we have to be explicit about it. + expectStderr 1 nix eval --raw --no-require-sigs --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $nonCaPath; + } + " | grepQuiet -E "The .fromPath. value .* is input-addressed, but .inputAddressed. is set to .false." + + # TODO: Should the closure be rejected, despite single user mode? + # [ ! -e $nonCaPath ] + + [ ! -e $caPath ] + + # We can use non-CA paths when we ask explicitly. + [[ $(nix eval --raw --no-require-sigs --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $nonCaPath; + inputAddressed = true; + } + ") = $nonCaPath ]] + + [ -e $nonCaPath ] + [ ! -e $caPath ] + + +fi + +[ ! -e $caPath ] + +# 'toPath' set to empty string should fail but print the expected path. +expectStderr 1 nix eval -v --json --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $nonCaPath; + toPath = \"\"; + } +" | grep "error: rewriting.*$nonCaPath.*yielded.*$caPath" + +# If fromPath is CA, then toPath isn't needed. +nix copy --to file://$cacheDir $caPath + +clearStore + +[ ! -e $caPath ] + +[[ $(nix eval -v --raw --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $caPath; + } +") = $caPath ]] + +[ -e $caPath ] + +# Check that URL query parameters aren't allowed. +clearStore +narCache=$TEST_ROOT/nar-cache +rm -rf $narCache +(! nix eval -v --raw --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir?local-nar-cache=$narCache\"; + fromPath = $caPath; + } +") +(! [ -e $narCache ]) + +# If toPath is specified but wrong, we check it (only) when the path is missing. +clearStore + +badPath=$(echo $caPath | sed -e 's!/store/................................-!/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-!') + +[ ! -e $badPath ] + +expectStderr 1 nix eval -v --raw --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $nonCaPath; + toPath = $badPath; + } +" | grep "error: rewriting.*$nonCaPath.*yielded.*$caPath.*while.*$badPath.*was expected" + +[ ! -e $badPath ] + +# We only check it when missing, as a performance optimization similar to what we do for fixed output derivations. So if it's already there, we don't check it. +# It would be nice for this to fail, but checking it would be too(?) slow. +[ -e $caPath ] + +[[ $(nix eval -v --raw --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $badPath; + toPath = $caPath; + } +") = $caPath ]] + + +# However, if the output address is unexpected, we can report it + + +expectStderr 1 nix eval -v --raw --expr " + builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $caPath; + inputAddressed = true; + } +" | grepQuiet 'error.*The store object referred to by.*fromPath.* at .* is not input-addressed, but .*inputAddressed.* is set to .*true.*' + diff --git a/tests/fetchGit.sh b/tests/functional/fetchGit.sh similarity index 96% rename from tests/fetchGit.sh rename to tests/functional/fetchGit.sh index 418b4f63fc2a..4985c7764f66 100644 --- a/tests/fetchGit.sh +++ b/tests/functional/fetchGit.sh @@ -35,6 +35,8 @@ unset _NIX_FORCE_HTTP path0=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$TEST_ROOT/worktree).outPath") path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = file://$TEST_ROOT/worktree; }).outPath") [[ $path0 = $path0_ ]] +path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree git+file://$TEST_ROOT/worktree).outPath") +[[ $path0 = $path0_ ]] export _NIX_FORCE_HTTP=1 [[ $(tail -n 1 $path0/hello) = "hello" ]] @@ -49,9 +51,7 @@ git -C $repo add differentbranch git -C $repo commit -m 'Test2' git -C $repo checkout master devrev=$(git -C $repo rev-parse devtest) -out=$(nix eval --impure --raw --expr "builtins.fetchGit { url = file://$repo; rev = \"$devrev\"; }" 2>&1) || status=$? -[[ $status == 1 ]] -[[ $out =~ 'Cannot find Git revision' ]] +nix eval --impure --raw --expr "builtins.fetchGit { url = file://$repo; rev = \"$devrev\"; }" [[ $(nix eval --raw --expr "builtins.readFile (builtins.fetchGit { url = file://$repo; rev = \"$devrev\"; allRefs = true; } + \"/differentbranch\")") = 'different file' ]] @@ -183,11 +183,7 @@ path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = # Nuke the cache rm -rf $TEST_HOME/.cache/nix -# Try again, but without 'git' on PATH. This should fail. -NIX=$(command -v nix) -(! PATH= $NIX eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath" ) - -# Try again, with 'git' available. This should work. +# Try again. This should work. path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath") [[ $path3 = $path5 ]] @@ -239,6 +235,7 @@ rm -rf $repo/.git # should succeed for a repo without commits git init $repo +git -C $repo add hello # need to add at least one file to cause the root of the repo to be visible path10=$(nix eval --impure --raw --expr "(builtins.fetchGit \"file://$repo\").outPath") # should succeed for a path with a space diff --git a/tests/fetchGitRefs.sh b/tests/functional/fetchGitRefs.sh similarity index 100% rename from tests/fetchGitRefs.sh rename to tests/functional/fetchGitRefs.sh diff --git a/tests/fetchGitSubmodules.sh b/tests/functional/fetchGitSubmodules.sh similarity index 93% rename from tests/fetchGitSubmodules.sh rename to tests/functional/fetchGitSubmodules.sh index df81232e5997..369cdc5db436 100644 --- a/tests/fetchGitSubmodules.sh +++ b/tests/functional/fetchGitSubmodules.sh @@ -118,11 +118,3 @@ cloneRepo=$TEST_ROOT/a/b/gitSubmodulesClone # NB /a/b to make the relative path git clone $rootRepo $cloneRepo pathIndirect=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$cloneRepo; rev = \"$rev2\"; submodules = true; }).outPath") [[ $pathIndirect = $pathWithRelative ]] - -# Test that if the clone has the submodule already, we're not fetching -# it again. -git -C $cloneRepo submodule update --init -rm $TEST_HOME/.cache/nix/fetcher-cache* -rm -rf $subRepo -pathSubmoduleGone=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$cloneRepo; rev = \"$rev2\"; submodules = true; }).outPath") -[[ $pathSubmoduleGone = $pathWithRelative ]] diff --git a/tests/functional/fetchGitVerification.sh b/tests/functional/fetchGitVerification.sh new file mode 100644 index 000000000000..b80e061b5f55 --- /dev/null +++ b/tests/functional/fetchGitVerification.sh @@ -0,0 +1,82 @@ +source common.sh + +requireGit +[[ $(type -p ssh-keygen) ]] || skipTest "ssh-keygen not installed" # require ssh-keygen + +enableFeatures "verified-fetches" + +clearStore + +repo="$TEST_ROOT/git" + +# generate signing keys +keysDir=$TEST_ROOT/.ssh +mkdir -p "$keysDir" +ssh-keygen -f "$keysDir/testkey1" -t ed25519 -P "" -C "test key 1" +key1File="$keysDir/testkey1.pub" +publicKey1=$(awk '{print $2}' "$key1File") +ssh-keygen -f "$keysDir/testkey2" -t rsa -P "" -C "test key 2" +key2File="$keysDir/testkey2.pub" +publicKey2=$(awk '{print $2}' "$key2File") + +git init $repo +git -C $repo config user.email "foobar@example.com" +git -C $repo config user.name "Foobar" +git -C $repo config gpg.format ssh + +echo 'hello' > $repo/text +git -C $repo add text +git -C $repo -c "user.signingkey=$key1File" commit -S -m 'initial commit' + +out=$(nix eval --impure --raw --expr "builtins.fetchGit { url = \"file://$repo\"; keytype = \"ssh-rsa\"; publicKey = \"$publicKey2\"; }" 2>&1) || status=$? +[[ $status == 1 ]] +[[ $out =~ 'No principal matched.' ]] +[[ $(nix eval --impure --raw --expr "builtins.readFile (builtins.fetchGit { url = \"file://$repo\"; publicKey = \"$publicKey1\"; } + \"/text\")") = 'hello' ]] + +echo 'hello world' > $repo/text + +# Verification on a dirty repo should fail. +out=$(nix eval --impure --raw --expr "builtins.fetchGit { url = \"file://$repo\"; keytype = \"ssh-rsa\"; publicKey = \"$publicKey2\"; }" 2>&1) || status=$? +[[ $status == 1 ]] +[[ $out =~ 'dirty' ]] + +git -C $repo add text +git -C $repo -c "user.signingkey=$key2File" commit -S -m 'second commit' + +[[ $(nix eval --impure --raw --expr "builtins.readFile (builtins.fetchGit { url = \"file://$repo\"; publicKeys = [{key = \"$publicKey1\";} {type = \"ssh-rsa\"; key = \"$publicKey2\";}]; } + \"/text\")") = 'hello world' ]] + +# Flake input test +flakeDir="$TEST_ROOT/flake" +mkdir -p "$flakeDir" +cat > "$flakeDir/flake.nix" < "$flakeDir/flake.nix" <&1) || status=$? +[[ $status == 1 ]] +[[ $out =~ 'No principal matched.' ]] diff --git a/tests/fetchMercurial.sh b/tests/functional/fetchMercurial.sh similarity index 100% rename from tests/fetchMercurial.sh rename to tests/functional/fetchMercurial.sh diff --git a/tests/fetchPath.sh b/tests/functional/fetchPath.sh similarity index 100% rename from tests/fetchPath.sh rename to tests/functional/fetchPath.sh diff --git a/tests/fetchTree-file.sh b/tests/functional/fetchTree-file.sh similarity index 86% rename from tests/fetchTree-file.sh rename to tests/functional/fetchTree-file.sh index fe569cfb8ac4..6395c133d8a8 100644 --- a/tests/fetchTree-file.sh +++ b/tests/functional/fetchTree-file.sh @@ -27,6 +27,7 @@ test_file_flake_input () { mkdir inputs echo foo > inputs/test_input_file + echo '{ outputs = { self }: { }; }' > inputs/flake.nix tar cfa test_input.tar.gz inputs cp test_input.tar.gz test_input_no_ext input_tarball_hash="$(nix hash path test_input.tar.gz)" @@ -50,6 +51,9 @@ test_file_flake_input () { url = "file+file://$PWD/test_input.tar.gz"; flake = false; }; + inputs.flake_no_ext = { + url = "file://$PWD/test_input_no_ext"; + }; outputs = { ... }: {}; } EOF @@ -58,7 +62,7 @@ EOF nix eval --file - < $flake1Dir/flake.nix < $rootRepo/root.nix git -C $rootRepo add root.nix git -C $rootRepo commit -m "Add root.nix" +flakeref=git+file://$rootRepo\?submodules=1\&dir=submodule + # Flake can live inside a submodule and can be accessed via ?dir=submodule -[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#sub ) = '"expression in submodule"' ]] +[[ $(nix eval --json $flakeref#sub ) = '"expression in submodule"' ]] + # The flake can access content outside of the submodule -[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#root ) = '"expression in root repo"' ]] +[[ $(nix eval --json $flakeref#root ) = '"expression in root repo"' ]] + +# Check that dirtying a submodule makes the entire thing dirty. +[[ $(nix flake metadata --json $flakeref | jq -r .locked.rev) != null ]] +echo '"foo"' > $rootRepo/submodule/sub.nix +[[ $(nix eval --json $flakeref#sub ) = '"foo"' ]] +[[ $(nix flake metadata --json $flakeref | jq -r .locked.rev) = null ]] diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh new file mode 100644 index 000000000000..ccf1699f9f0d --- /dev/null +++ b/tests/functional/flakes/flakes.sh @@ -0,0 +1,594 @@ +source ./common.sh + +requireGit + +clearStore +rm -rf $TEST_HOME/.cache $TEST_HOME/.config + +flake1Dir=$TEST_ROOT/flake1 +flake2Dir=$TEST_ROOT/flake\ 2 +percentEncodedFlake2Dir=$TEST_ROOT/flake%202 +flake3Dir=$TEST_ROOT/flake%20 +percentEncodedFlake3Dir=$TEST_ROOT/flake%2520 +flake5Dir=$TEST_ROOT/flake5 +flake7Dir=$TEST_ROOT/flake7 +nonFlakeDir=$TEST_ROOT/nonFlake +badFlakeDir=$TEST_ROOT/badFlake +flakeGitBare=$TEST_ROOT/flakeGitBare + +for repo in "$flake1Dir" "$flake2Dir" "$flake3Dir" "$flake7Dir" "$nonFlakeDir"; do + # Give one repo a non-main initial branch. + extraArgs= + if [[ "$repo" == "$flake2Dir" ]]; then + extraArgs="--initial-branch=main" + fi + + createGitRepo "$repo" "$extraArgs" +done + +createSimpleGitFlake "$flake1Dir" + +cat > "$flake2Dir/flake.nix" < "$flake3Dir/flake.nix" < "$flake3Dir/default.nix" < "$nonFlakeDir/README.md" < "$nonFlakeDir/shebang.sh" < $nonFlakeDir/shebang-comments.sh < $nonFlakeDir/shebang-reject.sh < $nonFlakeDir/shebang-inline-expr.sh <> $nonFlakeDir/shebang-inline-expr.sh <<"EOF" +#! nix --offline shell +#! nix --impure --expr `` +#! nix let flake = (builtins.getFlake (toString ../flake1)).packages; +#! nix fooScript = flake.${builtins.currentSystem}.fooScript; +#! nix /* just a comment !@#$%^&*()__+ # */ +#! nix in fooScript +#! nix `` +#! nix --no-write-lock-file --command bash +set -ex +foo +echo "$@" +EOF +chmod +x $nonFlakeDir/shebang-inline-expr.sh + +cat > $nonFlakeDir/fooScript.nix <<"EOF" +let flake = (builtins.getFlake (toString ../flake1)).packages; + fooScript = flake.${builtins.currentSystem}.fooScript; + in fooScript +EOF + +cat > $nonFlakeDir/shebang-file.sh <> $nonFlakeDir/shebang-file.sh <<"EOF" +#! nix --offline shell +#! nix --impure --file ./fooScript.nix +#! nix --no-write-lock-file --command bash +set -ex +foo +echo "$@" +EOF +chmod +x $nonFlakeDir/shebang-file.sh + +# Construct a custom registry, additionally test the --registry flag +nix registry add --registry "$registry" flake1 "git+file://$flake1Dir" +nix registry add --registry "$registry" flake2 "git+file://$percentEncodedFlake2Dir" +nix registry add --registry "$registry" flake3 "git+file://$percentEncodedFlake3Dir" +nix registry add --registry "$registry" flake4 flake3 +nix registry add --registry "$registry" nixpkgs flake1 + +# Test 'nix registry list'. +[[ $(nix registry list | wc -l) == 5 ]] +nix registry list | grep '^global' +nix registry list | grepInverse '^user' # nothing in user registry + +# Test 'nix flake metadata'. +nix flake metadata flake1 +nix flake metadata flake1 | grepQuiet 'Locked URL:.*flake1.*' + +# Test 'nix flake metadata' on a local flake. +(cd "$flake1Dir" && nix flake metadata) | grepQuiet 'URL:.*flake1.*' +(cd "$flake1Dir" && nix flake metadata .) | grepQuiet 'URL:.*flake1.*' +nix flake metadata "$flake1Dir" | grepQuiet 'URL:.*flake1.*' + +# Test 'nix flake metadata --json'. +json=$(nix flake metadata flake1 --json | jq .) +[[ $(echo "$json" | jq -r .description) = 'Bla bla' ]] +[[ -d $(echo "$json" | jq -r .path) ]] +[[ $(echo "$json" | jq -r .lastModified) = $(git -C "$flake1Dir" log -n1 --format=%ct) ]] +hash1=$(echo "$json" | jq -r .revision) + +echo foo > "$flake1Dir/foo" +git -C "$flake1Dir" add $flake1Dir/foo +[[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "$hash1-dirty" ]] + +echo -n '# foo' >> "$flake1Dir/flake.nix" +flake1OriginalCommit=$(git -C "$flake1Dir" rev-parse HEAD) +git -C "$flake1Dir" commit -a -m 'Foo' +flake1NewCommit=$(git -C "$flake1Dir" rev-parse HEAD) +hash2=$(nix flake metadata flake1 --json --refresh | jq -r .revision) +[[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "null" ]] +[[ $hash1 != $hash2 ]] + +# Test 'nix build' on a flake. +nix build -o "$TEST_ROOT/result" flake1#foo +[[ -e "$TEST_ROOT/result/hello" ]] + +# Test packages.default. +nix build -o "$TEST_ROOT/result" flake1 +[[ -e "$TEST_ROOT/result/hello" ]] + +nix build -o "$TEST_ROOT/result" "$flake1Dir" +nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir" + +# Check that store symlinks inside a flake are not interpreted as flakes. +nix build -o "$flake1Dir/result" "git+file://$flake1Dir" +nix path-info "$flake1Dir/result" + +# 'getFlake' on an unlocked flakeref should fail in pure mode, but +# succeed in impure mode. +(! nix build -o "$TEST_ROOT/result" --expr "(builtins.getFlake \"$flake1Dir\").packages.$system.default") +nix build -o "$TEST_ROOT/result" --expr "(builtins.getFlake \"$flake1Dir\").packages.$system.default" --impure + +# 'getFlake' on a locked flakeref should succeed even in pure mode. +nix build -o "$TEST_ROOT/result" --expr "(builtins.getFlake \"git+file://$flake1Dir?rev=$hash2\").packages.$system.default" + +# Building a flake with an unlocked dependency should fail in pure mode. +(! nix build -o "$TEST_ROOT/result" flake2#bar --no-registries) +(! nix build -o "$TEST_ROOT/result" flake2#bar --no-use-registries) +(! nix eval --expr "builtins.getFlake \"$flake2Dir\"") + +# But should succeed in impure mode. +(! nix build -o "$TEST_ROOT/result" flake2#bar --impure) +nix build -o "$TEST_ROOT/result" flake2#bar --impure --no-write-lock-file +nix eval --expr "builtins.getFlake \"$flake2Dir\"" --impure + +# Building a local flake with an unlocked dependency should fail with --no-update-lock-file. +expect 1 nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --no-update-lock-file 2>&1 | grep 'requires lock file changes' + +# But it should succeed without that flag. +nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --no-write-lock-file +expect 1 nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --no-update-lock-file 2>&1 | grep 'requires lock file changes' +nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --commit-lock-file +[[ -e "$flake2Dir/flake.lock" ]] +[[ -z $(git -C "$flake2Dir" diff main || echo failed) ]] + +# Rerunning the build should not change the lockfile. +nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" +[[ -z $(git -C "$flake2Dir" diff main || echo failed) ]] + +# Building with a lockfile should not require a fetch of the registry. +nix build -o "$TEST_ROOT/result" --flake-registry file:///no-registry.json "$flake2Dir#bar" --refresh +nix build -o "$TEST_ROOT/result" --no-registries "$flake2Dir#bar" --refresh +nix build -o "$TEST_ROOT/result" --no-use-registries "$flake2Dir#bar" --refresh + +# Updating the flake should not change the lockfile. +nix flake lock "$flake2Dir" +[[ -z $(git -C "$flake2Dir" diff main || echo failed) ]] + +# Now we should be able to build the flake in pure mode. +nix build -o "$TEST_ROOT/result" flake2#bar + +# Or without a registry. +nix build -o "$TEST_ROOT/result" --no-registries "git+file://$percentEncodedFlake2Dir#bar" --refresh +nix build -o "$TEST_ROOT/result" --no-use-registries "git+file://$percentEncodedFlake2Dir#bar" --refresh + +# Test whether indirect dependencies work. +nix build -o "$TEST_ROOT/result" "$flake3Dir#xyzzy" +git -C "$flake3Dir" add flake.lock + +# Add dependency to flake3. +rm "$flake3Dir/flake.nix" + +cat > "$flake3Dir/flake.nix" < "$flake3Dir/flake.nix" < \$out + [[ \$(cat \${inputs.nonFlake}/README.md) = \$(cat \${inputs.nonFlakeFile}) ]] + [[ \${inputs.nonFlakeFile} = \${inputs.nonFlakeFile2} ]] + ''; + }; + }; +} +EOF + +cp ../config.nix "$flake3Dir" + +git -C "$flake3Dir" add flake.nix config.nix +git -C "$flake3Dir" commit -m 'Add nonFlakeInputs' + +# Check whether `nix build` works with a lockfile which is missing a +# nonFlakeInputs. +nix build -o "$TEST_ROOT/result" "$flake3Dir#sth" --commit-lock-file + +nix build -o "$TEST_ROOT/result" flake3#fnord +[[ $(cat $TEST_ROOT/result) = FNORD ]] + +# Check whether flake input fetching is lazy: flake3#sth does not +# depend on flake2, so this shouldn't fail. +rm -rf "$TEST_HOME/.cache" +clearStore +mv "$flake2Dir" "$flake2Dir.tmp" +mv "$nonFlakeDir" "$nonFlakeDir.tmp" +nix build -o "$TEST_ROOT/result" flake3#sth +(! nix build -o "$TEST_ROOT/result" flake3#xyzzy) +(! nix build -o "$TEST_ROOT/result" flake3#fnord) +mv "$flake2Dir.tmp" "$flake2Dir" +mv "$nonFlakeDir.tmp" "$nonFlakeDir" +nix build -o "$TEST_ROOT/result" flake3#xyzzy flake3#fnord + +# Test doing multiple `lookupFlake`s +nix build -o "$TEST_ROOT/result" flake4#xyzzy + +# Test 'nix flake update' and --override-flake. +nix flake lock "$flake3Dir" +[[ -z $(git -C "$flake3Dir" diff master || echo failed) ]] + +nix flake update --flake "$flake3Dir" --override-flake flake2 nixpkgs +[[ ! -z $(git -C "$flake3Dir" diff master || echo failed) ]] + +# Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore +git -C "$flake3Dir" checkout -b removeXyzzy +rm "$flake3Dir/flake.nix" + +cat > "$flake3Dir/flake.nix" < \$out + ''; + }; + }; +} +EOF +nix flake lock "$flake3Dir" +git -C "$flake3Dir" add flake.nix flake.lock +git -C "$flake3Dir" commit -m 'Remove packages.xyzzy' +git -C "$flake3Dir" checkout master + +# Test whether fuzzy-matching works for registry entries. +(! nix build -o "$TEST_ROOT/result" flake4/removeXyzzy#xyzzy) +nix build -o "$TEST_ROOT/result" flake4/removeXyzzy#sth + +# Testing the nix CLI +nix registry add flake1 flake3 +[[ $(nix registry list | wc -l) == 6 ]] +nix registry pin flake1 +[[ $(nix registry list | wc -l) == 6 ]] +nix registry pin flake1 flake3 +[[ $(nix registry list | wc -l) == 6 ]] +nix registry remove flake1 +[[ $(nix registry list | wc -l) == 5 ]] + +# Test 'nix registry list' with a disabled global registry. +nix registry add user-flake1 git+file://$flake1Dir +nix registry add user-flake2 "git+file://$percentEncodedFlake2Dir" +[[ $(nix --flake-registry "" registry list | wc -l) == 2 ]] +nix --flake-registry "" registry list | grepQuietInverse '^global' # nothing in global registry +nix --flake-registry "" registry list | grepQuiet '^user' +nix registry remove user-flake1 +nix registry remove user-flake2 +[[ $(nix registry list | wc -l) == 5 ]] + +# Test 'nix flake clone'. +rm -rf $TEST_ROOT/flake1-v2 +nix flake clone flake1 --dest $TEST_ROOT/flake1-v2 +[ -e $TEST_ROOT/flake1-v2/flake.nix ] + +# Test 'follows' inputs. +cat > "$flake3Dir/flake.nix" < "$flake3Dir/flake.nix" < "$flake3Dir/flake.nix" < "$flake3Dir/flake.nix" < "$flake3Dir/flake.nix" < $badFlakeDir/flake.nix +nix store delete $(nix store add-path $badFlakeDir) + +[[ $(nix path-info $(nix store add-path $flake1Dir)) =~ flake1 ]] +[[ $(nix path-info path:$(nix store add-path $flake1Dir)) =~ simple ]] + +# Test fetching flakerefs in the legacy CLI. +[[ $(nix-instantiate --eval flake:flake3 -A x) = 123 ]] +[[ $(nix-instantiate --eval "flake:git+file://$percentEncodedFlake3Dir" -A x) = 123 ]] +[[ $(nix-instantiate -I flake3=flake:flake3 --eval '' -A x) = 123 ]] +[[ $(NIX_PATH=flake3=flake:flake3 nix-instantiate --eval '' -A x) = 123 ]] + +# Test alternate lockfile paths. +nix flake lock "$flake2Dir" --output-lock-file $TEST_ROOT/flake2.lock +cmp "$flake2Dir/flake.lock" $TEST_ROOT/flake2.lock >/dev/null # lockfiles should be identical, since we're referencing flake2's original one + +nix flake lock "$flake2Dir" --output-lock-file $TEST_ROOT/flake2-overridden.lock --override-input flake1 git+file://$flake1Dir?rev=$flake1OriginalCommit +expectStderr 1 cmp "$flake2Dir/flake.lock" $TEST_ROOT/flake2-overridden.lock +nix flake metadata "$flake2Dir" --reference-lock-file $TEST_ROOT/flake2-overridden.lock | grepQuiet $flake1OriginalCommit + +# reference-lock-file can only be used if allow-dirty is set. +expectStderr 1 nix flake metadata "$flake2Dir" --no-allow-dirty --reference-lock-file $TEST_ROOT/flake2-overridden.lock + +# Test shebang +[[ $($nonFlakeDir/shebang.sh) = "foo" ]] +[[ $($nonFlakeDir/shebang.sh "bar") = "foo"$'\n'"bar" ]] +[[ $($nonFlakeDir/shebang-comments.sh ) = "foo" ]] +[[ $($nonFlakeDir/shebang-inline-expr.sh baz) = "foo"$'\n'"baz" ]] +[[ $($nonFlakeDir/shebang-file.sh baz) = "foo"$'\n'"baz" ]] +expect 1 $nonFlakeDir/shebang-reject.sh 2>&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?' diff --git a/tests/functional/flakes/follow-paths.sh b/tests/functional/flakes/follow-paths.sh new file mode 100644 index 000000000000..1afd91bd2f8e --- /dev/null +++ b/tests/functional/flakes/follow-paths.sh @@ -0,0 +1,338 @@ +source ./common.sh + +requireGit + +flakeFollowsA=$TEST_ROOT/follows/flakeA +flakeFollowsB=$TEST_ROOT/follows/flakeA/flakeB +flakeFollowsC=$TEST_ROOT/follows/flakeA/flakeB/flakeC +flakeFollowsD=$TEST_ROOT/follows/flakeA/flakeD +flakeFollowsE=$TEST_ROOT/follows/flakeA/flakeE + +# Test following path flakerefs. +createGitRepo $flakeFollowsA +mkdir -p $flakeFollowsB +mkdir -p $flakeFollowsC +mkdir -p $flakeFollowsD +mkdir -p $flakeFollowsE + +cat > $flakeFollowsA/flake.nix < $flakeFollowsB/flake.nix < $flakeFollowsC/flake.nix < $flakeFollowsD/flake.nix < $flakeFollowsE/flake.nix < $flakeFollowsA/flake.nix < $flakeFollowsA/flake.nix <&1 | grep 'points outside' + +# Non-existant follows should print a warning. +cat >$flakeFollowsA/flake.nix <&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid'" +nix flake lock "$flakeFollowsA" 2>&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid2'" + +# Now test follow path overloading +# This tests a lockfile checking regression https://github.com/NixOS/nix/pull/8819 +# +# We construct the following graph, where p->q means p has input q. +# A double edge means that the edge gets overridden using `follows`. +# +# A +# / \ +# / \ +# v v +# B ==> C --- follows declared in A +# \\ / +# \\/ --- follows declared in B +# v +# D +# +# The message was +# error: input 'B/D' follows a non-existent input 'B/C/D' +# +# Note that for `B` to resolve its follow for `D`, it needs `C/D`, for which it needs to resolve the follow on `C` first. +flakeFollowsOverloadA="$TEST_ROOT/follows/overload/flakeA" +flakeFollowsOverloadB="$TEST_ROOT/follows/overload/flakeA/flakeB" +flakeFollowsOverloadC="$TEST_ROOT/follows/overload/flakeA/flakeB/flakeC" +flakeFollowsOverloadD="$TEST_ROOT/follows/overload/flakeA/flakeB/flakeC/flakeD" + +# Test following path flakerefs. +createGitRepo "$flakeFollowsOverloadA" +mkdir -p "$flakeFollowsOverloadB" +mkdir -p "$flakeFollowsOverloadC" +mkdir -p "$flakeFollowsOverloadD" + +cat > "$flakeFollowsOverloadD/flake.nix" < "$flakeFollowsOverloadC/flake.nix" < "$flakeFollowsOverloadB/flake.nix" < "$flakeFollowsOverloadA/flake.nix" < baz +# The message was +# error: follow cycle detected: [baz -> foo -> bar -> baz] +flakeFollowCycle="$TEST_ROOT/follows/followCycle" + +# Test following path flakerefs. +mkdir -p "$flakeFollowCycle" + +cat > $flakeFollowCycle/flake.nix <&1 && fail "nix flake lock should have failed." || true) +echo $checkRes | grep -F "error: follow cycle detected: [baz -> foo -> bar -> baz]" + + +# Test transitive input url locking +# This tests the following lockfile issue: https://github.com/NixOS/nix/issues/9143 +# +# We construct the following graph, where p->q means p has input q. +# +# A -> B -> C +# +# And override B/C to flake D, first in A's flake.nix and then with --override-input. +# +# A -> B -> D +flakeFollowsCustomUrlA="$TEST_ROOT/follows/custom-url/flakeA" +flakeFollowsCustomUrlB="$TEST_ROOT/follows/custom-url/flakeA/flakeB" +flakeFollowsCustomUrlC="$TEST_ROOT/follows/custom-url/flakeA/flakeB/flakeC" +flakeFollowsCustomUrlD="$TEST_ROOT/follows/custom-url/flakeA/flakeB/flakeD" + + +createGitRepo "$flakeFollowsCustomUrlA" +mkdir -p "$flakeFollowsCustomUrlB" +mkdir -p "$flakeFollowsCustomUrlC" +mkdir -p "$flakeFollowsCustomUrlD" + +cat > "$flakeFollowsCustomUrlD/flake.nix" < "$flakeFollowsCustomUrlC/flake.nix" < "$flakeFollowsCustomUrlB/flake.nix" < "$flakeFollowsCustomUrlA/flake.nix" < $out + ''; + }; + +in + + mkDerivation { + shell = busybox; + name = "hermetic"; + passthru = { inherit input1 input2 input3; }; + buildCommand = + '' + read x < ${input1} + read y < ${input3} + echo "$x $y" > $out + ''; + } diff --git a/tests/import-derivation.nix b/tests/functional/import-derivation.nix similarity index 100% rename from tests/import-derivation.nix rename to tests/functional/import-derivation.nix diff --git a/tests/import-derivation.sh b/tests/functional/import-derivation.sh similarity index 100% rename from tests/import-derivation.sh rename to tests/functional/import-derivation.sh diff --git a/tests/impure-derivations.nix b/tests/functional/impure-derivations.nix similarity index 100% rename from tests/impure-derivations.nix rename to tests/functional/impure-derivations.nix diff --git a/tests/impure-derivations.sh b/tests/functional/impure-derivations.sh similarity index 100% rename from tests/impure-derivations.sh rename to tests/functional/impure-derivations.sh diff --git a/tests/functional/impure-env.nix b/tests/functional/impure-env.nix new file mode 100644 index 000000000000..2b0380ed7296 --- /dev/null +++ b/tests/functional/impure-env.nix @@ -0,0 +1,16 @@ +{ var, value }: + +with import ./config.nix; + +mkDerivation { + name = "test"; + buildCommand = '' + echo ${var} = "''$${var}" + echo -n "''$${var}" > "$out" + ''; + + impureEnvVars = [ var ]; + + outputHashAlgo = "sha256"; + outputHash = builtins.hashString "sha256" value; +} diff --git a/tests/functional/impure-env.sh b/tests/functional/impure-env.sh new file mode 100644 index 000000000000..d9e4a34a2299 --- /dev/null +++ b/tests/functional/impure-env.sh @@ -0,0 +1,33 @@ +source common.sh + +# Needs the config option 'impure-env' to work +requireDaemonNewerThan "2.18.0pre20230816" + +enableFeatures "configurable-impure-env" +restartDaemon + +varTest() { + local var="$1"; shift + local value="$1"; shift + nix build --no-link -vL --argstr var "$var" --argstr value "$value" --impure "$@" --file impure-env.nix + clearStore +} + +clearStore +startDaemon + +varTest env_name value --impure-env env_name=value + +echo 'impure-env = set_in_config=config_value' >> "$NIX_CONF_DIR/nix.conf" +set_in_config=daemon_value restartDaemon + +varTest set_in_config config_value +varTest set_in_config client_value --impure-env set_in_config=client_value + +sed -i -e '/^trusted-users =/d' "$NIX_CONF_DIR/nix.conf" + +env_name=daemon_value restartDaemon + +varTest env_name daemon_value --impure-env env_name=client_value + +killDaemon diff --git a/tests/init.sh b/tests/functional/init.sh similarity index 100% rename from tests/init.sh rename to tests/functional/init.sh diff --git a/tests/install-darwin.sh b/tests/functional/install-darwin.sh similarity index 100% rename from tests/install-darwin.sh rename to tests/functional/install-darwin.sh diff --git a/tests/functional/lang-test-infra.sh b/tests/functional/lang-test-infra.sh new file mode 100644 index 000000000000..30da8977b6a0 --- /dev/null +++ b/tests/functional/lang-test-infra.sh @@ -0,0 +1,86 @@ +# Test the function for lang.sh +source common.sh + +source lang/framework.sh + +# We are testing this, so don't want outside world to affect us. +unset _NIX_TEST_ACCEPT + +# We'll only modify this in subshells so we don't need to reset it. +badDiff=0 + +# matches non-empty +echo Hi! > "$TEST_ROOT/got" +cp "$TEST_ROOT/got" "$TEST_ROOT/expected" +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 0 )) +) + +# matches empty, non-existant file is the same as empty file +echo -n > "$TEST_ROOT/got" +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/does-not-exist" + (( "$badDiff" == 0 )) +) + +# doesn't matches non-empty, non-existant file is the same as empty file +echo Hi! > "$TEST_ROOT/got" +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/does-not-exist" + (( "$badDiff" == 1 )) +) + +# doesn't match, `badDiff` set, file unchanged +echo Hi! > "$TEST_ROOT/got" +echo Bye! > "$TEST_ROOT/expected" +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 1 )) +) +[[ "$(echo Bye! )" == $(< "$TEST_ROOT/expected") ]] + +# _NIX_TEST_ACCEPT=1 matches non-empty +echo Hi! > "$TEST_ROOT/got" +cp "$TEST_ROOT/got" "$TEST_ROOT/expected" +( + _NIX_TEST_ACCEPT=1 diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 0 )) +) + +# _NIX_TEST_ACCEPT doesn't match, `badDiff=1` set, file changed (was previously non-empty) +echo Hi! > "$TEST_ROOT/got" +echo Bye! > "$TEST_ROOT/expected" +( + _NIX_TEST_ACCEPT=1 diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 1 )) +) +[[ "$(echo Hi! )" == $(< "$TEST_ROOT/expected") ]] +# second time succeeds +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 0 )) +) + +# _NIX_TEST_ACCEPT matches empty, non-existant file not created +echo -n > "$TEST_ROOT/got" +( + _NIX_TEST_ACCEPT=1 diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/does-not-exists" + (( "$badDiff" == 0 )) +) +[[ ! -f "$TEST_ROOT/does-not-exist" ]] + +# _NIX_TEST_ACCEPT doesn't match, output empty, file deleted +echo -n > "$TEST_ROOT/got" +echo Bye! > "$TEST_ROOT/expected" +badDiff=0 +( + _NIX_TEST_ACCEPT=1 diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 1 )) +) +[[ ! -f "$TEST_ROOT/expected" ]] +# second time succeeds +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 0 )) +) diff --git a/tests/functional/lang.sh b/tests/functional/lang.sh new file mode 100755 index 000000000000..12df32c87704 --- /dev/null +++ b/tests/functional/lang.sh @@ -0,0 +1,147 @@ +source common.sh + +set -o pipefail + +source lang/framework.sh + +# specialize function a bit +function diffAndAccept() { + local -r testName="$1" + local -r got="lang/$testName.$2" + local -r expected="lang/$testName.$3" + diffAndAcceptInner "$testName" "$got" "$expected" +} + +export TEST_VAR=foo # for eval-okay-getenv.nix +export NIX_REMOTE=dummy:// +export NIX_STORE_DIR=/nix/store + +nix-instantiate --eval -E 'builtins.trace "Hello" 123' 2>&1 | grepQuiet Hello +nix-instantiate --eval -E 'builtins.trace "Hello" 123' 2>/dev/null | grepQuiet 123 +nix-instantiate --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 +nix-instantiate --trace-verbose --eval -E 'builtins.traceVerbose "Hello" 123' 2>&1 | grepQuiet Hello +nix-instantiate --eval -E 'builtins.traceVerbose "Hello" 123' 2>&1 | grepQuietInverse Hello +nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 | grepQuietInverse Hello +expectStderr 1 nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" (throw "Foo")' | grepQuiet Hello +expectStderr 1 nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello %" (throw "Foo")' | grepQuiet 'Hello %' + +nix-instantiate --eval -E 'let x = builtins.trace { x = x; } true; in x' \ + 2>&1 | grepQuiet -E 'trace: { x = «potential infinite recursion»; }' + +nix-instantiate --eval -E 'let x = { repeating = x; tracing = builtins.trace x true; }; in x.tracing'\ + 2>&1 | grepQuiet -F 'trace: { repeating = «repeated»; tracing = «potential infinite recursion»; }' + +set +x + +badDiff=0 +badExitCode=0 + +for i in lang/parse-fail-*.nix; do + echo "parsing $i (should fail)"; + i=$(basename "$i" .nix) + if expectStderr 1 nix-instantiate --parse - < "lang/$i.nix" > "lang/$i.err" + then + diffAndAccept "$i" err err.exp + else + echo "FAIL: $i shouldn't parse" + badExitCode=1 + fi +done + +for i in lang/parse-okay-*.nix; do + echo "parsing $i (should succeed)"; + i=$(basename "$i" .nix) + if + expect 0 nix-instantiate --parse - < "lang/$i.nix" \ + 1> "lang/$i.out" \ + 2> "lang/$i.err" + then + sed "s!$(pwd)!/pwd!g" "lang/$i.out" "lang/$i.err" + diffAndAccept "$i" out exp + diffAndAccept "$i" err err.exp + else + echo "FAIL: $i should parse" + badExitCode=1 + fi +done + +for i in lang/eval-fail-*.nix; do + echo "evaluating $i (should fail)"; + i=$(basename "$i" .nix) + if + expectStderr 1 nix-instantiate --eval --strict --show-trace "lang/$i.nix" \ + | sed "s!$(pwd)!/pwd!g" > "lang/$i.err" + then + diffAndAccept "$i" err err.exp + else + echo "FAIL: $i shouldn't evaluate" + badExitCode=1 + fi +done + +for i in lang/eval-okay-*.nix; do + echo "evaluating $i (should succeed)"; + i=$(basename "$i" .nix) + + if test -e "lang/$i.exp.xml"; then + if expect 0 nix-instantiate --eval --xml --no-location --strict \ + "lang/$i.nix" > "lang/$i.out.xml" + then + diffAndAccept "$i" out.xml exp.xml + else + echo "FAIL: $i should evaluate" + badExitCode=1 + fi + elif test ! -e "lang/$i.exp-disabled"; then + declare -a flags=() + if test -e "lang/$i.flags"; then + read -r -a flags < "lang/$i.flags" + fi + + if + expect 0 env \ + NIX_PATH=lang/dir3:lang/dir4 \ + HOME=/fake-home \ + nix-instantiate "${flags[@]}" --eval --strict "lang/$i.nix" \ + 1> "lang/$i.out" \ + 2> "lang/$i.err" + then + sed -i "s!$(pwd)!/pwd!g" "lang/$i.out" "lang/$i.err" + diffAndAccept "$i" out exp + diffAndAccept "$i" err err.exp + else + echo "FAIL: $i should evaluate" + badExitCode=1 + fi + fi +done + +if test -n "${_NIX_TEST_ACCEPT-}"; then + if (( "$badDiff" )); then + echo 'Output did mot match, but accepted output as the persisted expected output.' + echo 'That means the next time the tests are run, they should pass.' + else + echo 'NOTE: Environment variable _NIX_TEST_ACCEPT is defined,' + echo 'indicating the unexpected output should be accepted as the expected output going forward,' + echo 'but no tests had unexpected output so there was no expected output to update.' + fi + if (( "$badExitCode" )); then + exit "$badExitCode" + else + skipTest "regenerating golden masters" + fi +else + if (( "$badDiff" )); then + echo '' + echo 'You can rerun this test with:' + echo '' + echo ' _NIX_TEST_ACCEPT=1 make tests/functional/lang.sh.test' + echo '' + echo 'to regenerate the files containing the expected output,' + echo 'and then view the git diff to decide whether a change is' + echo 'good/intentional or bad/unintentional.' + echo 'If the diff contains arbitrary or impure information,' + echo 'please improve the normalization that the test applies to the output.' + fi + exit $(( "$badExitCode" + "$badDiff" )) +fi diff --git a/tests/lang/binary-data b/tests/functional/lang/binary-data similarity index 100% rename from tests/lang/binary-data rename to tests/functional/lang/binary-data diff --git a/tests/lang/data b/tests/functional/lang/data similarity index 100% rename from tests/lang/data rename to tests/functional/lang/data diff --git a/tests/lang/dir1/a.nix b/tests/functional/lang/dir1/a.nix similarity index 100% rename from tests/lang/dir1/a.nix rename to tests/functional/lang/dir1/a.nix diff --git a/tests/lang/dir2/a.nix b/tests/functional/lang/dir2/a.nix similarity index 100% rename from tests/lang/dir2/a.nix rename to tests/functional/lang/dir2/a.nix diff --git a/tests/lang/dir2/b.nix b/tests/functional/lang/dir2/b.nix similarity index 100% rename from tests/lang/dir2/b.nix rename to tests/functional/lang/dir2/b.nix diff --git a/tests/lang/dir3/a.nix b/tests/functional/lang/dir3/a.nix similarity index 100% rename from tests/lang/dir3/a.nix rename to tests/functional/lang/dir3/a.nix diff --git a/tests/lang/dir3/b.nix b/tests/functional/lang/dir3/b.nix similarity index 100% rename from tests/lang/dir3/b.nix rename to tests/functional/lang/dir3/b.nix diff --git a/tests/lang/dir3/c.nix b/tests/functional/lang/dir3/c.nix similarity index 100% rename from tests/lang/dir3/c.nix rename to tests/functional/lang/dir3/c.nix diff --git a/tests/lang/dir4/a.nix b/tests/functional/lang/dir4/a.nix similarity index 100% rename from tests/lang/dir4/a.nix rename to tests/functional/lang/dir4/a.nix diff --git a/tests/lang/dir4/c.nix b/tests/functional/lang/dir4/c.nix similarity index 100% rename from tests/lang/dir4/c.nix rename to tests/functional/lang/dir4/c.nix diff --git a/tests/lang/eval-okay-hash.exp b/tests/functional/lang/empty.exp similarity index 100% rename from tests/lang/eval-okay-hash.exp rename to tests/functional/lang/empty.exp diff --git a/tests/functional/lang/eval-fail-abort.err.exp b/tests/functional/lang/eval-fail-abort.err.exp new file mode 100644 index 000000000000..345232d3f343 --- /dev/null +++ b/tests/functional/lang/eval-fail-abort.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'abort' builtin + + at /pwd/lang/eval-fail-abort.nix:1:14: + + 1| if true then abort "this should fail" else 1 + | ^ + 2| + + error: evaluation aborted with the following error message: 'this should fail' diff --git a/tests/lang/eval-fail-abort.nix b/tests/functional/lang/eval-fail-abort.nix similarity index 100% rename from tests/lang/eval-fail-abort.nix rename to tests/functional/lang/eval-fail-abort.nix diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.err.exp b/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.err.exp new file mode 100644 index 000000000000..ad91a22aa5b3 --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'addDrvOutputDependencies' builtin + + at /pwd/lang/eval-fail-addDrvOutputDependencies-empty-context.nix:1:1: + + 1| builtins.addDrvOutputDependencies "" + | ^ + 2| + + error: context of string '' must have exactly one element, but has 0 diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.nix b/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.nix new file mode 100644 index 000000000000..dc9ee3ba2e59 --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-empty-context.nix @@ -0,0 +1 @@ +builtins.addDrvOutputDependencies "" diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.err.exp b/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.err.exp new file mode 100644 index 000000000000..bb389db4e4c6 --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.err.exp @@ -0,0 +1,11 @@ +error: + … while calling the 'addDrvOutputDependencies' builtin + + at /pwd/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.nix:18:4: + + 17| + 18| in builtins.addDrvOutputDependencies combo-path + | ^ + 19| + + error: context of string '/nix/store/pg9yqs4yd85yhdm3f4i5dyaqp5jahrsz-fail.drv/nix/store/2dxd5frb715z451vbf7s8birlf3argbk-fail-2.drv' must have exactly one element, but has 2 diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.nix b/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.nix new file mode 100644 index 000000000000..dbde264dfaeb --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-multi-elem-context.nix @@ -0,0 +1,18 @@ +let + drv0 = derivation { + name = "fail"; + builder = "/bin/false"; + system = "x86_64-linux"; + outputs = [ "out" "foo" ]; + }; + + drv1 = derivation { + name = "fail-2"; + builder = "/bin/false"; + system = "x86_64-linux"; + outputs = [ "out" "foo" ]; + }; + + combo-path = "${drv0.drvPath}${drv1.drvPath}"; + +in builtins.addDrvOutputDependencies combo-path diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.err.exp b/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.err.exp new file mode 100644 index 000000000000..07038111822f --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.err.exp @@ -0,0 +1,11 @@ +error: + … while calling the 'addDrvOutputDependencies' builtin + + at /pwd/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.nix:9:4: + + 8| + 9| in builtins.addDrvOutputDependencies drv.outPath + | ^ + 10| + + error: `addDrvOutputDependencies` can only act on derivations, not on a derivation output such as 'out' diff --git a/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.nix b/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.nix new file mode 100644 index 000000000000..e379e1d9598b --- /dev/null +++ b/tests/functional/lang/eval-fail-addDrvOutputDependencies-wrong-element-kind.nix @@ -0,0 +1,9 @@ +let + drv = derivation { + name = "fail"; + builder = "/bin/false"; + system = "x86_64-linux"; + outputs = [ "out" "foo" ]; + }; + +in builtins.addDrvOutputDependencies drv.outPath diff --git a/tests/functional/lang/eval-fail-assert.err.exp b/tests/functional/lang/eval-fail-assert.err.exp new file mode 100644 index 000000000000..aeecd8167fc8 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert.err.exp @@ -0,0 +1,36 @@ +error: + … while evaluating the attribute 'body' + + at /pwd/lang/eval-fail-assert.nix:4:3: + + 3| + 4| body = x "x"; + | ^ + 5| } + + … from call site + + at /pwd/lang/eval-fail-assert.nix:4:10: + + 3| + 4| body = x "x"; + | ^ + 5| } + + … while calling 'x' + + at /pwd/lang/eval-fail-assert.nix:2:7: + + 1| let { + 2| x = arg: assert arg == "y"; 123; + | ^ + 3| + + error: assertion '(arg == "y")' failed + + at /pwd/lang/eval-fail-assert.nix:2:12: + + 1| let { + 2| x = arg: assert arg == "y"; 123; + | ^ + 3| diff --git a/tests/lang/eval-fail-assert.nix b/tests/functional/lang/eval-fail-assert.nix similarity index 100% rename from tests/lang/eval-fail-assert.nix rename to tests/functional/lang/eval-fail-assert.nix diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp new file mode 100644 index 000000000000..eb73e9a52986 --- /dev/null +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp @@ -0,0 +1,10 @@ +error: + … while evaluating a path segment + + at /pwd/lang/eval-fail-bad-string-interpolation-1.nix:1:2: + + 1| "${x: x}" + | ^ + 2| + + error: cannot coerce a function to a string diff --git a/tests/lang/eval-fail-bad-string-interpolation-1.nix b/tests/functional/lang/eval-fail-bad-string-interpolation-1.nix similarity index 100% rename from tests/lang/eval-fail-bad-string-interpolation-1.nix rename to tests/functional/lang/eval-fail-bad-string-interpolation-1.nix diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp new file mode 100644 index 000000000000..a287067cd1cd --- /dev/null +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp @@ -0,0 +1 @@ +error: path '/pwd/lang/fnord' does not exist diff --git a/tests/lang/eval-fail-bad-string-interpolation-2.nix b/tests/functional/lang/eval-fail-bad-string-interpolation-2.nix similarity index 100% rename from tests/lang/eval-fail-bad-string-interpolation-2.nix rename to tests/functional/lang/eval-fail-bad-string-interpolation-2.nix diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp new file mode 100644 index 000000000000..ac14f329b787 --- /dev/null +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp @@ -0,0 +1,10 @@ +error: + … while evaluating a path segment + + at /pwd/lang/eval-fail-bad-string-interpolation-3.nix:1:3: + + 1| ''${x: x}'' + | ^ + 2| + + error: cannot coerce a function to a string diff --git a/tests/lang/eval-fail-bad-string-interpolation-3.nix b/tests/functional/lang/eval-fail-bad-string-interpolation-3.nix similarity index 100% rename from tests/lang/eval-fail-bad-string-interpolation-3.nix rename to tests/functional/lang/eval-fail-bad-string-interpolation-3.nix diff --git a/tests/functional/lang/eval-fail-blackhole.err.exp b/tests/functional/lang/eval-fail-blackhole.err.exp new file mode 100644 index 000000000000..f0618d8ac7d2 --- /dev/null +++ b/tests/functional/lang/eval-fail-blackhole.err.exp @@ -0,0 +1,18 @@ +error: + … while evaluating the attribute 'body' + + at /pwd/lang/eval-fail-blackhole.nix:2:3: + + 1| let { + 2| body = x; + | ^ + 3| x = y; + + error: infinite recursion encountered + + at /pwd/lang/eval-fail-blackhole.nix:3:7: + + 2| body = x; + 3| x = y; + | ^ + 4| y = x; diff --git a/tests/lang/eval-fail-blackhole.nix b/tests/functional/lang/eval-fail-blackhole.nix similarity index 100% rename from tests/lang/eval-fail-blackhole.nix rename to tests/functional/lang/eval-fail-blackhole.nix diff --git a/tests/functional/lang/eval-fail-deepseq.err.exp b/tests/functional/lang/eval-fail-deepseq.err.exp new file mode 100644 index 000000000000..5e204ba73f6f --- /dev/null +++ b/tests/functional/lang/eval-fail-deepseq.err.exp @@ -0,0 +1,26 @@ +error: + … while calling the 'deepSeq' builtin + + at /pwd/lang/eval-fail-deepseq.nix:1:1: + + 1| builtins.deepSeq { x = abort "foo"; } 456 + | ^ + 2| + + … while evaluating the attribute 'x' + + at /pwd/lang/eval-fail-deepseq.nix:1:20: + + 1| builtins.deepSeq { x = abort "foo"; } 456 + | ^ + 2| + + … while calling the 'abort' builtin + + at /pwd/lang/eval-fail-deepseq.nix:1:24: + + 1| builtins.deepSeq { x = abort "foo"; } 456 + | ^ + 2| + + error: evaluation aborted with the following error message: 'foo' diff --git a/tests/lang/eval-fail-deepseq.nix b/tests/functional/lang/eval-fail-deepseq.nix similarity index 100% rename from tests/lang/eval-fail-deepseq.nix rename to tests/functional/lang/eval-fail-deepseq.nix diff --git a/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp b/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp new file mode 100644 index 000000000000..c5fa67523b98 --- /dev/null +++ b/tests/functional/lang/eval-fail-dup-dynamic-attrs.err.exp @@ -0,0 +1,18 @@ +error: + … while evaluating the attribute 'set' + + at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:2:3: + + 1| { + 2| set = { "${"" + "b"}" = 1; }; + | ^ + 3| set = { "${"b" + ""}" = 2; }; + + error: dynamic attribute 'b' already defined at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:2:11 + + at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:3:11: + + 2| set = { "${"" + "b"}" = 1; }; + 3| set = { "${"b" + ""}" = 2; }; + | ^ + 4| } diff --git a/tests/functional/lang/eval-fail-dup-dynamic-attrs.nix b/tests/functional/lang/eval-fail-dup-dynamic-attrs.nix new file mode 100644 index 000000000000..7ea17f6c8780 --- /dev/null +++ b/tests/functional/lang/eval-fail-dup-dynamic-attrs.nix @@ -0,0 +1,4 @@ +{ + set = { "${"" + "b"}" = 1; }; + set = { "${"b" + ""}" = 2; }; +} diff --git a/tests/functional/lang/eval-fail-foldlStrict-strict-op-application.err.exp b/tests/functional/lang/eval-fail-foldlStrict-strict-op-application.err.exp new file mode 100644 index 000000000000..0069285fb7e5 --- /dev/null +++ b/tests/functional/lang/eval-fail-foldlStrict-strict-op-application.err.exp @@ -0,0 +1,38 @@ +error: + … while calling the 'foldl'' builtin + + at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:2:1: + + 1| # Tests that the result of applying op is forced even if the value is never used + 2| builtins.foldl' + | ^ + 3| (_: f: f null) + + … while calling anonymous lambda + + at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:3:7: + + 2| builtins.foldl' + 3| (_: f: f null) + | ^ + 4| null + + … from call site + + at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:3:10: + + 2| builtins.foldl' + 3| (_: f: f null) + | ^ + 4| null + + … while calling anonymous lambda + + at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:5:6: + + 4| null + 5| [ (_: throw "Not the final value, but is still forced!") (_: 23) ] + | ^ + 6| + + error: Not the final value, but is still forced! diff --git a/tests/lang/eval-fail-foldlStrict-strict-op-application.nix b/tests/functional/lang/eval-fail-foldlStrict-strict-op-application.nix similarity index 100% rename from tests/lang/eval-fail-foldlStrict-strict-op-application.nix rename to tests/functional/lang/eval-fail-foldlStrict-strict-op-application.nix diff --git a/tests/functional/lang/eval-fail-fromTOML-timestamps.err.exp b/tests/functional/lang/eval-fail-fromTOML-timestamps.err.exp new file mode 100644 index 000000000000..5b60d253de33 --- /dev/null +++ b/tests/functional/lang/eval-fail-fromTOML-timestamps.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'fromTOML' builtin + + at /pwd/lang/eval-fail-fromTOML-timestamps.nix:1:1: + + 1| builtins.fromTOML '' + | ^ + 2| key = "value" + + error: while parsing a TOML string: Dates and times are not supported diff --git a/tests/lang/eval-fail-fromTOML-timestamps.nix b/tests/functional/lang/eval-fail-fromTOML-timestamps.nix similarity index 100% rename from tests/lang/eval-fail-fromTOML-timestamps.nix rename to tests/functional/lang/eval-fail-fromTOML-timestamps.nix diff --git a/tests/functional/lang/eval-fail-hashfile-missing.err.exp b/tests/functional/lang/eval-fail-hashfile-missing.err.exp new file mode 100644 index 000000000000..6d38608c0c5a --- /dev/null +++ b/tests/functional/lang/eval-fail-hashfile-missing.err.exp @@ -0,0 +1,15 @@ +error: + … while calling the 'toString' builtin + + at /pwd/lang/eval-fail-hashfile-missing.nix:4:3: + + 3| in + 4| toString (builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"])) + | ^ + 5| + + … while evaluating the first argument passed to builtins.toString + + … while calling the 'hashFile' builtin + + error: opening file '/pwd/lang/this-file-is-definitely-not-there-7392097': No such file or directory diff --git a/tests/lang/eval-fail-hashfile-missing.nix b/tests/functional/lang/eval-fail-hashfile-missing.nix similarity index 100% rename from tests/lang/eval-fail-hashfile-missing.nix rename to tests/functional/lang/eval-fail-hashfile-missing.nix diff --git a/tests/functional/lang/eval-fail-list.err.exp b/tests/functional/lang/eval-fail-list.err.exp new file mode 100644 index 000000000000..24d682118376 --- /dev/null +++ b/tests/functional/lang/eval-fail-list.err.exp @@ -0,0 +1,10 @@ +error: + … while evaluating one of the elements to concatenate + + at /pwd/lang/eval-fail-list.nix:1:2: + + 1| 8++1 + | ^ + 2| + + error: value is an integer while a list was expected diff --git a/tests/functional/lang/eval-fail-list.nix b/tests/functional/lang/eval-fail-list.nix new file mode 100644 index 000000000000..fa749f2f7403 --- /dev/null +++ b/tests/functional/lang/eval-fail-list.nix @@ -0,0 +1 @@ +8++1 diff --git a/tests/functional/lang/eval-fail-missing-arg.err.exp b/tests/functional/lang/eval-fail-missing-arg.err.exp new file mode 100644 index 000000000000..61fabf0d5f13 --- /dev/null +++ b/tests/functional/lang/eval-fail-missing-arg.err.exp @@ -0,0 +1,16 @@ +error: + … from call site + + at /pwd/lang/eval-fail-missing-arg.nix:1:1: + + 1| ({x, y, z}: x + y + z) {x = "foo"; z = "bar";} + | ^ + 2| + + error: function 'anonymous lambda' called without required argument 'y' + + at /pwd/lang/eval-fail-missing-arg.nix:1:2: + + 1| ({x, y, z}: x + y + z) {x = "foo"; z = "bar";} + | ^ + 2| diff --git a/tests/lang/eval-fail-missing-arg.nix b/tests/functional/lang/eval-fail-missing-arg.nix similarity index 100% rename from tests/lang/eval-fail-missing-arg.nix rename to tests/functional/lang/eval-fail-missing-arg.nix diff --git a/tests/functional/lang/eval-fail-nonexist-path.err.exp b/tests/functional/lang/eval-fail-nonexist-path.err.exp new file mode 100644 index 000000000000..a287067cd1cd --- /dev/null +++ b/tests/functional/lang/eval-fail-nonexist-path.err.exp @@ -0,0 +1 @@ +error: path '/pwd/lang/fnord' does not exist diff --git a/tests/lang/eval-fail-nonexist-path.nix b/tests/functional/lang/eval-fail-nonexist-path.nix similarity index 100% rename from tests/lang/eval-fail-nonexist-path.nix rename to tests/functional/lang/eval-fail-nonexist-path.nix diff --git a/tests/functional/lang/eval-fail-path-slash.err.exp b/tests/functional/lang/eval-fail-path-slash.err.exp new file mode 100644 index 000000000000..f0011c97fd59 --- /dev/null +++ b/tests/functional/lang/eval-fail-path-slash.err.exp @@ -0,0 +1,8 @@ +error: path has a trailing slash + + at /pwd/lang/eval-fail-path-slash.nix:6:12: + + 5| # and https://nixos.org/nix-dev/2016-June/020829.html + 6| /nix/store/ + | ^ + 7| diff --git a/tests/lang/eval-fail-path-slash.nix b/tests/functional/lang/eval-fail-path-slash.nix similarity index 100% rename from tests/lang/eval-fail-path-slash.nix rename to tests/functional/lang/eval-fail-path-slash.nix diff --git a/tests/functional/lang/eval-fail-recursion.err.exp b/tests/functional/lang/eval-fail-recursion.err.exp new file mode 100644 index 000000000000..af64133cb882 --- /dev/null +++ b/tests/functional/lang/eval-fail-recursion.err.exp @@ -0,0 +1,16 @@ +error: + … in the right operand of the update (//) operator + + at /pwd/lang/eval-fail-recursion.nix:1:12: + + 1| let a = {} // a; in a.foo + | ^ + 2| + + error: infinite recursion encountered + + at /pwd/lang/eval-fail-recursion.nix:1:15: + + 1| let a = {} // a; in a.foo + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-recursion.nix b/tests/functional/lang/eval-fail-recursion.nix new file mode 100644 index 000000000000..075b5ed066b6 --- /dev/null +++ b/tests/functional/lang/eval-fail-recursion.nix @@ -0,0 +1 @@ +let a = {} // a; in a.foo diff --git a/tests/functional/lang/eval-fail-remove.err.exp b/tests/functional/lang/eval-fail-remove.err.exp new file mode 100644 index 000000000000..e82cdac98654 --- /dev/null +++ b/tests/functional/lang/eval-fail-remove.err.exp @@ -0,0 +1,19 @@ +error: + … while evaluating the attribute 'body' + + at /pwd/lang/eval-fail-remove.nix:4:3: + + 3| + 4| body = (removeAttrs attrs ["x"]).x; + | ^ + 5| } + + error: attribute 'x' missing + + at /pwd/lang/eval-fail-remove.nix:4:10: + + 3| + 4| body = (removeAttrs attrs ["x"]).x; + | ^ + 5| } + Did you mean y? diff --git a/tests/lang/eval-fail-remove.nix b/tests/functional/lang/eval-fail-remove.nix similarity index 100% rename from tests/lang/eval-fail-remove.nix rename to tests/functional/lang/eval-fail-remove.nix diff --git a/tests/functional/lang/eval-fail-scope-5.err.exp b/tests/functional/lang/eval-fail-scope-5.err.exp new file mode 100644 index 000000000000..22b6166f8031 --- /dev/null +++ b/tests/functional/lang/eval-fail-scope-5.err.exp @@ -0,0 +1,36 @@ +error: + … while evaluating the attribute 'body' + + at /pwd/lang/eval-fail-scope-5.nix:8:3: + + 7| + 8| body = f {}; + | ^ + 9| + + … from call site + + at /pwd/lang/eval-fail-scope-5.nix:8:10: + + 7| + 8| body = f {}; + | ^ + 9| + + … while calling 'f' + + at /pwd/lang/eval-fail-scope-5.nix:6:7: + + 5| + 6| f = {x ? y, y ? x}: x + y; + | ^ + 7| + + error: infinite recursion encountered + + at /pwd/lang/eval-fail-scope-5.nix:6:12: + + 5| + 6| f = {x ? y, y ? x}: x + y; + | ^ + 7| diff --git a/tests/lang/eval-fail-scope-5.nix b/tests/functional/lang/eval-fail-scope-5.nix similarity index 100% rename from tests/lang/eval-fail-scope-5.nix rename to tests/functional/lang/eval-fail-scope-5.nix diff --git a/tests/functional/lang/eval-fail-seq.err.exp b/tests/functional/lang/eval-fail-seq.err.exp new file mode 100644 index 000000000000..33a7e9491aa3 --- /dev/null +++ b/tests/functional/lang/eval-fail-seq.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'seq' builtin + + at /pwd/lang/eval-fail-seq.nix:1:1: + + 1| builtins.seq (abort "foo") 2 + | ^ + 2| + + … while calling the 'abort' builtin + + at /pwd/lang/eval-fail-seq.nix:1:15: + + 1| builtins.seq (abort "foo") 2 + | ^ + 2| + + error: evaluation aborted with the following error message: 'foo' diff --git a/tests/lang/eval-fail-seq.nix b/tests/functional/lang/eval-fail-seq.nix similarity index 100% rename from tests/lang/eval-fail-seq.nix rename to tests/functional/lang/eval-fail-seq.nix diff --git a/tests/functional/lang/eval-fail-set-override.err.exp b/tests/functional/lang/eval-fail-set-override.err.exp new file mode 100644 index 000000000000..71481683db1f --- /dev/null +++ b/tests/functional/lang/eval-fail-set-override.err.exp @@ -0,0 +1,4 @@ +error: + … while evaluating the `__overrides` attribute + + error: value is an integer while a set was expected diff --git a/tests/functional/lang/eval-fail-set-override.nix b/tests/functional/lang/eval-fail-set-override.nix new file mode 100644 index 000000000000..03551c1869c1 --- /dev/null +++ b/tests/functional/lang/eval-fail-set-override.nix @@ -0,0 +1 @@ +rec { __overrides = 1; } diff --git a/tests/functional/lang/eval-fail-set.err.exp b/tests/functional/lang/eval-fail-set.err.exp new file mode 100644 index 000000000000..0d014050854a --- /dev/null +++ b/tests/functional/lang/eval-fail-set.err.exp @@ -0,0 +1,7 @@ +error: undefined variable 'x' + + at /pwd/lang/eval-fail-set.nix:1:3: + + 1| 8.x + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-set.nix b/tests/functional/lang/eval-fail-set.nix new file mode 100644 index 000000000000..c6b7980b681f --- /dev/null +++ b/tests/functional/lang/eval-fail-set.nix @@ -0,0 +1 @@ +8.x diff --git a/tests/functional/lang/eval-fail-substring.err.exp b/tests/functional/lang/eval-fail-substring.err.exp new file mode 100644 index 000000000000..5c58be29a66a --- /dev/null +++ b/tests/functional/lang/eval-fail-substring.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'substring' builtin + + at /pwd/lang/eval-fail-substring.nix:1:1: + + 1| builtins.substring (builtins.sub 0 1) 1 "x" + | ^ + 2| + + error: negative start position in 'substring' diff --git a/tests/lang/eval-fail-substring.nix b/tests/functional/lang/eval-fail-substring.nix similarity index 100% rename from tests/lang/eval-fail-substring.nix rename to tests/functional/lang/eval-fail-substring.nix diff --git a/tests/functional/lang/eval-fail-to-path.err.exp b/tests/functional/lang/eval-fail-to-path.err.exp new file mode 100644 index 000000000000..4ffa2cf6dbcb --- /dev/null +++ b/tests/functional/lang/eval-fail-to-path.err.exp @@ -0,0 +1,12 @@ +error: + … while calling the 'toPath' builtin + + at /pwd/lang/eval-fail-to-path.nix:1:1: + + 1| builtins.toPath "foo/bar" + | ^ + 2| + + … while evaluating the first argument passed to builtins.toPath + + error: string 'foo/bar' doesn't represent an absolute path diff --git a/tests/lang/eval-fail-to-path.nix b/tests/functional/lang/eval-fail-to-path.nix similarity index 100% rename from tests/lang/eval-fail-to-path.nix rename to tests/functional/lang/eval-fail-to-path.nix diff --git a/tests/functional/lang/eval-fail-toJSON.err.exp b/tests/functional/lang/eval-fail-toJSON.err.exp new file mode 100644 index 000000000000..4e618c203947 --- /dev/null +++ b/tests/functional/lang/eval-fail-toJSON.err.exp @@ -0,0 +1,57 @@ +error: + … while calling the 'toJSON' builtin + + at /pwd/lang/eval-fail-toJSON.nix:1:1: + + 1| builtins.toJSON { + | ^ + 2| a.b = [ + + … while evaluating attribute 'a' + + at /pwd/lang/eval-fail-toJSON.nix:2:3: + + 1| builtins.toJSON { + 2| a.b = [ + | ^ + 3| true + + … while evaluating attribute 'b' + + at /pwd/lang/eval-fail-toJSON.nix:2:3: + + 1| builtins.toJSON { + 2| a.b = [ + | ^ + 3| true + + … while evaluating list element at index 3 + + … while evaluating attribute 'c' + + at /pwd/lang/eval-fail-toJSON.nix:7:7: + + 6| { + 7| c.d = throw "hah no"; + | ^ + 8| } + + … while evaluating attribute 'd' + + at /pwd/lang/eval-fail-toJSON.nix:7:7: + + 6| { + 7| c.d = throw "hah no"; + | ^ + 8| } + + … while calling the 'throw' builtin + + at /pwd/lang/eval-fail-toJSON.nix:7:13: + + 6| { + 7| c.d = throw "hah no"; + | ^ + 8| } + + error: hah no diff --git a/tests/functional/lang/eval-fail-toJSON.nix b/tests/functional/lang/eval-fail-toJSON.nix new file mode 100644 index 000000000000..8112e1c1f2fb --- /dev/null +++ b/tests/functional/lang/eval-fail-toJSON.nix @@ -0,0 +1,10 @@ +builtins.toJSON { + a.b = [ + true + false + "it's a bird" + { + c.d = throw "hah no"; + } + ]; +} diff --git a/tests/functional/lang/eval-fail-undeclared-arg.err.exp b/tests/functional/lang/eval-fail-undeclared-arg.err.exp new file mode 100644 index 000000000000..30db743c7ee1 --- /dev/null +++ b/tests/functional/lang/eval-fail-undeclared-arg.err.exp @@ -0,0 +1,17 @@ +error: + … from call site + + at /pwd/lang/eval-fail-undeclared-arg.nix:1:1: + + 1| ({x, z}: x + z) {x = "foo"; y = "bla"; z = "bar";} + | ^ + 2| + + error: function 'anonymous lambda' called with unexpected argument 'y' + + at /pwd/lang/eval-fail-undeclared-arg.nix:1:2: + + 1| ({x, z}: x + z) {x = "foo"; y = "bla"; z = "bar";} + | ^ + 2| + Did you mean one of x or z? diff --git a/tests/lang/eval-fail-undeclared-arg.nix b/tests/functional/lang/eval-fail-undeclared-arg.nix similarity index 100% rename from tests/lang/eval-fail-undeclared-arg.nix rename to tests/functional/lang/eval-fail-undeclared-arg.nix diff --git a/tests/lang/eval-okay-any-all.exp b/tests/functional/lang/eval-okay-any-all.exp similarity index 100% rename from tests/lang/eval-okay-any-all.exp rename to tests/functional/lang/eval-okay-any-all.exp diff --git a/tests/lang/eval-okay-any-all.nix b/tests/functional/lang/eval-okay-any-all.nix similarity index 100% rename from tests/lang/eval-okay-any-all.nix rename to tests/functional/lang/eval-okay-any-all.nix diff --git a/tests/lang/eval-okay-arithmetic.exp b/tests/functional/lang/eval-okay-arithmetic.exp similarity index 100% rename from tests/lang/eval-okay-arithmetic.exp rename to tests/functional/lang/eval-okay-arithmetic.exp diff --git a/tests/lang/eval-okay-arithmetic.nix b/tests/functional/lang/eval-okay-arithmetic.nix similarity index 100% rename from tests/lang/eval-okay-arithmetic.nix rename to tests/functional/lang/eval-okay-arithmetic.nix diff --git a/tests/lang/eval-okay-attrnames.exp b/tests/functional/lang/eval-okay-attrnames.exp similarity index 100% rename from tests/lang/eval-okay-attrnames.exp rename to tests/functional/lang/eval-okay-attrnames.exp diff --git a/tests/lang/eval-okay-attrnames.nix b/tests/functional/lang/eval-okay-attrnames.nix similarity index 100% rename from tests/lang/eval-okay-attrnames.nix rename to tests/functional/lang/eval-okay-attrnames.nix diff --git a/tests/lang/eval-okay-attrs.exp b/tests/functional/lang/eval-okay-attrs.exp similarity index 100% rename from tests/lang/eval-okay-attrs.exp rename to tests/functional/lang/eval-okay-attrs.exp diff --git a/tests/lang/eval-okay-attrs.nix b/tests/functional/lang/eval-okay-attrs.nix similarity index 100% rename from tests/lang/eval-okay-attrs.nix rename to tests/functional/lang/eval-okay-attrs.nix diff --git a/tests/lang/eval-okay-attrs2.exp b/tests/functional/lang/eval-okay-attrs2.exp similarity index 100% rename from tests/lang/eval-okay-attrs2.exp rename to tests/functional/lang/eval-okay-attrs2.exp diff --git a/tests/lang/eval-okay-attrs2.nix b/tests/functional/lang/eval-okay-attrs2.nix similarity index 100% rename from tests/lang/eval-okay-attrs2.nix rename to tests/functional/lang/eval-okay-attrs2.nix diff --git a/tests/lang/eval-okay-attrs3.exp b/tests/functional/lang/eval-okay-attrs3.exp similarity index 100% rename from tests/lang/eval-okay-attrs3.exp rename to tests/functional/lang/eval-okay-attrs3.exp diff --git a/tests/lang/eval-okay-attrs3.nix b/tests/functional/lang/eval-okay-attrs3.nix similarity index 100% rename from tests/lang/eval-okay-attrs3.nix rename to tests/functional/lang/eval-okay-attrs3.nix diff --git a/tests/lang/eval-okay-attrs4.exp b/tests/functional/lang/eval-okay-attrs4.exp similarity index 100% rename from tests/lang/eval-okay-attrs4.exp rename to tests/functional/lang/eval-okay-attrs4.exp diff --git a/tests/lang/eval-okay-attrs4.nix b/tests/functional/lang/eval-okay-attrs4.nix similarity index 100% rename from tests/lang/eval-okay-attrs4.nix rename to tests/functional/lang/eval-okay-attrs4.nix diff --git a/tests/lang/eval-okay-attrs5.exp b/tests/functional/lang/eval-okay-attrs5.exp similarity index 100% rename from tests/lang/eval-okay-attrs5.exp rename to tests/functional/lang/eval-okay-attrs5.exp diff --git a/tests/lang/eval-okay-attrs5.nix b/tests/functional/lang/eval-okay-attrs5.nix similarity index 100% rename from tests/lang/eval-okay-attrs5.nix rename to tests/functional/lang/eval-okay-attrs5.nix diff --git a/tests/lang/eval-okay-attrs6.exp b/tests/functional/lang/eval-okay-attrs6.exp similarity index 100% rename from tests/lang/eval-okay-attrs6.exp rename to tests/functional/lang/eval-okay-attrs6.exp diff --git a/tests/lang/eval-okay-attrs6.nix b/tests/functional/lang/eval-okay-attrs6.nix similarity index 100% rename from tests/lang/eval-okay-attrs6.nix rename to tests/functional/lang/eval-okay-attrs6.nix diff --git a/tests/lang/eval-okay-autoargs.exp b/tests/functional/lang/eval-okay-autoargs.exp similarity index 100% rename from tests/lang/eval-okay-autoargs.exp rename to tests/functional/lang/eval-okay-autoargs.exp diff --git a/tests/lang/eval-okay-autoargs.flags b/tests/functional/lang/eval-okay-autoargs.flags similarity index 100% rename from tests/lang/eval-okay-autoargs.flags rename to tests/functional/lang/eval-okay-autoargs.flags diff --git a/tests/lang/eval-okay-autoargs.nix b/tests/functional/lang/eval-okay-autoargs.nix similarity index 100% rename from tests/lang/eval-okay-autoargs.nix rename to tests/functional/lang/eval-okay-autoargs.nix diff --git a/tests/lang/eval-okay-backslash-newline-1.exp b/tests/functional/lang/eval-okay-backslash-newline-1.exp similarity index 100% rename from tests/lang/eval-okay-backslash-newline-1.exp rename to tests/functional/lang/eval-okay-backslash-newline-1.exp diff --git a/tests/lang/eval-okay-backslash-newline-1.nix b/tests/functional/lang/eval-okay-backslash-newline-1.nix similarity index 100% rename from tests/lang/eval-okay-backslash-newline-1.nix rename to tests/functional/lang/eval-okay-backslash-newline-1.nix diff --git a/tests/lang/eval-okay-backslash-newline-2.exp b/tests/functional/lang/eval-okay-backslash-newline-2.exp similarity index 100% rename from tests/lang/eval-okay-backslash-newline-2.exp rename to tests/functional/lang/eval-okay-backslash-newline-2.exp diff --git a/tests/lang/eval-okay-backslash-newline-2.nix b/tests/functional/lang/eval-okay-backslash-newline-2.nix similarity index 100% rename from tests/lang/eval-okay-backslash-newline-2.nix rename to tests/functional/lang/eval-okay-backslash-newline-2.nix diff --git a/tests/lang/eval-okay-builtins-add.exp b/tests/functional/lang/eval-okay-builtins-add.exp similarity index 100% rename from tests/lang/eval-okay-builtins-add.exp rename to tests/functional/lang/eval-okay-builtins-add.exp diff --git a/tests/lang/eval-okay-builtins-add.nix b/tests/functional/lang/eval-okay-builtins-add.nix similarity index 100% rename from tests/lang/eval-okay-builtins-add.nix rename to tests/functional/lang/eval-okay-builtins-add.nix diff --git a/tests/lang/eval-okay-builtins.exp b/tests/functional/lang/eval-okay-builtins.exp similarity index 100% rename from tests/lang/eval-okay-builtins.exp rename to tests/functional/lang/eval-okay-builtins.exp diff --git a/tests/lang/eval-okay-builtins.nix b/tests/functional/lang/eval-okay-builtins.nix similarity index 100% rename from tests/lang/eval-okay-builtins.nix rename to tests/functional/lang/eval-okay-builtins.nix diff --git a/tests/lang/eval-okay-callable-attrs.exp b/tests/functional/lang/eval-okay-callable-attrs.exp similarity index 100% rename from tests/lang/eval-okay-callable-attrs.exp rename to tests/functional/lang/eval-okay-callable-attrs.exp diff --git a/tests/lang/eval-okay-callable-attrs.nix b/tests/functional/lang/eval-okay-callable-attrs.nix similarity index 100% rename from tests/lang/eval-okay-callable-attrs.nix rename to tests/functional/lang/eval-okay-callable-attrs.nix diff --git a/tests/lang/eval-okay-catattrs.exp b/tests/functional/lang/eval-okay-catattrs.exp similarity index 100% rename from tests/lang/eval-okay-catattrs.exp rename to tests/functional/lang/eval-okay-catattrs.exp diff --git a/tests/lang/eval-okay-catattrs.nix b/tests/functional/lang/eval-okay-catattrs.nix similarity index 100% rename from tests/lang/eval-okay-catattrs.nix rename to tests/functional/lang/eval-okay-catattrs.nix diff --git a/tests/lang/eval-okay-closure.exp b/tests/functional/lang/eval-okay-closure.exp similarity index 100% rename from tests/lang/eval-okay-closure.exp rename to tests/functional/lang/eval-okay-closure.exp diff --git a/tests/lang/eval-okay-closure.exp.xml b/tests/functional/lang/eval-okay-closure.exp.xml similarity index 100% rename from tests/lang/eval-okay-closure.exp.xml rename to tests/functional/lang/eval-okay-closure.exp.xml diff --git a/tests/lang/eval-okay-closure.nix b/tests/functional/lang/eval-okay-closure.nix similarity index 100% rename from tests/lang/eval-okay-closure.nix rename to tests/functional/lang/eval-okay-closure.nix diff --git a/tests/lang/eval-okay-comments.exp b/tests/functional/lang/eval-okay-comments.exp similarity index 100% rename from tests/lang/eval-okay-comments.exp rename to tests/functional/lang/eval-okay-comments.exp diff --git a/tests/lang/eval-okay-comments.nix b/tests/functional/lang/eval-okay-comments.nix similarity index 100% rename from tests/lang/eval-okay-comments.nix rename to tests/functional/lang/eval-okay-comments.nix diff --git a/tests/lang/eval-okay-concat.exp b/tests/functional/lang/eval-okay-concat.exp similarity index 100% rename from tests/lang/eval-okay-concat.exp rename to tests/functional/lang/eval-okay-concat.exp diff --git a/tests/lang/eval-okay-concat.nix b/tests/functional/lang/eval-okay-concat.nix similarity index 100% rename from tests/lang/eval-okay-concat.nix rename to tests/functional/lang/eval-okay-concat.nix diff --git a/tests/lang/eval-okay-concatmap.exp b/tests/functional/lang/eval-okay-concatmap.exp similarity index 100% rename from tests/lang/eval-okay-concatmap.exp rename to tests/functional/lang/eval-okay-concatmap.exp diff --git a/tests/lang/eval-okay-concatmap.nix b/tests/functional/lang/eval-okay-concatmap.nix similarity index 100% rename from tests/lang/eval-okay-concatmap.nix rename to tests/functional/lang/eval-okay-concatmap.nix diff --git a/tests/lang/eval-okay-concatstringssep.exp b/tests/functional/lang/eval-okay-concatstringssep.exp similarity index 100% rename from tests/lang/eval-okay-concatstringssep.exp rename to tests/functional/lang/eval-okay-concatstringssep.exp diff --git a/tests/lang/eval-okay-concatstringssep.nix b/tests/functional/lang/eval-okay-concatstringssep.nix similarity index 100% rename from tests/lang/eval-okay-concatstringssep.nix rename to tests/functional/lang/eval-okay-concatstringssep.nix diff --git a/tests/functional/lang/eval-okay-context-introspection.exp b/tests/functional/lang/eval-okay-context-introspection.exp new file mode 100644 index 000000000000..a136b0035e0a --- /dev/null +++ b/tests/functional/lang/eval-okay-context-introspection.exp @@ -0,0 +1 @@ +[ true true true true true true true true true true true true true ] diff --git a/tests/lang/eval-okay-context-introspection.nix b/tests/functional/lang/eval-okay-context-introspection.nix similarity index 60% rename from tests/lang/eval-okay-context-introspection.nix rename to tests/functional/lang/eval-okay-context-introspection.nix index 50a78d946e76..8886cf32e94b 100644 --- a/tests/lang/eval-okay-context-introspection.nix +++ b/tests/functional/lang/eval-okay-context-introspection.nix @@ -31,11 +31,29 @@ let (builtins.unsafeDiscardStringContext str) (builtins.getContext str); + # Only holds true if string context contains both a `DrvDeep` and + # `Opaque` element. + almostEtaRule = str: + str == builtins.addDrvOutputDependencies + (builtins.unsafeDiscardOutputDependency str); + + addDrvOutputDependencies_idempotent = str: + builtins.addDrvOutputDependencies str == + builtins.addDrvOutputDependencies (builtins.addDrvOutputDependencies str); + + rules = str: [ + (etaRule str) + (almostEtaRule str) + (addDrvOutputDependencies_idempotent str) + ]; + in [ (legit-context == desired-context) (reconstructed-path == combo-path) (etaRule "foo") - (etaRule drv.drvPath) (etaRule drv.foo.outPath) - (etaRule (builtins.unsafeDiscardOutputDependency drv.drvPath)) +] ++ builtins.concatMap rules [ + drv.drvPath + (builtins.addDrvOutputDependencies drv.drvPath) + (builtins.unsafeDiscardOutputDependency drv.drvPath) ] diff --git a/tests/lang/eval-okay-context.exp b/tests/functional/lang/eval-okay-context.exp similarity index 100% rename from tests/lang/eval-okay-context.exp rename to tests/functional/lang/eval-okay-context.exp diff --git a/tests/lang/eval-okay-context.nix b/tests/functional/lang/eval-okay-context.nix similarity index 100% rename from tests/lang/eval-okay-context.nix rename to tests/functional/lang/eval-okay-context.nix diff --git a/tests/functional/lang/eval-okay-convertHash.exp b/tests/functional/lang/eval-okay-convertHash.exp new file mode 100644 index 000000000000..60e0a3c494a1 --- /dev/null +++ b/tests/functional/lang/eval-okay-convertHash.exp @@ -0,0 +1 @@ +{ hashesBase16 = [ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ]; hashesBase32 = [ "3y8bwfr609h3lh9ch0izcqq7fl" "26mrvc0v1nslch8r0w45zywsbc" "1v4gi57l97pmnylq6lmgxkhd5v" "143xibwh31h9bvxzalr0sjvbbvpa6ffs" "i4hj30pkrfdpgc5dbcgcydqviibfhm6d" "fxz2p030yba2bza71qhss79k3l5y24kd" "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73" "0qy6iz9yh6a079757mxdmypx0gcmnzjd3ij5q78bzk00vxll82lh" "0mkygpci4r4yb8zz5rs2kxcgvw0a2yf5zlj6r8qgfll6pnrqf0xd" "0zdl9zrg8r3i9c1g90lgg9ip5ijzv3yhz91i0zzn3r8ap9ws784gkp9dk9j3aglhgf1amqb0pj21mh7h1nxcl18akqvvf7ggqsy30yg" "19ncrpp37dx0nzzjw4k6zaqkb9mzaq2myhgpzh5aff7qqcj5wwdxslg6ixwncm7gyq8l761gwf87fgsh2bwfyr52s53k2dkqvw8c24x" "2kz74snvckxldmmbisz9ikmy031d28cs6xfdbl6rhxx42glpyz4vww4lajrc5akklxwixl0js4g84233pxvmbykiic5m7i5m9r4nr11" ]; hashesBase64 = [ "1B2M2Y8AsgTpgAmY7PhCfg==" "bGnufyEcZAQZ1TZswHauRg==" "uzQ4+6vUYOptvSfRU+IjOw==" "2jmj7l5rSw0yVb/vlWAYkK/YBwk=" "zVToVowbN88eW62wd5vL84IhIYk=" "bRLhCx0zHa0hDkf9JdTyYIArfnc=" "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" "kApEad8AzL/QwUXG0eS3lT3Qr6+t11NOOkAZ6NOPxmM=" "rQOHs72GUvcwykbSX5wXCvD9WJ9C5/I/Wp5kEtl9flY=" "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" "nQiG+MaziTmKFiV7x5eA+rmDHH/BHIqwf6cyy3s0j+reOC+SYXycUwX++6CvAqtf05pYfTMJl/9b0NsZ92ZmUw==" "IWRLcqolnlpYjNOvuvsdQxD0iJaA9sg7nVMVlqWihPNNvr/0CdI7zIau5rrRDIkWBvB1xvR1XLU22ifbVpPzpw==" ]; hashesSRI = [ "md5-1B2M2Y8AsgTpgAmY7PhCfg==" "md5-bGnufyEcZAQZ1TZswHauRg==" "md5-uzQ4+6vUYOptvSfRU+IjOw==" "sha1-2jmj7l5rSw0yVb/vlWAYkK/YBwk=" "sha1-zVToVowbN88eW62wd5vL84IhIYk=" "sha1-bRLhCx0zHa0hDkf9JdTyYIArfnc=" "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" "sha256-kApEad8AzL/QwUXG0eS3lT3Qr6+t11NOOkAZ6NOPxmM=" "sha256-rQOHs72GUvcwykbSX5wXCvD9WJ9C5/I/Wp5kEtl9flY=" "sha512-z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" "sha512-nQiG+MaziTmKFiV7x5eA+rmDHH/BHIqwf6cyy3s0j+reOC+SYXycUwX++6CvAqtf05pYfTMJl/9b0NsZ92ZmUw==" "sha512-IWRLcqolnlpYjNOvuvsdQxD0iJaA9sg7nVMVlqWihPNNvr/0CdI7zIau5rrRDIkWBvB1xvR1XLU22ifbVpPzpw==" ]; } diff --git a/tests/functional/lang/eval-okay-convertHash.nix b/tests/functional/lang/eval-okay-convertHash.nix new file mode 100644 index 000000000000..cf4909aaf01c --- /dev/null +++ b/tests/functional/lang/eval-okay-convertHash.nix @@ -0,0 +1,31 @@ +let + hashAlgos = [ "md5" "md5" "md5" "sha1" "sha1" "sha1" "sha256" "sha256" "sha256" "sha512" "sha512" "sha512" ]; + hashesBase16 = import ./eval-okay-hashstring.exp; + map2 = f: { fsts, snds }: if fsts == [ ] then [ ] else [ (f (builtins.head fsts) (builtins.head snds)) ] ++ map2 f { fsts = builtins.tail fsts; snds = builtins.tail snds; }; + map2' = f: fsts: snds: map2 f { inherit fsts snds; }; + getOutputHashes = hashes: { + hashesBase16 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base16";}) hashAlgos hashes; + hashesBase32 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";}) hashAlgos hashes; + hashesBase64 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base64";}) hashAlgos hashes; + hashesSRI = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "sri" ;}) hashAlgos hashes; + }; + getOutputHashesColon = hashes: { + hashesBase16 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base16";}) hashAlgos hashes; + hashesBase32 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base32";}) hashAlgos hashes; + hashesBase64 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base64";}) hashAlgos hashes; + hashesSRI = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "sri" ;}) hashAlgos hashes; + }; + outputHashes = getOutputHashes hashesBase16; +in +# map2'` +assert map2' (s1: s2: s1 + s2) [ "a" "b" ] [ "c" "d" ] == [ "ac" "bd" ]; +# hashesBase16 +assert outputHashes.hashesBase16 == hashesBase16; +# standard SRI hashes +assert outputHashes.hashesSRI == (map2' (hashAlgo: hashBody: hashAlgo + "-" + hashBody) hashAlgos outputHashes.hashesBase64); +# without prefix +assert builtins.all (x: getOutputHashes x == outputHashes) (builtins.attrValues outputHashes); +# colon-separated. +# Note that colon prefix must not be applied to the standard SRI. e.g. "sha256:sha256-..." is illegal. +assert builtins.all (x: getOutputHashesColon x == outputHashes) (with outputHashes; [ hashesBase16 hashesBase32 hashesBase64 ]); +outputHashes diff --git a/tests/lang/eval-okay-curpos.exp b/tests/functional/lang/eval-okay-curpos.exp similarity index 100% rename from tests/lang/eval-okay-curpos.exp rename to tests/functional/lang/eval-okay-curpos.exp diff --git a/tests/lang/eval-okay-curpos.nix b/tests/functional/lang/eval-okay-curpos.nix similarity index 100% rename from tests/lang/eval-okay-curpos.nix rename to tests/functional/lang/eval-okay-curpos.nix diff --git a/tests/lang/eval-okay-deepseq.exp b/tests/functional/lang/eval-okay-deepseq.exp similarity index 100% rename from tests/lang/eval-okay-deepseq.exp rename to tests/functional/lang/eval-okay-deepseq.exp diff --git a/tests/lang/eval-okay-deepseq.nix b/tests/functional/lang/eval-okay-deepseq.nix similarity index 100% rename from tests/lang/eval-okay-deepseq.nix rename to tests/functional/lang/eval-okay-deepseq.nix diff --git a/tests/lang/eval-okay-delayed-with-inherit.exp b/tests/functional/lang/eval-okay-delayed-with-inherit.exp similarity index 100% rename from tests/lang/eval-okay-delayed-with-inherit.exp rename to tests/functional/lang/eval-okay-delayed-with-inherit.exp diff --git a/tests/lang/eval-okay-delayed-with-inherit.nix b/tests/functional/lang/eval-okay-delayed-with-inherit.nix similarity index 100% rename from tests/lang/eval-okay-delayed-with-inherit.nix rename to tests/functional/lang/eval-okay-delayed-with-inherit.nix diff --git a/tests/lang/eval-okay-delayed-with.exp b/tests/functional/lang/eval-okay-delayed-with.exp similarity index 100% rename from tests/lang/eval-okay-delayed-with.exp rename to tests/functional/lang/eval-okay-delayed-with.exp diff --git a/tests/lang/eval-okay-delayed-with.nix b/tests/functional/lang/eval-okay-delayed-with.nix similarity index 100% rename from tests/lang/eval-okay-delayed-with.nix rename to tests/functional/lang/eval-okay-delayed-with.nix diff --git a/tests/lang/eval-okay-dynamic-attrs-2.exp b/tests/functional/lang/eval-okay-dynamic-attrs-2.exp similarity index 100% rename from tests/lang/eval-okay-dynamic-attrs-2.exp rename to tests/functional/lang/eval-okay-dynamic-attrs-2.exp diff --git a/tests/lang/eval-okay-dynamic-attrs-2.nix b/tests/functional/lang/eval-okay-dynamic-attrs-2.nix similarity index 100% rename from tests/lang/eval-okay-dynamic-attrs-2.nix rename to tests/functional/lang/eval-okay-dynamic-attrs-2.nix diff --git a/tests/lang/eval-okay-dynamic-attrs-bare.exp b/tests/functional/lang/eval-okay-dynamic-attrs-bare.exp similarity index 100% rename from tests/lang/eval-okay-dynamic-attrs-bare.exp rename to tests/functional/lang/eval-okay-dynamic-attrs-bare.exp diff --git a/tests/lang/eval-okay-dynamic-attrs-bare.nix b/tests/functional/lang/eval-okay-dynamic-attrs-bare.nix similarity index 100% rename from tests/lang/eval-okay-dynamic-attrs-bare.nix rename to tests/functional/lang/eval-okay-dynamic-attrs-bare.nix diff --git a/tests/lang/eval-okay-dynamic-attrs.exp b/tests/functional/lang/eval-okay-dynamic-attrs.exp similarity index 100% rename from tests/lang/eval-okay-dynamic-attrs.exp rename to tests/functional/lang/eval-okay-dynamic-attrs.exp diff --git a/tests/lang/eval-okay-dynamic-attrs.nix b/tests/functional/lang/eval-okay-dynamic-attrs.nix similarity index 100% rename from tests/lang/eval-okay-dynamic-attrs.nix rename to tests/functional/lang/eval-okay-dynamic-attrs.nix diff --git a/tests/lang/eval-okay-elem.exp b/tests/functional/lang/eval-okay-elem.exp similarity index 100% rename from tests/lang/eval-okay-elem.exp rename to tests/functional/lang/eval-okay-elem.exp diff --git a/tests/lang/eval-okay-elem.nix b/tests/functional/lang/eval-okay-elem.nix similarity index 100% rename from tests/lang/eval-okay-elem.nix rename to tests/functional/lang/eval-okay-elem.nix diff --git a/tests/lang/eval-okay-empty-args.exp b/tests/functional/lang/eval-okay-empty-args.exp similarity index 100% rename from tests/lang/eval-okay-empty-args.exp rename to tests/functional/lang/eval-okay-empty-args.exp diff --git a/tests/lang/eval-okay-empty-args.nix b/tests/functional/lang/eval-okay-empty-args.nix similarity index 100% rename from tests/lang/eval-okay-empty-args.nix rename to tests/functional/lang/eval-okay-empty-args.nix diff --git a/tests/lang/eval-okay-eq-derivations.exp b/tests/functional/lang/eval-okay-eq-derivations.exp similarity index 100% rename from tests/lang/eval-okay-eq-derivations.exp rename to tests/functional/lang/eval-okay-eq-derivations.exp diff --git a/tests/lang/eval-okay-eq-derivations.nix b/tests/functional/lang/eval-okay-eq-derivations.nix similarity index 100% rename from tests/lang/eval-okay-eq-derivations.nix rename to tests/functional/lang/eval-okay-eq-derivations.nix diff --git a/tests/lang/eval-okay-eq.exp b/tests/functional/lang/eval-okay-eq.exp similarity index 100% rename from tests/lang/eval-okay-eq.exp rename to tests/functional/lang/eval-okay-eq.exp diff --git a/tests/lang/eval-okay-eq.nix b/tests/functional/lang/eval-okay-eq.nix similarity index 100% rename from tests/lang/eval-okay-eq.nix rename to tests/functional/lang/eval-okay-eq.nix diff --git a/tests/lang/eval-okay-filter.exp b/tests/functional/lang/eval-okay-filter.exp similarity index 100% rename from tests/lang/eval-okay-filter.exp rename to tests/functional/lang/eval-okay-filter.exp diff --git a/tests/lang/eval-okay-filter.nix b/tests/functional/lang/eval-okay-filter.nix similarity index 100% rename from tests/lang/eval-okay-filter.nix rename to tests/functional/lang/eval-okay-filter.nix diff --git a/tests/functional/lang/eval-okay-flake-ref-to-string.exp b/tests/functional/lang/eval-okay-flake-ref-to-string.exp new file mode 100644 index 000000000000..110f8442d0ae --- /dev/null +++ b/tests/functional/lang/eval-okay-flake-ref-to-string.exp @@ -0,0 +1 @@ +"github:NixOS/nixpkgs/23.05?dir=lib" diff --git a/tests/functional/lang/eval-okay-flake-ref-to-string.nix b/tests/functional/lang/eval-okay-flake-ref-to-string.nix new file mode 100644 index 000000000000..dbb4e5b2af40 --- /dev/null +++ b/tests/functional/lang/eval-okay-flake-ref-to-string.nix @@ -0,0 +1,7 @@ +builtins.flakeRefToString { + type = "github"; + owner = "NixOS"; + repo = "nixpkgs"; + ref = "23.05"; + dir = "lib"; +} diff --git a/tests/lang/eval-okay-flatten.exp b/tests/functional/lang/eval-okay-flatten.exp similarity index 100% rename from tests/lang/eval-okay-flatten.exp rename to tests/functional/lang/eval-okay-flatten.exp diff --git a/tests/lang/eval-okay-flatten.nix b/tests/functional/lang/eval-okay-flatten.nix similarity index 100% rename from tests/lang/eval-okay-flatten.nix rename to tests/functional/lang/eval-okay-flatten.nix diff --git a/tests/lang/eval-okay-float.exp b/tests/functional/lang/eval-okay-float.exp similarity index 100% rename from tests/lang/eval-okay-float.exp rename to tests/functional/lang/eval-okay-float.exp diff --git a/tests/lang/eval-okay-float.nix b/tests/functional/lang/eval-okay-float.nix similarity index 100% rename from tests/lang/eval-okay-float.nix rename to tests/functional/lang/eval-okay-float.nix diff --git a/tests/lang/eval-okay-floor-ceil.exp b/tests/functional/lang/eval-okay-floor-ceil.exp similarity index 100% rename from tests/lang/eval-okay-floor-ceil.exp rename to tests/functional/lang/eval-okay-floor-ceil.exp diff --git a/tests/lang/eval-okay-floor-ceil.nix b/tests/functional/lang/eval-okay-floor-ceil.nix similarity index 100% rename from tests/lang/eval-okay-floor-ceil.nix rename to tests/functional/lang/eval-okay-floor-ceil.nix diff --git a/tests/lang/eval-okay-foldlStrict-lazy-elements.exp b/tests/functional/lang/eval-okay-foldlStrict-lazy-elements.exp similarity index 100% rename from tests/lang/eval-okay-foldlStrict-lazy-elements.exp rename to tests/functional/lang/eval-okay-foldlStrict-lazy-elements.exp diff --git a/tests/lang/eval-okay-foldlStrict-lazy-elements.nix b/tests/functional/lang/eval-okay-foldlStrict-lazy-elements.nix similarity index 100% rename from tests/lang/eval-okay-foldlStrict-lazy-elements.nix rename to tests/functional/lang/eval-okay-foldlStrict-lazy-elements.nix diff --git a/tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp b/tests/functional/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp similarity index 100% rename from tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp rename to tests/functional/lang/eval-okay-foldlStrict-lazy-initial-accumulator.exp diff --git a/tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix b/tests/functional/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix similarity index 100% rename from tests/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix rename to tests/functional/lang/eval-okay-foldlStrict-lazy-initial-accumulator.nix diff --git a/tests/lang/eval-okay-foldlStrict.exp b/tests/functional/lang/eval-okay-foldlStrict.exp similarity index 100% rename from tests/lang/eval-okay-foldlStrict.exp rename to tests/functional/lang/eval-okay-foldlStrict.exp diff --git a/tests/lang/eval-okay-foldlStrict.nix b/tests/functional/lang/eval-okay-foldlStrict.nix similarity index 100% rename from tests/lang/eval-okay-foldlStrict.nix rename to tests/functional/lang/eval-okay-foldlStrict.nix diff --git a/tests/lang/eval-okay-fromTOML-timestamps.exp b/tests/functional/lang/eval-okay-fromTOML-timestamps.exp similarity index 100% rename from tests/lang/eval-okay-fromTOML-timestamps.exp rename to tests/functional/lang/eval-okay-fromTOML-timestamps.exp diff --git a/tests/lang/eval-okay-fromTOML-timestamps.flags b/tests/functional/lang/eval-okay-fromTOML-timestamps.flags similarity index 100% rename from tests/lang/eval-okay-fromTOML-timestamps.flags rename to tests/functional/lang/eval-okay-fromTOML-timestamps.flags diff --git a/tests/lang/eval-okay-fromTOML-timestamps.nix b/tests/functional/lang/eval-okay-fromTOML-timestamps.nix similarity index 100% rename from tests/lang/eval-okay-fromTOML-timestamps.nix rename to tests/functional/lang/eval-okay-fromTOML-timestamps.nix diff --git a/tests/lang/eval-okay-fromTOML.exp b/tests/functional/lang/eval-okay-fromTOML.exp similarity index 100% rename from tests/lang/eval-okay-fromTOML.exp rename to tests/functional/lang/eval-okay-fromTOML.exp diff --git a/tests/lang/eval-okay-fromTOML.nix b/tests/functional/lang/eval-okay-fromTOML.nix similarity index 100% rename from tests/lang/eval-okay-fromTOML.nix rename to tests/functional/lang/eval-okay-fromTOML.nix diff --git a/tests/lang/eval-okay-fromjson-escapes.exp b/tests/functional/lang/eval-okay-fromjson-escapes.exp similarity index 100% rename from tests/lang/eval-okay-fromjson-escapes.exp rename to tests/functional/lang/eval-okay-fromjson-escapes.exp diff --git a/tests/lang/eval-okay-fromjson-escapes.nix b/tests/functional/lang/eval-okay-fromjson-escapes.nix similarity index 100% rename from tests/lang/eval-okay-fromjson-escapes.nix rename to tests/functional/lang/eval-okay-fromjson-escapes.nix diff --git a/tests/lang/eval-okay-fromjson.exp b/tests/functional/lang/eval-okay-fromjson.exp similarity index 100% rename from tests/lang/eval-okay-fromjson.exp rename to tests/functional/lang/eval-okay-fromjson.exp diff --git a/tests/lang/eval-okay-fromjson.nix b/tests/functional/lang/eval-okay-fromjson.nix similarity index 64% rename from tests/lang/eval-okay-fromjson.nix rename to tests/functional/lang/eval-okay-fromjson.nix index e1c0f86cc4e4..4c526b9ae5d6 100644 --- a/tests/lang/eval-okay-fromjson.nix +++ b/tests/functional/lang/eval-okay-fromjson.nix @@ -11,9 +11,12 @@ builtins.fromJSON "Width": 200, "Height": 250 }, + "Animated" : false, + "IDs": [116, 943, 234, 38793, true ,false,null, -100], + "Escapes": "\"\\\/\t\n\r\t", "Subtitle" : false, - "Latitude": 46.2051, - "Longitude": 6.0723 + "Latitude": 37.7668, + "Longitude": -122.3959 } } '' @@ -28,8 +31,11 @@ builtins.fromJSON Width = 200; Height = 250; }; + Animated = false; + IDs = [ 116 943 234 38793 true false null (0-100) ]; + Escapes = "\"\\\/\t\n\r\t"; # supported in JSON but not Nix: \b\f Subtitle = false; - Latitude = 46.2051; - Longitude = 6.0723; + Latitude = 37.7668; + Longitude = -122.3959; }; } diff --git a/tests/lang/eval-okay-functionargs.exp b/tests/functional/lang/eval-okay-functionargs.exp similarity index 100% rename from tests/lang/eval-okay-functionargs.exp rename to tests/functional/lang/eval-okay-functionargs.exp diff --git a/tests/lang/eval-okay-functionargs.exp.xml b/tests/functional/lang/eval-okay-functionargs.exp.xml similarity index 100% rename from tests/lang/eval-okay-functionargs.exp.xml rename to tests/functional/lang/eval-okay-functionargs.exp.xml diff --git a/tests/lang/eval-okay-functionargs.nix b/tests/functional/lang/eval-okay-functionargs.nix similarity index 100% rename from tests/lang/eval-okay-functionargs.nix rename to tests/functional/lang/eval-okay-functionargs.nix diff --git a/tests/lang/eval-okay-getattrpos-functionargs.exp b/tests/functional/lang/eval-okay-getattrpos-functionargs.exp similarity index 100% rename from tests/lang/eval-okay-getattrpos-functionargs.exp rename to tests/functional/lang/eval-okay-getattrpos-functionargs.exp diff --git a/tests/lang/eval-okay-getattrpos-functionargs.nix b/tests/functional/lang/eval-okay-getattrpos-functionargs.nix similarity index 100% rename from tests/lang/eval-okay-getattrpos-functionargs.nix rename to tests/functional/lang/eval-okay-getattrpos-functionargs.nix diff --git a/tests/lang/eval-okay-getattrpos-undefined.exp b/tests/functional/lang/eval-okay-getattrpos-undefined.exp similarity index 100% rename from tests/lang/eval-okay-getattrpos-undefined.exp rename to tests/functional/lang/eval-okay-getattrpos-undefined.exp diff --git a/tests/lang/eval-okay-getattrpos-undefined.nix b/tests/functional/lang/eval-okay-getattrpos-undefined.nix similarity index 100% rename from tests/lang/eval-okay-getattrpos-undefined.nix rename to tests/functional/lang/eval-okay-getattrpos-undefined.nix diff --git a/tests/lang/eval-okay-getattrpos.exp b/tests/functional/lang/eval-okay-getattrpos.exp similarity index 100% rename from tests/lang/eval-okay-getattrpos.exp rename to tests/functional/lang/eval-okay-getattrpos.exp diff --git a/tests/lang/eval-okay-getattrpos.nix b/tests/functional/lang/eval-okay-getattrpos.nix similarity index 100% rename from tests/lang/eval-okay-getattrpos.nix rename to tests/functional/lang/eval-okay-getattrpos.nix diff --git a/tests/lang/eval-okay-getenv.exp b/tests/functional/lang/eval-okay-getenv.exp similarity index 100% rename from tests/lang/eval-okay-getenv.exp rename to tests/functional/lang/eval-okay-getenv.exp diff --git a/tests/lang/eval-okay-getenv.nix b/tests/functional/lang/eval-okay-getenv.nix similarity index 100% rename from tests/lang/eval-okay-getenv.nix rename to tests/functional/lang/eval-okay-getenv.nix diff --git a/tests/lang/eval-okay-groupBy.exp b/tests/functional/lang/eval-okay-groupBy.exp similarity index 100% rename from tests/lang/eval-okay-groupBy.exp rename to tests/functional/lang/eval-okay-groupBy.exp diff --git a/tests/lang/eval-okay-groupBy.nix b/tests/functional/lang/eval-okay-groupBy.nix similarity index 100% rename from tests/lang/eval-okay-groupBy.nix rename to tests/functional/lang/eval-okay-groupBy.nix diff --git a/tests/lang/readDir/bar b/tests/functional/lang/eval-okay-hash.exp similarity index 100% rename from tests/lang/readDir/bar rename to tests/functional/lang/eval-okay-hash.exp diff --git a/tests/lang/eval-okay-hashfile.exp b/tests/functional/lang/eval-okay-hashfile.exp similarity index 100% rename from tests/lang/eval-okay-hashfile.exp rename to tests/functional/lang/eval-okay-hashfile.exp diff --git a/tests/lang/eval-okay-hashfile.nix b/tests/functional/lang/eval-okay-hashfile.nix similarity index 100% rename from tests/lang/eval-okay-hashfile.nix rename to tests/functional/lang/eval-okay-hashfile.nix diff --git a/tests/lang/eval-okay-hashstring.exp b/tests/functional/lang/eval-okay-hashstring.exp similarity index 100% rename from tests/lang/eval-okay-hashstring.exp rename to tests/functional/lang/eval-okay-hashstring.exp diff --git a/tests/lang/eval-okay-hashstring.nix b/tests/functional/lang/eval-okay-hashstring.nix similarity index 100% rename from tests/lang/eval-okay-hashstring.nix rename to tests/functional/lang/eval-okay-hashstring.nix diff --git a/tests/lang/eval-okay-if.exp b/tests/functional/lang/eval-okay-if.exp similarity index 100% rename from tests/lang/eval-okay-if.exp rename to tests/functional/lang/eval-okay-if.exp diff --git a/tests/lang/eval-okay-if.nix b/tests/functional/lang/eval-okay-if.nix similarity index 100% rename from tests/lang/eval-okay-if.nix rename to tests/functional/lang/eval-okay-if.nix diff --git a/tests/lang/eval-okay-import.exp b/tests/functional/lang/eval-okay-import.exp similarity index 100% rename from tests/lang/eval-okay-import.exp rename to tests/functional/lang/eval-okay-import.exp diff --git a/tests/lang/eval-okay-import.nix b/tests/functional/lang/eval-okay-import.nix similarity index 100% rename from tests/lang/eval-okay-import.nix rename to tests/functional/lang/eval-okay-import.nix diff --git a/tests/lang/eval-okay-ind-string.exp b/tests/functional/lang/eval-okay-ind-string.exp similarity index 100% rename from tests/lang/eval-okay-ind-string.exp rename to tests/functional/lang/eval-okay-ind-string.exp diff --git a/tests/lang/eval-okay-ind-string.nix b/tests/functional/lang/eval-okay-ind-string.nix similarity index 100% rename from tests/lang/eval-okay-ind-string.nix rename to tests/functional/lang/eval-okay-ind-string.nix diff --git a/tests/lang/eval-okay-intersectAttrs.exp b/tests/functional/lang/eval-okay-intersectAttrs.exp similarity index 100% rename from tests/lang/eval-okay-intersectAttrs.exp rename to tests/functional/lang/eval-okay-intersectAttrs.exp diff --git a/tests/lang/eval-okay-intersectAttrs.nix b/tests/functional/lang/eval-okay-intersectAttrs.nix similarity index 100% rename from tests/lang/eval-okay-intersectAttrs.nix rename to tests/functional/lang/eval-okay-intersectAttrs.nix diff --git a/tests/lang/eval-okay-let.exp b/tests/functional/lang/eval-okay-let.exp similarity index 100% rename from tests/lang/eval-okay-let.exp rename to tests/functional/lang/eval-okay-let.exp diff --git a/tests/lang/eval-okay-let.nix b/tests/functional/lang/eval-okay-let.nix similarity index 100% rename from tests/lang/eval-okay-let.nix rename to tests/functional/lang/eval-okay-let.nix diff --git a/tests/lang/eval-okay-list.exp b/tests/functional/lang/eval-okay-list.exp similarity index 100% rename from tests/lang/eval-okay-list.exp rename to tests/functional/lang/eval-okay-list.exp diff --git a/tests/lang/eval-okay-list.nix b/tests/functional/lang/eval-okay-list.nix similarity index 100% rename from tests/lang/eval-okay-list.nix rename to tests/functional/lang/eval-okay-list.nix diff --git a/tests/lang/eval-okay-listtoattrs.exp b/tests/functional/lang/eval-okay-listtoattrs.exp similarity index 100% rename from tests/lang/eval-okay-listtoattrs.exp rename to tests/functional/lang/eval-okay-listtoattrs.exp diff --git a/tests/lang/eval-okay-listtoattrs.nix b/tests/functional/lang/eval-okay-listtoattrs.nix similarity index 100% rename from tests/lang/eval-okay-listtoattrs.nix rename to tests/functional/lang/eval-okay-listtoattrs.nix diff --git a/tests/lang/eval-okay-logic.exp b/tests/functional/lang/eval-okay-logic.exp similarity index 100% rename from tests/lang/eval-okay-logic.exp rename to tests/functional/lang/eval-okay-logic.exp diff --git a/tests/lang/eval-okay-logic.nix b/tests/functional/lang/eval-okay-logic.nix similarity index 100% rename from tests/lang/eval-okay-logic.nix rename to tests/functional/lang/eval-okay-logic.nix diff --git a/tests/lang/eval-okay-map.exp b/tests/functional/lang/eval-okay-map.exp similarity index 100% rename from tests/lang/eval-okay-map.exp rename to tests/functional/lang/eval-okay-map.exp diff --git a/tests/lang/eval-okay-map.nix b/tests/functional/lang/eval-okay-map.nix similarity index 100% rename from tests/lang/eval-okay-map.nix rename to tests/functional/lang/eval-okay-map.nix diff --git a/tests/lang/eval-okay-mapattrs.exp b/tests/functional/lang/eval-okay-mapattrs.exp similarity index 100% rename from tests/lang/eval-okay-mapattrs.exp rename to tests/functional/lang/eval-okay-mapattrs.exp diff --git a/tests/lang/eval-okay-mapattrs.nix b/tests/functional/lang/eval-okay-mapattrs.nix similarity index 100% rename from tests/lang/eval-okay-mapattrs.nix rename to tests/functional/lang/eval-okay-mapattrs.nix diff --git a/tests/functional/lang/eval-okay-merge-dynamic-attrs.exp b/tests/functional/lang/eval-okay-merge-dynamic-attrs.exp new file mode 100644 index 000000000000..157d677ce8d0 --- /dev/null +++ b/tests/functional/lang/eval-okay-merge-dynamic-attrs.exp @@ -0,0 +1 @@ +{ set1 = { a = 1; b = 2; }; set2 = { a = 1; b = 2; }; set3 = { a = 1; b = 2; }; set4 = { a = 1; b = 2; }; } diff --git a/tests/functional/lang/eval-okay-merge-dynamic-attrs.nix b/tests/functional/lang/eval-okay-merge-dynamic-attrs.nix new file mode 100644 index 000000000000..f459a554f341 --- /dev/null +++ b/tests/functional/lang/eval-okay-merge-dynamic-attrs.nix @@ -0,0 +1,13 @@ +{ + set1 = { a = 1; }; + set1 = { "${"b" + ""}" = 2; }; + + set2 = { "${"b" + ""}" = 2; }; + set2 = { a = 1; }; + + set3.a = 1; + set3."${"b" + ""}" = 2; + + set4."${"b" + ""}" = 2; + set4.a = 1; +} diff --git a/tests/lang/eval-okay-nested-with.exp b/tests/functional/lang/eval-okay-nested-with.exp similarity index 100% rename from tests/lang/eval-okay-nested-with.exp rename to tests/functional/lang/eval-okay-nested-with.exp diff --git a/tests/lang/eval-okay-nested-with.nix b/tests/functional/lang/eval-okay-nested-with.nix similarity index 100% rename from tests/lang/eval-okay-nested-with.nix rename to tests/functional/lang/eval-okay-nested-with.nix diff --git a/tests/lang/eval-okay-new-let.exp b/tests/functional/lang/eval-okay-new-let.exp similarity index 100% rename from tests/lang/eval-okay-new-let.exp rename to tests/functional/lang/eval-okay-new-let.exp diff --git a/tests/lang/eval-okay-new-let.nix b/tests/functional/lang/eval-okay-new-let.nix similarity index 100% rename from tests/lang/eval-okay-new-let.nix rename to tests/functional/lang/eval-okay-new-let.nix diff --git a/tests/lang/eval-okay-null-dynamic-attrs.exp b/tests/functional/lang/eval-okay-null-dynamic-attrs.exp similarity index 100% rename from tests/lang/eval-okay-null-dynamic-attrs.exp rename to tests/functional/lang/eval-okay-null-dynamic-attrs.exp diff --git a/tests/lang/eval-okay-null-dynamic-attrs.nix b/tests/functional/lang/eval-okay-null-dynamic-attrs.nix similarity index 100% rename from tests/lang/eval-okay-null-dynamic-attrs.nix rename to tests/functional/lang/eval-okay-null-dynamic-attrs.nix diff --git a/tests/lang/eval-okay-overrides.exp b/tests/functional/lang/eval-okay-overrides.exp similarity index 100% rename from tests/lang/eval-okay-overrides.exp rename to tests/functional/lang/eval-okay-overrides.exp diff --git a/tests/lang/eval-okay-overrides.nix b/tests/functional/lang/eval-okay-overrides.nix similarity index 66% rename from tests/lang/eval-okay-overrides.nix rename to tests/functional/lang/eval-okay-overrides.nix index 358742b36e22..719bdc9c05e5 100644 --- a/tests/lang/eval-okay-overrides.nix +++ b/tests/functional/lang/eval-okay-overrides.nix @@ -1,6 +1,6 @@ let - overrides = { a = 2; }; + overrides = { a = 2; b = 3; }; in (rec { __overrides = overrides; diff --git a/tests/functional/lang/eval-okay-parse-flake-ref.exp b/tests/functional/lang/eval-okay-parse-flake-ref.exp new file mode 100644 index 000000000000..fc17ba085731 --- /dev/null +++ b/tests/functional/lang/eval-okay-parse-flake-ref.exp @@ -0,0 +1 @@ +{ dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github"; } diff --git a/tests/functional/lang/eval-okay-parse-flake-ref.nix b/tests/functional/lang/eval-okay-parse-flake-ref.nix new file mode 100644 index 000000000000..db4ed2742cdc --- /dev/null +++ b/tests/functional/lang/eval-okay-parse-flake-ref.nix @@ -0,0 +1 @@ + builtins.parseFlakeRef "github:NixOS/nixpkgs/23.05?dir=lib" diff --git a/tests/lang/eval-okay-partition.exp b/tests/functional/lang/eval-okay-partition.exp similarity index 100% rename from tests/lang/eval-okay-partition.exp rename to tests/functional/lang/eval-okay-partition.exp diff --git a/tests/lang/eval-okay-partition.nix b/tests/functional/lang/eval-okay-partition.nix similarity index 100% rename from tests/lang/eval-okay-partition.nix rename to tests/functional/lang/eval-okay-partition.nix diff --git a/tests/lang/eval-okay-path-string-interpolation.exp b/tests/functional/lang/eval-okay-path-string-interpolation.exp similarity index 100% rename from tests/lang/eval-okay-path-string-interpolation.exp rename to tests/functional/lang/eval-okay-path-string-interpolation.exp diff --git a/tests/lang/eval-okay-path-string-interpolation.nix b/tests/functional/lang/eval-okay-path-string-interpolation.nix similarity index 100% rename from tests/lang/eval-okay-path-string-interpolation.nix rename to tests/functional/lang/eval-okay-path-string-interpolation.nix diff --git a/tests/functional/lang/eval-okay-path.exp b/tests/functional/lang/eval-okay-path.exp new file mode 100644 index 000000000000..635e2243a2ab --- /dev/null +++ b/tests/functional/lang/eval-okay-path.exp @@ -0,0 +1 @@ +[ "/nix/store/ya937r4ydw0l6kayq8jkyqaips9c75jm-output" "/nix/store/m7y372g6jb0g4hh1dzmj847rd356fhnz-output" ] diff --git a/tests/functional/lang/eval-okay-path.nix b/tests/functional/lang/eval-okay-path.nix new file mode 100644 index 000000000000..599b3354147c --- /dev/null +++ b/tests/functional/lang/eval-okay-path.nix @@ -0,0 +1,15 @@ +[ + (builtins.path + { path = ./.; + filter = path: _: baseNameOf path == "data"; + recursive = true; + sha256 = "1yhm3gwvg5a41yylymgblsclk95fs6jy72w0wv925mmidlhcq4sw"; + name = "output"; + }) + (builtins.path + { path = ./data; + recursive = false; + sha256 = "0k4lwj58f2w5yh92ilrwy9917pycipbrdrr13vbb3yd02j09vfxm"; + name = "output"; + }) +] diff --git a/tests/lang/eval-okay-pathexists.exp b/tests/functional/lang/eval-okay-pathexists.exp similarity index 100% rename from tests/lang/eval-okay-pathexists.exp rename to tests/functional/lang/eval-okay-pathexists.exp diff --git a/tests/functional/lang/eval-okay-pathexists.nix b/tests/functional/lang/eval-okay-pathexists.nix new file mode 100644 index 000000000000..31697f66a909 --- /dev/null +++ b/tests/functional/lang/eval-okay-pathexists.nix @@ -0,0 +1,31 @@ +builtins.pathExists (./lib.nix) +&& builtins.pathExists (builtins.toPath ./lib.nix) +&& builtins.pathExists (builtins.toString ./lib.nix) +&& !builtins.pathExists (builtins.toString ./lib.nix + "/") +&& !builtins.pathExists (builtins.toString ./lib.nix + "/.") +# FIXME +# && !builtins.pathExists (builtins.toString ./lib.nix + "/..") +# && !builtins.pathExists (builtins.toString ./lib.nix + "/a/..") +# && !builtins.pathExists (builtins.toString ./lib.nix + "/../lib.nix") +&& !builtins.pathExists (builtins.toString ./lib.nix + "/./") +&& !builtins.pathExists (builtins.toString ./lib.nix + "/./.") +&& builtins.pathExists (builtins.toString ./.. + "/lang/lib.nix") +&& !builtins.pathExists (builtins.toString ./.. + "lang/lib.nix") +&& builtins.pathExists (builtins.toString ./. + "/../lang/lib.nix") +&& builtins.pathExists (builtins.toString ./. + "/../lang/./lib.nix") +&& builtins.pathExists (builtins.toString ./.) +&& builtins.pathExists (builtins.toString ./. + "/") +&& builtins.pathExists (builtins.toString ./. + "/../lang") +&& builtins.pathExists (builtins.toString ./. + "/../lang/") +&& builtins.pathExists (builtins.toString ./. + "/../lang/.") +&& builtins.pathExists (builtins.toString ./. + "/../lang/./") +&& builtins.pathExists (builtins.toString ./. + "/../lang//./") +&& builtins.pathExists (builtins.toString ./. + "/../lang/..") +&& builtins.pathExists (builtins.toString ./. + "/../lang/../") +&& builtins.pathExists (builtins.toString ./. + "/../lang/..//") +&& builtins.pathExists (builtins.toPath (builtins.toString ./lib.nix)) +&& !builtins.pathExists (builtins.toPath (builtins.toString ./bla.nix)) +&& builtins.pathExists (builtins.toPath { __toString = x: builtins.toString ./lib.nix; }) +&& builtins.pathExists (builtins.toPath { outPath = builtins.toString ./lib.nix; }) +&& builtins.pathExists ./lib.nix +&& !builtins.pathExists ./bla.nix diff --git a/tests/lang/eval-okay-patterns.exp b/tests/functional/lang/eval-okay-patterns.exp similarity index 100% rename from tests/lang/eval-okay-patterns.exp rename to tests/functional/lang/eval-okay-patterns.exp diff --git a/tests/lang/eval-okay-patterns.nix b/tests/functional/lang/eval-okay-patterns.nix similarity index 100% rename from tests/lang/eval-okay-patterns.nix rename to tests/functional/lang/eval-okay-patterns.nix diff --git a/tests/functional/lang/eval-okay-print.err.exp b/tests/functional/lang/eval-okay-print.err.exp new file mode 100644 index 000000000000..3fc99be3efad --- /dev/null +++ b/tests/functional/lang/eval-okay-print.err.exp @@ -0,0 +1 @@ +trace: [ ] diff --git a/tests/functional/lang/eval-okay-print.exp b/tests/functional/lang/eval-okay-print.exp new file mode 100644 index 000000000000..0d960fb705b4 --- /dev/null +++ b/tests/functional/lang/eval-okay-print.exp @@ -0,0 +1 @@ +[ null [ [ «repeated» ] ] ] diff --git a/tests/functional/lang/eval-okay-print.nix b/tests/functional/lang/eval-okay-print.nix new file mode 100644 index 000000000000..d36ba4da31cd --- /dev/null +++ b/tests/functional/lang/eval-okay-print.nix @@ -0,0 +1 @@ +with builtins; trace [(1+1)] [ null toString (deepSeq "x") (a: a) (let x=[x]; in x) ] diff --git a/tests/lang/eval-okay-readDir.exp b/tests/functional/lang/eval-okay-readDir.exp similarity index 100% rename from tests/lang/eval-okay-readDir.exp rename to tests/functional/lang/eval-okay-readDir.exp diff --git a/tests/lang/eval-okay-readDir.nix b/tests/functional/lang/eval-okay-readDir.nix similarity index 100% rename from tests/lang/eval-okay-readDir.nix rename to tests/functional/lang/eval-okay-readDir.nix diff --git a/tests/lang/eval-okay-readFileType.exp b/tests/functional/lang/eval-okay-readFileType.exp similarity index 100% rename from tests/lang/eval-okay-readFileType.exp rename to tests/functional/lang/eval-okay-readFileType.exp diff --git a/tests/lang/eval-okay-readFileType.nix b/tests/functional/lang/eval-okay-readFileType.nix similarity index 100% rename from tests/lang/eval-okay-readFileType.nix rename to tests/functional/lang/eval-okay-readFileType.nix diff --git a/tests/lang/eval-okay-readfile.exp b/tests/functional/lang/eval-okay-readfile.exp similarity index 100% rename from tests/lang/eval-okay-readfile.exp rename to tests/functional/lang/eval-okay-readfile.exp diff --git a/tests/lang/eval-okay-readfile.nix b/tests/functional/lang/eval-okay-readfile.nix similarity index 100% rename from tests/lang/eval-okay-readfile.nix rename to tests/functional/lang/eval-okay-readfile.nix diff --git a/tests/lang/eval-okay-redefine-builtin.exp b/tests/functional/lang/eval-okay-redefine-builtin.exp similarity index 100% rename from tests/lang/eval-okay-redefine-builtin.exp rename to tests/functional/lang/eval-okay-redefine-builtin.exp diff --git a/tests/lang/eval-okay-redefine-builtin.nix b/tests/functional/lang/eval-okay-redefine-builtin.nix similarity index 100% rename from tests/lang/eval-okay-redefine-builtin.nix rename to tests/functional/lang/eval-okay-redefine-builtin.nix diff --git a/tests/lang/eval-okay-regex-match.exp b/tests/functional/lang/eval-okay-regex-match.exp similarity index 100% rename from tests/lang/eval-okay-regex-match.exp rename to tests/functional/lang/eval-okay-regex-match.exp diff --git a/tests/lang/eval-okay-regex-match.nix b/tests/functional/lang/eval-okay-regex-match.nix similarity index 100% rename from tests/lang/eval-okay-regex-match.nix rename to tests/functional/lang/eval-okay-regex-match.nix diff --git a/tests/lang/eval-okay-regex-split.exp b/tests/functional/lang/eval-okay-regex-split.exp similarity index 100% rename from tests/lang/eval-okay-regex-split.exp rename to tests/functional/lang/eval-okay-regex-split.exp diff --git a/tests/lang/eval-okay-regex-split.nix b/tests/functional/lang/eval-okay-regex-split.nix similarity index 100% rename from tests/lang/eval-okay-regex-split.nix rename to tests/functional/lang/eval-okay-regex-split.nix diff --git a/tests/lang/eval-okay-regression-20220122.exp b/tests/functional/lang/eval-okay-regression-20220122.exp similarity index 100% rename from tests/lang/eval-okay-regression-20220122.exp rename to tests/functional/lang/eval-okay-regression-20220122.exp diff --git a/tests/lang/eval-okay-regression-20220122.nix b/tests/functional/lang/eval-okay-regression-20220122.nix similarity index 100% rename from tests/lang/eval-okay-regression-20220122.nix rename to tests/functional/lang/eval-okay-regression-20220122.nix diff --git a/tests/lang/eval-okay-regression-20220125.exp b/tests/functional/lang/eval-okay-regression-20220125.exp similarity index 100% rename from tests/lang/eval-okay-regression-20220125.exp rename to tests/functional/lang/eval-okay-regression-20220125.exp diff --git a/tests/lang/eval-okay-regression-20220125.nix b/tests/functional/lang/eval-okay-regression-20220125.nix similarity index 100% rename from tests/lang/eval-okay-regression-20220125.nix rename to tests/functional/lang/eval-okay-regression-20220125.nix diff --git a/tests/lang/eval-okay-remove.exp b/tests/functional/lang/eval-okay-remove.exp similarity index 100% rename from tests/lang/eval-okay-remove.exp rename to tests/functional/lang/eval-okay-remove.exp diff --git a/tests/lang/eval-okay-remove.nix b/tests/functional/lang/eval-okay-remove.nix similarity index 100% rename from tests/lang/eval-okay-remove.nix rename to tests/functional/lang/eval-okay-remove.nix diff --git a/tests/lang/eval-okay-replacestrings.exp b/tests/functional/lang/eval-okay-replacestrings.exp similarity index 100% rename from tests/lang/eval-okay-replacestrings.exp rename to tests/functional/lang/eval-okay-replacestrings.exp diff --git a/tests/lang/eval-okay-replacestrings.nix b/tests/functional/lang/eval-okay-replacestrings.nix similarity index 100% rename from tests/lang/eval-okay-replacestrings.nix rename to tests/functional/lang/eval-okay-replacestrings.nix diff --git a/tests/lang/eval-okay-scope-1.exp b/tests/functional/lang/eval-okay-scope-1.exp similarity index 100% rename from tests/lang/eval-okay-scope-1.exp rename to tests/functional/lang/eval-okay-scope-1.exp diff --git a/tests/lang/eval-okay-scope-1.nix b/tests/functional/lang/eval-okay-scope-1.nix similarity index 100% rename from tests/lang/eval-okay-scope-1.nix rename to tests/functional/lang/eval-okay-scope-1.nix diff --git a/tests/lang/eval-okay-scope-2.exp b/tests/functional/lang/eval-okay-scope-2.exp similarity index 100% rename from tests/lang/eval-okay-scope-2.exp rename to tests/functional/lang/eval-okay-scope-2.exp diff --git a/tests/lang/eval-okay-scope-2.nix b/tests/functional/lang/eval-okay-scope-2.nix similarity index 100% rename from tests/lang/eval-okay-scope-2.nix rename to tests/functional/lang/eval-okay-scope-2.nix diff --git a/tests/lang/eval-okay-scope-3.exp b/tests/functional/lang/eval-okay-scope-3.exp similarity index 100% rename from tests/lang/eval-okay-scope-3.exp rename to tests/functional/lang/eval-okay-scope-3.exp diff --git a/tests/lang/eval-okay-scope-3.nix b/tests/functional/lang/eval-okay-scope-3.nix similarity index 100% rename from tests/lang/eval-okay-scope-3.nix rename to tests/functional/lang/eval-okay-scope-3.nix diff --git a/tests/lang/eval-okay-scope-4.exp b/tests/functional/lang/eval-okay-scope-4.exp similarity index 100% rename from tests/lang/eval-okay-scope-4.exp rename to tests/functional/lang/eval-okay-scope-4.exp diff --git a/tests/lang/eval-okay-scope-4.nix b/tests/functional/lang/eval-okay-scope-4.nix similarity index 100% rename from tests/lang/eval-okay-scope-4.nix rename to tests/functional/lang/eval-okay-scope-4.nix diff --git a/tests/lang/eval-okay-scope-6.exp b/tests/functional/lang/eval-okay-scope-6.exp similarity index 100% rename from tests/lang/eval-okay-scope-6.exp rename to tests/functional/lang/eval-okay-scope-6.exp diff --git a/tests/lang/eval-okay-scope-6.nix b/tests/functional/lang/eval-okay-scope-6.nix similarity index 100% rename from tests/lang/eval-okay-scope-6.nix rename to tests/functional/lang/eval-okay-scope-6.nix diff --git a/tests/lang/eval-okay-scope-7.exp b/tests/functional/lang/eval-okay-scope-7.exp similarity index 100% rename from tests/lang/eval-okay-scope-7.exp rename to tests/functional/lang/eval-okay-scope-7.exp diff --git a/tests/lang/eval-okay-scope-7.nix b/tests/functional/lang/eval-okay-scope-7.nix similarity index 100% rename from tests/lang/eval-okay-scope-7.nix rename to tests/functional/lang/eval-okay-scope-7.nix diff --git a/tests/lang/eval-okay-search-path.exp b/tests/functional/lang/eval-okay-search-path.exp similarity index 100% rename from tests/lang/eval-okay-search-path.exp rename to tests/functional/lang/eval-okay-search-path.exp diff --git a/tests/functional/lang/eval-okay-search-path.flags b/tests/functional/lang/eval-okay-search-path.flags new file mode 100644 index 000000000000..dfad1c611877 --- /dev/null +++ b/tests/functional/lang/eval-okay-search-path.flags @@ -0,0 +1 @@ +-I lang/dir1 -I lang/dir2 -I dir5=lang/dir3 diff --git a/tests/lang/eval-okay-search-path.nix b/tests/functional/lang/eval-okay-search-path.nix similarity index 100% rename from tests/lang/eval-okay-search-path.nix rename to tests/functional/lang/eval-okay-search-path.nix diff --git a/tests/lang/eval-okay-seq.exp b/tests/functional/lang/eval-okay-seq.exp similarity index 100% rename from tests/lang/eval-okay-seq.exp rename to tests/functional/lang/eval-okay-seq.exp diff --git a/tests/lang/eval-okay-seq.nix b/tests/functional/lang/eval-okay-seq.nix similarity index 100% rename from tests/lang/eval-okay-seq.nix rename to tests/functional/lang/eval-okay-seq.nix diff --git a/tests/lang/eval-okay-sort.exp b/tests/functional/lang/eval-okay-sort.exp similarity index 100% rename from tests/lang/eval-okay-sort.exp rename to tests/functional/lang/eval-okay-sort.exp diff --git a/tests/lang/eval-okay-sort.nix b/tests/functional/lang/eval-okay-sort.nix similarity index 100% rename from tests/lang/eval-okay-sort.nix rename to tests/functional/lang/eval-okay-sort.nix diff --git a/tests/lang/eval-okay-splitversion.exp b/tests/functional/lang/eval-okay-splitversion.exp similarity index 100% rename from tests/lang/eval-okay-splitversion.exp rename to tests/functional/lang/eval-okay-splitversion.exp diff --git a/tests/lang/eval-okay-splitversion.nix b/tests/functional/lang/eval-okay-splitversion.nix similarity index 100% rename from tests/lang/eval-okay-splitversion.nix rename to tests/functional/lang/eval-okay-splitversion.nix diff --git a/tests/lang/eval-okay-string.exp b/tests/functional/lang/eval-okay-string.exp similarity index 100% rename from tests/lang/eval-okay-string.exp rename to tests/functional/lang/eval-okay-string.exp diff --git a/tests/lang/eval-okay-string.nix b/tests/functional/lang/eval-okay-string.nix similarity index 100% rename from tests/lang/eval-okay-string.nix rename to tests/functional/lang/eval-okay-string.nix diff --git a/tests/lang/eval-okay-strings-as-attrs-names.exp b/tests/functional/lang/eval-okay-strings-as-attrs-names.exp similarity index 100% rename from tests/lang/eval-okay-strings-as-attrs-names.exp rename to tests/functional/lang/eval-okay-strings-as-attrs-names.exp diff --git a/tests/lang/eval-okay-strings-as-attrs-names.nix b/tests/functional/lang/eval-okay-strings-as-attrs-names.nix similarity index 100% rename from tests/lang/eval-okay-strings-as-attrs-names.nix rename to tests/functional/lang/eval-okay-strings-as-attrs-names.nix diff --git a/tests/lang/eval-okay-substring.exp b/tests/functional/lang/eval-okay-substring.exp similarity index 100% rename from tests/lang/eval-okay-substring.exp rename to tests/functional/lang/eval-okay-substring.exp diff --git a/tests/lang/eval-okay-substring.nix b/tests/functional/lang/eval-okay-substring.nix similarity index 100% rename from tests/lang/eval-okay-substring.nix rename to tests/functional/lang/eval-okay-substring.nix diff --git a/tests/functional/lang/eval-okay-symlink-resolution.exp b/tests/functional/lang/eval-okay-symlink-resolution.exp new file mode 100644 index 000000000000..8b8441b91da7 --- /dev/null +++ b/tests/functional/lang/eval-okay-symlink-resolution.exp @@ -0,0 +1 @@ +"test" diff --git a/tests/functional/lang/eval-okay-symlink-resolution.nix b/tests/functional/lang/eval-okay-symlink-resolution.nix new file mode 100644 index 000000000000..ffb1818bde0d --- /dev/null +++ b/tests/functional/lang/eval-okay-symlink-resolution.nix @@ -0,0 +1 @@ +import symlink-resolution/foo/overlays/overlay.nix diff --git a/tests/lang/eval-okay-tail-call-1.exp-disabled b/tests/functional/lang/eval-okay-tail-call-1.exp-disabled similarity index 100% rename from tests/lang/eval-okay-tail-call-1.exp-disabled rename to tests/functional/lang/eval-okay-tail-call-1.exp-disabled diff --git a/tests/lang/eval-okay-tail-call-1.nix b/tests/functional/lang/eval-okay-tail-call-1.nix similarity index 100% rename from tests/lang/eval-okay-tail-call-1.nix rename to tests/functional/lang/eval-okay-tail-call-1.nix diff --git a/tests/lang/eval-okay-tojson.exp b/tests/functional/lang/eval-okay-tojson.exp similarity index 100% rename from tests/lang/eval-okay-tojson.exp rename to tests/functional/lang/eval-okay-tojson.exp diff --git a/tests/lang/eval-okay-tojson.nix b/tests/functional/lang/eval-okay-tojson.nix similarity index 100% rename from tests/lang/eval-okay-tojson.nix rename to tests/functional/lang/eval-okay-tojson.nix diff --git a/tests/lang/eval-okay-toxml.exp b/tests/functional/lang/eval-okay-toxml.exp similarity index 100% rename from tests/lang/eval-okay-toxml.exp rename to tests/functional/lang/eval-okay-toxml.exp diff --git a/tests/lang/eval-okay-toxml.nix b/tests/functional/lang/eval-okay-toxml.nix similarity index 100% rename from tests/lang/eval-okay-toxml.nix rename to tests/functional/lang/eval-okay-toxml.nix diff --git a/tests/lang/eval-okay-toxml2.exp b/tests/functional/lang/eval-okay-toxml2.exp similarity index 100% rename from tests/lang/eval-okay-toxml2.exp rename to tests/functional/lang/eval-okay-toxml2.exp diff --git a/tests/lang/eval-okay-toxml2.nix b/tests/functional/lang/eval-okay-toxml2.nix similarity index 100% rename from tests/lang/eval-okay-toxml2.nix rename to tests/functional/lang/eval-okay-toxml2.nix diff --git a/tests/lang/eval-okay-tryeval.exp b/tests/functional/lang/eval-okay-tryeval.exp similarity index 100% rename from tests/lang/eval-okay-tryeval.exp rename to tests/functional/lang/eval-okay-tryeval.exp diff --git a/tests/lang/eval-okay-tryeval.nix b/tests/functional/lang/eval-okay-tryeval.nix similarity index 100% rename from tests/lang/eval-okay-tryeval.nix rename to tests/functional/lang/eval-okay-tryeval.nix diff --git a/tests/lang/eval-okay-types.exp b/tests/functional/lang/eval-okay-types.exp similarity index 100% rename from tests/lang/eval-okay-types.exp rename to tests/functional/lang/eval-okay-types.exp diff --git a/tests/lang/eval-okay-types.nix b/tests/functional/lang/eval-okay-types.nix similarity index 100% rename from tests/lang/eval-okay-types.nix rename to tests/functional/lang/eval-okay-types.nix diff --git a/tests/lang/eval-okay-versions.exp b/tests/functional/lang/eval-okay-versions.exp similarity index 100% rename from tests/lang/eval-okay-versions.exp rename to tests/functional/lang/eval-okay-versions.exp diff --git a/tests/lang/eval-okay-versions.nix b/tests/functional/lang/eval-okay-versions.nix similarity index 100% rename from tests/lang/eval-okay-versions.nix rename to tests/functional/lang/eval-okay-versions.nix diff --git a/tests/lang/eval-okay-with.exp b/tests/functional/lang/eval-okay-with.exp similarity index 100% rename from tests/lang/eval-okay-with.exp rename to tests/functional/lang/eval-okay-with.exp diff --git a/tests/lang/eval-okay-with.nix b/tests/functional/lang/eval-okay-with.nix similarity index 100% rename from tests/lang/eval-okay-with.nix rename to tests/functional/lang/eval-okay-with.nix diff --git a/tests/lang/eval-okay-xml.exp.xml b/tests/functional/lang/eval-okay-xml.exp.xml similarity index 100% rename from tests/lang/eval-okay-xml.exp.xml rename to tests/functional/lang/eval-okay-xml.exp.xml diff --git a/tests/lang/eval-okay-xml.nix b/tests/functional/lang/eval-okay-xml.nix similarity index 100% rename from tests/lang/eval-okay-xml.nix rename to tests/functional/lang/eval-okay-xml.nix diff --git a/tests/lang/eval-okay-zipAttrsWith.exp b/tests/functional/lang/eval-okay-zipAttrsWith.exp similarity index 100% rename from tests/lang/eval-okay-zipAttrsWith.exp rename to tests/functional/lang/eval-okay-zipAttrsWith.exp diff --git a/tests/lang/eval-okay-zipAttrsWith.nix b/tests/functional/lang/eval-okay-zipAttrsWith.nix similarity index 100% rename from tests/lang/eval-okay-zipAttrsWith.nix rename to tests/functional/lang/eval-okay-zipAttrsWith.nix diff --git a/tests/functional/lang/framework.sh b/tests/functional/lang/framework.sh new file mode 100644 index 000000000000..516bff8ad1dd --- /dev/null +++ b/tests/functional/lang/framework.sh @@ -0,0 +1,33 @@ +# Golden test support +# +# Test that the output of the given test matches what is expected. If +# `_NIX_TEST_ACCEPT` is non-empty also update the expected output so +# that next time the test succeeds. +function diffAndAcceptInner() { + local -r testName=$1 + local -r got="$2" + local -r expected="$3" + + # Absence of expected file indicates empty output expected. + if test -e "$expected"; then + local -r expectedOrEmpty="$expected" + else + local -r expectedOrEmpty=lang/empty.exp + fi + + # Diff so we get a nice message + if ! diff --unified "$got" "$expectedOrEmpty"; then + echo "FAIL: evaluation result of $testName not as expected" + badDiff=1 + fi + + # Update expected if `_NIX_TEST_ACCEPT` is non-empty. + if test -n "${_NIX_TEST_ACCEPT-}"; then + cp "$got" "$expected" + # Delete empty expected files to avoid bloating the repo with + # empty files. + if ! test -s "$expected"; then + rm "$expected" + fi + fi +} diff --git a/tests/lang/imported.nix b/tests/functional/lang/imported.nix similarity index 100% rename from tests/lang/imported.nix rename to tests/functional/lang/imported.nix diff --git a/tests/lang/imported2.nix b/tests/functional/lang/imported2.nix similarity index 100% rename from tests/lang/imported2.nix rename to tests/functional/lang/imported2.nix diff --git a/tests/lang/lib.nix b/tests/functional/lang/lib.nix similarity index 100% rename from tests/lang/lib.nix rename to tests/functional/lang/lib.nix diff --git a/tests/functional/lang/parse-fail-dup-attrs-1.err.exp b/tests/functional/lang/parse-fail-dup-attrs-1.err.exp new file mode 100644 index 000000000000..4fe6b7a1f0ed --- /dev/null +++ b/tests/functional/lang/parse-fail-dup-attrs-1.err.exp @@ -0,0 +1,7 @@ +error: attribute 'x' already defined at «stdin»:1:3 + + at «stdin»:3:3: + + 2| y = 456; + 3| x = 789; + | ^ diff --git a/tests/lang/parse-fail-dup-attrs-1.nix b/tests/functional/lang/parse-fail-dup-attrs-1.nix similarity index 100% rename from tests/lang/parse-fail-dup-attrs-1.nix rename to tests/functional/lang/parse-fail-dup-attrs-1.nix diff --git a/tests/functional/lang/parse-fail-dup-attrs-2.err.exp b/tests/functional/lang/parse-fail-dup-attrs-2.err.exp new file mode 100644 index 000000000000..3aba2891f62b --- /dev/null +++ b/tests/functional/lang/parse-fail-dup-attrs-2.err.exp @@ -0,0 +1,7 @@ +error: attribute 'x' already defined at «stdin»:9:5 + + at «stdin»:10:17: + + 9| x = 789; + 10| inherit (as) x; + | ^ diff --git a/tests/lang/parse-fail-dup-attrs-2.nix b/tests/functional/lang/parse-fail-dup-attrs-2.nix similarity index 100% rename from tests/lang/parse-fail-dup-attrs-2.nix rename to tests/functional/lang/parse-fail-dup-attrs-2.nix diff --git a/tests/functional/lang/parse-fail-dup-attrs-3.err.exp b/tests/functional/lang/parse-fail-dup-attrs-3.err.exp new file mode 100644 index 000000000000..3aba2891f62b --- /dev/null +++ b/tests/functional/lang/parse-fail-dup-attrs-3.err.exp @@ -0,0 +1,7 @@ +error: attribute 'x' already defined at «stdin»:9:5 + + at «stdin»:10:17: + + 9| x = 789; + 10| inherit (as) x; + | ^ diff --git a/tests/lang/parse-fail-dup-attrs-3.nix b/tests/functional/lang/parse-fail-dup-attrs-3.nix similarity index 100% rename from tests/lang/parse-fail-dup-attrs-3.nix rename to tests/functional/lang/parse-fail-dup-attrs-3.nix diff --git a/tests/functional/lang/parse-fail-dup-attrs-4.err.exp b/tests/functional/lang/parse-fail-dup-attrs-4.err.exp new file mode 100644 index 000000000000..ff68446a1a86 --- /dev/null +++ b/tests/functional/lang/parse-fail-dup-attrs-4.err.exp @@ -0,0 +1,7 @@ +error: attribute 'services.ssh.port' already defined at «stdin»:2:3 + + at «stdin»:3:3: + + 2| services.ssh.port = 22; + 3| services.ssh.port = 23; + | ^ diff --git a/tests/lang/parse-fail-dup-attrs-4.nix b/tests/functional/lang/parse-fail-dup-attrs-4.nix similarity index 100% rename from tests/lang/parse-fail-dup-attrs-4.nix rename to tests/functional/lang/parse-fail-dup-attrs-4.nix diff --git a/tests/functional/lang/parse-fail-dup-attrs-7.err.exp b/tests/functional/lang/parse-fail-dup-attrs-7.err.exp new file mode 100644 index 000000000000..512a499ca9b3 --- /dev/null +++ b/tests/functional/lang/parse-fail-dup-attrs-7.err.exp @@ -0,0 +1,7 @@ +error: attribute 'x' already defined at «stdin»:6:12 + + at «stdin»:7:12: + + 6| inherit x; + 7| inherit x; + | ^ diff --git a/tests/lang/parse-fail-dup-attrs-7.nix b/tests/functional/lang/parse-fail-dup-attrs-7.nix similarity index 100% rename from tests/lang/parse-fail-dup-attrs-7.nix rename to tests/functional/lang/parse-fail-dup-attrs-7.nix diff --git a/tests/functional/lang/parse-fail-dup-formals.err.exp b/tests/functional/lang/parse-fail-dup-formals.err.exp new file mode 100644 index 000000000000..1d566fb33420 --- /dev/null +++ b/tests/functional/lang/parse-fail-dup-formals.err.exp @@ -0,0 +1,6 @@ +error: duplicate formal function argument 'x' + + at «stdin»:1:8: + + 1| {x, y, x}: x + | ^ diff --git a/tests/lang/parse-fail-dup-formals.nix b/tests/functional/lang/parse-fail-dup-formals.nix similarity index 100% rename from tests/lang/parse-fail-dup-formals.nix rename to tests/functional/lang/parse-fail-dup-formals.nix diff --git a/tests/functional/lang/parse-fail-eof-in-string.err.exp b/tests/functional/lang/parse-fail-eof-in-string.err.exp new file mode 100644 index 000000000000..f9fa72312928 --- /dev/null +++ b/tests/functional/lang/parse-fail-eof-in-string.err.exp @@ -0,0 +1,7 @@ +error: syntax error, unexpected end of file, expecting '"' + + at «stdin»:3:5: + + 2| # Note that this file must not end with a newline. + 3| a 1"$ + | ^ diff --git a/tests/lang/parse-fail-eof-in-string.nix b/tests/functional/lang/parse-fail-eof-in-string.nix similarity index 100% rename from tests/lang/parse-fail-eof-in-string.nix rename to tests/functional/lang/parse-fail-eof-in-string.nix diff --git a/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp b/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp new file mode 100644 index 000000000000..32f77679584a --- /dev/null +++ b/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp @@ -0,0 +1,8 @@ +error: attribute 'z' already defined at «stdin»:3:16 + + at «stdin»:2:3: + + 1| { + 2| x.z = 3; + | ^ + 3| x = { y = 3; z = 3; }; diff --git a/tests/lang/parse-fail-mixed-nested-attrs1.nix b/tests/functional/lang/parse-fail-mixed-nested-attrs1.nix similarity index 100% rename from tests/lang/parse-fail-mixed-nested-attrs1.nix rename to tests/functional/lang/parse-fail-mixed-nested-attrs1.nix diff --git a/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp b/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp new file mode 100644 index 000000000000..0437cd50c8ab --- /dev/null +++ b/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp @@ -0,0 +1,8 @@ +error: attribute 'y' already defined at «stdin»:3:9 + + at «stdin»:2:3: + + 1| { + 2| x.y.y = 3; + | ^ + 3| x = { y.y= 3; z = 3; }; diff --git a/tests/lang/parse-fail-mixed-nested-attrs2.nix b/tests/functional/lang/parse-fail-mixed-nested-attrs2.nix similarity index 100% rename from tests/lang/parse-fail-mixed-nested-attrs2.nix rename to tests/functional/lang/parse-fail-mixed-nested-attrs2.nix diff --git a/tests/functional/lang/parse-fail-patterns-1.err.exp b/tests/functional/lang/parse-fail-patterns-1.err.exp new file mode 100644 index 000000000000..634a04aaa4be --- /dev/null +++ b/tests/functional/lang/parse-fail-patterns-1.err.exp @@ -0,0 +1,7 @@ +error: duplicate formal function argument 'args' + + at «stdin»:1:1: + + 1| args@{args, x, y, z}: x + | ^ + 2| diff --git a/tests/lang/parse-fail-patterns-1.nix b/tests/functional/lang/parse-fail-patterns-1.nix similarity index 100% rename from tests/lang/parse-fail-patterns-1.nix rename to tests/functional/lang/parse-fail-patterns-1.nix diff --git a/tests/functional/lang/parse-fail-regression-20060610.err.exp b/tests/functional/lang/parse-fail-regression-20060610.err.exp new file mode 100644 index 000000000000..167d01e85d32 --- /dev/null +++ b/tests/functional/lang/parse-fail-regression-20060610.err.exp @@ -0,0 +1,8 @@ +error: undefined variable 'gcc' + + at «stdin»:8:12: + + 7| + 8| body = ({ + | ^ + 9| inherit gcc; diff --git a/tests/lang/parse-fail-regression-20060610.nix b/tests/functional/lang/parse-fail-regression-20060610.nix similarity index 100% rename from tests/lang/parse-fail-regression-20060610.nix rename to tests/functional/lang/parse-fail-regression-20060610.nix diff --git a/tests/functional/lang/parse-fail-undef-var-2.err.exp b/tests/functional/lang/parse-fail-undef-var-2.err.exp new file mode 100644 index 000000000000..77c96bbd23a1 --- /dev/null +++ b/tests/functional/lang/parse-fail-undef-var-2.err.exp @@ -0,0 +1,7 @@ +error: syntax error, unexpected ':', expecting '}' + + at «stdin»:3:13: + + 2| + 3| f = {x, y : + | ^ diff --git a/tests/lang/parse-fail-undef-var-2.nix b/tests/functional/lang/parse-fail-undef-var-2.nix similarity index 100% rename from tests/lang/parse-fail-undef-var-2.nix rename to tests/functional/lang/parse-fail-undef-var-2.nix diff --git a/tests/functional/lang/parse-fail-undef-var.err.exp b/tests/functional/lang/parse-fail-undef-var.err.exp new file mode 100644 index 000000000000..48e88747fafe --- /dev/null +++ b/tests/functional/lang/parse-fail-undef-var.err.exp @@ -0,0 +1,7 @@ +error: undefined variable 'y' + + at «stdin»:1:4: + + 1| x: y + | ^ + 2| diff --git a/tests/lang/parse-fail-undef-var.nix b/tests/functional/lang/parse-fail-undef-var.nix similarity index 100% rename from tests/lang/parse-fail-undef-var.nix rename to tests/functional/lang/parse-fail-undef-var.nix diff --git a/tests/functional/lang/parse-fail-utf8.err.exp b/tests/functional/lang/parse-fail-utf8.err.exp new file mode 100644 index 000000000000..6087479a37ee --- /dev/null +++ b/tests/functional/lang/parse-fail-utf8.err.exp @@ -0,0 +1,6 @@ +error: syntax error, unexpected invalid token, expecting end of file + + at «stdin»:1:5: + + 1| 123 + | ^ diff --git a/tests/lang/parse-fail-uft8.nix b/tests/functional/lang/parse-fail-utf8.nix similarity index 100% rename from tests/lang/parse-fail-uft8.nix rename to tests/functional/lang/parse-fail-utf8.nix diff --git a/tests/functional/lang/parse-okay-1.exp b/tests/functional/lang/parse-okay-1.exp new file mode 100644 index 000000000000..d5ab5f18ab94 --- /dev/null +++ b/tests/functional/lang/parse-okay-1.exp @@ -0,0 +1 @@ +({ x, y, z }: ((x + y) + z)) diff --git a/tests/lang/parse-okay-1.nix b/tests/functional/lang/parse-okay-1.nix similarity index 100% rename from tests/lang/parse-okay-1.nix rename to tests/functional/lang/parse-okay-1.nix diff --git a/tests/functional/lang/parse-okay-crlf.exp b/tests/functional/lang/parse-okay-crlf.exp new file mode 100644 index 000000000000..4213609fc251 --- /dev/null +++ b/tests/functional/lang/parse-okay-crlf.exp @@ -0,0 +1 @@ +rec { foo = "multi\nline\n string\n test\r"; x = y; y = 123; z = 456; } diff --git a/tests/lang/parse-okay-crlf.nix b/tests/functional/lang/parse-okay-crlf.nix similarity index 100% rename from tests/lang/parse-okay-crlf.nix rename to tests/functional/lang/parse-okay-crlf.nix diff --git a/tests/functional/lang/parse-okay-dup-attrs-5.exp b/tests/functional/lang/parse-okay-dup-attrs-5.exp new file mode 100644 index 000000000000..88b0b036faee --- /dev/null +++ b/tests/functional/lang/parse-okay-dup-attrs-5.exp @@ -0,0 +1 @@ +{ services = { ssh = { enable = true; port = 23; }; }; } diff --git a/tests/lang/parse-okay-dup-attrs-5.nix b/tests/functional/lang/parse-okay-dup-attrs-5.nix similarity index 100% rename from tests/lang/parse-okay-dup-attrs-5.nix rename to tests/functional/lang/parse-okay-dup-attrs-5.nix diff --git a/tests/functional/lang/parse-okay-dup-attrs-6.exp b/tests/functional/lang/parse-okay-dup-attrs-6.exp new file mode 100644 index 000000000000..88b0b036faee --- /dev/null +++ b/tests/functional/lang/parse-okay-dup-attrs-6.exp @@ -0,0 +1 @@ +{ services = { ssh = { enable = true; port = 23; }; }; } diff --git a/tests/lang/parse-okay-dup-attrs-6.nix b/tests/functional/lang/parse-okay-dup-attrs-6.nix similarity index 100% rename from tests/lang/parse-okay-dup-attrs-6.nix rename to tests/functional/lang/parse-okay-dup-attrs-6.nix diff --git a/tests/functional/lang/parse-okay-mixed-nested-attrs-1.exp b/tests/functional/lang/parse-okay-mixed-nested-attrs-1.exp new file mode 100644 index 000000000000..89c66f76018e --- /dev/null +++ b/tests/functional/lang/parse-okay-mixed-nested-attrs-1.exp @@ -0,0 +1 @@ +{ x = { q = 3; y = 3; z = 3; }; } diff --git a/tests/lang/parse-okay-mixed-nested-attrs-1.nix b/tests/functional/lang/parse-okay-mixed-nested-attrs-1.nix similarity index 100% rename from tests/lang/parse-okay-mixed-nested-attrs-1.nix rename to tests/functional/lang/parse-okay-mixed-nested-attrs-1.nix diff --git a/tests/functional/lang/parse-okay-mixed-nested-attrs-2.exp b/tests/functional/lang/parse-okay-mixed-nested-attrs-2.exp new file mode 100644 index 000000000000..89c66f76018e --- /dev/null +++ b/tests/functional/lang/parse-okay-mixed-nested-attrs-2.exp @@ -0,0 +1 @@ +{ x = { q = 3; y = 3; z = 3; }; } diff --git a/tests/lang/parse-okay-mixed-nested-attrs-2.nix b/tests/functional/lang/parse-okay-mixed-nested-attrs-2.nix similarity index 100% rename from tests/lang/parse-okay-mixed-nested-attrs-2.nix rename to tests/functional/lang/parse-okay-mixed-nested-attrs-2.nix diff --git a/tests/functional/lang/parse-okay-mixed-nested-attrs-3.exp b/tests/functional/lang/parse-okay-mixed-nested-attrs-3.exp new file mode 100644 index 000000000000..b89a59734b53 --- /dev/null +++ b/tests/functional/lang/parse-okay-mixed-nested-attrs-3.exp @@ -0,0 +1 @@ +{ services = { httpd = { enable = true; }; ssh = { enable = true; port = 123; }; }; } diff --git a/tests/lang/parse-okay-mixed-nested-attrs-3.nix b/tests/functional/lang/parse-okay-mixed-nested-attrs-3.nix similarity index 100% rename from tests/lang/parse-okay-mixed-nested-attrs-3.nix rename to tests/functional/lang/parse-okay-mixed-nested-attrs-3.nix diff --git a/tests/functional/lang/parse-okay-regression-20041027.exp b/tests/functional/lang/parse-okay-regression-20041027.exp new file mode 100644 index 000000000000..9df7219e4b4b --- /dev/null +++ b/tests/functional/lang/parse-okay-regression-20041027.exp @@ -0,0 +1 @@ +({ fetchurl, stdenv }: ((stdenv).mkDerivation { name = "libXi-6.0.1"; src = (fetchurl { md5 = "7e935a42428d63a387b3c048be0f2756"; url = "http://freedesktop.org/~xlibs/release/libXi-6.0.1.tar.bz2"; }); })) diff --git a/tests/lang/parse-okay-regression-20041027.nix b/tests/functional/lang/parse-okay-regression-20041027.nix similarity index 100% rename from tests/lang/parse-okay-regression-20041027.nix rename to tests/functional/lang/parse-okay-regression-20041027.nix diff --git a/tests/functional/lang/parse-okay-regression-751.exp b/tests/functional/lang/parse-okay-regression-751.exp new file mode 100644 index 000000000000..e2ed886fe35c --- /dev/null +++ b/tests/functional/lang/parse-okay-regression-751.exp @@ -0,0 +1 @@ +(let const = (a: "const"); in ((const { x = "q"; }))) diff --git a/tests/lang/parse-okay-regression-751.nix b/tests/functional/lang/parse-okay-regression-751.nix similarity index 100% rename from tests/lang/parse-okay-regression-751.nix rename to tests/functional/lang/parse-okay-regression-751.nix diff --git a/tests/functional/lang/parse-okay-subversion.exp b/tests/functional/lang/parse-okay-subversion.exp new file mode 100644 index 000000000000..4168ee8bf095 --- /dev/null +++ b/tests/functional/lang/parse-okay-subversion.exp @@ -0,0 +1 @@ +({ fetchurl, localServer ? false, httpServer ? false, sslSupport ? false, pythonBindings ? false, javaSwigBindings ? false, javahlBindings ? false, stdenv, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null }: assert (expat != null); assert (localServer -> (db4 != null)); assert (httpServer -> ((httpd != null) && ((httpd).expat == expat))); assert (sslSupport -> ((openssl != null) && (httpServer -> ((httpd).openssl == openssl)))); assert (pythonBindings -> ((swig != null) && (swig).pythonSupport)); assert (javaSwigBindings -> ((swig != null) && (swig).javaSupport)); assert (javahlBindings -> (j2sdk != null)); ((stdenv).mkDerivation { builder = /foo/bar; db4 = (if localServer then db4 else null); inherit expat ; inherit httpServer ; httpd = (if httpServer then httpd else null); j2sdk = (if javaSwigBindings then (swig).j2sdk else (if javahlBindings then j2sdk else null)); inherit javaSwigBindings ; inherit javahlBindings ; inherit localServer ; name = "subversion-1.1.1"; openssl = (if sslSupport then openssl else null); patches = (if javahlBindings then [ (/javahl.patch) ] else [ ]); python = (if pythonBindings then (swig).python else null); inherit pythonBindings ; src = (fetchurl { md5 = "a180c3fe91680389c210c99def54d9e0"; url = "http://subversion.tigris.org/tarballs/subversion-1.1.1.tar.bz2"; }); inherit sslSupport ; swig = (if (pythonBindings || javaSwigBindings) then swig else null); })) diff --git a/tests/lang/parse-okay-subversion.nix b/tests/functional/lang/parse-okay-subversion.nix similarity index 100% rename from tests/lang/parse-okay-subversion.nix rename to tests/functional/lang/parse-okay-subversion.nix diff --git a/tests/functional/lang/parse-okay-url.exp b/tests/functional/lang/parse-okay-url.exp new file mode 100644 index 000000000000..e5f0829b0ce1 --- /dev/null +++ b/tests/functional/lang/parse-okay-url.exp @@ -0,0 +1 @@ +[ ("x:x") ("https://svn.cs.uu.nl:12443/repos/trace/trunk") ("http://www2.mplayerhq.hu/MPlayer/releases/fonts/font-arial-iso-8859-1.tar.bz2") ("http://losser.st-lab.cs.uu.nl/~armijn/.nix/gcc-3.3.4-static-nix.tar.gz") ("http://fpdownload.macromedia.com/get/shockwave/flash/english/linux/7.0r25/install_flash_player_7_linux.tar.gz") ("https://ftp5.gwdg.de/pub/linux/archlinux/extra/os/x86_64/unzip-6.0-14-x86_64.pkg.tar.zst") ("ftp://ftp.gtk.org/pub/gtk/v1.2/gtk+-1.2.10.tar.gz") ] diff --git a/tests/lang/parse-okay-url.nix b/tests/functional/lang/parse-okay-url.nix similarity index 100% rename from tests/lang/parse-okay-url.nix rename to tests/functional/lang/parse-okay-url.nix diff --git a/tests/lang/readDir/foo/git-hates-directories b/tests/functional/lang/readDir/bar similarity index 100% rename from tests/lang/readDir/foo/git-hates-directories rename to tests/functional/lang/readDir/bar diff --git a/tests/functional/lang/readDir/foo/git-hates-directories b/tests/functional/lang/readDir/foo/git-hates-directories new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/lang/readDir/ldir b/tests/functional/lang/readDir/ldir similarity index 100% rename from tests/lang/readDir/ldir rename to tests/functional/lang/readDir/ldir diff --git a/tests/lang/readDir/linked b/tests/functional/lang/readDir/linked similarity index 100% rename from tests/lang/readDir/linked rename to tests/functional/lang/readDir/linked diff --git a/tests/functional/lang/symlink-resolution/foo/lib/default.nix b/tests/functional/lang/symlink-resolution/foo/lib/default.nix new file mode 100644 index 000000000000..8b8441b91da7 --- /dev/null +++ b/tests/functional/lang/symlink-resolution/foo/lib/default.nix @@ -0,0 +1 @@ +"test" diff --git a/tests/functional/lang/symlink-resolution/foo/overlays b/tests/functional/lang/symlink-resolution/foo/overlays new file mode 120000 index 000000000000..0d44a21c5085 --- /dev/null +++ b/tests/functional/lang/symlink-resolution/foo/overlays @@ -0,0 +1 @@ +../overlays \ No newline at end of file diff --git a/tests/functional/lang/symlink-resolution/overlays/overlay.nix b/tests/functional/lang/symlink-resolution/overlays/overlay.nix new file mode 100644 index 000000000000..b0368308e29b --- /dev/null +++ b/tests/functional/lang/symlink-resolution/overlays/overlay.nix @@ -0,0 +1 @@ +import ../lib diff --git a/tests/functional/legacy-ssh-store.sh b/tests/functional/legacy-ssh-store.sh new file mode 100644 index 000000000000..894efccd46b0 --- /dev/null +++ b/tests/functional/legacy-ssh-store.sh @@ -0,0 +1,4 @@ +source common.sh + +# Check that store info trusted doesn't yet work with ssh:// +nix --store ssh://localhost?remote-store=$TEST_ROOT/other-store store info --json | jq -e 'has("trusted") | not' diff --git a/tests/linux-sandbox-cert-test.nix b/tests/functional/linux-sandbox-cert-test.nix similarity index 100% rename from tests/linux-sandbox-cert-test.nix rename to tests/functional/linux-sandbox-cert-test.nix diff --git a/tests/linux-sandbox.sh b/tests/functional/linux-sandbox.sh similarity index 100% rename from tests/linux-sandbox.sh rename to tests/functional/linux-sandbox.sh diff --git a/tests/local-store.sh b/tests/functional/local-store.sh similarity index 79% rename from tests/local-store.sh rename to tests/functional/local-store.sh index 89502f864b98..f7c8eb3f1a87 100644 --- a/tests/local-store.sh +++ b/tests/functional/local-store.sh @@ -18,5 +18,5 @@ PATH2=$(nix path-info --store "$PWD/x" $CORRECT_PATH) PATH3=$(nix path-info --store "local?root=$PWD/x" $CORRECT_PATH) [ $CORRECT_PATH == $PATH3 ] -# Ensure store ping trusted works with local store -nix --store ./x store ping --json | jq -e '.trusted' +# Ensure store info trusted works with local store +nix --store ./x store info --json | jq -e '.trusted' diff --git a/tests/local.mk b/tests/functional/local.mk similarity index 75% rename from tests/local.mk rename to tests/functional/local.mk index b7bdb37e9585..a636cbe43c13 100644 --- a/tests/local.mk +++ b/tests/functional/local.mk @@ -12,14 +12,15 @@ nix_tests = \ flakes/check.sh \ flakes/unlocked-override.sh \ flakes/absolute-paths.sh \ + flakes/absolute-attr-paths.sh \ flakes/build-paths.sh \ flakes/flake-in-submodule.sh \ - ca/gc.sh \ gc.sh \ nix-collect-garbage-d.sh \ remote-store.sh \ legacy-ssh-store.sh \ lang.sh \ + lang-test-infra.sh \ experimental-features.sh \ fetchMercurial.sh \ gc-auto.sh \ @@ -27,8 +28,6 @@ nix_tests = \ user-envs-migration.sh \ binary-cache.sh \ multiple-outputs.sh \ - ca/build.sh \ - ca/new-build-cmd.sh \ nix-build.sh \ gc-concurrent.sh \ repair.sh \ @@ -46,24 +45,18 @@ nix_tests = \ referrers.sh \ optimise-store.sh \ substitute-with-invalid-ca.sh \ - ca/concurrent-builds.sh \ signing.sh \ - ca/build-with-garbage-path.sh \ hash.sh \ gc-non-blocking.sh \ check.sh \ - ca/substitute.sh \ nix-shell.sh \ - ca/signatures.sh \ - ca/nix-shell.sh \ - ca/nix-copy.sh \ check-refs.sh \ build-remote-input-addressed.sh \ secure-drv-outputs.sh \ restricted.sh \ fetchGitSubmodules.sh \ + fetchGitVerification.sh \ flakes/search-root.sh \ - ca/duplicate-realisation-in-closure.sh \ readfile-context.sh \ nix-channel.sh \ recursive.sh \ @@ -76,13 +69,11 @@ nix_tests = \ build-remote-trustless-should-pass-2.sh \ build-remote-trustless-should-pass-3.sh \ build-remote-trustless-should-fail-0.sh \ + build-remote-with-mounted-ssh-ng.sh \ nar-access.sh \ pure-eval.sh \ eval.sh \ - ca/post-hook.sh \ repl.sh \ - ca/repl.sh \ - ca/recursive.sh \ binary-cache-build-remote.sh \ search.sh \ logging.sh \ @@ -94,6 +85,7 @@ nix_tests = \ misc.sh \ dump-db.sh \ linux-sandbox.sh \ + supplementary-groups.sh \ build-dry.sh \ structured-attrs.sh \ shell.sh \ @@ -101,44 +93,40 @@ nix_tests = \ zstd.sh \ compression-levels.sh \ nix-copy-ssh.sh \ + nix-copy-ssh-ng.sh \ post-hook.sh \ function-trace.sh \ flakes/config.sh \ fmt.sh \ eval-store.sh \ why-depends.sh \ - ca/why-depends.sh \ derivation-json.sh \ - ca/derivation-json.sh \ import-derivation.sh \ - ca/import-derivation.sh \ - dyn-drv/text-hashed-output.sh \ - dyn-drv/recursive-mod-json.sh \ nix_path.sh \ case-hack.sh \ placeholders.sh \ ssh-relay.sh \ - plugins.sh \ build.sh \ build-delete.sh \ output-normalization.sh \ - ca/nix-run.sh \ - selfref-gc.sh ca/selfref-gc.sh \ + selfref-gc.sh \ db-migration.sh \ bash-profile.sh \ pass-as-file.sh \ nix-profile.sh \ suggestions.sh \ - store-ping.sh \ + store-info.sh \ fetchClosure.sh \ completions.sh \ flakes/show.sh \ impure-derivations.sh \ path-from-hash-part.sh \ - test-libstoreconsumer.sh \ + path-info.sh \ toString-path.sh \ read-only-store.sh \ - acls.sh + acls.sh \ + nested-sandboxing.sh \ + impure-env.sh acls_test = \ acls.sh \ @@ -147,26 +135,28 @@ acls_test = \ # nix_tests += $(acls_test) - ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh endif +ifeq ($(ENABLE_BUILD), yes) + nix_tests += test-libstoreconsumer.sh + + ifeq ($(BUILD_SHARED_LIBS), 1) + nix_tests += plugins.sh + endif +endif + +$(d)/test-libstoreconsumer.sh.test $(d)/test-libstoreconsumer.sh.test-debug: \ + $(buildprefix)$(d)/test-libstoreconsumer/test-libstoreconsumer +$(d)/plugins.sh.test $(d)/plugins.sh.test-debug: \ + $(buildprefix)$(d)/plugins/libplugintest.$(SO_EXT) + install-tests += $(foreach x, $(nix_tests), $(d)/$(x)) -clean-files += \ +test-clean-files := \ $(d)/common/vars-and-functions.sh \ - $(d)/config.nix \ - $(d)/ca/config.nix \ - $(d)/dyn-drv/config.nix + $(d)/config.nix -test-deps += \ - tests/common/vars-and-functions.sh \ - tests/config.nix \ - tests/ca/config.nix \ - tests/test-libstoreconsumer/test-libstoreconsumer \ - tests/dyn-drv/config.nix - -ifeq ($(BUILD_SHARED_LIBS), 1) - test-deps += tests/plugins/libplugintest.$(SO_EXT) -endif +clean-files += $(test-clean-files) +test-deps += $(test-clean-files) diff --git a/tests/logging.sh b/tests/functional/logging.sh similarity index 100% rename from tests/logging.sh rename to tests/functional/logging.sh diff --git a/tests/misc.sh b/tests/functional/misc.sh similarity index 64% rename from tests/misc.sh rename to tests/functional/misc.sh index 60d58310ec3a..af96d20bd4af 100644 --- a/tests/misc.sh +++ b/tests/functional/misc.sh @@ -24,3 +24,9 @@ eval_stdin_res=$(echo 'let a = {} // a; in a.foo' | nix-instantiate --eval -E - echo $eval_stdin_res | grep "at «stdin»:1:15:" echo $eval_stdin_res | grep "infinite recursion encountered" +# Attribute path errors +expectStderr 1 nix-instantiate --eval -E '{}' -A '"x' | grepQuiet "missing closing quote in selection path" +expectStderr 1 nix-instantiate --eval -E '[]' -A 'x' | grepQuiet "should be a set" +expectStderr 1 nix-instantiate --eval -E '{}' -A '1' | grepQuiet "should be a list" +expectStderr 1 nix-instantiate --eval -E '{}' -A '.' | grepQuiet "empty attribute name" +expectStderr 1 nix-instantiate --eval -E '[]' -A '1' | grepQuiet "out of range" diff --git a/tests/multiple-outputs.nix b/tests/functional/multiple-outputs.nix similarity index 100% rename from tests/multiple-outputs.nix rename to tests/functional/multiple-outputs.nix diff --git a/tests/multiple-outputs.sh b/tests/functional/multiple-outputs.sh similarity index 100% rename from tests/multiple-outputs.sh rename to tests/functional/multiple-outputs.sh diff --git a/tests/nar-access.nix b/tests/functional/nar-access.nix similarity index 100% rename from tests/nar-access.nix rename to tests/functional/nar-access.nix diff --git a/tests/nar-access.sh b/tests/functional/nar-access.sh similarity index 90% rename from tests/nar-access.sh rename to tests/functional/nar-access.sh index d487d58d2391..87981e7d9024 100644 --- a/tests/nar-access.sh +++ b/tests/functional/nar-access.sh @@ -25,6 +25,11 @@ diff -u baz.cat-nar $storePath/foo/baz nix store cat $storePath/foo/baz > baz.cat-nar diff -u baz.cat-nar $storePath/foo/baz +# Check that 'nix store cat' fails on invalid store paths. +invalidPath="$(dirname $storePath)/99999999999999999999999999999999-foo" +cp -r $storePath $invalidPath +expect 1 nix store cat $invalidPath/foo/baz + # Test --json. diff -u \ <(nix nar ls --json $narFile / | jq -S) \ @@ -46,7 +51,7 @@ diff -u \ <(echo '{"type":"regular","size":0}' | jq -S) # Test missing files. -expect 1 nix store ls --json -R $storePath/xyzzy 2>&1 | grep 'does not exist in NAR' +expect 1 nix store ls --json -R $storePath/xyzzy 2>&1 | grep 'does not exist' expect 1 nix store ls $storePath/xyzzy 2>&1 | grep 'does not exist' # Test failure to dump. diff --git a/tests/functional/nested-sandboxing.sh b/tests/functional/nested-sandboxing.sh new file mode 100644 index 000000000000..61fe043c6a80 --- /dev/null +++ b/tests/functional/nested-sandboxing.sh @@ -0,0 +1,11 @@ +source common.sh +# This test is run by `tests/functional/nested-sandboxing/runner.nix` in an extra layer of sandboxing. +[[ -d /nix/store ]] || skipTest "running this test without Nix's deps being drawn from /nix/store is not yet supported" + +requireSandboxSupport + +source ./nested-sandboxing/command.sh + +expectStderr 100 runNixBuild badStoreUrl 2 | grepQuiet '`sandbox-build-dir` must not contain' + +runNixBuild goodStoreUrl 5 diff --git a/tests/functional/nested-sandboxing/command.sh b/tests/functional/nested-sandboxing/command.sh new file mode 100644 index 000000000000..69366486cec0 --- /dev/null +++ b/tests/functional/nested-sandboxing/command.sh @@ -0,0 +1,29 @@ +export NIX_BIN_DIR=$(dirname $(type -p nix)) +# TODO Get Nix and its closure more flexibly +export EXTRA_SANDBOX="/nix/store $(dirname $NIX_BIN_DIR)" + +badStoreUrl () { + local altitude=$1 + echo $TEST_ROOT/store-$altitude +} + +goodStoreUrl () { + local altitude=$1 + echo $("badStoreUrl" "$altitude")?store=/foo-$altitude +} + +# The non-standard sandbox-build-dir helps ensure that we get the same behavior +# whether this test is being run in a derivation as part of the nix build or +# being manually run by a developer outside a derivation +runNixBuild () { + local storeFun=$1 + local altitude=$2 + nix-build \ + --no-substitute --no-out-link \ + --store "$("$storeFun" "$altitude")" \ + --extra-sandbox-paths "$EXTRA_SANDBOX" \ + ./nested-sandboxing/runner.nix \ + --arg altitude "$((altitude - 1))" \ + --argstr storeFun "$storeFun" \ + --sandbox-build-dir /build-non-standard +} diff --git a/tests/functional/nested-sandboxing/runner.nix b/tests/functional/nested-sandboxing/runner.nix new file mode 100644 index 000000000000..9a5822c881ac --- /dev/null +++ b/tests/functional/nested-sandboxing/runner.nix @@ -0,0 +1,24 @@ +{ altitude, storeFun }: + +with import ../config.nix; + +mkDerivation { + name = "nested-sandboxing"; + busybox = builtins.getEnv "busybox"; + EXTRA_SANDBOX = builtins.getEnv "EXTRA_SANDBOX"; + buildCommand = if altitude == 0 then '' + echo Deep enough! > $out + '' else '' + cp -r ${../common} ./common + cp ${../common.sh} ./common.sh + cp ${../config.nix} ./config.nix + cp -r ${./.} ./nested-sandboxing + + export PATH=${builtins.getEnv "NIX_BIN_DIR"}:$PATH + + source common.sh + source ./nested-sandboxing/command.sh + + runNixBuild ${storeFun} ${toString altitude} >> $out + ''; +} diff --git a/tests/nix-build-examples.nix b/tests/functional/nix-build-examples.nix similarity index 100% rename from tests/nix-build-examples.nix rename to tests/functional/nix-build-examples.nix diff --git a/tests/nix-build.sh b/tests/functional/nix-build.sh similarity index 100% rename from tests/nix-build.sh rename to tests/functional/nix-build.sh diff --git a/tests/nix-channel.sh b/tests/functional/nix-channel.sh similarity index 100% rename from tests/nix-channel.sh rename to tests/functional/nix-channel.sh diff --git a/tests/nix-collect-garbage-d.sh b/tests/functional/nix-collect-garbage-d.sh similarity index 100% rename from tests/nix-collect-garbage-d.sh rename to tests/functional/nix-collect-garbage-d.sh diff --git a/tests/functional/nix-copy-ssh-ng.sh b/tests/functional/nix-copy-ssh-ng.sh new file mode 100644 index 000000000000..463b5e0c4bbd --- /dev/null +++ b/tests/functional/nix-copy-ssh-ng.sh @@ -0,0 +1,18 @@ +source common.sh + +clearStore +clearCache + +remoteRoot=$TEST_ROOT/store2 +chmod -R u+w "$remoteRoot" || true +rm -rf "$remoteRoot" + +outPath=$(nix-build --no-out-link dependencies.nix) + +nix store info --store "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" + +# Regression test for https://github.com/NixOS/nix/issues/6253 +nix copy --to "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath --no-check-sigs & +nix copy --to "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath --no-check-sigs + +[ -f $remoteRoot$outPath/foobar ] diff --git a/tests/nix-copy-ssh.sh b/tests/functional/nix-copy-ssh.sh similarity index 100% rename from tests/nix-copy-ssh.sh rename to tests/functional/nix-copy-ssh.sh diff --git a/tests/nix-daemon-untrusting.sh b/tests/functional/nix-daemon-untrusting.sh similarity index 100% rename from tests/nix-daemon-untrusting.sh rename to tests/functional/nix-daemon-untrusting.sh diff --git a/tests/nix-profile.sh b/tests/functional/nix-profile.sh similarity index 97% rename from tests/nix-profile.sh rename to tests/functional/nix-profile.sh index 9da3f802be41..7c478a0cdd1f 100644 --- a/tests/nix-profile.sh +++ b/tests/functional/nix-profile.sh @@ -47,8 +47,9 @@ cp ./config.nix $flake1Dir/ # Test upgrading from nix-env. nix-env -f ./user-envs.nix -i foo-1.0 -nix profile list | grep '0 - - .*-foo-1.0' +nix profile list | grep -A2 'Index:.*0' | grep 'Store paths:.*foo-1.0' nix profile install $flake1Dir -L +nix profile list | grep -A4 'Index:.*1' | grep 'Locked flake URL:.*narHash' [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] [ -e $TEST_HOME/.nix-profile/share/man ] (! [ -e $TEST_HOME/.nix-profile/include ]) diff --git a/tests/nix-shell.sh b/tests/functional/nix-shell.sh similarity index 96% rename from tests/nix-shell.sh rename to tests/functional/nix-shell.sh index edaa1249bce2..13403fadb829 100644 --- a/tests/nix-shell.sh +++ b/tests/functional/nix-shell.sh @@ -84,6 +84,11 @@ chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.rb abc ruby) [ "$output" = '-e load(ARGV.shift) -- '"$TEST_ROOT"'/spaced \'\''"shell.shebang.rb abc ruby' ] +# Test nix-shell shebang quoting +sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.nix > $TEST_ROOT/shell.shebang.nix +chmod a+rx $TEST_ROOT/shell.shebang.nix +$TEST_ROOT/shell.shebang.nix + # Test 'nix develop'. nix develop -f "$shellDotNix" shellDrv -c bash -c '[[ -n $stdenv ]]' diff --git a/tests/nix_path.sh b/tests/functional/nix_path.sh similarity index 100% rename from tests/nix_path.sh rename to tests/functional/nix_path.sh diff --git a/tests/optimise-store.sh b/tests/functional/optimise-store.sh similarity index 100% rename from tests/optimise-store.sh rename to tests/functional/optimise-store.sh diff --git a/tests/output-normalization.sh b/tests/functional/output-normalization.sh similarity index 100% rename from tests/output-normalization.sh rename to tests/functional/output-normalization.sh diff --git a/tests/parallel.builder.sh b/tests/functional/parallel.builder.sh similarity index 100% rename from tests/parallel.builder.sh rename to tests/functional/parallel.builder.sh diff --git a/tests/parallel.nix b/tests/functional/parallel.nix similarity index 100% rename from tests/parallel.nix rename to tests/functional/parallel.nix diff --git a/tests/parallel.sh b/tests/functional/parallel.sh similarity index 100% rename from tests/parallel.sh rename to tests/functional/parallel.sh diff --git a/tests/pass-as-file.sh b/tests/functional/pass-as-file.sh similarity index 100% rename from tests/pass-as-file.sh rename to tests/functional/pass-as-file.sh diff --git a/tests/path-from-hash-part.sh b/tests/functional/path-from-hash-part.sh similarity index 100% rename from tests/path-from-hash-part.sh rename to tests/functional/path-from-hash-part.sh diff --git a/tests/functional/path-info.sh b/tests/functional/path-info.sh new file mode 100644 index 000000000000..763935eb71ed --- /dev/null +++ b/tests/functional/path-info.sh @@ -0,0 +1,23 @@ +source common.sh + +echo foo > $TEST_ROOT/foo +foo=$(nix store add-file $TEST_ROOT/foo) + +echo bar > $TEST_ROOT/bar +bar=$(nix store add-file $TEST_ROOT/bar) + +echo baz > $TEST_ROOT/baz +baz=$(nix store add-file $TEST_ROOT/baz) +nix-store --delete "$baz" + +diff --unified --color=always \ + <(nix path-info --json "$foo" "$bar" "$baz" | + jq --sort-keys 'map_values(.narHash)') \ + <(jq --sort-keys <<-EOF + { + "$foo": "sha256-QvtAMbUl/uvi+LCObmqOhvNOapHdA2raiI4xG5zI5pA=", + "$bar": "sha256-9fhYGu9fqxcQC2Kc81qh2RMo1QcLBUBo8U+pPn+jthQ=", + "$baz": null + } +EOF + ) diff --git a/tests/path.nix b/tests/functional/path.nix similarity index 100% rename from tests/path.nix rename to tests/functional/path.nix diff --git a/tests/placeholders.sh b/tests/functional/placeholders.sh similarity index 100% rename from tests/placeholders.sh rename to tests/functional/placeholders.sh diff --git a/tests/plugins.sh b/tests/functional/plugins.sh similarity index 100% rename from tests/plugins.sh rename to tests/functional/plugins.sh diff --git a/tests/plugins/local.mk b/tests/functional/plugins/local.mk similarity index 100% rename from tests/plugins/local.mk rename to tests/functional/plugins/local.mk diff --git a/tests/plugins/plugintest.cc b/tests/functional/plugins/plugintest.cc similarity index 100% rename from tests/plugins/plugintest.cc rename to tests/functional/plugins/plugintest.cc diff --git a/tests/post-hook.sh b/tests/functional/post-hook.sh similarity index 100% rename from tests/post-hook.sh rename to tests/functional/post-hook.sh diff --git a/tests/pure-eval.nix b/tests/functional/pure-eval.nix similarity index 100% rename from tests/pure-eval.nix rename to tests/functional/pure-eval.nix diff --git a/tests/pure-eval.sh b/tests/functional/pure-eval.sh similarity index 100% rename from tests/pure-eval.sh rename to tests/functional/pure-eval.sh diff --git a/tests/push-to-store-old.sh b/tests/functional/push-to-store-old.sh similarity index 100% rename from tests/push-to-store-old.sh rename to tests/functional/push-to-store-old.sh diff --git a/tests/push-to-store.sh b/tests/functional/push-to-store.sh similarity index 100% rename from tests/push-to-store.sh rename to tests/functional/push-to-store.sh diff --git a/tests/read-only-store.sh b/tests/functional/read-only-store.sh similarity index 100% rename from tests/read-only-store.sh rename to tests/functional/read-only-store.sh diff --git a/tests/readfile-context.nix b/tests/functional/readfile-context.nix similarity index 100% rename from tests/readfile-context.nix rename to tests/functional/readfile-context.nix diff --git a/tests/readfile-context.sh b/tests/functional/readfile-context.sh similarity index 100% rename from tests/readfile-context.sh rename to tests/functional/readfile-context.sh diff --git a/tests/recursive.nix b/tests/functional/recursive.nix similarity index 100% rename from tests/recursive.nix rename to tests/functional/recursive.nix diff --git a/tests/recursive.sh b/tests/functional/recursive.sh similarity index 100% rename from tests/recursive.sh rename to tests/functional/recursive.sh diff --git a/tests/referrers.sh b/tests/functional/referrers.sh similarity index 100% rename from tests/referrers.sh rename to tests/functional/referrers.sh diff --git a/tests/remote-store.sh b/tests/functional/remote-store.sh similarity index 78% rename from tests/remote-store.sh rename to tests/functional/remote-store.sh index ea32a20d30ea..5c7bfde46bae 100644 --- a/tests/remote-store.sh +++ b/tests/functional/remote-store.sh @@ -5,17 +5,17 @@ clearStore # Ensure "fake ssh" remote store works just as legacy fake ssh would. nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store doctor -# Ensure that store ping trusted works with ssh-ng:// -nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store store ping --json | jq -e '.trusted' +# Ensure that store info trusted works with ssh-ng:// +nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store store info --json | jq -e '.trusted' startDaemon if isDaemonNewer "2.15pre0"; then # Ensure that ping works trusted with new daemon - nix store ping --json | jq -e '.trusted' + nix store info --json | jq -e '.trusted' else # And the the field is absent with the old daemon - nix store ping --json | jq -e 'has("trusted") | not' + nix store info --json | jq -e 'has("trusted") | not' fi # Test import-from-derivation through the daemon. @@ -24,7 +24,7 @@ fi import ( mkDerivation { name = "foo"; - bla = import ./dependencies.nix; + bla = import ./dependencies.nix {}; buildCommand = " echo \\\"hi\\\" > $out "; diff --git a/tests/repair.sh b/tests/functional/repair.sh similarity index 100% rename from tests/repair.sh rename to tests/functional/repair.sh diff --git a/tests/repl.sh b/tests/functional/repl.sh similarity index 62% rename from tests/repl.sh rename to tests/functional/repl.sh index f0790cbc73c4..991da4214dca 100644 --- a/tests/repl.sh +++ b/tests/functional/repl.sh @@ -54,11 +54,17 @@ testRepl # Same thing (kind-of), but with a remote store. testRepl --store "$TEST_ROOT/repl-store?real=$NIX_STORE_DIR" -testReplResponse () { +# Remove ANSI escape sequences. They can prevent grep from finding a match. +stripColors () { + sed -E 's/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g' +} + +testReplResponseGeneral () { + local grepMode="$1"; shift local commands="$1"; shift local expectedResponse="$1"; shift - local response="$(nix repl "$@" <<< "$commands")" - echo "$response" | grepQuiet -s "$expectedResponse" \ + local response="$(nix repl "$@" <<< "$commands" | stripColors)" + echo "$response" | grepQuiet "$grepMode" -s "$expectedResponse" \ || fail "repl command set: $commands @@ -69,7 +75,16 @@ $expectedResponse but with: -$response" +$response +" +} + +testReplResponse () { + testReplResponseGeneral --basic-regexp "$@" +} + +testReplResponseNoRegex () { + testReplResponseGeneral --fixed-strings "$@" } # :a uses the newest version of a symbol @@ -83,25 +98,20 @@ testReplResponse ' # note the escaped \, # \\ # because the second argument is a regex -testReplResponse ' +testReplResponseNoRegex ' "$" + "{hi}" -' '"\\${hi}"' +' '"\${hi}"' testReplResponse ' drvPath ' '".*-simple.drv"' \ -$testDir/simple.nix +--file $testDir/simple.nix testReplResponse ' drvPath ' '".*-simple.drv"' \ --file $testDir/simple.nix --experimental-features 'ca-derivations' -testReplResponse ' -drvPath -' '".*-simple.drv"' \ ---file $testDir/simple.nix --extra-experimental-features 'repl-flake ca-derivations' - mkdir -p flake && cat < flake/flake.nix { outputs = { self }: { @@ -115,7 +125,7 @@ EOF testReplResponse ' foo + baz ' "3" \ - ./flake ./flake\#bar --experimental-features 'flakes repl-flake' + ./flake ./flake\#bar --experimental-features 'flakes' # Test the `:reload` mechansim with flakes: # - Eval `./flake#changingThing` @@ -128,6 +138,37 @@ sleep 1 # Leave the repl the time to eval 'foo' sed -i 's/beforeChange/afterChange/' flake/flake.nix echo ":reload" echo "changingThing" -) | nix repl ./flake --experimental-features 'flakes repl-flake') +) | nix repl ./flake --experimental-features 'flakes') echo "$replResult" | grepQuiet -s beforeChange echo "$replResult" | grepQuiet -s afterChange + +# Test recursive printing and formatting +# Normal output should print attributes in lexicographical order non-recursively +testReplResponseNoRegex ' +{ a = { b = 2; }; l = [ 1 2 3 ]; s = "string"; n = 1234; x = rec { y = { z = { inherit y; }; }; }; } +' '{ a = { ... }; l = [ ... ]; n = 1234; s = "string"; x = { ... }; }' + +# Same for lists, but order is preserved +testReplResponseNoRegex ' +[ 42 1 "thingy" ({ a = 1; }) ([ 1 2 3 ]) ] +' '[ 42 1 "thingy" { ... } [ ... ] ]' + +# Same for let expressions +testReplResponseNoRegex ' +let x = { y = { a = 1; }; inherit x; }; in x +' '{ x = { ... }; y = { ... }; }' + +# The :p command should recursively print sets, but prevent infinite recursion +testReplResponseNoRegex ' +:p { a = { b = 2; }; s = "string"; n = 1234; x = rec { y = { z = { inherit y; }; }; }; } +' '{ a = { b = 2; }; n = 1234; s = "string"; x = { y = { z = { y = «repeated»; }; }; }; }' + +# Same for lists +testReplResponseNoRegex ' +:p [ 42 1 "thingy" (rec { a = 1; b = { inherit a; inherit b; }; }) ([ 1 2 3 ]) ] +' '[ 42 1 "thingy" { a = 1; b = { a = 1; b = «repeated»; }; } [ 1 2 3 ] ]' + +# Same for let expressions +testReplResponseNoRegex ' +:p let x = { y = { a = 1; }; inherit x; }; in x +' '{ x = { x = «repeated»; y = { a = 1; }; }; y = «repeated»; }' diff --git a/tests/restricted.nix b/tests/functional/restricted.nix similarity index 100% rename from tests/restricted.nix rename to tests/functional/restricted.nix diff --git a/tests/restricted.sh b/tests/functional/restricted.sh similarity index 91% rename from tests/restricted.sh rename to tests/functional/restricted.sh index 776893a561f7..197ae7a10fbf 100644 --- a/tests/restricted.sh +++ b/tests/functional/restricted.sh @@ -9,10 +9,10 @@ nix-instantiate --restrict-eval ./simple.nix -I src=. nix-instantiate --restrict-eval ./simple.nix -I src1=simple.nix -I src2=config.nix -I src3=./simple.builder.sh (! nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix') -nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I src=.. +nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I src=../.. -(! nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../src/nix-channel') -nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../src/nix-channel' -I src=../src +(! nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel') +nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel' -I src=../../src (! nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in ') nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in ' -I src=. @@ -49,3 +49,5 @@ output="$(nix eval --raw --restrict-eval -I "$traverseDir" \ 2>&1 || :)" echo "$output" | grep "is forbidden" echo "$output" | grepInverse -F restricted-secret + +expectStderr 1 nix-instantiate --restrict-eval true ./dependencies.nix | grepQuiet "forbidden in restricted mode" diff --git a/tests/search.nix b/tests/functional/search.nix similarity index 100% rename from tests/search.nix rename to tests/functional/search.nix diff --git a/tests/search.sh b/tests/functional/search.sh similarity index 100% rename from tests/search.sh rename to tests/functional/search.sh diff --git a/tests/secure-drv-outputs.nix b/tests/functional/secure-drv-outputs.nix similarity index 100% rename from tests/secure-drv-outputs.nix rename to tests/functional/secure-drv-outputs.nix diff --git a/tests/secure-drv-outputs.sh b/tests/functional/secure-drv-outputs.sh similarity index 100% rename from tests/secure-drv-outputs.sh rename to tests/functional/secure-drv-outputs.sh diff --git a/tests/selfref-gc.sh b/tests/functional/selfref-gc.sh similarity index 100% rename from tests/selfref-gc.sh rename to tests/functional/selfref-gc.sh diff --git a/tests/shell-hello.nix b/tests/functional/shell-hello.nix similarity index 100% rename from tests/shell-hello.nix rename to tests/functional/shell-hello.nix diff --git a/tests/shell.nix b/tests/functional/shell.nix similarity index 100% rename from tests/shell.nix rename to tests/functional/shell.nix diff --git a/tests/shell.sh b/tests/functional/shell.sh similarity index 100% rename from tests/shell.sh rename to tests/functional/shell.sh diff --git a/tests/functional/shell.shebang.nix b/tests/functional/shell.shebang.nix new file mode 100755 index 000000000000..08e43d53c468 --- /dev/null +++ b/tests/functional/shell.shebang.nix @@ -0,0 +1,10 @@ +#! @ENV_PROG@ nix-shell +#! nix-shell -I nixpkgs=shell.nix --no-substitute +#! nix-shell --argstr s1 'foo "bar" \baz'"'"'qux' --argstr s2 "foo 'bar' \"\baz" --argstr s3 \foo\ bar\'baz --argstr s4 '' +#! nix-shell shell.shebang.nix --command true +{ s1, s2, s3, s4 }: +assert s1 == ''foo "bar" \baz'qux''; +assert s2 == "foo 'bar' \"baz"; +assert s3 == "foo bar'baz"; +assert s4 == ""; +(import {}).runCommand "nix-shell" {} "" diff --git a/tests/shell.shebang.rb b/tests/functional/shell.shebang.rb similarity index 100% rename from tests/shell.shebang.rb rename to tests/functional/shell.shebang.rb diff --git a/tests/shell.shebang.sh b/tests/functional/shell.shebang.sh similarity index 100% rename from tests/shell.shebang.sh rename to tests/functional/shell.shebang.sh diff --git a/tests/signing.sh b/tests/functional/signing.sh similarity index 94% rename from tests/signing.sh rename to tests/functional/signing.sh index 9b673c609690..942b516306d7 100644 --- a/tests/signing.sh +++ b/tests/functional/signing.sh @@ -84,6 +84,10 @@ info=$(nix path-info --store file://$cacheDir --json $outPath2) # Copying to a diverted store should fail due to a lack of signatures by trusted keys. chmod -R u+w $TEST_ROOT/store0 || true rm -rf $TEST_ROOT/store0 + +# Fails or very flaky only on GHA + macOS: +# expectStderr 1 nix copy --to $TEST_ROOT/store0 $outPath | grepQuiet -E 'cannot add path .* because it lacks a signature by a trusted key' +# but this works: (! nix copy --to $TEST_ROOT/store0 $outPath) # But succeed if we supply the public keys. diff --git a/tests/simple-failing.nix b/tests/functional/simple-failing.nix similarity index 100% rename from tests/simple-failing.nix rename to tests/functional/simple-failing.nix diff --git a/tests/simple.builder.sh b/tests/functional/simple.builder.sh similarity index 100% rename from tests/simple.builder.sh rename to tests/functional/simple.builder.sh diff --git a/tests/simple.nix b/tests/functional/simple.nix similarity index 100% rename from tests/simple.nix rename to tests/functional/simple.nix diff --git a/tests/simple.sh b/tests/functional/simple.sh similarity index 100% rename from tests/simple.sh rename to tests/functional/simple.sh diff --git a/tests/ssh-relay.sh b/tests/functional/ssh-relay.sh similarity index 100% rename from tests/ssh-relay.sh rename to tests/functional/ssh-relay.sh diff --git a/tests/store-ping.sh b/tests/functional/store-info.sh similarity index 69% rename from tests/store-ping.sh rename to tests/functional/store-info.sh index 9846c7d3d120..c002e50bebae 100644 --- a/tests/store-ping.sh +++ b/tests/functional/store-info.sh @@ -1,7 +1,7 @@ source common.sh -STORE_INFO=$(nix store ping 2>&1) -STORE_INFO_JSON=$(nix store ping --json) +STORE_INFO=$(nix store info 2>&1) +STORE_INFO_JSON=$(nix store info --json) echo "$STORE_INFO" | grep "Store URL: ${NIX_REMOTE}" @@ -11,7 +11,7 @@ if [[ -v NIX_DAEMON_PACKAGE ]] && isDaemonNewer "2.7.0pre20220126"; then [[ "$(echo "$STORE_INFO_JSON" | jq -r ".version")" == "$DAEMON_VERSION" ]] fi -expect 127 NIX_REMOTE=unix:$PWD/store nix store ping || \ - fail "nix store ping on a non-existent store should fail" +expect 127 NIX_REMOTE=unix:$PWD/store nix store info || \ + fail "nix store info on a non-existent store should fail" [[ "$(echo "$STORE_INFO_JSON" | jq -r ".url")" == "${NIX_REMOTE:-local}" ]] diff --git a/tests/structured-attrs-shell.nix b/tests/functional/structured-attrs-shell.nix similarity index 100% rename from tests/structured-attrs-shell.nix rename to tests/functional/structured-attrs-shell.nix diff --git a/tests/structured-attrs.nix b/tests/functional/structured-attrs.nix similarity index 100% rename from tests/structured-attrs.nix rename to tests/functional/structured-attrs.nix diff --git a/tests/structured-attrs.sh b/tests/functional/structured-attrs.sh similarity index 50% rename from tests/structured-attrs.sh rename to tests/functional/structured-attrs.sh index 378dbc73548c..f11992dcd444 100644 --- a/tests/structured-attrs.sh +++ b/tests/functional/structured-attrs.sh @@ -15,9 +15,21 @@ nix-build structured-attrs.nix -A all -o $TEST_ROOT/result export NIX_BUILD_SHELL=$SHELL env NIX_PATH=nixpkgs=shell.nix nix-shell structured-attrs-shell.nix \ - --run 'test -e .attrs.json; test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"' + --run 'test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"' + +nix develop -f structured-attrs-shell.nix -c bash -c 'test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"' # `nix develop` is a slightly special way of dealing with environment vars, it parses # these from a shell-file exported from a derivation. This is to test especially `outputs` # (which is an associative array in thsi case) being fine. nix develop -f structured-attrs-shell.nix -c bash -c 'test -n "$out"' + +nix print-dev-env -f structured-attrs-shell.nix | grepQuiet 'NIX_ATTRS_JSON_FILE=' +nix print-dev-env -f structured-attrs-shell.nix | grepQuiet 'NIX_ATTRS_SH_FILE=' +nix print-dev-env -f shell.nix shellDrv | grepQuietInverse 'NIX_ATTRS_SH_FILE' + +jsonOut="$(nix print-dev-env -f structured-attrs-shell.nix --json)" + +test "$(<<<"$jsonOut" jq '.structuredAttrs|keys|.[]' -r)" = "$(printf ".attrs.json\n.attrs.sh")" + +test "$(<<<"$jsonOut" jq '.variables.out.value' -r)" = "$(<<<"$jsonOut" jq '.structuredAttrs.".attrs.json"' -r | jq -r '.outputs.out')" diff --git a/tests/substitute-with-invalid-ca.sh b/tests/functional/substitute-with-invalid-ca.sh similarity index 100% rename from tests/substitute-with-invalid-ca.sh rename to tests/functional/substitute-with-invalid-ca.sh diff --git a/tests/suggestions.sh b/tests/functional/suggestions.sh similarity index 100% rename from tests/suggestions.sh rename to tests/functional/suggestions.sh diff --git a/tests/functional/supplementary-groups.sh b/tests/functional/supplementary-groups.sh new file mode 100644 index 000000000000..d18fb24147cf --- /dev/null +++ b/tests/functional/supplementary-groups.sh @@ -0,0 +1,37 @@ +source common.sh + +requireSandboxSupport +[[ $busybox =~ busybox ]] || skipTest "no busybox" +if ! command -p -v unshare; then skipTest "Need unshare"; fi +needLocalStore "The test uses --store always so we would just be bypassing the daemon" + +unshare --mount --map-root-user bash < $tarball + (cd $TEST_ROOT && GNUTAR_REPRODUCIBLE= tar --mtime=$tarroot/default.nix --owner=0 --group=0 --numeric-owner --sort=name -c -f - tarball) | $compressor > $tarball nix-env -f file://$tarball -qa --out-path | grepQuiet dependencies @@ -36,6 +37,8 @@ test_tarball() { nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file:///does-not-exist/must-remain-unused/$tarball; narHash = \"$hash\"; })" expectStderr 102 nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" | grep 'NAR hash mismatch in input' + [[ $(nix eval --impure --expr "(fetchTree file://$tarball).lastModified") = 1000000000 ]] + nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" >&2 nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" 2>&1 | grep 'true' diff --git a/tests/test-infra.sh b/tests/functional/test-infra.sh similarity index 100% rename from tests/test-infra.sh rename to tests/functional/test-infra.sh diff --git a/tests/test-libstoreconsumer.sh b/tests/functional/test-libstoreconsumer.sh similarity index 100% rename from tests/test-libstoreconsumer.sh rename to tests/functional/test-libstoreconsumer.sh diff --git a/tests/test-libstoreconsumer/README.md b/tests/functional/test-libstoreconsumer/README.md similarity index 100% rename from tests/test-libstoreconsumer/README.md rename to tests/functional/test-libstoreconsumer/README.md diff --git a/tests/test-libstoreconsumer/local.mk b/tests/functional/test-libstoreconsumer/local.mk similarity index 100% rename from tests/test-libstoreconsumer/local.mk rename to tests/functional/test-libstoreconsumer/local.mk diff --git a/tests/test-libstoreconsumer/main.cc b/tests/functional/test-libstoreconsumer/main.cc similarity index 92% rename from tests/test-libstoreconsumer/main.cc rename to tests/functional/test-libstoreconsumer/main.cc index 31b6d8ef16aa..c61489af69a6 100644 --- a/tests/test-libstoreconsumer/main.cc +++ b/tests/functional/test-libstoreconsumer/main.cc @@ -23,7 +23,7 @@ int main (int argc, char **argv) std::vector paths { DerivedPath::Built { - .drvPath = store->parseStorePath(drvPath), + .drvPath = makeConstantStorePathRef(store->parseStorePath(drvPath)), .outputs = OutputsSpec::Names{"out"} } }; diff --git a/tests/timeout.nix b/tests/functional/timeout.nix similarity index 100% rename from tests/timeout.nix rename to tests/functional/timeout.nix diff --git a/tests/timeout.sh b/tests/functional/timeout.sh similarity index 100% rename from tests/timeout.sh rename to tests/functional/timeout.sh diff --git a/tests/toString-path.sh b/tests/functional/toString-path.sh similarity index 100% rename from tests/toString-path.sh rename to tests/functional/toString-path.sh diff --git a/tests/undefined-variable.nix b/tests/functional/undefined-variable.nix similarity index 100% rename from tests/undefined-variable.nix rename to tests/functional/undefined-variable.nix diff --git a/tests/user-envs-migration.sh b/tests/functional/user-envs-migration.sh similarity index 100% rename from tests/user-envs-migration.sh rename to tests/functional/user-envs-migration.sh diff --git a/tests/user-envs.builder.sh b/tests/functional/user-envs.builder.sh similarity index 100% rename from tests/user-envs.builder.sh rename to tests/functional/user-envs.builder.sh diff --git a/tests/user-envs.nix b/tests/functional/user-envs.nix similarity index 100% rename from tests/user-envs.nix rename to tests/functional/user-envs.nix diff --git a/tests/user-envs.sh b/tests/functional/user-envs.sh similarity index 97% rename from tests/user-envs.sh rename to tests/functional/user-envs.sh index d1260ba0450c..dcd6b1b97a64 100644 --- a/tests/user-envs.sh +++ b/tests/functional/user-envs.sh @@ -26,6 +26,7 @@ nix-env -f ./user-envs.nix -qa --json --out-path | jq -e '.[] | select(.name == .outputName == "out", (.outputs.out | test("'$NIX_STORE_DIR'.*-0\\.1")) ] | all' +nix-env -f ./user-envs.nix -qa --json --drv-path | jq -e '.[] | select(.name == "bar-0.1") | (.drvPath | test("'$NIX_STORE_DIR'.*-0\\.1\\.drv"))' # Query descriptions. nix-env -f ./user-envs.nix -qa '*' --description | grepQuiet silly diff --git a/tests/why-depends.sh b/tests/functional/why-depends.sh similarity index 64% rename from tests/why-depends.sh rename to tests/functional/why-depends.sh index b35a0d1cf39d..9680bf80ebe5 100644 --- a/tests/why-depends.sh +++ b/tests/functional/why-depends.sh @@ -22,3 +22,8 @@ echo "$PRECISE_WHY_DEPENDS_OUTPUT" | grepQuiet input-2 # But only the “precise” one should refer to `reference-to-input-2` echo "$FAST_WHY_DEPENDS_OUTPUT" | grepQuietInverse reference-to-input-2 echo "$PRECISE_WHY_DEPENDS_OUTPUT" | grepQuiet reference-to-input-2 + +<<<"$PRECISE_WHY_DEPENDS_OUTPUT" sed -n '2p' | grepQuiet "└───reference-to-input-2 -> " +<<<"$PRECISE_WHY_DEPENDS_OUTPUT" sed -n '3p' | grep " →" | grepQuiet "dependencies-input-2" +<<<"$PRECISE_WHY_DEPENDS_OUTPUT" sed -n '4p' | grepQuiet " └───input0: …" # in input-2, file input0 +<<<"$PRECISE_WHY_DEPENDS_OUTPUT" sed -n '5p' | grep " →" | grepQuiet "dependencies-input-0" # is dependencies-input-0 referenced diff --git a/tests/zstd.sh b/tests/functional/zstd.sh similarity index 100% rename from tests/zstd.sh rename to tests/functional/zstd.sh diff --git a/tests/installer/default.nix b/tests/installer/default.nix index 49cfd2bccba3..238c6ac8e8d2 100644 --- a/tests/installer/default.nix +++ b/tests/installer/default.nix @@ -213,7 +213,7 @@ let source /etc/bashrc || true nix-env --version - nix --extra-experimental-features nix-command store ping + nix --extra-experimental-features nix-command store info out=\$(nix-build --no-substitute -E 'derivation { name = "foo"; system = "x86_64-linux"; builder = "/bin/sh"; args = ["-c" "echo foobar > \$out"]; }') [[ \$(cat \$out) = foobar ]] diff --git a/tests/lang.sh b/tests/lang.sh deleted file mode 100644 index 8170cb39d378..000000000000 --- a/tests/lang.sh +++ /dev/null @@ -1,82 +0,0 @@ -source common.sh - -export TEST_VAR=foo # for eval-okay-getenv.nix -export NIX_REMOTE=dummy:// -export NIX_STORE_DIR=/nix/store - -nix-instantiate --eval -E 'builtins.trace "Hello" 123' 2>&1 | grepQuiet Hello -nix-instantiate --eval -E 'builtins.trace "Hello" 123' 2>/dev/null | grepQuiet 123 -nix-instantiate --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 -nix-instantiate --trace-verbose --eval -E 'builtins.traceVerbose "Hello" 123' 2>&1 | grepQuiet Hello -nix-instantiate --eval -E 'builtins.traceVerbose "Hello" 123' 2>&1 | grepQuietInverse Hello -nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 | grepQuietInverse Hello -expectStderr 1 nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" (throw "Foo")' | grepQuiet Hello - -nix-instantiate --eval -E 'let x = builtins.trace { x = x; } true; in x' \ - 2>&1 | grepQuiet -E 'trace: { x = «potential infinite recursion»; }' - -nix-instantiate --eval -E 'let x = { repeating = x; tracing = builtins.trace x true; }; in x.tracing'\ - 2>&1 | grepQuiet -F 'trace: { repeating = «repeated»; tracing = «potential infinite recursion»; }' - -set +x - -fail=0 - -for i in lang/parse-fail-*.nix; do - echo "parsing $i (should fail)"; - i=$(basename $i .nix) - if ! expect 1 nix-instantiate --parse - < lang/$i.nix; then - echo "FAIL: $i shouldn't parse" - fail=1 - fi -done - -for i in lang/parse-okay-*.nix; do - echo "parsing $i (should succeed)"; - i=$(basename $i .nix) - if ! expect 0 nix-instantiate --parse - < lang/$i.nix > lang/$i.out; then - echo "FAIL: $i should parse" - fail=1 - fi -done - -for i in lang/eval-fail-*.nix; do - echo "evaluating $i (should fail)"; - i=$(basename $i .nix) - if ! expect 1 nix-instantiate --eval lang/$i.nix; then - echo "FAIL: $i shouldn't evaluate" - fail=1 - fi -done - -for i in lang/eval-okay-*.nix; do - echo "evaluating $i (should succeed)"; - i=$(basename $i .nix) - - if test -e lang/$i.exp; then - flags= - if test -e lang/$i.flags; then - flags=$(cat lang/$i.flags) - fi - if ! expect 0 env NIX_PATH=lang/dir3:lang/dir4 HOME=/fake-home nix-instantiate $flags --eval --strict lang/$i.nix > lang/$i.out; then - echo "FAIL: $i should evaluate" - fail=1 - elif ! diff <(< lang/$i.out sed -e "s|$(pwd)|/pwd|g") lang/$i.exp; then - echo "FAIL: evaluation result of $i not as expected" - fail=1 - fi - fi - - if test -e lang/$i.exp.xml; then - if ! expect 0 nix-instantiate --eval --xml --no-location --strict \ - lang/$i.nix > lang/$i.out.xml; then - echo "FAIL: $i should evaluate" - fail=1 - elif ! cmp -s lang/$i.out.xml lang/$i.exp.xml; then - echo "FAIL: XML evaluation result of $i not as expected" - fail=1 - fi - fi -done - -exit $fail diff --git a/tests/lang/eval-okay-context-introspection.exp b/tests/lang/eval-okay-context-introspection.exp deleted file mode 100644 index 03b400cc8862..000000000000 --- a/tests/lang/eval-okay-context-introspection.exp +++ /dev/null @@ -1 +0,0 @@ -[ true true true true true true ] diff --git a/tests/lang/eval-okay-path.exp b/tests/lang/eval-okay-path.exp deleted file mode 100644 index 3ce7f828305d..000000000000 --- a/tests/lang/eval-okay-path.exp +++ /dev/null @@ -1 +0,0 @@ -"/nix/store/ya937r4ydw0l6kayq8jkyqaips9c75jm-output" diff --git a/tests/lang/eval-okay-path.nix b/tests/lang/eval-okay-path.nix deleted file mode 100644 index e67168cf3edf..000000000000 --- a/tests/lang/eval-okay-path.nix +++ /dev/null @@ -1,7 +0,0 @@ -builtins.path - { path = ./.; - filter = path: _: baseNameOf path == "data"; - recursive = true; - sha256 = "1yhm3gwvg5a41yylymgblsclk95fs6jy72w0wv925mmidlhcq4sw"; - name = "output"; - } diff --git a/tests/lang/eval-okay-pathexists.nix b/tests/lang/eval-okay-pathexists.nix deleted file mode 100644 index 50c28ee0cd30..000000000000 --- a/tests/lang/eval-okay-pathexists.nix +++ /dev/null @@ -1,5 +0,0 @@ -builtins.pathExists (builtins.toPath ./lib.nix) -&& builtins.pathExists (builtins.toPath (builtins.toString ./lib.nix)) -&& !builtins.pathExists (builtins.toPath (builtins.toString ./bla.nix)) -&& builtins.pathExists ./lib.nix -&& !builtins.pathExists ./bla.nix diff --git a/tests/lang/eval-okay-search-path.flags b/tests/lang/eval-okay-search-path.flags deleted file mode 100644 index a28e6821004a..000000000000 --- a/tests/lang/eval-okay-search-path.flags +++ /dev/null @@ -1 +0,0 @@ --I lang/dir1 -I lang/dir2 -I dir5=lang/dir3 \ No newline at end of file diff --git a/tests/legacy-ssh-store.sh b/tests/legacy-ssh-store.sh deleted file mode 100644 index 71b716b8453b..000000000000 --- a/tests/legacy-ssh-store.sh +++ /dev/null @@ -1,4 +0,0 @@ -source common.sh - -# Check that store ping trusted doesn't yet work with ssh:// -nix --store ssh://localhost?remote-store=$TEST_ROOT/other-store store ping --json | jq -e 'has("trusted") | not' diff --git a/tests/nixos/containers/systemd-nspawn.nix b/tests/nixos/containers/systemd-nspawn.nix index f54f32f2af5f..1dad4ebd7545 100644 --- a/tests/nixos/containers/systemd-nspawn.nix +++ b/tests/nixos/containers/systemd-nspawn.nix @@ -73,6 +73,8 @@ runCommand "test" --resolv-conf=off \ --bind-ro=/nix/store \ --bind=$out \ + --bind=/proc:/run/host/proc \ + --bind=/sys:/run/host/sys \ --private-network \ $toplevel/init '' diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix new file mode 100644 index 000000000000..5b6197dad066 --- /dev/null +++ b/tests/nixos/default.nix @@ -0,0 +1,45 @@ +{ lib, nixpkgs, nixpkgsFor }: + +let + + nixos-lib = import (nixpkgs + "/nixos/lib") { }; + + # https://nixos.org/manual/nixos/unstable/index.html#sec-calling-nixos-tests + runNixOSTestFor = system: test: nixos-lib.runTest { + imports = [ test ]; + hostPkgs = nixpkgsFor.${system}.native; + defaults = { + nixpkgs.pkgs = nixpkgsFor.${system}.native; + }; + _module.args.nixpkgs = nixpkgs; + }; + +in + +{ + authorization = runNixOSTestFor "x86_64-linux" ./authorization.nix; + + acls = runNixOSTestFor "x86_64-linux" ./acls.nix; + + remoteBuilds = runNixOSTestFor "x86_64-linux" ./remote-builds.nix; + + remoteBuildsSshNg = runNixOSTestFor "x86_64-linux" ./remote-builds-ssh-ng.nix; + + nix-copy-closure = runNixOSTestFor "x86_64-linux" ./nix-copy-closure.nix; + + nix-copy = runNixOSTestFor "x86_64-linux" ./nix-copy.nix; + + nssPreload = runNixOSTestFor "x86_64-linux" ./nss-preload.nix; + + githubFlakes = runNixOSTestFor "x86_64-linux" ./github-flakes.nix; + + sourcehutFlakes = runNixOSTestFor "x86_64-linux" ./sourcehut-flakes.nix; + + tarballFlakes = runNixOSTestFor "x86_64-linux" ./tarball-flakes.nix; + + containers = runNixOSTestFor "x86_64-linux" ./containers/containers.nix; + + setuid = lib.genAttrs + ["i686-linux" "x86_64-linux"] + (system: runNixOSTestFor system ./setuid.nix); +} diff --git a/tests/nixos/github-flakes.nix b/tests/nixos/github-flakes.nix index e4d34769180e..62ae8871b87a 100644 --- a/tests/nixos/github-flakes.nix +++ b/tests/nixos/github-flakes.nix @@ -82,7 +82,7 @@ let dir=NixOS-nixpkgs-${nixpkgs.shortRev} cp -prd ${nixpkgs} $dir # Set the correct timestamp in the tarball. - find $dir -print0 | xargs -0 touch -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} -- + find $dir -print0 | xargs -0 touch -h -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} -- tar cfz $out/archive/${nixpkgs.rev}.tar.gz $dir --hard-dereference ''; in @@ -186,6 +186,10 @@ in client.succeed("nix registry pin nixpkgs") client.succeed("nix flake metadata nixpkgs --tarball-ttl 0 >&2") + # Test fetchTree on a github URL. + hash = client.succeed(f"nix eval --raw --expr '(fetchTree {info['url']}).narHash'") + assert hash == info['locked']['narHash'] + # Shut down the web server. The flake should be cached on the client. github.succeed("systemctl stop httpd.service") diff --git a/tests/nixos/nix-copy.nix b/tests/nixos/nix-copy.nix index 16c477bf9908..2981cc2b80d5 100644 --- a/tests/nixos/nix-copy.nix +++ b/tests/nixos/nix-copy.nix @@ -1,4 +1,6 @@ # Test that ‘nix copy’ works over ssh. +# Run interactively with: +# rm key key.pub; nix run .#hydraJobs.tests.nix-copy.driverInteractive { lib, config, nixpkgs, hostPkgs, ... }: @@ -55,7 +57,9 @@ in { server.wait_for_unit("sshd") client.wait_for_unit("network.target") client.wait_for_unit("getty@tty1.service") - client.wait_for_text("]#") + # Either the prompt: ]# + # or an OCR misreading of it: 1# + client.wait_for_text("[]1]#") # Copy the closure of package A from the client to the server using password authentication, # and check that all prompts are visible @@ -79,6 +83,15 @@ in { server.copy_from_host("key.pub", "/root/.ssh/authorized_keys") server.succeed("systemctl restart sshd") client.succeed(f"ssh -o StrictHostKeyChecking=no {server.name} 'echo hello world'") + client.succeed(f"ssh -O check {server.name}") + client.succeed(f"ssh -O exit {server.name}") + client.fail(f"ssh -O check {server.name}") + + # Check that an explicit master will work + client.succeed(f"ssh -MNfS /tmp/master {server.name}") + client.succeed(f"ssh -S /tmp/master -O check {server.name}") + client.succeed("NIX_SSHOPTS='-oControlPath=/tmp/master' nix copy --to ssh://server ${pkgA} >&2") + client.succeed(f"ssh -S /tmp/master -O exit {server.name}") # Copy the closure of package B from the server to the client, using ssh-ng. client.fail("nix-store --check-validity ${pkgB}") diff --git a/tests/nixos/remote-builds-ssh-ng.nix b/tests/nixos/remote-builds-ssh-ng.nix new file mode 100644 index 000000000000..b59dde9bf1db --- /dev/null +++ b/tests/nixos/remote-builds-ssh-ng.nix @@ -0,0 +1,108 @@ +{ config, lib, hostPkgs, ... }: + +let + pkgs = config.nodes.client.nixpkgs.pkgs; + + # Trivial Nix expression to build remotely. + expr = config: nr: pkgs.writeText "expr.nix" + '' + let utils = builtins.storePath ${config.system.build.extraUtils}; in + derivation { + name = "hello-${toString nr}"; + system = "i686-linux"; + PATH = "''${utils}/bin"; + builder = "''${utils}/bin/sh"; + args = [ "-c" "${ + lib.concatStringsSep "; " [ + ''if [[ -n $NIX_LOG_FD ]]'' + ''then echo '@nix {\"action\":\"setPhase\",\"phase\":\"buildPhase\"}' >&''$NIX_LOG_FD'' + "fi" + "echo Hello" + "mkdir $out" + "cat /proc/sys/kernel/hostname > $out/host" + ] + }" ]; + outputs = [ "out" ]; + } + ''; +in + +{ + name = "remote-builds-ssh-ng"; + + nodes = + { builder = + { config, pkgs, ... }: + { services.openssh.enable = true; + virtualisation.writableStore = true; + nix.settings.sandbox = true; + nix.settings.substituters = lib.mkForce [ ]; + }; + + client = + { config, lib, pkgs, ... }: + { nix.settings.max-jobs = 0; # force remote building + nix.distributedBuilds = true; + nix.buildMachines = + [ { hostName = "builder"; + sshUser = "root"; + sshKey = "/root/.ssh/id_ed25519"; + system = "i686-linux"; + maxJobs = 1; + protocol = "ssh-ng"; + } + ]; + virtualisation.writableStore = true; + virtualisation.additionalPaths = [ config.system.build.extraUtils ]; + nix.settings.substituters = lib.mkForce [ ]; + programs.ssh.extraConfig = "ConnectTimeout 30"; + }; + }; + + testScript = { nodes }: '' + # fmt: off + import subprocess + + start_all() + + # Create an SSH key on the client. + subprocess.run([ + "${hostPkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" + ], capture_output=True, check=True) + client.succeed("mkdir -p -m 700 /root/.ssh") + client.copy_from_host("key", "/root/.ssh/id_ed25519") + client.succeed("chmod 600 /root/.ssh/id_ed25519") + + # Install the SSH key on the builder. + client.wait_for_unit("network.target") + builder.succeed("mkdir -p -m 700 /root/.ssh") + builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys") + builder.wait_for_unit("sshd") + client.succeed(f"ssh -o StrictHostKeyChecking=no {builder.name} 'echo hello world'") + + # Perform a build + out = client.succeed("nix-build ${expr nodes.client.config 1} 2> build-output") + + # Verify that the build was done on the builder + builder.succeed(f"test -e {out.strip()}") + + # Print the build log, prefix the log lines to avoid nix intercepting lines starting with @nix + buildOutput = client.succeed("sed -e 's/^/build-output:/' build-output") + print(buildOutput) + + # Make sure that we get the expected build output + client.succeed("grep -qF Hello build-output") + + # We don't want phase reporting in the build output + client.fail("grep -qF '@nix' build-output") + + # Get the log file + client.succeed(f"nix-store --read-log {out.strip()} > log-output") + # Prefix the log lines to avoid nix intercepting lines starting with @nix + logOutput = client.succeed("sed -e 's/^/log-file:/' log-output") + print(logOutput) + + # Check that we get phase reporting in the log file + client.succeed("grep -q '@nix {\"action\":\"setPhase\",\"phase\":\"buildPhase\"}' log-output") + ''; +} diff --git a/tests/nixos/sourcehut-flakes.nix b/tests/nixos/sourcehut-flakes.nix index a76fed0209f3..6e8d884a01b7 100644 --- a/tests/nixos/sourcehut-flakes.nix +++ b/tests/nixos/sourcehut-flakes.nix @@ -47,7 +47,7 @@ let cp -prd ${nixpkgs} $dir # Set the correct timestamp in the tarball. - find $dir -print0 | xargs -0 touch -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} -- + find $dir -print0 | xargs -0 touch -h -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} -- mkdir -p $out/archive tar cfz $out/archive/${nixpkgs.rev}.tar.gz $dir --hard-dereference diff --git a/tests/nixos/tarball-flakes.nix b/tests/nixos/tarball-flakes.nix index 1d43a5d04d2e..e30d15739e61 100644 --- a/tests/nixos/tarball-flakes.nix +++ b/tests/nixos/tarball-flakes.nix @@ -11,7 +11,7 @@ let dir=nixpkgs-${nixpkgs.shortRev} cp -prd ${nixpkgs} $dir # Set the correct timestamp in the tarball. - find $dir -print0 | xargs -0 touch -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} -- + find $dir -print0 | xargs -0 touch -h -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} -- tar cfz $out/stable/${nixpkgs.rev}.tar.gz $dir --hard-dereference echo 'Redirect "/latest.tar.gz" "/stable/${nixpkgs.rev}.tar.gz"' > $out/.htaccess diff --git a/unit-test-data/libstore/common-protocol/content-address.bin b/unit-test-data/libstore/common-protocol/content-address.bin new file mode 100644 index 000000000000..8f14bcdb3e50 Binary files /dev/null and b/unit-test-data/libstore/common-protocol/content-address.bin differ diff --git a/unit-test-data/libstore/common-protocol/drv-output.bin b/unit-test-data/libstore/common-protocol/drv-output.bin new file mode 100644 index 000000000000..800a45fd8757 Binary files /dev/null and b/unit-test-data/libstore/common-protocol/drv-output.bin differ diff --git a/unit-test-data/libstore/common-protocol/optional-content-address.bin b/unit-test-data/libstore/common-protocol/optional-content-address.bin new file mode 100644 index 000000000000..f8cfe65ba27f Binary files /dev/null and b/unit-test-data/libstore/common-protocol/optional-content-address.bin differ diff --git a/unit-test-data/libstore/common-protocol/optional-store-path.bin b/unit-test-data/libstore/common-protocol/optional-store-path.bin new file mode 100644 index 000000000000..4fbca5576b6d Binary files /dev/null and b/unit-test-data/libstore/common-protocol/optional-store-path.bin differ diff --git a/unit-test-data/libstore/common-protocol/realisation.bin b/unit-test-data/libstore/common-protocol/realisation.bin new file mode 100644 index 000000000000..2176c6c4afd9 Binary files /dev/null and b/unit-test-data/libstore/common-protocol/realisation.bin differ diff --git a/unit-test-data/libstore/common-protocol/set.bin b/unit-test-data/libstore/common-protocol/set.bin new file mode 100644 index 000000000000..ce11ede7fe7a Binary files /dev/null and b/unit-test-data/libstore/common-protocol/set.bin differ diff --git a/unit-test-data/libstore/common-protocol/store-path.bin b/unit-test-data/libstore/common-protocol/store-path.bin new file mode 100644 index 000000000000..3fc05f2981d3 Binary files /dev/null and b/unit-test-data/libstore/common-protocol/store-path.bin differ diff --git a/unit-test-data/libstore/common-protocol/string.bin b/unit-test-data/libstore/common-protocol/string.bin new file mode 100644 index 000000000000..aa7b5a604745 Binary files /dev/null and b/unit-test-data/libstore/common-protocol/string.bin differ diff --git a/unit-test-data/libstore/common-protocol/vector.bin b/unit-test-data/libstore/common-protocol/vector.bin new file mode 100644 index 000000000000..7a37c8cd1093 Binary files /dev/null and b/unit-test-data/libstore/common-protocol/vector.bin differ diff --git a/unit-test-data/libstore/derivation/bad-old-version-dyn-deps.drv b/unit-test-data/libstore/derivation/bad-old-version-dyn-deps.drv new file mode 100644 index 000000000000..3cd1ded029d3 --- /dev/null +++ b/unit-test-data/libstore/derivation/bad-old-version-dyn-deps.drv @@ -0,0 +1 @@ +Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]) \ No newline at end of file diff --git a/unit-test-data/libstore/derivation/bad-version.drv b/unit-test-data/libstore/derivation/bad-version.drv new file mode 100644 index 000000000000..bbf75c114de6 --- /dev/null +++ b/unit-test-data/libstore/derivation/bad-version.drv @@ -0,0 +1 @@ +DrvWithVersion("invalid-version",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]) \ No newline at end of file diff --git a/unit-test-data/libstore/derivation/dynDerivationDeps.drv b/unit-test-data/libstore/derivation/dynDerivationDeps.drv new file mode 100644 index 000000000000..cfffe48ec904 --- /dev/null +++ b/unit-test-data/libstore/derivation/dynDerivationDeps.drv @@ -0,0 +1 @@ +DrvWithVersion("xp-dyn-drv",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]) \ No newline at end of file diff --git a/unit-test-data/libstore/derivation/dynDerivationDeps.json b/unit-test-data/libstore/derivation/dynDerivationDeps.json new file mode 100644 index 000000000000..9dbeb1f15aff --- /dev/null +++ b/unit-test-data/libstore/derivation/dynDerivationDeps.json @@ -0,0 +1,38 @@ +{ + "args": [ + "bar", + "baz" + ], + "builder": "foo", + "env": { + "BIG_BAD": "WOLF" + }, + "inputDrvs": { + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { + "dynamicOutputs": { + "cat": { + "dynamicOutputs": {}, + "outputs": [ + "kitten" + ] + }, + "goose": { + "dynamicOutputs": {}, + "outputs": [ + "gosling" + ] + } + }, + "outputs": [ + "cat", + "dog" + ] + } + }, + "inputSrcs": [ + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + ], + "name": "dyn-dep-derivation", + "outputs": {}, + "system": "wasm-sel4" +} diff --git a/unit-test-data/libstore/derivation/output-caFixedFlat.json b/unit-test-data/libstore/derivation/output-caFixedFlat.json new file mode 100644 index 000000000000..fe000ea36e3d --- /dev/null +++ b/unit-test-data/libstore/derivation/output-caFixedFlat.json @@ -0,0 +1,5 @@ +{ + "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", + "hashAlgo": "sha256", + "path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name" +} diff --git a/unit-test-data/libstore/derivation/output-caFixedNAR.json b/unit-test-data/libstore/derivation/output-caFixedNAR.json new file mode 100644 index 000000000000..1afd602236bd --- /dev/null +++ b/unit-test-data/libstore/derivation/output-caFixedNAR.json @@ -0,0 +1,5 @@ +{ + "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", + "hashAlgo": "r:sha256", + "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" +} diff --git a/unit-test-data/libstore/derivation/output-caFixedText.json b/unit-test-data/libstore/derivation/output-caFixedText.json new file mode 100644 index 000000000000..0b2cc8bbc423 --- /dev/null +++ b/unit-test-data/libstore/derivation/output-caFixedText.json @@ -0,0 +1,5 @@ +{ + "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", + "hashAlgo": "text:sha256", + "path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name" +} diff --git a/unit-test-data/libstore/derivation/output-caFloating.json b/unit-test-data/libstore/derivation/output-caFloating.json new file mode 100644 index 000000000000..9115de851a11 --- /dev/null +++ b/unit-test-data/libstore/derivation/output-caFloating.json @@ -0,0 +1,3 @@ +{ + "hashAlgo": "r:sha256" +} diff --git a/unit-test-data/libstore/derivation/output-deferred.json b/unit-test-data/libstore/derivation/output-deferred.json new file mode 100644 index 000000000000..0967ef424bce --- /dev/null +++ b/unit-test-data/libstore/derivation/output-deferred.json @@ -0,0 +1 @@ +{} diff --git a/unit-test-data/libstore/derivation/output-impure.json b/unit-test-data/libstore/derivation/output-impure.json new file mode 100644 index 000000000000..62b61cdcae71 --- /dev/null +++ b/unit-test-data/libstore/derivation/output-impure.json @@ -0,0 +1,4 @@ +{ + "hashAlgo": "r:sha256", + "impure": true +} diff --git a/unit-test-data/libstore/derivation/output-inputAddressed.json b/unit-test-data/libstore/derivation/output-inputAddressed.json new file mode 100644 index 000000000000..86c7f3a05ce8 --- /dev/null +++ b/unit-test-data/libstore/derivation/output-inputAddressed.json @@ -0,0 +1,3 @@ +{ + "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" +} diff --git a/unit-test-data/libstore/derivation/simple.drv b/unit-test-data/libstore/derivation/simple.drv new file mode 100644 index 000000000000..bda74ad25c01 --- /dev/null +++ b/unit-test-data/libstore/derivation/simple.drv @@ -0,0 +1 @@ +Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]) \ No newline at end of file diff --git a/unit-test-data/libstore/derivation/simple.json b/unit-test-data/libstore/derivation/simple.json new file mode 100644 index 000000000000..20d0f8933e6b --- /dev/null +++ b/unit-test-data/libstore/derivation/simple.json @@ -0,0 +1,25 @@ +{ + "args": [ + "bar", + "baz" + ], + "builder": "foo", + "env": { + "BIG_BAD": "WOLF" + }, + "inputDrvs": { + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { + "dynamicOutputs": {}, + "outputs": [ + "cat", + "dog" + ] + } + }, + "inputSrcs": [ + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + ], + "name": "simple-derivation", + "outputs": {}, + "system": "wasm-sel4" +} diff --git a/unit-test-data/libstore/nar-info/impure.json b/unit-test-data/libstore/nar-info/impure.json new file mode 100644 index 000000000000..bb9791a6ace2 --- /dev/null +++ b/unit-test-data/libstore/nar-info/impure.json @@ -0,0 +1,20 @@ +{ + "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "compression": "xz", + "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "downloadHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "downloadSize": 4029176, + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "references": [ + "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "registrationTime": 23423, + "signatures": [ + "asdf", + "qwer" + ], + "ultimate": true, + "url": "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz" +} diff --git a/unit-test-data/libstore/nar-info/pure.json b/unit-test-data/libstore/nar-info/pure.json new file mode 100644 index 000000000000..955baec3107b --- /dev/null +++ b/unit-test-data/libstore/nar-info/pure.json @@ -0,0 +1,9 @@ +{ + "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "references": [ + "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ] +} diff --git a/unit-test-data/libstore/path-info/impure.json b/unit-test-data/libstore/path-info/impure.json new file mode 100644 index 000000000000..0c452cc49300 --- /dev/null +++ b/unit-test-data/libstore/path-info/impure.json @@ -0,0 +1,16 @@ +{ + "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "references": [ + "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "registrationTime": 23423, + "signatures": [ + "asdf", + "qwer" + ], + "ultimate": true +} diff --git a/unit-test-data/libstore/path-info/pure.json b/unit-test-data/libstore/path-info/pure.json new file mode 100644 index 000000000000..955baec3107b --- /dev/null +++ b/unit-test-data/libstore/path-info/pure.json @@ -0,0 +1,9 @@ +{ + "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "references": [ + "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ] +} diff --git a/unit-test-data/libstore/serve-protocol/build-result-2.2.bin b/unit-test-data/libstore/serve-protocol/build-result-2.2.bin new file mode 100644 index 000000000000..ae684778bc26 Binary files /dev/null and b/unit-test-data/libstore/serve-protocol/build-result-2.2.bin differ diff --git a/unit-test-data/libstore/serve-protocol/build-result-2.3.bin b/unit-test-data/libstore/serve-protocol/build-result-2.3.bin new file mode 100644 index 000000000000..d51e08dfc0d1 Binary files /dev/null and b/unit-test-data/libstore/serve-protocol/build-result-2.3.bin differ diff --git a/unit-test-data/libstore/serve-protocol/build-result-2.6.bin b/unit-test-data/libstore/serve-protocol/build-result-2.6.bin new file mode 100644 index 000000000000..b02c706eab8d Binary files /dev/null and b/unit-test-data/libstore/serve-protocol/build-result-2.6.bin differ diff --git a/unit-test-data/libstore/serve-protocol/content-address.bin b/unit-test-data/libstore/serve-protocol/content-address.bin new file mode 100644 index 000000000000..8f14bcdb3e50 Binary files /dev/null and b/unit-test-data/libstore/serve-protocol/content-address.bin differ diff --git a/unit-test-data/libstore/serve-protocol/drv-output.bin b/unit-test-data/libstore/serve-protocol/drv-output.bin new file mode 100644 index 000000000000..800a45fd8757 Binary files /dev/null and b/unit-test-data/libstore/serve-protocol/drv-output.bin differ diff --git a/unit-test-data/libstore/serve-protocol/optional-content-address.bin b/unit-test-data/libstore/serve-protocol/optional-content-address.bin new file mode 100644 index 000000000000..f8cfe65ba27f Binary files /dev/null and b/unit-test-data/libstore/serve-protocol/optional-content-address.bin differ diff --git a/unit-test-data/libstore/serve-protocol/optional-store-path.bin b/unit-test-data/libstore/serve-protocol/optional-store-path.bin new file mode 100644 index 000000000000..4fbca5576b6d Binary files /dev/null and b/unit-test-data/libstore/serve-protocol/optional-store-path.bin differ diff --git a/unit-test-data/libstore/serve-protocol/realisation.bin b/unit-test-data/libstore/serve-protocol/realisation.bin new file mode 100644 index 000000000000..2176c6c4afd9 Binary files /dev/null and b/unit-test-data/libstore/serve-protocol/realisation.bin differ diff --git a/unit-test-data/libstore/serve-protocol/set.bin b/unit-test-data/libstore/serve-protocol/set.bin new file mode 100644 index 000000000000..ce11ede7fe7a Binary files /dev/null and b/unit-test-data/libstore/serve-protocol/set.bin differ diff --git a/unit-test-data/libstore/serve-protocol/store-path.bin b/unit-test-data/libstore/serve-protocol/store-path.bin new file mode 100644 index 000000000000..3fc05f2981d3 Binary files /dev/null and b/unit-test-data/libstore/serve-protocol/store-path.bin differ diff --git a/unit-test-data/libstore/serve-protocol/string.bin b/unit-test-data/libstore/serve-protocol/string.bin new file mode 100644 index 000000000000..aa7b5a604745 Binary files /dev/null and b/unit-test-data/libstore/serve-protocol/string.bin differ diff --git a/unit-test-data/libstore/serve-protocol/vector.bin b/unit-test-data/libstore/serve-protocol/vector.bin new file mode 100644 index 000000000000..7a37c8cd1093 Binary files /dev/null and b/unit-test-data/libstore/serve-protocol/vector.bin differ diff --git a/unit-test-data/libstore/worker-protocol/build-result-1.27.bin b/unit-test-data/libstore/worker-protocol/build-result-1.27.bin new file mode 100644 index 000000000000..ae684778bc26 Binary files /dev/null and b/unit-test-data/libstore/worker-protocol/build-result-1.27.bin differ diff --git a/unit-test-data/libstore/worker-protocol/build-result-1.28.bin b/unit-test-data/libstore/worker-protocol/build-result-1.28.bin new file mode 100644 index 000000000000..74bcd5cf98b8 Binary files /dev/null and b/unit-test-data/libstore/worker-protocol/build-result-1.28.bin differ diff --git a/unit-test-data/libstore/worker-protocol/build-result-1.29.bin b/unit-test-data/libstore/worker-protocol/build-result-1.29.bin new file mode 100644 index 000000000000..b02c706eab8d Binary files /dev/null and b/unit-test-data/libstore/worker-protocol/build-result-1.29.bin differ diff --git a/unit-test-data/libstore/worker-protocol/content-address.bin b/unit-test-data/libstore/worker-protocol/content-address.bin new file mode 100644 index 000000000000..8f14bcdb3e50 Binary files /dev/null and b/unit-test-data/libstore/worker-protocol/content-address.bin differ diff --git a/unit-test-data/libstore/worker-protocol/derived-path-1.29.bin b/unit-test-data/libstore/worker-protocol/derived-path-1.29.bin new file mode 100644 index 000000000000..05ea7678aa05 Binary files /dev/null and b/unit-test-data/libstore/worker-protocol/derived-path-1.29.bin differ diff --git a/unit-test-data/libstore/worker-protocol/derived-path-1.30.bin b/unit-test-data/libstore/worker-protocol/derived-path-1.30.bin new file mode 100644 index 000000000000..0729b2690e2f Binary files /dev/null and b/unit-test-data/libstore/worker-protocol/derived-path-1.30.bin differ diff --git a/unit-test-data/libstore/worker-protocol/drv-output.bin b/unit-test-data/libstore/worker-protocol/drv-output.bin new file mode 100644 index 000000000000..800a45fd8757 Binary files /dev/null and b/unit-test-data/libstore/worker-protocol/drv-output.bin differ diff --git a/unit-test-data/libstore/worker-protocol/keyed-build-result-1.29.bin b/unit-test-data/libstore/worker-protocol/keyed-build-result-1.29.bin new file mode 100644 index 000000000000..c5b3c7f3669a Binary files /dev/null and b/unit-test-data/libstore/worker-protocol/keyed-build-result-1.29.bin differ diff --git a/unit-test-data/libstore/worker-protocol/optional-content-address.bin b/unit-test-data/libstore/worker-protocol/optional-content-address.bin new file mode 100644 index 000000000000..f8cfe65ba27f Binary files /dev/null and b/unit-test-data/libstore/worker-protocol/optional-content-address.bin differ diff --git a/unit-test-data/libstore/worker-protocol/optional-store-path.bin b/unit-test-data/libstore/worker-protocol/optional-store-path.bin new file mode 100644 index 000000000000..4fbca5576b6d Binary files /dev/null and b/unit-test-data/libstore/worker-protocol/optional-store-path.bin differ diff --git a/unit-test-data/libstore/worker-protocol/optional-trusted-flag.bin b/unit-test-data/libstore/worker-protocol/optional-trusted-flag.bin new file mode 100644 index 000000000000..51b239409bc8 Binary files /dev/null and b/unit-test-data/libstore/worker-protocol/optional-trusted-flag.bin differ diff --git a/unit-test-data/libstore/worker-protocol/realisation.bin b/unit-test-data/libstore/worker-protocol/realisation.bin new file mode 100644 index 000000000000..2176c6c4afd9 Binary files /dev/null and b/unit-test-data/libstore/worker-protocol/realisation.bin differ diff --git a/unit-test-data/libstore/worker-protocol/set.bin b/unit-test-data/libstore/worker-protocol/set.bin new file mode 100644 index 000000000000..ce11ede7fe7a Binary files /dev/null and b/unit-test-data/libstore/worker-protocol/set.bin differ diff --git a/unit-test-data/libstore/worker-protocol/store-path.bin b/unit-test-data/libstore/worker-protocol/store-path.bin new file mode 100644 index 000000000000..3fc05f2981d3 Binary files /dev/null and b/unit-test-data/libstore/worker-protocol/store-path.bin differ diff --git a/unit-test-data/libstore/worker-protocol/string.bin b/unit-test-data/libstore/worker-protocol/string.bin new file mode 100644 index 000000000000..aa7b5a604745 Binary files /dev/null and b/unit-test-data/libstore/worker-protocol/string.bin differ diff --git a/unit-test-data/libstore/worker-protocol/unkeyed-valid-path-info-1.15.bin b/unit-test-data/libstore/worker-protocol/unkeyed-valid-path-info-1.15.bin new file mode 100644 index 000000000000..e69ccbe83862 Binary files /dev/null and b/unit-test-data/libstore/worker-protocol/unkeyed-valid-path-info-1.15.bin differ diff --git a/unit-test-data/libstore/worker-protocol/valid-path-info-1.15.bin b/unit-test-data/libstore/worker-protocol/valid-path-info-1.15.bin new file mode 100644 index 000000000000..7adc8dd4472e Binary files /dev/null and b/unit-test-data/libstore/worker-protocol/valid-path-info-1.15.bin differ diff --git a/unit-test-data/libstore/worker-protocol/valid-path-info-1.16.bin b/unit-test-data/libstore/worker-protocol/valid-path-info-1.16.bin new file mode 100644 index 000000000000..a72de6bd62ab Binary files /dev/null and b/unit-test-data/libstore/worker-protocol/valid-path-info-1.16.bin differ diff --git a/unit-test-data/libstore/worker-protocol/vector.bin b/unit-test-data/libstore/worker-protocol/vector.bin new file mode 100644 index 000000000000..7a37c8cd1093 Binary files /dev/null and b/unit-test-data/libstore/worker-protocol/vector.bin differ diff --git a/unit-test-data/libutil/git/check-data.sh b/unit-test-data/libutil/git/check-data.sh new file mode 100644 index 000000000000..68b705c95c6e --- /dev/null +++ b/unit-test-data/libutil/git/check-data.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/git-hashing/unit-test-data +mkdir -p $TEST_ROOT + +repo="$TEST_ROOT/scratch" +git init "$repo" + +git -C "$repo" config user.email "you@example.com" +git -C "$repo" config user.name "Your Name" + +# `-w` to write for tree test +freshlyAddedHash=$(git -C "$repo" hash-object -w -t blob --stdin < "./hello-world.bin") +encodingHash=$(sha1sum -b < "./hello-world-blob.bin" | head -c 40) + +# If the hashes match, then `hello-world-blob.bin` must be the encoding +# of `hello-world.bin`. +[[ "$encodingHash" == "$freshlyAddedHash" ]] + +# Create empty directory object for tree test +echo -n | git -C "$repo" hash-object -w -t tree --stdin + +# Relies on both child hashes already existing in the git store +freshlyAddedHash=$(git -C "$repo" mktree < "./tree.txt") +encodingHash=$(sha1sum -b < "./tree.bin" | head -c 40) + +# If the hashes match, then `tree.bin` must be the encoding of the +# directory denoted by `tree.txt` interpreted as git directory listing. +[[ "$encodingHash" == "$freshlyAddedHash" ]] diff --git a/unit-test-data/libutil/git/hello-world-blob.bin b/unit-test-data/libutil/git/hello-world-blob.bin new file mode 100644 index 000000000000..255f5df55cce Binary files /dev/null and b/unit-test-data/libutil/git/hello-world-blob.bin differ diff --git a/unit-test-data/libutil/git/hello-world.bin b/unit-test-data/libutil/git/hello-world.bin new file mode 100644 index 000000000000..63ddb340119b Binary files /dev/null and b/unit-test-data/libutil/git/hello-world.bin differ diff --git a/unit-test-data/libutil/git/tree.bin b/unit-test-data/libutil/git/tree.bin new file mode 100644 index 000000000000..5256ec140702 Binary files /dev/null and b/unit-test-data/libutil/git/tree.bin differ diff --git a/unit-test-data/libutil/git/tree.txt b/unit-test-data/libutil/git/tree.txt new file mode 100644 index 000000000000..be3d02920c91 --- /dev/null +++ b/unit-test-data/libutil/git/tree.txt @@ -0,0 +1,3 @@ +100644 blob 63ddb340119baf8492d2da53af47e8c7cfcd5eb2 Foo +100755 blob 63ddb340119baf8492d2da53af47e8c7cfcd5eb2 bAr +040000 tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904 baZ