From 49f21a860f946e9a857bc32a1835a73bde5912d2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Sep 2024 21:43:59 +0200 Subject: [PATCH 01/11] NAR parser: Fix check for duplicate / incorrectly sorted entries "prevName" was always empty because it was declared in the wrong scope. (cherry picked from commit 495d32e1b8e5d5143f048d1be755a96bea822b19) --- src/libutil/archive.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index d20936de4b4..078a2c41c07 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -214,11 +214,13 @@ static void parse(FileSystemObjectSink & sink, Source & source, const Path & pat else if (t == "directory") { sink.createDirectory(path); + std::string prevName; + while (1) { s = getString(); if (s == "entry") { - std::string name, prevName; + std::string name; s = getString(); if (s != "(") throw badArchive("expected open tag"); From f9b3defe7f1a25f9def3e0744443e01dd2dc90a9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 5 Sep 2024 16:41:15 +0200 Subject: [PATCH 02/11] Add test case for NARs with duplicate directory entries This test was made by @puckipedia. (cherry picked from commit 83d5b32803e5b828967a27b1ea93c5728d3a4d0a) --- tests/functional/case-hack.sh | 21 --------------------- tests/functional/duplicate.nar | Bin 0 -> 1400 bytes tests/functional/local.mk | 2 +- tests/functional/nars.sh | 23 +++++++++++++++++++++++ 4 files changed, 24 insertions(+), 22 deletions(-) delete mode 100755 tests/functional/case-hack.sh create mode 100644 tests/functional/duplicate.nar create mode 100755 tests/functional/nars.sh diff --git a/tests/functional/case-hack.sh b/tests/functional/case-hack.sh deleted file mode 100755 index fbc8242ff8f..00000000000 --- a/tests/functional/case-hack.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash - -source common.sh - -clearStore - -rm -rf $TEST_ROOT/case - -opts="--option use-case-hack true" - -# Check whether restoring and dumping a NAR that contains case -# collisions is round-tripping, even on a case-insensitive system. -nix-store $opts --restore $TEST_ROOT/case < case.nar -nix-store $opts --dump $TEST_ROOT/case > $TEST_ROOT/case.nar -cmp case.nar $TEST_ROOT/case.nar -[ "$(nix-hash $opts --type sha256 $TEST_ROOT/case)" = "$(nix-hash --flat --type sha256 case.nar)" ] - -# Check whether we detect true collisions (e.g. those remaining after -# removal of the suffix). -touch "$TEST_ROOT/case/xt_CONNMARK.h~nix~case~hack~3" -(! nix-store $opts --dump $TEST_ROOT/case > /dev/null) diff --git a/tests/functional/duplicate.nar b/tests/functional/duplicate.nar new file mode 100644 index 0000000000000000000000000000000000000000..1d0993ed4cab41a6d45907ac0c17026afd5471a2 GIT binary patch literal 1400 zcmdT@+it=z49zZ#4T*h25D#ojRW~kz9 z$BsP}-LYn0DAbktf#N+v9qTBW&+onV;7jX2S0C@V9t<{lr}pt&I-XgF4v29E z3g3EyMu?&G+_E0O>ztu< "$TEST_ROOT/case.nar" +cmp case.nar "$TEST_ROOT/case.nar" +[ "$(nix-hash "${opts[@]}" --type sha256 "$TEST_ROOT/case")" = "$(nix-hash --flat --type sha256 case.nar)" ] + +# Check whether we detect true collisions (e.g. those remaining after +# removal of the suffix). +touch "$TEST_ROOT/case/xt_CONNMARK.h~nix~case~hack~3" +(! nix-store $opts --dump $TEST_ROOT/case > /dev/null) From 7358d217cf50fe5763e8bd638c7344443f44e133 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 5 Sep 2024 16:48:43 +0200 Subject: [PATCH 03/11] Test that nix-store --restore fails if the output already exists This restores the behaviour from before the std::filesystem refactorings. (cherry picked from commit da1ad28912334bb57f923afb4745273fd68f695c) --- tests/functional/nars.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/functional/nars.sh b/tests/functional/nars.sh index ba5a69bd2c8..eaf9cd4e5a9 100755 --- a/tests/functional/nars.sh +++ b/tests/functional/nars.sh @@ -8,6 +8,9 @@ clearStore rm -rf "$TEST_ROOT/out" expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet "NAR directory is not sorted" +# Check that nix-store --restore fails if the output already exists. +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet "path '.*/out/' already exists" + # Check whether restoring and dumping a NAR that contains case # collisions is round-tripping, even on a case-insensitive system. rm -rf "$TEST_ROOT/case" From 3ab521f8e0f3dd1c1a1be0d95ee79878f97dc3f8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 5 Sep 2024 16:54:12 +0200 Subject: [PATCH 04/11] More tests (cherry picked from commit 77c090cdbd56220895a2447efae79f68ed7861c5) --- tests/functional/nars.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/functional/nars.sh b/tests/functional/nars.sh index eaf9cd4e5a9..376ef069b4d 100755 --- a/tests/functional/nars.sh +++ b/tests/functional/nars.sh @@ -11,6 +11,17 @@ expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet # Check that nix-store --restore fails if the output already exists. expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet "path '.*/out/' already exists" +rm -rf "$TEST_ROOT/out" +echo foo > "$TEST_ROOT/out" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet "cannot create directory.*File exists" + +rm -rf "$TEST_ROOT/out" +ln -s "$TEST_ROOT/out2" "$TEST_ROOT/out" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet "cannot create directory.*File exists" + +mkdir -p "$TEST_ROOT/out2" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet "path '.*/out/' already exists" + # Check whether restoring and dumping a NAR that contains case # collisions is round-tripping, even on a case-insensitive system. rm -rf "$TEST_ROOT/case" From b51c0915eed53f712af8903a3b688b7f85d6fbcf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 5 Sep 2024 19:26:10 +0200 Subject: [PATCH 05/11] Detect NAR directory entries that collide with another path after case-hacking The test was made by @puckipedia. (cherry picked from commit 35575873813f60fff26f27a65e09038986f17cb5) --- src/libutil/archive.cc | 3 +++ tests/functional/case-collision.nar | Bin 0 -> 1928 bytes tests/functional/nars.sh | 8 +++++++- 3 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 tests/functional/case-collision.nar diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 078a2c41c07..5825900363e 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -243,6 +243,9 @@ static void parse(FileSystemObjectSink & sink, Source & source, const Path & pat debug("case collision between '%1%' and '%2%'", i->first, name); name += caseHackSuffix; name += std::to_string(++i->second); + auto j = names.find(name); + if (j != names.end()) + throw Error("NAR contains file name '%s' that collides with case-hacked file name '%s'", prevName, j->first); } else names[name] = 0; } diff --git a/tests/functional/case-collision.nar b/tests/functional/case-collision.nar new file mode 100644 index 0000000000000000000000000000000000000000..2eff86901c617be2a830d23074923cb5b3b69aa3 GIT binary patch literal 1928 zcmd^9%}&EG3@&2)Y!WvfAc(_YXsQr5o`XF=mU?TnHklH4TQ7Zf(qMC#G>KJ{av&Gy za}?+EXU7lO&ocTjmrj*>2lMyfx+4Dz*%4W6x6p6LgbVFJp>-|c8?s<9`cB0$vW{^$ z?iYCMuQE2ai07y7GmkrZ&%wH>q|5FJD{C-t@C1MJc_jzOWqdC0M~c()?t*xok{-HJ zs!i9+H#iU9)|ED!?3UuAbZZF8FyEZ~jG6y2J~toM9S7FoQvGmE`2|Vij(PpHA1=*f z7ka8+sd=Qc8V} DaOkrB literal 0 HcmV?d00001 diff --git a/tests/functional/nars.sh b/tests/functional/nars.sh index 376ef069b4d..237955fbeb4 100755 --- a/tests/functional/nars.sh +++ b/tests/functional/nars.sh @@ -34,4 +34,10 @@ cmp case.nar "$TEST_ROOT/case.nar" # Check whether we detect true collisions (e.g. those remaining after # removal of the suffix). touch "$TEST_ROOT/case/xt_CONNMARK.h~nix~case~hack~3" -(! nix-store $opts --dump $TEST_ROOT/case > /dev/null) +(! nix-store "${opts[@]}" --dump "$TEST_ROOT/case" > /dev/null) + +# Detect NARs that have a directory entry that after case-hacking +# collides with another entry (e.g. a directory containing 'Test', +# 'Test~nix~case~hack~1' and 'test'). +rm -rf "$TEST_ROOT/case" +expectStderr 1 nix-store "${opts[@]}" --restore "$TEST_ROOT/case" < case-collision.nar | grepQuiet "NAR contains file name 'test' that collides with case-hacked file name 'Test~nix~case~hack~1'" From 2a25c1d9b4004e404af824a5a822c63a888d0e8b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 5 Sep 2024 20:37:26 +0200 Subject: [PATCH 06/11] Test that deserializing NARs with names with equal Unicode normal forms fails on macOS The test is based on the one by @puckipedia but with the file names swapped to make them sorted. (cherry picked from commit 7a765a6aafa27267659eb7339cf7039990f30caa) --- tests/functional/nars.sh | 11 +++++++++++ tests/functional/unnormalized.nar | Bin 0 -> 1728 bytes 2 files changed, 11 insertions(+) create mode 100644 tests/functional/unnormalized.nar diff --git a/tests/functional/nars.sh b/tests/functional/nars.sh index 237955fbeb4..583f132cbd0 100755 --- a/tests/functional/nars.sh +++ b/tests/functional/nars.sh @@ -41,3 +41,14 @@ touch "$TEST_ROOT/case/xt_CONNMARK.h~nix~case~hack~3" # 'Test~nix~case~hack~1' and 'test'). rm -rf "$TEST_ROOT/case" expectStderr 1 nix-store "${opts[@]}" --restore "$TEST_ROOT/case" < case-collision.nar | grepQuiet "NAR contains file name 'test' that collides with case-hacked file name 'Test~nix~case~hack~1'" + +# Deserializing a NAR that contains file names that Unicode-normalize +# to the same name should fail on macOS but succeed on Linux. +rm -rf "$TEST_ROOT/out" +if [[ $(uname) = Darwin ]]; then + expectStderr 1 nix-store --restore "$TEST_ROOT/out" < unnormalized.nar | grepQuiet "cannot create directory.*File exists" +else + nix-store --restore "$TEST_ROOT/out" < unnormalized.nar + [[ -e $TEST_ROOT/out/â ]] + [[ -e $TEST_ROOT/out/â ]] +fi diff --git a/tests/functional/unnormalized.nar b/tests/functional/unnormalized.nar new file mode 100644 index 0000000000000000000000000000000000000000..4b7edb17e0b4a9b75cf2958e9f12cceca22d267c GIT binary patch literal 1728 zcmd^9&2GXl4DNo}ka&koJMc51YTAwW-~mEvXhfQz#07fgQFxVI_fQML(N5J=2`NbQ zV*7LLe6bx5vh%0qe#)&Vu$+Qc|z8XR?vo72w}Ja>8T2af{uR|2^gTKAx4X{4ZTc z-^V~CHIFT~SHUB7Jzi)&Hr6bq0+*^U@tqW~ Date: Thu, 5 Sep 2024 20:55:24 +0200 Subject: [PATCH 07/11] Fix test on macOS (cherry picked from commit 21dcbd7e83929fbf8b6c666d743afa0a9ea73d83) --- tests/functional/nars.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/nars.sh b/tests/functional/nars.sh index 583f132cbd0..d591ac52198 100755 --- a/tests/functional/nars.sh +++ b/tests/functional/nars.sh @@ -46,7 +46,7 @@ expectStderr 1 nix-store "${opts[@]}" --restore "$TEST_ROOT/case" < case-collisi # to the same name should fail on macOS but succeed on Linux. rm -rf "$TEST_ROOT/out" if [[ $(uname) = Darwin ]]; then - expectStderr 1 nix-store --restore "$TEST_ROOT/out" < unnormalized.nar | grepQuiet "cannot create directory.*File exists" + expectStderr 1 nix-store --restore "$TEST_ROOT/out" < unnormalized.nar | grepQuiet "path '.*/out/â' already exists" else nix-store --restore "$TEST_ROOT/out" < unnormalized.nar [[ -e $TEST_ROOT/out/â ]] From b88d78f98dfa6fbd266320cf6f55a9149e7c1c7b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 6 Sep 2024 16:28:09 +0200 Subject: [PATCH 08/11] Test that deserializing regular files / symlinks is exclusive (cherry picked from commit 52ba3cc5eac0418218a90c0cddb06688d4c7b5d3) --- tests/functional/nars.sh | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/functional/nars.sh b/tests/functional/nars.sh index d591ac52198..747662f952d 100755 --- a/tests/functional/nars.sh +++ b/tests/functional/nars.sh @@ -22,6 +22,44 @@ expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet mkdir -p "$TEST_ROOT/out2" expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet "path '.*/out/' already exists" +# The same, but for a regular file. +nix-store --dump ./nars.sh > "$TEST_ROOT/tmp.nar" + +rm -rf "$TEST_ROOT/out" +nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" | grepQuiet "File exists" + +rm -rf "$TEST_ROOT/out" +mkdir -p "$TEST_ROOT/out" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" | grepQuiet "File exists" + +rm -rf "$TEST_ROOT/out" +ln -s "$TEST_ROOT/out2" "$TEST_ROOT/out" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" | grepQuiet "File exists" + +mkdir -p "$TEST_ROOT/out2" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" | grepQuiet "File exists" + +# The same, but for a symlink +ln -sfn foo "$TEST_ROOT/symlink" +nix-store --dump "$TEST_ROOT/symlink" > "$TEST_ROOT/tmp.nar" + +rm -rf "$TEST_ROOT/out" +nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" +[[ -L "$TEST_ROOT/out" ]] +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" | grepQuiet "File exists" + +rm -rf "$TEST_ROOT/out" +mkdir -p "$TEST_ROOT/out" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" | grepQuiet "File exists" + +rm -rf "$TEST_ROOT/out" +ln -s "$TEST_ROOT/out2" "$TEST_ROOT/out" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" | grepQuiet "File exists" + +mkdir -p "$TEST_ROOT/out2" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" | grepQuiet "File exists" + # Check whether restoring and dumping a NAR that contains case # collisions is round-tripping, even on a case-insensitive system. rm -rf "$TEST_ROOT/case" From 8ce0c8226880d18e4ccfce4db951e24917eb3da6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 9 Sep 2024 14:11:35 +0200 Subject: [PATCH 09/11] Typo (cherry picked from commit 4cfa59fdb32aa4fcc58b735d8843ce308692a652) --- tests/functional/nars.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/nars.sh b/tests/functional/nars.sh index 747662f952d..5e109b80738 100755 --- a/tests/functional/nars.sh +++ b/tests/functional/nars.sh @@ -40,7 +40,7 @@ expectStderr 1 nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" | gre mkdir -p "$TEST_ROOT/out2" expectStderr 1 nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" | grepQuiet "File exists" -# The same, but for a symlink +# The same, but for a symlink. ln -sfn foo "$TEST_ROOT/symlink" nix-store --dump "$TEST_ROOT/symlink" > "$TEST_ROOT/tmp.nar" From 78eb84595022e9bedbc25130b2acbdf0f69ac39a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 9 Sep 2024 14:29:05 +0200 Subject: [PATCH 10/11] Improve use-case-hack description slightly (cherry picked from commit 5ca2f58798e6f514b5194c16c0fea0d8ec128171) --- src/libutil/archive.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 5825900363e..39af6dc8360 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -23,7 +23,7 @@ struct ArchiveSettings : Config false, #endif "use-case-hack", - "Whether to enable a Darwin-specific hack for dealing with file name collisions."}; + "Whether to enable a macOS-specific hack for dealing with file name case collisions."}; }; static ArchiveSettings archiveSettings; From a0df9295753a0110d16819ad9dc7d7580ecaf21a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 10 Sep 2024 10:10:10 +0200 Subject: [PATCH 11/11] Fix test --- tests/functional/nars.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/functional/nars.sh b/tests/functional/nars.sh index 5e109b80738..eb375701635 100755 --- a/tests/functional/nars.sh +++ b/tests/functional/nars.sh @@ -9,18 +9,18 @@ rm -rf "$TEST_ROOT/out" expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet "NAR directory is not sorted" # Check that nix-store --restore fails if the output already exists. -expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet "path '.*/out/' already exists" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet "creating directory '.*/out': File exists" rm -rf "$TEST_ROOT/out" echo foo > "$TEST_ROOT/out" -expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet "cannot create directory.*File exists" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet "creating directory '.*/out': File exists" rm -rf "$TEST_ROOT/out" ln -s "$TEST_ROOT/out2" "$TEST_ROOT/out" -expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet "cannot create directory.*File exists" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet "creating directory '.*/out': File exists" mkdir -p "$TEST_ROOT/out2" -expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet "path '.*/out/' already exists" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet "creating directory '.*/out': File exists" # The same, but for a regular file. nix-store --dump ./nars.sh > "$TEST_ROOT/tmp.nar" @@ -84,7 +84,7 @@ expectStderr 1 nix-store "${opts[@]}" --restore "$TEST_ROOT/case" < case-collisi # to the same name should fail on macOS but succeed on Linux. rm -rf "$TEST_ROOT/out" if [[ $(uname) = Darwin ]]; then - expectStderr 1 nix-store --restore "$TEST_ROOT/out" < unnormalized.nar | grepQuiet "path '.*/out/â' already exists" + expectStderr 1 nix-store --restore "$TEST_ROOT/out" < unnormalized.nar | grepQuiet "File exists" else nix-store --restore "$TEST_ROOT/out" < unnormalized.nar [[ -e $TEST_ROOT/out/â ]]