diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 4534cf54c3ee..19018aaff9d0 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -8,6 +8,9 @@ #include "nix/util/url.hh" #include "nix/util/forwarding-source-accessor.hh" #include "nix/util/archive.hh" +#include "nix/util/users.hh" +#include "nix/store/pathlocks.hh" +#include "nix/util/environment-variables.hh" #include @@ -367,6 +370,18 @@ std::pair, Input> Input::getAccessorUnchecked(const Settings return {accessor, result}; }; + /* Acquire a path lock on this input. Note that fetching the same input in parallel is supposed to be safe (it's up + * to the fetchers to guarantee this), so this is merely intended to avoid work duplication. */ + auto lockFilePath = + getCacheDir() / "fetcher-locks" + / hashString(HashAlgorithm::SHA256, attrsToJSON(toAttrs()).dump()).to_string(HashFormat::Base16, false); + std::filesystem::create_directories(lockFilePath.parent_path()); + PathLocks lock( + {lockFilePath.string()}, fmt("waiting for another Nix process to finish fetching input '%s'...", to_string())); + + if (getEnv("_NIX_TEST_CONCURRENT_FETCHES")) + std::this_thread::sleep_for(std::chrono::seconds(1)); + /* See if the input is in the cache of the fetcher. */ try { if (auto res = scheme->getAccessor(settings, store, *this, true)) diff --git a/tests/functional/tarball.sh b/tests/functional/tarball.sh index 451ee879a5b3..1900f4b00a46 100755 --- a/tests/functional/tarball.sh +++ b/tests/functional/tarball.sh @@ -115,3 +115,17 @@ path="$(nix flake prefetch --refresh --json "tarball+file://$TEST_ROOT/tar.tar" [[ $(cat "$path/a/b/xyzzy") = xyzzy ]] [[ $(cat "$path/a/b/foo") = foo ]] [[ $(cat "$path/bla") = abc ]] + +# Test that concurrent invocations of Nix will fetch the tarball only once. +rm -rf "$TEST_HOME/.cache" +store="$TEST_ROOT/prefetch-store" +nix-store --store "$store" --init # needed because concurrent creation of the store can give SQLite errors +_NIX_TEST_CONCURRENT_FETCHES=1 _NIX_FORCE_HTTP=1 nix flake prefetch --store "$store" -v "tarball+file://$TEST_ROOT/tar.tar" 2> "$TEST_ROOT/log1" & +pid1="$!" +_NIX_TEST_CONCURRENT_FETCHES=1 _NIX_FORCE_HTTP=1 nix flake prefetch --store "$store" -v "tarball+file://$TEST_ROOT/tar.tar" 2> "$TEST_ROOT/log2" & +pid2="$!" +wait "$pid1" +wait "$pid2" +[[ $(cat "$TEST_ROOT/log1" "$TEST_ROOT/log2" | grep -c "Download.*to") -eq 2 ]] +[[ $(cat "$TEST_ROOT/log1" "$TEST_ROOT/log2" | grep -c "downloading.*tar.tar") -eq 1 ]] +[[ $(cat "$TEST_ROOT/log1" "$TEST_ROOT/log2" | grep -c "waiting for another Nix process to finish fetching input") -eq 1 ]]