Skip to content

Input::getAccessor(): Try substitution before fetching #380

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
edolstra merged 2 commits into from
Mar 11, 2026
Merged

Input::getAccessor(): Try substitution before fetching #380

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a6af221
Input::getAccessor(): Try substitution before fetching
edolstra Mar 10, 2026
41746d0
Handle the tarball fetcher
edolstra Mar 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 25 additions & 21 deletions src/libfetchers/fetchers.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -356,36 +356,40 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings
return makeStoreAccessor();
}

try {
auto [accessor, result] = scheme->getAccessor(settings, store, *this);

auto fixupAccessor = [&](ref<SourceAccessor> accessor, Input result) -> std::pair<ref<SourceAccessor>, Input> {
if (auto fp = accessor->getFingerprint(CanonPath::root).second)
result.cachedFingerprint = *fp;
else
accessor->fingerprint = result.getFingerprint(store);

accessor->provenance = std::make_shared<TreeProvenance>(result);

return {accessor, std::move(result)};
} catch (Error & e) {
if (storePath) {
// Fall back to substitution.
try {
store.ensurePath(*storePath);
warn(
"Successfully substituted input '%s' after failing to fetch it from its original location: %s",
to_string(),
e.info().msg);
return makeStoreAccessor();
}
// Ignore any substitution error, rethrow the original error.
catch (Error & e2) {
debug("substitution of input '%s' failed: %s", to_string(), e2.info().msg);
} catch (...) {
}
return {accessor, result};
};

/* See if the input is in the cache of the fetcher. */
try {
if (auto res = scheme->getAccessor(settings, store, *this, true))
return fixupAccessor(res->first, std::move(res->second));
} catch (...) {
}
Comment thread
edolstra marked this conversation as resolved.

/* If not, try to substitute the input. */
if (storePath) {
try {
store.ensurePath(*storePath);
return makeStoreAccessor();
}
// Ignore any substitution error.
catch (Error & e2) {
debug("substitution of input '%s' failed: %s", to_string(), e2.info().msg);
} catch (...) {
}
throw;
}

/* If we can't substitute, then fetch normally. */
auto [accessor, result] = scheme->getAccessor(settings, store, *this);
return fixupAccessor(accessor, result);
}

Input Input::applyOverrides(std::optional<std::string> ref, std::optional<Hash> rev) const
Expand Down
21 changes: 11 additions & 10 deletions src/libfetchers/git.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -849,8 +849,8 @@ struct GitInputScheme : InputScheme
return ref{accessor};
}

std::pair<ref<SourceAccessor>, Input>
getAccessorFromCommit(const Settings & settings, Store & store, RepoInfo & repoInfo, Input && input) const
std::optional<std::pair<ref<SourceAccessor>, Input>> getAccessorFromCommit(
const Settings & settings, Store & store, RepoInfo & repoInfo, Input && input, bool fastOnly) const
{
assert(!repoInfo.workdirInfo.isDirty);

Expand Down Expand Up @@ -918,6 +918,9 @@ struct GitInputScheme : InputScheme
}

if (doFetch) {
if (fastOnly)
return std::nullopt;

try {
auto fetchRef = getAllRefsAttr(input) ? "refs/*:refs/*"
: input.getRev() ? input.getRev()->gitRev()
Expand Down Expand Up @@ -1086,7 +1089,7 @@ struct GitInputScheme : InputScheme

assert(!origRev || origRev == rev);

return {accessor, std::move(input)};
return {{accessor, std::move(input)}};
}

std::pair<ref<SourceAccessor>, Input>
Expand Down Expand Up @@ -1173,8 +1176,8 @@ struct GitInputScheme : InputScheme
return {accessor, std::move(input)};
}

std::pair<ref<SourceAccessor>, Input>
getAccessor(const Settings & settings, Store & store, const Input & _input) const override
std::optional<std::pair<ref<SourceAccessor>, Input>>
getAccessor(const Settings & settings, Store & store, const Input & _input, bool fastOnly) const override
{
Input input(_input);

Expand All @@ -1189,11 +1192,9 @@ struct GitInputScheme : InputScheme
throw UnimplementedError("exportIgnore and submodules are not supported together yet");
}

auto [accessor, final] = input.getRef() || input.getRev() || !repoInfo.getPath()
? getAccessorFromCommit(settings, store, repoInfo, std::move(input))
: getAccessorFromWorkdir(settings, store, repoInfo, std::move(input));

return {accessor, std::move(final)};
return input.getRef() || input.getRev() || !repoInfo.getPath()
? getAccessorFromCommit(settings, store, repoInfo, std::move(input), fastOnly)
: std::optional{getAccessorFromWorkdir(settings, store, repoInfo, std::move(input))};
}

std::optional<std::string> getFingerprint(Store & store, const Input & input) const override
Expand Down
23 changes: 16 additions & 7 deletions src/libfetchers/github.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,8 @@ struct GitArchiveInputScheme : InputScheme
time_t lastModified;
};

std::pair<Input, TarballInfo> downloadArchive(const Settings & settings, Store & store, Input input) const
std::optional<std::pair<Input, TarballInfo>>
downloadArchive(const Settings & settings, Store & store, Input input, bool fastOnly) const
{
if (!maybeGetStrAttr(input.attrs, "ref"))
input.attrs.insert_or_assign("ref", "HEAD");
Expand Down Expand Up @@ -299,12 +300,16 @@ struct GitArchiveInputScheme : InputScheme
auto treeHash = getRevAttr(*treeHashAttrs, "treeHash");
auto lastModified = getIntAttr(*lastModifiedAttrs, "lastModified");
if (settings.getTarballCache()->hasObject(treeHash))
return {std::move(input), TarballInfo{.treeHash = treeHash, .lastModified = (time_t) lastModified}};
return {
{std::move(input), TarballInfo{.treeHash = treeHash, .lastModified = (time_t) lastModified}}};
else
debug("Git tree with hash '%s' has disappeared from the cache, refetching...", treeHash.gitRev());
}
}

if (fastOnly)
return std::nullopt;

/* Stream the tarball into the tarball cache. */
auto url = getDownloadUrl(settings, input);

Expand Down Expand Up @@ -340,13 +345,17 @@ struct GitArchiveInputScheme : InputScheme
rev->gitRev(), input.to_string(), upstreamTreeHash->gitRev(), tarballInfo.treeHash.gitRev());
#endif

return {std::move(input), tarballInfo};
return {{std::move(input), tarballInfo}};
}

std::pair<ref<SourceAccessor>, Input>
getAccessor(const Settings & settings, Store & store, const Input & _input) const override
std::optional<std::pair<ref<SourceAccessor>, Input>>
getAccessor(const Settings & settings, Store & store, const Input & _input, bool fastOnly) const override
{
auto [input, tarballInfo] = downloadArchive(settings, store, _input);
auto res = downloadArchive(settings, store, _input, fastOnly);
if (fastOnly && !res)
return std::nullopt;
assert(res);
auto [input, tarballInfo] = *res;

#if 0
input.attrs.insert_or_assign("treeHash", tarballInfo.treeHash.gitRev());
Expand All @@ -363,7 +372,7 @@ struct GitArchiveInputScheme : InputScheme
input.attrs.insert_or_assign(
"narHash", accessor->hashPath(CanonPath::root).to_string(HashFormat::SRI, true));

return {accessor, input};
return {{accessor, input}};
}

bool isLocked(const Settings & settings, const Input & input) const override
Expand Down
13 changes: 12 additions & 1 deletion src/libfetchers/include/nix/fetchers/fetchers.hh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,8 +240,19 @@ struct InputScheme
std::string_view contents,
std::optional<std::string> commitMsg) const;

virtual std::optional<std::pair<ref<SourceAccessor>, Input>>
getAccessor(const Settings & settings, Store & store, const Input & input, bool fastOnly) const
{
if (fastOnly)
return std::nullopt;
return getAccessor(settings, store, input);
}

virtual std::pair<ref<SourceAccessor>, Input>
getAccessor(const Settings & settings, Store & store, const Input & input) const = 0;
getAccessor(const Settings & settings, Store & store, const Input & input) const
{
return getAccessor(settings, store, input, false).value();
}
Comment thread
edolstra marked this conversation as resolved.

/**
* Is this `InputScheme` part of an experimental feature?
Expand Down
8 changes: 5 additions & 3 deletions src/libfetchers/path.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,9 +138,11 @@ struct PathInputScheme : InputScheme
throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string());
}

std::pair<ref<SourceAccessor>, Input>
getAccessor(const Settings & settings, Store & store, const Input & _input) const override
std::optional<std::pair<ref<SourceAccessor>, Input>>
getAccessor(const Settings & settings, Store & store, const Input & _input, bool fastOnly) const override
{
// Note: fastOnly is ignored because the path fetcher is always fast.

Input input(_input);

auto absPath = getAbsPath(input);
Expand All @@ -165,7 +167,7 @@ struct PathInputScheme : InputScheme
}
}

return {accessor, std::move(input)};
return {{accessor, std::move(input)}};
}
};

Expand Down
23 changes: 17 additions & 6 deletions src/libfetchers/tarball.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,12 @@ DownloadFileResult downloadFile(
};
}

static DownloadTarballResult downloadTarball_(
const Settings & settings, const std::string & urlS, const Headers & headers, const std::string & displayPrefix)
static std::optional<DownloadTarballResult> downloadTarball_(
const Settings & settings,
const std::string & urlS,
const Headers & headers,
const std::string & displayPrefix,
bool fastOnly)
{
ParsedURL url = parseURL(urlS);

Expand Down Expand Up @@ -153,6 +157,9 @@ static DownloadTarballResult downloadTarball_(
`tarballTtl`, so no need to check the server. */
return attrsToResult(cached->value);

if (fastOnly)
return std::nullopt;

auto _res = std::make_shared<Sync<FileTransferResult>>();

auto source = sinkToSource([&](Sink & sink) {
Expand Down Expand Up @@ -484,12 +491,16 @@ struct TarballInputScheme : CurlInputScheme
: (requireTree || hasTarballExtension(url)));
}

std::pair<ref<SourceAccessor>, Input>
getAccessor(const Settings & settings, Store & store, const Input & _input) const override
std::optional<std::pair<ref<SourceAccessor>, Input>>
getAccessor(const Settings & settings, Store & store, const Input & _input, bool fastOnly) const override
{
auto input(_input);

auto result = downloadTarball_(settings, getStrAttr(input.attrs, "url"), {}, "«" + input.to_string(true) + "»");
auto res =
downloadTarball_(settings, getStrAttr(input.attrs, "url"), {}, "«" + input.to_string(true) + "»", fastOnly);
if (!res)
return std::nullopt;
auto & result = *res;

if (result.immutableUrl) {
auto immutableInput = Input::fromURL(settings, *result.immutableUrl);
Expand All @@ -507,7 +518,7 @@ struct TarballInputScheme : CurlInputScheme
"narHash",
settings.getTarballCache()->treeHashToNarHash(settings, result.treeHash).to_string(HashFormat::SRI, true));

return {result.accessor, input};
return {{result.accessor, input}};
}

std::optional<std::string> getFingerprint(Store & store, const Input & input) const override
Expand Down
Loading