Nix: cross fingers


After a build has completed, Nix sometimes needs to rewrite hashes in the result, e.g.,:
rewriting hashes in '/nix/store/2sp9hma57rj2f0drhyikab993jkqa3px-bionix-query-tmb-csv'; cross fingers
This is currently done in ram, which causes issues when using BioNix as some of these outputs are too large to fit in ram, such as BAM files. I patched Nix to instead do the hash rewriting on disk via a memory map:

diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index dd932cee9..c1e9f8ac3 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -3705,18 +3705,45 @@ void DerivationGoal::registerOutputs()
.hint = hintfmt("rewriting hashes in '%1%'; cross fingers", path)
});

+
+
/* Canonicalise first. This ensures that the path we're
rewriting doesn't contain a hard link to /etc/shadow or
something like that. */
canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen);

- /* FIXME: this is in-memory. */
- StringSink sink;
- dumpPath(actualPath, sink);
- deletePath(actualPath);
- sink.s = make_ref(rewriteStrings(*sink.s, outputRewrites));
- StringSource source(*sink.s);
- restorePath(actualPath, source);
+ AutoDelete tmpDir(createTempDir(), true);
+ Path tmpFile = (Path) tmpDir + "/rewrite";
+ {
+ AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0600);
+ if(!fd) throw SysError("creating temporary file '%s'", tmpFile);
+ FdSink sink(fd.get());
+ dumpPath(actualPath, sink);
+ deletePath(actualPath);
+ }
+
+ {
+ AutoCloseFD fd = open(tmpFile.c_str(), O_RDWR);
+ if(!fd) throw SysError("Opening temporary file '%s'", tmpFile);
+ struct stat stat_buf;
+ if(fstat(fd.get(), &stat_buf) == -1) throw SysError("fstat: '%s'", tmpFile);
+ void *ptr = mmap(NULL, stat_buf.st_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd.get(), 0);
+ if(!ptr) throw SysError("mmap: '%s'", tmpFile);
+ for(auto & i : outputRewrites) {
+ if(i.first == i.second) continue;
+ void *j;
+ while((j = memmem(ptr, stat_buf.st_size, i.first.c_str(), i.first.size())))
+ memcpy(j, i.second.c_str(), i.first.size());
+ }
+ munmap(ptr, stat_buf.st_size);
+ }
+
+ {
+ AutoCloseFD fd = open(tmpFile.c_str(), O_RDONLY);
+ if(!fd) throw SysError("Opening temporary file '%s'", tmpFile);
+ FdSource source(fd.get());
+ restorePath(actualPath, source);
+ }

rewritten = true;
}