Ignore temporary directories created when unpacking tarballs

Author: xzfc

.github/workflows/test.yml

@@ -20,6 +20,9 @@ jobs:
         signingKey: ${{ secrets.CACHIX_SIGNING_KEY }}
     - name: Build and install cached-nix-shell
       run: nix-env -i -f default.nix
+    - name: Workaround for Darwin
+      if: matrix.os == 'macos-latest'
+      run: cached-nix-shell -p --run true
     - name: Test cached-nix-shell
       run: ./tests/run.sh
     - name: Test nix-trace

nix-trace/README.md

@@ -29,6 +29,9 @@ open() != -1 && read error:  `f` FILENAME `\0` `e` `\0`
 
 opendir() == NULL:           `d` FILENAME `\0` `-` `\0`
 opendir() != NULL:           `d` FILENAME `\0` b3sum(directory listing) `\0`
+
+mkdir() == 0                 `t` FILENAME `\0` `+` `\0`
+unlinkat() == NULL           `t` FILENAME `\0` `-` `\0`
 ```
 
 Directory listing:

nix-trace/test.sh

@@ -2,14 +2,13 @@
 
 run() {
 	rm -f test-tmp/log
-	DYLD_INSERT_LIBRARIES=$PWD/build/trace-nix.so LD_PRELOAD=$PWD/build/trace-nix.so TRACE_NIX=test-tmp/log \
-		nix-shell --run : -p -- "$@" 2>/dev/null
-}
-
-run_without_p() {
-	rm -f test-tmp/log
-	DYLD_INSERT_LIBRARIES=$PWD/build/trace-nix.so LD_PRELOAD=$PWD/build/trace-nix.so TRACE_NIX=test-tmp/log \
-		nix-shell --run : -- "$@" 2>/dev/null
+	env \
+		DYLD_INSERT_LIBRARIES="$PWD"/build/trace-nix.so \
+		LD_PRELOAD="$PWD"/build/trace-nix.so \
+		TRACE_NIX=test-tmp/log \
+		nix-shell --run : "$@" 2>/dev/null &
+	NIX_PID=$!
+	wait
 }
 
 result=0
@@ -23,6 +22,8 @@ dir_b3sum() {
 }
 
 check() {
+	local grep_opts=
+	[ "$1" != "--first" ] || { shift; grep_opts=-m1; }
 	local name="$1" key="$2" val="$3"
 
 	if ! grep -qzFx -- "$key" test-tmp/log; then
@@ -31,7 +32,11 @@ check() {
 		result=1
 	fi
 
-	local actual_val="$(grep -zFx -A1 -- "$key" test-tmp/log | tail -zn1 | tr -d '\0')"
+	local actual_val=$(
+		grep $grep_opts -zFx -A1 -- "$key" test-tmp/log |
+		tail -zn1 |
+		tr -d '\0'
+	)
 	if [ "$val" != "$actual_val" ]; then
 		printf "\33[31mFail: %s: expected '%s', got '%s'\33[m\n" \
 			"$name" "$val" "$actual_val"
@@ -48,71 +53,92 @@ echo '"foo"' > test-tmp/test.nix
 : > test-tmp/empty
 ln -s empty test-tmp/link
 
+mkdir -p test-tmp/repo/data test-tmp/tmpdir
+echo '{}' > test-tmp/repo/data/default.nix
+tar -C test-tmp/repo -cf test-tmp/repo.tar .
+
 x=""
 for i in {1..64};do
 	x=x$x
 	mkdir -p test-tmp/many-dirs/$x
 done
 
-run 'with import <unstable> {}; bash'
+export XDG_CACHE_HOME="$PWD/test-tmp/xdg-cache"
+export TMPDIR="$PWD/test-tmp/tmpdir"
+
+
+run -p 'with import <unstable> {}; bash'
 check import-channel \
 	"s/nix/var/nix/profiles/per-user/root/channels/unstable" \
 	"l$(readlink /nix/var/nix/profiles/per-user/root/channels/unstable)"
 
-run 'with import <nonexistentChannel> {}; bash'
+run -p 'with import <nonexistentChannel> {}; bash'
 check import-channel-ne \
 	"s/nix/var/nix/profiles/per-user/root/channels/nonexistentChannel" '-'
 
 
-run 'import ./test-tmp/test.nix'
+run -p 'import ./test-tmp/test.nix'
 check import-relative-nix \
 	"s$PWD/test-tmp/test.nix" "+"
 
-run 'import ./test-tmp'
+run -p 'import ./test-tmp'
 check import-relative-nix-dir \
 	"s$PWD/test-tmp" "d"
 
-run 'import ./nonexistent.nix'
+run -p 'import ./nonexistent.nix'
 check import-relative-nix-ne \
 	"s$PWD/nonexistent.nix" "-"
 
 
-run 'builtins.readFile ./test-tmp/test.nix'
+run -p 'builtins.readFile ./test-tmp/test.nix'
 check builtins.readFile \
 	"f$PWD/test-tmp/test.nix" \
 	"$(b3sum ./test-tmp/test.nix | head -c 32)"
 
-run 'builtins.readFile "/nonexistent/readFile"'
+run -p 'builtins.readFile "/nonexistent/readFile"'
 check builtins.readFile-ne \
 	"f/nonexistent/readFile" "-"
 
-run 'builtins.readFile ./test-tmp'
+run -p 'builtins.readFile ./test-tmp'
 check builtins.readFile-dir \
 	"f$PWD/test-tmp" "e"
 
-run 'builtins.readFile ./test-tmp/empty'
+run -p 'builtins.readFile ./test-tmp/empty'
 check builtins.readFile-empty \
 	"f$PWD/test-tmp/empty" \
 	"$(b3sum ./test-tmp/empty | head -c 32)"
 
 
-run 'builtins.readDir ./test-tmp'
+run -p 'builtins.readDir ./test-tmp'
 check builtins.readDir \
 	"d$PWD/test-tmp" "$(dir_b3sum ./test-tmp)"
 
-run 'builtins.readDir "/nonexistent/readDir"'
+run -p 'builtins.readDir "/nonexistent/readDir"'
 check builtins.readDir-ne \
 	"d/nonexistent/readDir" "-"
 
 
-run 'builtins.readDir ./test-tmp/many-dirs'
+run -p 'builtins.readDir ./test-tmp/many-dirs'
 check builtins.readDir-many-dirs \
 	"d$PWD/test-tmp/many-dirs" "$(dir_b3sum ./test-tmp/many-dirs)"
 
-run_without_p
+run
 check implicit:shell.nix \
 	"s$PWD/shell.nix" "-"
 check implicit:default.nix \
 	"s$PWD/default.nix" "-"
 
+
+run -p "fetchTarball file://$PWD/test-tmp/repo.tar?1"
+check --first fetchTarball:create \
+	"t$PWD/test-tmp/tmpdir/nix-$NIX_PID-1" "+"
+check fetchTarball:delete \
+	"t$PWD/test-tmp/tmpdir/nix-$NIX_PID-1" "-"
+
+run -I "q=file://$PWD/test-tmp/repo.tar?2" -p "import <q>"
+check --first tarball-I:create \
+	"t$PWD/test-tmp/tmpdir/nix-$NIX_PID-1" "+"
+check tarball-I:delete \
+	"t$PWD/test-tmp/tmpdir/nix-$NIX_PID-1" "-"
+
 exit $result

nix-trace/trace-nix.c

@@ -5,6 +5,7 @@
 #include <dlfcn.h>
 #include <errno.h>
 #include <fcntl.h>
+#include <inttypes.h>
 #include <limits.h>
 #include <stdarg.h>
 #include <stdio.h>
@@ -17,6 +18,9 @@
 
 static FILE *log_f = NULL;
 static const char *pwd = NULL;
+static char tmp_prefix[PATH_MAX];  // "$TMPDIR/nix-$$-"
+static size_t tmp_prefix_dirname_len = 0;  // Length of "$TMPDIR"
+static size_t tmp_prefix_basename_len = 0;  // Length of "nix-$$-"
 
 #define FATAL() \
 	do { \
@@ -88,6 +92,33 @@ static void __attribute__((constructor)) init() {
 
 	INIT_MUTEX(print_mutex);
 	INIT_MUTEX(buf_mutex);
+
+	// References:
+	//   https://github.com/NixOS/nix/blob/2.15.1/src/libutil/filesystem.cc#L18
+	//   https://github.com/NixOS/nix/blob/2.15.1/src/libutil/util.hh#L337-L338
+	const char *tmpdir = getenv("TMPDIR");
+	if (tmpdir == NULL)
+		tmpdir = "/tmp";
+	char tmpdir_real[PATH_MAX];
+	if (realpath(tmpdir, tmpdir_real) == NULL) {
+		fprintf(stderr, "trace-nix: cannot resolve TMPDIR: %s\n", strerror(errno));
+		tmp_prefix[0] = '\0';
+		return;
+	}
+	const char *tmpdirend = tmpdir_real + strlen(tmpdir_real);
+	while (tmpdirend > tmpdir_real && tmpdirend[-1] == '/')
+		tmpdirend--;
+	int len = snprintf(tmp_prefix, sizeof tmp_prefix,
+		"%.*s/nix-%" PRIu64 "-",
+		(int)(tmpdirend - tmpdir_real),
+		tmpdir_real,
+		(uint64_t)getpid());
+	tmp_prefix_dirname_len = tmpdirend - tmpdir_real;
+	tmp_prefix_basename_len = len - tmp_prefix_dirname_len - 1;
+	if (len < 0 || len >= sizeof tmp_prefix) {
+		fprintf(stderr, "trace-nix: TMPDIR too long\n");
+		tmp_prefix[0] = '\0';
+	}
 }
 
 #ifdef __APPLE__
@@ -158,13 +189,64 @@ WRAPPER(DIR *, opendir, (const char *path)) {
 	return dirp;
 }
 
+WRAPPER(int, mkdir, (const char *path, mode_t mode)) {
+	int result = REAL(mkdir)(path, mode);
+	if (result == 0 && *tmp_prefix && memcmp(path, tmp_prefix,
+				tmp_prefix_dirname_len + 1 + tmp_prefix_basename_len) == 0)
+		print_log('t', path, "+");
+	return result;
+}
+
+WRAPPER(int, unlinkat, (int dirfd, const char *path, int flags)) {
+	int result = REAL(unlinkat)(dirfd, path, flags);
+	if (result != 0 || *tmp_prefix == '\0' || flags != AT_REMOVEDIR)
+		return result;
+	size_t path_len = strlen(path);
+	if (path_len > 45) // 45 == len(f"nix-{2**64}-{2**64}")
+		return result;
+	// Check that the path starts with 'nix-$$-' and do not contain slash.
+	if (memcmp(path, tmp_prefix + tmp_prefix_dirname_len + 1, tmp_prefix_basename_len) != 0 ||
+		strchr(path + tmp_prefix_dirname_len + 1 + tmp_prefix_basename_len, '/'))
+		return result;
+
+	char dir_path[PATH_MAX];
+#ifdef __linux__
+	snprintf(dir_path, sizeof dir_path, "/proc/self/fd/%d", dirfd);
+#elif defined(__APPLE__)
+	if (fcntl(dirfd, F_GETPATH, dir_path) == -1) {
+		fprintf(stderr, "trace-nix: fcntl(%d, F_GETPATH): %s\n", dirfd, strerror(errno));
+		return result;
+	}
+#else
+#warning "Not implemented for this platform"
+	return result;
+#endif
+	char full_path[PATH_MAX];
+	if (realpath(dir_path, full_path) == NULL) {
+		fprintf(stderr, "trace-nix: realpath(%s): %s\n", dir_path, strerror(errno));
+		return result;
+	}
+
+	size_t dir_path_len = strlen(full_path);
+	if (dir_path_len + 1 + path_len + 1 > sizeof full_path) {
+		fprintf(stderr, "trace-nix: path too long: %s/%s\n", full_path, path);
+		return result;
+	}
+	full_path[dir_path_len] = '/';
+	memcpy(full_path + dir_path_len + 1, path, path_len + 1);
+
+	print_log('t', full_path, "-");
+	return result;
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 
 static int enable(const char *path) {
 	if (log_f == NULL || (*path != '/' && strcmp(path, "shell.nix")))
 		return 0;
 
 	static const char *ignored_paths[] = {
+		"/dev/urandom",
 		"/etc/ssl/certs/ca-certificates.crt",
 		"/nix/var/nix/daemon-socket/socket",
 		"/nix",

src/main.rs

@@ -279,7 +279,7 @@ fn run_nix_shell(inp: &NixShellInput) -> NixShellOutput {
     trace_file
         .read_to_end(&mut trace_data)
         .expect("Can't read trace file");
-    let trace = Trace::load(trace_data);
+    let trace = Trace::load_raw(trace_data);
     if trace.check_for_changes() {
         eprintln!("cached-nix-shell: some files are already updated, cache won't be reused");
     }
@@ -572,7 +572,7 @@ fn check_cache(hash: &str) -> Option<BTreeMap<OsString, OsString>> {
         return None;
     }
 
-    let trace = read(trace_fname).unwrap().pipe(Trace::load);
+    let trace = read(trace_fname).unwrap().pipe(Trace::load_sorted);
     if trace.check_for_changes() {
         return None;
     }