diff --git a/doc/manual/rl-next/darwin-runtime-roots.md b/doc/manual/rl-next/darwin-runtime-roots.md new file mode 100644 index 000000000000..ef255acf7f93 --- /dev/null +++ b/doc/manual/rl-next/darwin-runtime-roots.md @@ -0,0 +1,12 @@ +--- +synopsis: Nix now finds Darwin runtime garbage collector roots without `lsof` +issues: [3011, 13990] +--- + +On Darwin, the garbage collector now uses `libproc` to retain store paths +referenced by running processes through their working directories, open files, +mapped regions, or environment variables. This avoids the slow `lsof` fallback +and adds environment-variable roots, which `lsof` could not discover. + +macOS does not expose the environment variables of entitled binaries while +System Integrity Protection is enabled, so those roots remain unavailable. diff --git a/src/libstore/darwin/local-gc.cc b/src/libstore/darwin/local-gc.cc new file mode 100644 index 000000000000..7fa9ac0e9a2a --- /dev/null +++ b/src/libstore/darwin/local-gc.cc @@ -0,0 +1,202 @@ +#include "../local-gc-private.hh" + +#include "nix/util/error.hh" +#include "nix/util/fmt.hh" +#include "nix/util/signals.hh" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace nix { + +void findDarwinRuntimeRoots(const StoreDirConfig & config, UncheckedRoots & unchecked) +{ + auto storePathRegex = makeStorePathRegex(config); + + std::vector pids; + std::size_t pidBufSize = 1; + + while (pidBufSize > pids.size() * sizeof(int)) { + // Reserve some extra size so we don't fail too much. + pids.resize((pidBufSize + pidBufSize / 8) / sizeof(int)); + auto size = proc_listpids(PROC_ALL_PIDS, 0, pids.data(), pids.size() * sizeof(int)); + + if (size <= 0) + throw SysError("listing PIDs"); + pidBufSize = size; + } + + pids.resize(pidBufSize / sizeof(int)); + + for (auto pid : pids) { + checkInterrupt(); + + // It doesn't make sense to ask about the kernel. + if (pid == 0) + continue; + + try { + // Process cwd/root directory. + struct proc_vnodepathinfo vnodeInfo; + if (proc_pidinfo(pid, PROC_PIDVNODEPATHINFO, 0, &vnodeInfo, sizeof(vnodeInfo)) <= 0) + throw SysError("getting pid %1% working directory", pid); + + unchecked[std::string(vnodeInfo.pvi_cdir.vip_path)].emplace(fmt("{libproc/%d/cwd}", pid)); + unchecked[std::string(vnodeInfo.pvi_rdir.vip_path)].emplace(fmt("{libproc/%d/rootdir}", pid)); + + // File descriptors. + std::vector fds; + std::size_t fdBufSize = 1; + while (fdBufSize > fds.size() * sizeof(struct proc_fdinfo)) { + // Reserve some extra size so we don't fail too much. + fds.resize((fdBufSize + fdBufSize / 8) / sizeof(struct proc_fdinfo)); + errno = 0; + auto size = proc_pidinfo(pid, PROC_PIDLISTFDS, 0, fds.data(), fds.size() * sizeof(struct proc_fdinfo)); + + // The libproc wrapper converts a -1 syscall result to 0. A + // process with no file descriptors also returns 0, but leaves + // errno unchanged. + if (size <= 0) { + if (errno == 0) { + fdBufSize = 0; + break; + } + throw SysError("listing pid %1% file descriptors", pid); + } + fdBufSize = size; + } + fds.resize(fdBufSize / sizeof(struct proc_fdinfo)); + + for (auto fd : fds) { + // By definition, only a vnode is on the filesystem. + if (fd.proc_fdtype != PROX_FDTYPE_VNODE) + continue; + + struct vnode_fdinfowithpath fdInfo; + if (proc_pidfdinfo(pid, fd.proc_fd, PROC_PIDFDVNODEPATHINFO, &fdInfo, sizeof(fdInfo)) <= 0) { + // They probably just closed this fd, so continue looking + // at ranges and environment variables. + if (errno == EBADF) + continue; + throw SysError("getting pid %1% fd %2% path", pid, fd.proc_fd); + } + + unchecked[std::string(fdInfo.pvip.vip_path)].emplace(fmt("{libproc/%d/fd/%d}", pid, fd.proc_fd)); + } + + // Regions, including mmapped files, executables, and shared libraries. + uint64_t nextAddr = 0; + while (true) { + // PROC_PIDREGIONPATHINFO2 includes regions backed by a vnode + // and has been available since OS X 10.10, but is not exposed + // by the SDK headers. Its numeric value is 22. + struct proc_regionwithpathinfo regionInfo; + if (proc_pidinfo(pid, 22, nextAddr, ®ionInfo, sizeof(regionInfo)) <= 0) { + // The API signals the end of the region list as an error. + if (errno == ESRCH || errno == EINVAL) + break; + throw SysError("getting pid %1% region path", pid); + } + + unchecked[std::string(regionInfo.prp_vip.vip_path)].emplace(fmt("{libproc/%d/region}", pid)); + + nextAddr = regionInfo.prp_prinfo.pri_address + regionInfo.prp_prinfo.pri_size; + } + + // Arguments and environment variables. Environment variables of + // entitled binaries cannot be read unless Nix has the + // com.apple.private.read-environment-variables entitlement or SIP + // is disabled. Arguments remain readable, but must be ignored: a + // path passed to `nix-store --delete` must not root itself. + int sysctlName[3] = {CTL_KERN, KERN_PROCARGS2, pid}; + size_t argsSize = 0; + if (sysctl(sysctlName, 3, nullptr, &argsSize, nullptr, 0) < 0) + throw SysError("reading pid %1% arguments", pid); + + std::vector args(argsSize); + if (sysctl(sysctlName, 3, args.data(), &argsSize, nullptr, 0) < 0) + throw SysError("reading pid %1% arguments", pid); + + if (argsSize < args.size()) + args.resize(argsSize); + + // The first four bytes contain argc, followed by the executable, + // argc arguments, then the environment. Skip the executable and + // arguments before searching for store paths. + if (args.size() < sizeof(int)) + continue; + + int argc; + std::memcpy(&argc, args.data(), sizeof(argc)); + if (argc < 0) + continue; + + auto argsIter = args.begin() + sizeof(argc); + // Skip the executable and any padding before argv. + argsIter = std::find(argsIter, args.end(), '\0'); + argsIter = std::find_if(argsIter, args.end(), [](char ch) { return ch != '\0'; }); + + // Empty arguments are significant, so skip exactly one + // NUL-terminated entry for each argument. + for (int i = 0; argsIter != args.end() && i < argc; ++i) { + argsIter = std::find(argsIter, args.end(), '\0'); + if (argsIter != args.end()) + ++argsIter; + } + + if (argsIter != args.end()) { + using RegexIterator = boost::regex_iterator; + for (RegexIterator i(argsIter, args.end(), storePathRegex), end; i != end; ++i) + unchecked[i->str()].emplace(fmt("{libproc/%d/environ}", pid)); + } + + // Per-thread working directories. + struct proc_taskallinfo taskAllInfo; + if (proc_pidinfo(pid, PROC_PIDTASKALLINFO, 0, &taskAllInfo, sizeof(taskAllInfo)) <= 0) + throw SysError("reading pid %1% tasks", pid); + + // If the process doesn't have the per-thread cwd flag, the + // process-wide cwd above is sufficient. + if (taskAllInfo.pbsd.pbi_flags & PROC_FLAG_THCWD) { + std::vector tids(taskAllInfo.ptinfo.pti_threadnum); + auto tidBufSize = + proc_pidinfo(pid, PROC_PIDLISTTHREADS, 0, tids.data(), tids.size() * sizeof(uint64_t)); + if (tidBufSize <= 0) + throw SysError("listing pid %1% threads", pid); + + tids.resize(tidBufSize / sizeof(uint64_t)); + for (auto tid : tids) { + struct proc_threadwithpathinfo threadPathInfo; + if (proc_pidinfo(pid, PROC_PIDTHREADPATHINFO, tid, &threadPathInfo, sizeof(threadPathInfo)) <= 0) { + // The thread may have exited since it was listed. + if (errno == ESRCH) + continue; + throw SysError("reading pid %1% thread %2% cwd", pid, tid); + } + + unchecked[std::string(threadPathInfo.pvip.vip_path)].emplace( + fmt("{libproc/%d/thread/%d/cwd}", pid, tid)); + } + } + } catch (SysError & e) { + // Processes can exit or become inaccessible at any point while + // they are inspected. Protected processes also reject some of + // these calls. + if (e.is(std::errc::no_such_file_or_directory) || e.is(std::errc::no_such_process) + || e.is(std::errc::invalid_argument) || e.is(std::errc::permission_denied) + || e.is(std::errc::operation_not_permitted) || e.is(std::errc::io_error)) + continue; + throw; + } + } +} + +} // namespace nix diff --git a/src/libstore/darwin/meson.build b/src/libstore/darwin/meson.build index 72d94baac2be..beebfd54afbe 100644 --- a/src/libstore/darwin/meson.build +++ b/src/libstore/darwin/meson.build @@ -1,3 +1,6 @@ include_dirs += [ include_directories('build') ] -sources += files('build/darwin-derivation-builder.cc') +sources += files( + 'build/darwin-derivation-builder.cc', + 'local-gc.cc', +) diff --git a/src/libstore/include/nix/store/local-gc.hh b/src/libstore/include/nix/store/local-gc.hh index d609626e2f56..4dd8294c8c39 100644 --- a/src/libstore/include/nix/store/local-gc.hh +++ b/src/libstore/include/nix/store/local-gc.hh @@ -1,6 +1,4 @@ #include "nix/store/gc-store.hh" -#include -#include namespace nix { diff --git a/src/libstore/local-gc-private.hh b/src/libstore/local-gc-private.hh new file mode 100644 index 000000000000..05e735a2e3a2 --- /dev/null +++ b/src/libstore/local-gc-private.hh @@ -0,0 +1,31 @@ +#pragma once + +#include "nix/store/store-dir-config.hh" +#include "nix/util/types.hh" + +#include +#include +#include + +namespace nix { + +/** + * The key is a string because std::filesystem::path cannot be used here with + * macOS's libc++. + */ +using UncheckedRoots = boost::unordered_flat_map< + std::string, + boost::unordered_flat_set>, + StringViewHash, + std::equal_to<>>; + +inline boost::regex makeStorePathRegex(const StoreDirConfig & config) +{ + static auto specialRegex = boost::regex(R"([.^$\\*+?()\[\]{}|])"); + auto quotedStoreDir = boost::regex_replace(config.storeDir, specialRegex, R"(\\$&)"); + return boost::regex(quotedStoreDir + R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)"); +} + +void findDarwinRuntimeRoots(const StoreDirConfig & config, UncheckedRoots & unchecked); + +} // namespace nix diff --git a/src/libstore/local-gc.cc b/src/libstore/local-gc.cc index 8bbbdc8366ac..98df303cc9d1 100644 --- a/src/libstore/local-gc.cc +++ b/src/libstore/local-gc.cc @@ -3,10 +3,11 @@ #include "nix/util/file-system.hh" #include "nix/util/signals.hh" #include "nix/store/local-gc.hh" +#include "local-gc-private.hh" #include #include -#if !defined(__linux__) +#if !defined(__linux__) && !defined(__APPLE__) // For shelling out to lsof # include "store-config-private.hh" # include "nix/util/environment-variables.hh" @@ -15,16 +16,7 @@ namespace nix { -/** - * Key is a mere string because cannot has path with macOS's libc++ - */ -typedef boost::unordered_flat_map< - std::string, - boost::unordered_flat_set>, - StringViewHash, - std::equal_to<>> - UncheckedRoots; - +#ifdef __linux__ static void readProcLink(const std::filesystem::path & file, UncheckedRoots & roots) { std::filesystem::path buf; @@ -39,12 +31,7 @@ static void readProcLink(const std::filesystem::path & file, UncheckedRoots & ro if (buf.is_absolute()) roots[buf.string()].emplace(file.string()); } - -static std::string quoteRegexChars(const std::string & raw) -{ - static auto specialRegex = boost::regex(R"([.^$\\*+?()\[\]{}|])"); - return boost::regex_replace(raw, specialRegex, R"(\\$&)"); -} +#endif #ifdef __linux__ static void readFileRoots(const std::filesystem::path & path, UncheckedRoots & roots) @@ -62,12 +49,15 @@ Roots findRuntimeRootsUnchecked(const StoreDirConfig & config) { UncheckedRoots unchecked; +#ifdef __APPLE__ + findDarwinRuntimeRoots(config, unchecked); +#elif defined(__linux__) auto procDir = AutoCloseDir{opendir("/proc")}; if (procDir) { struct dirent * ent; static const auto digitsRegex = boost::regex(R"(^\d+$)"); static const auto mapRegex = boost::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)"); - auto storePathRegex = boost::regex(quoteRegexChars(config.storeDir) + R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)"); + auto storePathRegex = makeStorePathRegex(config); while (errno = 0, ent = readdir(procDir.get())) { checkInterrupt(); if (boost::regex_match(ent->d_name, digitsRegex)) { @@ -120,10 +110,11 @@ Roots findRuntimeRootsUnchecked(const StoreDirConfig & config) throw SysError("iterating /proc"); } -#if !defined(__linux__) - // lsof is really slow on OS X. This actually causes the gc-concurrent.sh test to fail. - // See: https://github.com/NixOS/nix/issues/3011 - // Because of this we disable lsof when running the tests. + readFileRoots("/proc/sys/kernel/modprobe", unchecked); + readFileRoots("/proc/sys/kernel/fbsplash", unchecked); + readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked); +#else + // lsof can be prohibitively slow, so allow tests to disable it. if (getEnv("_NIX_TEST_NO_LSOF") != "1") { try { boost::regex lsofRegex(R"(^n(/.*)$)"); @@ -140,12 +131,6 @@ Roots findRuntimeRootsUnchecked(const StoreDirConfig & config) } #endif -#ifdef __linux__ - readFileRoots("/proc/sys/kernel/modprobe", unchecked); - readFileRoots("/proc/sys/kernel/fbsplash", unchecked); - readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked); -#endif - Roots roots; for (auto & [target, links] : unchecked) { diff --git a/tests/functional/gc-runtime.nix b/tests/functional/gc-runtime.nix index 10dc5e29f56d..5d6f1ff9b555 100644 --- a/tests/functional/gc-runtime.nix +++ b/tests/functional/gc-runtime.nix @@ -20,7 +20,8 @@ with import ./config.nix; cat > $out/program << 'EOF' #! ${shell} - sleep 10000 < "$1" + ${path}/env -i ${path}/sleep 10000 < "$1" & + wait EOF chmod +x $out/program diff --git a/tests/functional/gc-runtime.sh b/tests/functional/gc-runtime.sh index ac788406dea1..ff82ab5275ae 100755 --- a/tests/functional/gc-runtime.sh +++ b/tests/functional/gc-runtime.sh @@ -3,10 +3,10 @@ source common.sh case $system in - *linux*) + *linux* | *darwin*) ;; *) - skipTest "Not running Linux"; + skipTest "Not running Linux or Darwin"; esac set -m # enable job control, needed for kill @@ -16,8 +16,8 @@ environPath=$(nix-build --no-link ./gc-runtime.nix -A environ) openPath=$(nix-build --no-link ./gc-runtime.nix -A open) echo "backgrounding program..." -export environPath -"$programPath"/program "$openPath"/open & +env -i "environPath=$environPath" \ + "$programPath"/program "$openPath"/open "" & sleep 2 # hack - wait for the program to get started child=$! echo PID=$child