Skip to content

fs: watch directories, not files, in recursive fs.watch fallback #65486

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
codebytere wants to merge 1 commit into
base: main
Choose a base branch
from
+282 −108
Open

fs: watch directories, not files, in recursive fs.watch fallback #65486

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions benchmark/fs/bench-watch-recursive.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
'use strict';

// Setting up (and tearing down) a recursive fs.watch() on a directory tree.
// On Linux and other platforms without a native recursive watcher this is
// implemented in JavaScript on top of per-directory watchers.

const common = require('../common');
const fs = require('fs');
const path = require('path');

const bench = common.createBenchmark(main, {
n: [5],
dir: ['lib', 'test/fixtures'],
});

function main({ n, dir }) {
const fullPath = path.resolve(__dirname, '../../', dir);
bench.start();
for (let i = 0; i < n; i++) {
fs.watch(fullPath, { recursive: true }).close();
}
bench.end(n);
}
259 changes: 151 additions & 108 deletions lib/internal/fs/recursive_watch.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ const {
},
} = require('internal/errors');
const { getValidatedPath } = require('internal/fs/utils');
const { createIgnoreMatcher, kFSWatchStart, StatWatcher } = require('internal/fs/watchers');
const { createIgnoreMatcher, kFSWatchStart } = require('internal/fs/watchers');
const { kEmptyObject } = require('internal/util');
const { validateBoolean, validateAbortSignal, validateIgnoreOption } = require('internal/validators');
const {
Expand All @@ -37,14 +37,22 @@ function lazyLoadFsSync() {

let kResistStopPropagation;

// Inotify reports changes to a directory's entries, with their names, on the
// directory's own watch, so one watcher per directory is enough on Linux.
// kqueue and event ports only report that the directory itself changed, so
// elsewhere every file keeps a watcher of its own as well.
const kDirectoryWatchReportsEntries = process.platform === 'linux';

class FSWatcher extends EventEmitter {
#options = null;
#closed = false;
#files = new SafeMap();
// Every path below the root that has been reported (or existed at start).
#entries = new SafeSet();
// One fs.watch() per directory and symbolic link (and per file where the
// directory watch does not report its entries).
#watchers = new SafeMap();
#symbolicFiles = new SafeSet();
#symbolicLinks = new SafeSet();
#rootPath = pathResolve();
#watchingFile = false;
#ignoreMatcher = null;

constructor(options = kEmptyObject) {
Expand Down Expand Up @@ -94,129 +102,168 @@ class FSWatcher extends EventEmitter {

this.#closed = true;

for (const file of this.#files.keys()) {
this.#watchers.get(file)?.close();
this.#watchers.delete(file);
for (const watcher of this.#watchers.values()) {
watcher.close();
}

this.#files.clear();
this.#symbolicFiles.clear();
this.#watchers.clear();
this.#entries.clear();
this.#symbolicLinks.clear();
this.emit('close');
}

#unwatchFiles(file) {
this.#symbolicFiles.delete(file);
#emit(eventType, file) {
this.emit('change', eventType, pathRelative(this.#rootPath, file));
}

#forget(file) {
const childPrefix = file + pathSep;
for (const filename of this.#files.keys()) {
if (filename === file ||
StringPrototypeStartsWith(filename, childPrefix)) {
this.#files.delete(filename);
this.#watchers.get(filename)?.close();
this.#watchers.delete(filename);
for (const entry of this.#entries) {
if (entry === file || StringPrototypeStartsWith(entry, childPrefix)) {
this.#entries.delete(entry);
this.#symbolicLinks.delete(entry);
const watcher = this.#watchers.get(entry);
if (watcher !== undefined) {
watcher.close();
this.#watchers.delete(entry);
}
}
}
}

#watchFolder(folder) {
const { readdirSync } = lazyLoadFsSync();

// An entry that vanished between being listed and being watched is left to
// the directory's own watcher to report.
#watch(file, onChange) {
if (this.#closed || this.#watchers.has(file)) {
return;
}
const { watch } = lazyLoadFsSync();
let watcher;
try {
const files = readdirSync(folder, {
withFileTypes: true,
});

for (const file of files) {
if (this.#closed) {
break;
}

const f = pathJoin(folder, file.name);
const relativePath = pathRelative(this.#rootPath, f);

// Skip watching ignored paths entirely to avoid kernel resource pressure
if (this.#ignoreMatcher?.(relativePath)) {
continue;
}

if (!this.#files.has(f)) {
this.emit('change', 'rename', relativePath);

if (file.isSymbolicLink()) {
this.#symbolicFiles.add(f);
}

try {
this.#watchFile(f);
if (file.isDirectory() && !file.isSymbolicLink()) {
this.#watchFolder(f);
}
} catch (err) {
// Ignore ENOENT
if (err.code !== 'ENOENT') {
throw err;
}
}
}
watcher = watch(file, { persistent: this.#options.persistent }, onChange);
} catch (err) {
if (err.code === 'ENOENT') {
return;
}
throw err;
}
this.#watchers.set(file, watcher);
}

// Registers the entries of `folder` that are not known yet (emitting
// 'rename' for them unless this is the initial scan) and arms one watcher
// for the directory; #addEntry() descends into subdirectories.
#scanFolder(folder, initial) {
const { readdirSync } = lazyLoadFsSync();
let entries;
try {
entries = readdirSync(folder, { withFileTypes: true });
} catch (error) {
if (error.code !== 'ENOENT') {
this.emit('error', error);
}
return;
}

this.#watch(folder, (eventType, filename) => this.#onFolderEvent(folder, filename));

for (const entry of entries) {
if (this.#closed) {
break;
}
const file = pathJoin(folder, entry.name);
if (!this.#entries.has(file) && !this.#ignoreMatcher?.(pathRelative(this.#rootPath, file))) {
this.#addEntry(file, entry, initial);
}
}
}

// `entry` is the Dirent or the lstat() Stats of `file`.
#addEntry(file, entry, initial) {
this.#entries.add(file);
if (!initial) {
this.#emit('rename', file);
}
if (entry.isSymbolicLink()) {
// The link target is watched so that changes behind the link surface
// as a 'rename' of the link, as they always have on this code path.
this.#symbolicLinks.add(file);
this.#watch(file, () => this.#emit('rename', file));
} else if (entry.isDirectory()) {
this.#scanFolder(file, initial);
} else if (!kDirectoryWatchReportsEntries) {
this.#watch(file, () => this.#onEntryEvent(file));
}
}

#watchFile(file) {
#onFolderEvent(folder, filename) {
if (this.#closed) {
return;
}
const { lstatSync, statSync } = lazyLoadFsSync();
if (!kDirectoryWatchReportsEntries || filename == null) {
// All that is known is that something about `folder` changed.
if (statSync(folder, { throwIfNoEntry: false }) === undefined) {
this.#emit('rename', folder);
this.#forget(folder);
} else {
this.#scanFolder(folder, false);
}
return;
}
// Events about the watched directory itself are reported under its own
// name; those take the "unknown entry" path and are resolved by the parent.
const file = pathJoin(folder, filename);

const { watch, statSync } = lazyLoadFsSync();

if (this.#files.has(file)) {
if (!this.#entries.has(file)) {
if (this.#ignoreMatcher?.(pathRelative(this.#rootPath, file))) {
return;
}
const entry = lstatSync(file, { throwIfNoEntry: false });
if (entry !== undefined) {
this.#addEntry(file, entry, false);
} else if (folder === this.#rootPath && statSync(folder, { throwIfNoEntry: false }) === undefined) {
this.#emit('rename', folder);
this.#forget(folder);
}
return;
}

{
const existingStat = statSync(file);
this.#files.set(file, existingStat);
this.#onEntryEvent(file);
}

// Something happened to a known entry: work out what from its current state.
#onEntryEvent(file) {
if (this.#closed) {
return;
}
const { statSync } = lazyLoadFsSync();
const stats = statSync(file, { throwIfNoEntry: false });
if (stats === undefined) {
this.#emit('rename', file);
this.#forget(file);
} else if (this.#symbolicLinks.has(file)) {
this.#emit('rename', file);
} else if (stats.isDirectory()) {
this.#scanFolder(file, false);
} else {
this.#emit('change', file);
}
}

const watcher = watch(file, {
persistent: this.#options.persistent,
}, (eventType, filename) => {
const existingStat = this.#files.get(file);
let currentStats;

try {
currentStats = statSync(file);
this.#files.set(file, currentStats);
} catch {
// This happens if the file was removed
#watchRootFile(file) {
const { statSync } = lazyLoadFsSync();
this.#entries.add(file);
this.#watch(file, () => {
if (this.#closed) {
return;
}

if (currentStats === undefined || (currentStats.birthtimeMs === 0 && existingStat.birthtimeMs !== 0)) {
// The file is now deleted
this.#files.delete(file);
this.#watchers.delete(file);
watcher.close();
this.emit('change', 'rename', pathRelative(this.#rootPath, file));
this.#unwatchFiles(file);
} else if (file === this.#rootPath && this.#watchingFile) {
// This case will only be triggered when watching a file with fs.watch
this.emit('change', 'change', pathBasename(file));
} else if (this.#symbolicFiles.has(file)) {
// Stats from watchFile does not return correct value for currentStats.isSymbolicLink()
// Since it is only valid when using fs.lstat(). Therefore, check the existing symbolic files.
this.emit('change', 'rename', pathRelative(this.#rootPath, file));
} else if (currentStats.isDirectory()) {
this.#watchFolder(file);
if (statSync(file, { throwIfNoEntry: false }) === undefined) {
this.#emit('rename', file);
this.#forget(file);
} else {
// Watching a directory will trigger a change event for child files)
this.emit('change', 'change', pathRelative(this.#rootPath, file));
this.emit('change', 'change', pathBasename(file));
}
});
this.#watchers.set(file, watcher);
}

[kFSWatchStart](filename) {
Expand All @@ -227,11 +274,11 @@ class FSWatcher extends EventEmitter {

this.#rootPath = filename;
this.#closed = false;
this.#watchingFile = file.isFile();

this.#watchFile(filename);
if (file.isDirectory()) {
this.#watchFolder(filename);
this.#scanFolder(filename, true);
} else {
this.#watchRootFile(filename);
}
} catch (error) {
if (!this.#options.throwIfNoEntry && error.code === 'ENOENT') {
Expand All @@ -243,19 +290,15 @@ class FSWatcher extends EventEmitter {
}

ref() {
this.#files.forEach((file) => {
if (file instanceof StatWatcher) {
file.ref();
}
});
for (const watcher of this.#watchers.values()) {
watcher.ref();
}
}

unref() {
this.#files.forEach((file) => {
if (file instanceof StatWatcher) {
file.unref();
}
});
for (const watcher of this.#watchers.values()) {
watcher.unref();
}
}

[SymbolAsyncIterator]() {
Expand Down
Loading
Loading