root
/
walk.js
1
0
Fork 0
Code Issues 3 Pull Requests Projects Releases Wiki Activity

Compare commits

base: root:main
Branches Tags
root:main
root:master
root:esmodule
root:v1.1.0
root:v1.0.0
..
compare: root:esmodule
Branches Tags
root:main
root:master
root:esmodule
root:v1.1.0
root:v1.0.0

No commits in common. "main" and "esmodule" have entirely different histories.
main ... esmodule

8 changed files with 142 additions and 259 deletions
Show Stats Expand all files Collapse all files

228
README.md
View File

@ -1,4 +1,4 @@
# [Walk.js](https://git.rootprojects.org/root/walk.js) (@root/walk)
# Walk.js (@root/walk)
Walk a directory recursively and handle each entity (files, directories, symlnks, etc).
@ -8,80 +8,27 @@ using Node.js v10+'s `fs.readdir`'s `withFileTypes` and ES 2021)
```js
await Walk.walk(pathname, walkFunc);
async function walkFunc(err, pathname, dirent) {
// err is failure to lstat a file or directory
// pathname is relative path, including the file or folder name
// dirent = { name, isDirectory(), isFile(), isSymbolicLink(), ... }
if (err) {
return false;
}
console.log(pathname);
}
```
# Table of Contents
- Install
- Usage
- CommonJS
- ES Modules
- API
- Walk.walk
- walkFunc
- Example: filter dotfiles
- Walk.create
- withFileStats
- sort (and filter)
- [Node walk in <50 Lines of Code](https://therootcompany.com/blog/fs-walk-for-node-js/)
- License (MPL-2.0)
# Install
```bash
npm install --save @root/walk
```
# Usage
You can use this with Node v12+ using Vanilla JS (CommonJS) or ES2021 (ES Modules).
## CommonJS (Vanilla JS / ES5)
```js
var Walk = require("@root/walk");
var path = require("path");
Walk.walk("./", walkFunc).then(function () {
console.log("Done");
});
// walkFunc must be async, or return a Promise
function walkFunc(err, pathname, dirent) {
if (err) {
// throw an error to stop walking
// (or return to ignore and keep going)
console.warn("fs stat error for %s: %s", pathname, err.message);
return Promise.resolve();
}
// return false to skip a directory
// (ex: skipping "dot file" directories)
if (dirent.isDirectory() && dirent.name.startsWith(".")) {
return Promise.resolve(false);
}
// fs.Dirent is a slimmed-down, faster version of fs.Stats
console.log("name:", dirent.name, "in", path.dirname(pathname));
// (only one of these will be true)
console.log("is file?", dirent.isFile());
console.log("is link?", dirent.isSymbolicLink());
return Promise.resolve();
// ...
}
```
## ECMAScript 2021 (ES Modules)
Where
- `err` is a failure to lstat a file or directory
- `pathname` may be relative
- `dirent` is an `fs.Dirent` that has
- `dirent.name`
- `dirent.isFile()`
- `dirent.isDirectory()`
- `dirent.isSymbolicLink()`
- etc
# Examples
You can use this with Node v12+ using Vanilla JS or ES2021.
## ES 2021 Modules
`@root/walk` can be used with async/await or Promises.
@ -89,28 +36,55 @@ function walkFunc(err, pathname, dirent) {
import { walk } from "@root/walk";
import path from "path";
const walkFunc = async (err, pathname, dirent) => {
await walk("./", async (err, pathname, dirent) => {
if (err) {
// throw an error to stop walking
// (or return to ignore and keep going)
console.warn("fs stat error for %s: %s", pathname, err.message);
return;
}
// return false to skip a directory
// (ex: skipping "dot files")
if (dirent.isDirectory() && dirent.name.startsWith(".")) {
return false;
}
// fs.Dirent is a slimmed-down, faster version of fs.Stat
console.log("name:", dirent.name, "in", path.dirname(pathname));
// (only one of these will be true)
console.log("is file?", dirent.isFile());
console.log("is link?", dirent.isSymbolicLink());
});
console.log("Done");
```
## Vanilla JS (ES5)
```js
var Walk = require("@root/walk");
var path = require("path");
Walk.walk("./", function walkFunc(err, pathname, dirent) {
if (err) {
throw err;
}
if (dirent.isDirectory() && dirent.name.startsWith(".")) {
return false;
return Promise.resolve(false);
}
console.log("name:", dirent.name, "in", path.dirname(pathname));
};
await walk("./", walkFunc);
return Promise.resolve();
}).then(function () {
console.log("Done");
});
```
# API Documentation
## Walk.walk(pathname, walkFunc)
`Walk.walk` walks `pathname` (inclusive) and calls `walkFunc` for each file system entity.
`Walk.walk` walks `pathname` (inclusive) and calls `walkFunc` for each file system entry.
It can be used with Promises:
@ -125,18 +99,21 @@ await Walk.walk(pathname, asyncWalker);
```
The behavior should exactly match Go's
[`filepath.Walk`](https://golang.org/pkg/path/filepath/#Walk) with a few exceptions:
[`filepath.Walk`](https://golang.org/pkg/path/filepath/#Walk) with 3 exceptions:
- uses JavaScript Promises/async/await
- receives `dirent` rather than `lstat` (for performance, see `withFileStats`)
- optional parameters to change stat behavior and sort order
- receives `dirent` rather than `lstat` (for performance)
### walkFunc
<!-- TODO
- can be created with `options` to change default behaviors
-->
## walkFunc
Handles each directory entry
```js
async function walkFunc(err, pathname, dirent) {
function walker(err, pathname, dirent) {
// `err` is a file system stat error
// `pathname` is the full pathname, including the file name
// `dirent` is an fs.Dirent with a `name`, `isDirectory`, `isFile`, etc
@ -144,82 +121,11 @@ async function walkFunc(err, pathname, dirent) {
}
```
## Walk.create(options)
<!-- TODO
## create(options)
Create a custom walker with these options:
- `withFileStats: true` walkFunc will receive fs.Stats[] from fs.lstat instead of fs.Dirent[]
- `sort: (entities) => entities.sort()` sort and/or filter entities before walking them
```js
const walk = Walk.create({
withFileStats: true,
sort: (entities) => entities.sort()),
});
```
## withFileStats
By default `walk` will use `fs.readdir(pathname, { withFileTypes: true })` which returns `fs.Dirent[]`,
which only has name and file type info, but is much faster when you don't need the complete `fs.Stats`.
Enable `withFileStats` to use get full `fs.Stats`. This will use `fs.readdir(pathname)` (returning `String[]`)
and then call `fs.lstat(pathname)` - including `mtime`, `birthtime`, `uid`, etc - right after.
```js
const walk = Walk.create({
withFileStats: true,
});
walk(".", async function (err, pathname, stat) {
console.log(stat.name, stat.uid, stat.birthtime, stat.isDirectory());
});
```
## sort (and filter)
Sometimes you want to give priority to walking certain directories first.
The `sort` option allows you to specify a funciton that modifies the `fs.Dirent[]` entities (default) or `String[]` filenames (`withFileStats: true`).
Since you must return the sorted array, you can also filter here if you'd prefer.
```js
const byNameWithoutDotFiles = (entities) => {
// sort by name
// filter dot files
return entities
.sort((a, b) => {
if (a.name > b.name) {
return 1;
}
if (a.name < b.name) {
return -1;
}
return 0;
})
.filter((ent) => !ent.name.startsWith("."));
};
const walk = Walk.create({ sort: byNameWithoutDotFiles });
walk(".", async function (err, pathname, stat) {
// each directories contents will be listed alphabetically
console.log(pathname);
});
```
Note: this gets the result of `fs.readdir()`. If `withFileStats` is `true` you will get a `String[]` of filenames - because this hapens BEFORE `fs.lstat()` is called - otherwise you will get `fs.Dirent[]`.
# node walk in 50 lines of code
If you're like me and you hate dependencies,
here's the bare minimum node fs walk function:
See [snippet.js](/snippet.js) or <https://therootcompany.com/blog/fs-walk-for-node-js/>.
# License
The main module, as published to NPM, is licensed the MPL-2.0.
The ~50 line snippet is licensed CC0-1.0 (Public Domain).
- `withFileTypes: false` walkFunc will receive String[] instead of fs.Dirent[]
- `sort: sortFunc`
-->

30
bin/walk.js
View File

@ -1,20 +1,14 @@
"use strict";
import path from "path";
//import { walk } from "../index.js";
var path = require("path");
var Walk = require("../index.js");
var walk = Walk.walk;
var alt = process.argv[2];
if (alt) {
walk = Walk.create({
import Walk from "../index.js";
var walk = Walk.create({
sort: function (ents) {
return ents.filter(function (ent) {
return !ent.name.startsWith(".");
});
},
});
}
var rootpath = process.argv[2] || ".";
@ -23,21 +17,23 @@ walk(rootpath, async function (err, pathname, dirent) {
throw err;
}
if (!alt) {
/*
if (dirent.name.startsWith(".")) {
return false;
}
}
*/
var entType = "?";
var entType;
if (dirent.isDirectory()) {
entType = "d";
entType = " dir";
} else if (dirent.isFile()) {
entType = "f";
entType = "file";
} else if (dirent.isSymbolicLink()) {
entType = "@";
entType = "link";
} else {
entType = "----";
}
console.info("%s %s", entType, path.dirname(path.resolve(pathname)), dirent.name);
console.info("[%s] %s", entType, path.dirname(path.resolve(pathname)), dirent.name);
}).catch(function (err) {
console.error(err.stack);
});

29
create.js
View File

@ -1,13 +1,16 @@
"use strict";
const fs = require("fs").promises;
const Walk = require("./walk.js");
const path = require("path");
const _withFileTypes = { withFileTypes: true };
const _noopts = {};
const _pass = (err) => err;
// a port of Go's filepath.Walk
import { promises as fs } from "fs";
import Walk from "./walk.js";
import path from "path";
var _withFileTypes = { withFileTypes: true };
var _noopts = {};
function pass(err) {
return err;
}
Walk.create = function (opts) {
if (!opts) {
opts = _noopts;
@ -25,7 +28,7 @@ Walk.create = function (opts) {
// the first run, or if false === withFileTypes
if ("string" === typeof _dirent) {
let _name = path.basename(path.resolve(pathname));
_dirent = await fs.lstat(pathname).catch(_pass);
_dirent = await fs.lstat(pathname).catch(pass);
if (_dirent instanceof Error) {
err = _dirent;
} else {
@ -34,7 +37,7 @@ Walk.create = function (opts) {
}
// run the user-supplied function and either skip, bail, or continue
err = await walkFunc(err, pathname, _dirent).catch(_pass);
err = await walkFunc(err, pathname, _dirent).catch(pass);
if (false === err || Walk.skipDir === err) {
return;
}
@ -55,7 +58,7 @@ Walk.create = function (opts) {
// TODO check if the error is "not a directory"
// (and thus allow false === opts.withFileTypes)
let result = await fs.readdir(pathname, _readdirOpts).catch(_pass);
let result = await fs.readdir(pathname, _readdirOpts).catch(pass);
if (result instanceof Error) {
return walkFunc(result, pathname, _dirent);
}
@ -70,4 +73,4 @@ Walk.create = function (opts) {
return _walk;
};
module.exports = Walk;
export default Walk;

8
index.js
View File

@ -1,4 +1,6 @@
"use strict";
import Walk from "./walk.js";
import Walk2 from "./create.js";
module.exports = require("./walk.js");
module.exports.create = require("./create.js").create;
Walk.create = Walk2.create;
export default Walk;

2
package-lock.json generated
View File

@ -1,5 +1,5 @@
{
"name": "@root/walk",
"version": "1.1.0",
"version": "1.0.0",
"lockfileVersion": 1
}

4
package.json
View File

@ -1,15 +1,15 @@
{
"name": "@root/walk",
"version": "1.1.0",
"version": "1.0.0",
"description": "fs.walk for node (as a port of Go's filepath.Walk)",
"homepage": "https://git.rootprojects.org/root/walk.js",
"main": "index.js",
"type": "module",
"files": [
"walk.js",
"create.js"
],
"scripts": {
"prettier": "npx prettier --write '**/*.{md,js,mjs,cjs}'",
"test": "echo \"Error: no test specified\" && exit 1"
},
"repository": {

72
snippet.js
View File

@ -1,77 +1,55 @@
/**
* @license
* walk.js - fs.walk for node.js (a port of Go's filepath.Walk)
*
* Written in 2020 by AJ ONeal <coolaj86@gmail.com>
* To the extent possible under law, the author(s) have dedicated all copyright
* and related and neighboring rights to this software to the public domain
* worldwide. This software is distributed without any warranty.
*
* You should have received a copy of the CC0 Public Domain Dedication along with
* this software. If not, see <https://creativecommons.org/publicdomain/zero/1.0/>.
*/
"use strict";
async function walk(pathname, walkFunc, dirent) {
const fs = require("fs").promises;
const path = require("path");
const _pass = (err) => err;
// ECMAScript 2021
// (or Vanilla JS)
import { promises as fs } from "fs";
// or let fs = require("fs").promises;
import path from "path";
// or let path = require("path");
// a port of Go's filepath.Walk
async function walk(pathname, walkFunc, _dirent) {
let err;
function pass(e) {
return e;
}
// special case: walk the very first file or folder
if (!dirent) {
let filename = path.basename(path.resolve(pathname));
dirent = await fs.lstat(pathname).catch(_pass);
if (dirent instanceof Error) {
err = dirent;
// special case of the very first run
if (!_dirent) {
let _name = path.basename(path.resolve(pathname));
_dirent = await fs.lstat(pathname).catch(pass);
if (_dirent instanceof Error) {
err = _dirent;
} else {
dirent.name = filename;
_dirent.name = _name;
}
}
// run the user-supplied function and either skip, bail, or continue
err = await walkFunc(err, pathname, dirent).catch(_pass);
err = await walkFunc(err, pathname, _dirent).catch(pass);
if (false === err) {
// walkFunc can return false to skip
return;
}
if (err instanceof Error) {
// if walkFunc throws, we throw
throw err;
}
// "walk does not follow symbolic links"
// (doing so could cause infinite loops)
if (!dirent.isDirectory()) {
if (!_dirent.isDirectory()) {
return;
}
let result = await fs.readdir(pathname, { withFileTypes: true }).catch(_pass);
let result = await fs.readdir(pathname, { withFileTypes: true }).catch(pass);
if (result instanceof Error) {
// notify on directory read error
return walkFunc(result, pathname, dirent);
return walkFunc(result, pathname, _dirent);
}
for (let entity of result) {
await walk(path.join(pathname, entity.name), walkFunc, entity);
for (let dirent of result) {
await walk(path.join(pathname, dirent.name), walkFunc, dirent);
}
}
// Example Usage:
const path = require("path");
walk("./", function (err, pathname, dirent) {
if (dirent.name.startsWith(".")) {
return Promise.resolve(false);
}
var typ = "-";
if (dirent.isFile()) {
typ = "f";
} else if (dirent.isDirectory()) {
typ = "d";
} else if (dirent.isSymbolicLink()) {
typ = "@";
}
console.info(typ, path.resolve(pathname));
console.log(path.resolve(pathname));
return Promise.resolve(true);
});

10
walk.js
View File

@ -1,10 +1,8 @@
"use strict";
import { promises as fs } from "fs";
import path from "path";
const fs = require("fs").promises;
const path = require("path");
const skipDir = new Error("skip this directory");
const _withFileTypes = { withFileTypes: true };
const skipDir = new Error("skip this directory");
const pass = (err) => err;
// a port of Go's filepath.Walk
@ -44,7 +42,7 @@ const walk = async (pathname, walkFunc, _dirent) => {
}
};
module.exports = {
export default {
walk,
skipDir,
};