From ee74fc446591e22a5c76951a2e9c4f75cc0eb494 Mon Sep 17 00:00:00 2001 From: AJ ONeal Date: Wed, 9 Dec 2020 15:58:53 -0700 Subject: [PATCH] cleanup --- README.md | 224 ++++++++++++++++++++++++++++++++++++--------------- bin/walk.js | 46 ++++++----- create.js | 29 +++---- index.js | 8 +- package.json | 2 +- snippet.js | 72 +++++++++++------ walk.js | 10 ++- 7 files changed, 254 insertions(+), 137 deletions(-) diff --git a/README.md b/README.md index 8718322..7070a2a 100644 --- a/README.md +++ b/README.md @@ -8,27 +8,80 @@ using Node.js v10+'s `fs.readdir`'s `withFileTypes` and ES 2021) ```js await Walk.walk(pathname, walkFunc); -function walkFunc(err, pathname, dirent) { - // ... +async function walkFunc(err, pathname, dirent) { + // err is failure to lstat a file or directory + // pathname is relative path, including the file or folder name + // dirent = { name, isDirectory(), isFile(), isSymbolicLink(), ... } + + if (err) { + return false; + } + console.log(pathname); } ``` -Where +# Table of Contents -- `err` is a failure to lstat a file or directory -- `pathname` may be relative -- `dirent` is an `fs.Dirent` that has - - `dirent.name` - - `dirent.isFile()` - - `dirent.isDirectory()` - - `dirent.isSymbolicLink()` - - etc +- Install +- Usage + - CommonJS + - ES Modules +- API + - Walk.walk + - walkFunc + - Example: filter dotfiles + - Walk.create + - withFileStats + - sort (and filter) +- [Node walk in <50 Lines of Code](https://therootcompany.com/blog/fs-walk-for-node-js/) +- License (MPL-2.0) -# Examples +# Install -You can use this with Node v12+ using Vanilla JS or ES2021. +```bash +npm install --save @root/walk +``` -## ES 2021 Modules +# Usage + +You can use this with Node v12+ using Vanilla JS (CommonJS) or ES2021 (ES Modules). + +## CommonJS (Vanilla JS / ES5) + +```js +var Walk = require("@root/walk"); +var path = require("path"); + +Walk.walk("./", walkFunc).then(function () { + console.log("Done"); +}); + +// walkFunc must be async, or return a Promise +function walkFunc(err, pathname, dirent) { + if (err) { + // throw an error to stop walking + // (or return to ignore and keep going) + console.warn("fs stat error for %s: %s", pathname, err.message); + return Promise.resolve(); + } + + // return false to skip a directory + // (ex: skipping "dot file" directories) + if (dirent.isDirectory() && dirent.name.startsWith(".")) { + return Promise.resolve(false); + } + + // fs.Dirent is a slimmed-down, faster version of fs.Stats + console.log("name:", dirent.name, "in", path.dirname(pathname)); + // (only one of these will be true) + console.log("is file?", dirent.isFile()); + console.log("is link?", dirent.isSymbolicLink()); + + return Promise.resolve(); +} +``` + +## ECMAScript 2021 (ES Modules) `@root/walk` can be used with async/await or Promises. @@ -36,55 +89,28 @@ You can use this with Node v12+ using Vanilla JS or ES2021. import { walk } from "@root/walk"; import path from "path"; -await walk("./", async (err, pathname, dirent) => { - if (err) { - // throw an error to stop walking - // (or return to ignore and keep going) - console.warn("fs stat error for %s: %s", pathname, err.message); - return; - } - - // return false to skip a directory - // (ex: skipping "dot files") - if (dirent.isDirectory() && dirent.name.startsWith(".")) { - return false; - } - - // fs.Dirent is a slimmed-down, faster version of fs.Stat - console.log("name:", dirent.name, "in", path.dirname(pathname)); - // (only one of these will be true) - console.log("is file?", dirent.isFile()); - console.log("is link?", dirent.isSymbolicLink()); -}); - -console.log("Done"); -``` - -## Vanilla JS (ES5) - -```js -var Walk = require("@root/walk"); -var path = require("path"); - -Walk.walk("./", function walkFunc(err, pathname, dirent) { +const walkFunc = async (err, pathname, dirent) => { if (err) { throw err; } if (dirent.isDirectory() && dirent.name.startsWith(".")) { - return Promise.resolve(false); + return false; } - console.log("name:", dirent.name, "in", path.dirname(pathname)); - return Promise.resolve(); -}).then(function () { - console.log("Done"); -}); + console.log("name:", dirent.name, "in", path.dirname(pathname)); +}; + +await walk("./", walkFunc); + +console.log("Done"); ``` # API Documentation -`Walk.walk` walks `pathname` (inclusive) and calls `walkFunc` for each file system entry. +## Walk.walk(pathname, walkFunc) + +`Walk.walk` walks `pathname` (inclusive) and calls `walkFunc` for each file system entity. It can be used with Promises: @@ -99,21 +125,18 @@ await Walk.walk(pathname, asyncWalker); ``` The behavior should exactly match Go's -[`filepath.Walk`](https://golang.org/pkg/path/filepath/#Walk) with 3 exceptions: +[`filepath.Walk`](https://golang.org/pkg/path/filepath/#Walk) with a few exceptions: - uses JavaScript Promises/async/await -- receives `dirent` rather than `lstat` (for performance) +- receives `dirent` rather than `lstat` (for performance, see `withFileStats`) +- optional parameters to change stat behavior and sort order - - -## walkFunc +### walkFunc Handles each directory entry ```js -function walker(err, pathname, dirent) { +async function walkFunc(err, pathname, dirent) { // `err` is a file system stat error // `pathname` is the full pathname, including the file name // `dirent` is an fs.Dirent with a `name`, `isDirectory`, `isFile`, etc @@ -121,11 +144,82 @@ function walker(err, pathname, dirent) { } ``` - +- `withFileStats: true` walkFunc will receive fs.Stats[] from fs.lstat instead of fs.Dirent[] +- `sort: (entities) => entities.sort()` sort and/or filter entities before walking them + +```js +const walk = Walk.create({ + withFileStats: true, + sort: (entities) => entities.sort()), +}); +``` + +## withFileStats + +By default `walk` will use `fs.readdir(pathname, { withFileTypes: true })` which returns `fs.Dirent[]`, +which only has name and file type info, but is much faster when you don't need the complete `fs.Stats`. + +Enable `withFileStats` to use get full `fs.Stats`. This will use `fs.readdir(pathname)` (returning `String[]`) +and then call `fs.lstat(pathname)` - including `mtime`, `birthtime`, `uid`, etc - right after. + +```js +const walk = Walk.create({ + withFileStats: true, +}); + +walk(".", async function (err, pathname, stat) { + console.log(stat.name, stat.uid, stat.birthtime, stat.isDirectory()); +}); +``` + +## sort (and filter) + +Sometimes you want to give priority to walking certain directories first. + +The `sort` option allows you to specify a funciton that modifies the `fs.Dirent[]` entities (default) or `String[]` filenames (`withFileStats: true`). + +Since you must return the sorted array, you can also filter here if you'd prefer. + +```js +const byNameWithoutDotFiles = (entities) => { + // sort by name + // filter dot files + return entities + .sort((a, b) => { + if (a.name > b.name) { + return 1; + } + if (a.name < b.name) { + return -1; + } + return 0; + }) + .filter((ent) => !ent.name.startsWith(".")); +}; + +const walk = Walk.create({ sort: byNameWithoutDotFiles }); + +walk(".", async function (err, pathname, stat) { + // each directories contents will be listed alphabetically + console.log(pathname); +}); +``` + +Note: this gets the result of `fs.readdir()`. If `withFileStats` is `true` you will get a `String[]` of filenames - because this hapens BEFORE `fs.lstat()` is called - otherwise you will get `fs.Dirent[]`. + +# node walk in 50 lines of code + +If you're like me and you hate dependencies, +here's the bare minimum node fs walk function: + +See [snippet.js](/snippet.js) or . + +# License + +The main module, as published to NPM, is licensed the MPL-2.0. + +The ~50 line snippet is licensed CC0-1.0 (Public Domain). diff --git a/bin/walk.js b/bin/walk.js index a506554..46a82d8 100644 --- a/bin/walk.js +++ b/bin/walk.js @@ -1,14 +1,20 @@ -import path from "path"; -//import { walk } from "../index.js"; +"use strict"; -import Walk from "../index.js"; -var walk = Walk.create({ - sort: function (ents) { - return ents.filter(function (ent) { - return !ent.name.startsWith("."); - }); - }, -}); +var path = require("path"); +var Walk = require("../index.js"); + +var walk = Walk.walk; +var alt = process.argv[2]; + +if (alt) { + walk = Walk.create({ + sort: function (ents) { + return ents.filter(function (ent) { + return !ent.name.startsWith("."); + }); + }, + }); +} var rootpath = process.argv[2] || "."; @@ -17,23 +23,21 @@ walk(rootpath, async function (err, pathname, dirent) { throw err; } - /* - if (dirent.name.startsWith(".")) { - return false; + if (!alt) { + if (dirent.name.startsWith(".")) { + return false; + } } - */ - var entType; + var entType = "?"; if (dirent.isDirectory()) { - entType = " dir"; + entType = "d"; } else if (dirent.isFile()) { - entType = "file"; + entType = "f"; } else if (dirent.isSymbolicLink()) { - entType = "link"; - } else { - entType = "----"; + entType = "@"; } - console.info("[%s] %s", entType, path.dirname(path.resolve(pathname)), dirent.name); + console.info("%s %s", entType, path.dirname(path.resolve(pathname)), dirent.name); }).catch(function (err) { console.error(err.stack); }); diff --git a/create.js b/create.js index e550194..db01904 100644 --- a/create.js +++ b/create.js @@ -1,16 +1,13 @@ +"use strict"; + +const fs = require("fs").promises; +const Walk = require("./walk.js"); +const path = require("path"); +const _withFileTypes = { withFileTypes: true }; +const _noopts = {}; +const _pass = (err) => err; + // a port of Go's filepath.Walk - -import { promises as fs } from "fs"; -import Walk from "./walk.js"; -import path from "path"; - -var _withFileTypes = { withFileTypes: true }; -var _noopts = {}; - -function pass(err) { - return err; -} - Walk.create = function (opts) { if (!opts) { opts = _noopts; @@ -28,7 +25,7 @@ Walk.create = function (opts) { // the first run, or if false === withFileTypes if ("string" === typeof _dirent) { let _name = path.basename(path.resolve(pathname)); - _dirent = await fs.lstat(pathname).catch(pass); + _dirent = await fs.lstat(pathname).catch(_pass); if (_dirent instanceof Error) { err = _dirent; } else { @@ -37,7 +34,7 @@ Walk.create = function (opts) { } // run the user-supplied function and either skip, bail, or continue - err = await walkFunc(err, pathname, _dirent).catch(pass); + err = await walkFunc(err, pathname, _dirent).catch(_pass); if (false === err || Walk.skipDir === err) { return; } @@ -58,7 +55,7 @@ Walk.create = function (opts) { // TODO check if the error is "not a directory" // (and thus allow false === opts.withFileTypes) - let result = await fs.readdir(pathname, _readdirOpts).catch(pass); + let result = await fs.readdir(pathname, _readdirOpts).catch(_pass); if (result instanceof Error) { return walkFunc(result, pathname, _dirent); } @@ -73,4 +70,4 @@ Walk.create = function (opts) { return _walk; }; -export default Walk; +module.exports = Walk; diff --git a/index.js b/index.js index 50f7f40..12660a5 100644 --- a/index.js +++ b/index.js @@ -1,6 +1,4 @@ -import Walk from "./walk.js"; -import Walk2 from "./create.js"; +"use strict"; -Walk.create = Walk2.create; - -export default Walk; +module.exports = require("./walk.js"); +module.exports.create = require("./create.js").create; diff --git a/package.json b/package.json index 46498e9..624005f 100644 --- a/package.json +++ b/package.json @@ -4,12 +4,12 @@ "description": "fs.walk for node (as a port of Go's filepath.Walk)", "homepage": "https://git.rootprojects.org/root/walk.js", "main": "index.js", - "type": "module", "files": [ "walk.js", "create.js" ], "scripts": { + "prettier": "npx prettier --write '**/*.{md,js,mjs,cjs}'", "test": "echo \"Error: no test specified\" && exit 1" }, "repository": { diff --git a/snippet.js b/snippet.js index 872043c..a8d80c1 100644 --- a/snippet.js +++ b/snippet.js @@ -1,55 +1,77 @@ -// ECMAScript 2021 -// (or Vanilla JS) -import { promises as fs } from "fs"; -// or let fs = require("fs").promises; -import path from "path"; -// or let path = require("path"); +/** + * @license + * walk.js - fs.walk for node.js (a port of Go's filepath.Walk) + * + * Written in 2020 by AJ ONeal + * To the extent possible under law, the author(s) have dedicated all copyright + * and related and neighboring rights to this software to the public domain + * worldwide. This software is distributed without any warranty. + * + * You should have received a copy of the CC0 Public Domain Dedication along with + * this software. If not, see . + */ + +"use strict"; + +async function walk(pathname, walkFunc, dirent) { + const fs = require("fs").promises; + const path = require("path"); + const _pass = (err) => err; -// a port of Go's filepath.Walk -async function walk(pathname, walkFunc, _dirent) { let err; - function pass(e) { - return e; - } - // special case of the very first run - if (!_dirent) { - let _name = path.basename(path.resolve(pathname)); - _dirent = await fs.lstat(pathname).catch(pass); - if (_dirent instanceof Error) { - err = _dirent; + // special case: walk the very first file or folder + if (!dirent) { + let filename = path.basename(path.resolve(pathname)); + dirent = await fs.lstat(pathname).catch(_pass); + if (dirent instanceof Error) { + err = dirent; } else { - _dirent.name = _name; + dirent.name = filename; } } // run the user-supplied function and either skip, bail, or continue - err = await walkFunc(err, pathname, _dirent).catch(pass); + err = await walkFunc(err, pathname, dirent).catch(_pass); if (false === err) { + // walkFunc can return false to skip return; } if (err instanceof Error) { + // if walkFunc throws, we throw throw err; } // "walk does not follow symbolic links" - if (!_dirent.isDirectory()) { + // (doing so could cause infinite loops) + if (!dirent.isDirectory()) { return; } - let result = await fs.readdir(pathname, { withFileTypes: true }).catch(pass); + let result = await fs.readdir(pathname, { withFileTypes: true }).catch(_pass); if (result instanceof Error) { - return walkFunc(result, pathname, _dirent); + // notify on directory read error + return walkFunc(result, pathname, dirent); } - for (let dirent of result) { - await walk(path.join(pathname, dirent.name), walkFunc, dirent); + for (let entity of result) { + await walk(path.join(pathname, entity.name), walkFunc, entity); } } +// Example Usage: +const path = require("path"); walk("./", function (err, pathname, dirent) { if (dirent.name.startsWith(".")) { return Promise.resolve(false); } - console.log(path.resolve(pathname)); + var typ = "-"; + if (dirent.isFile()) { + typ = "f"; + } else if (dirent.isDirectory()) { + typ = "d"; + } else if (dirent.isSymbolicLink()) { + typ = "@"; + } + console.info(typ, path.resolve(pathname)); return Promise.resolve(true); }); diff --git a/walk.js b/walk.js index 6ad60c2..e63ca58 100644 --- a/walk.js +++ b/walk.js @@ -1,8 +1,10 @@ -import { promises as fs } from "fs"; -import path from "path"; +"use strict"; + +const fs = require("fs").promises; +const path = require("path"); -const _withFileTypes = { withFileTypes: true }; const skipDir = new Error("skip this directory"); +const _withFileTypes = { withFileTypes: true }; const pass = (err) => err; // a port of Go's filepath.Walk @@ -42,7 +44,7 @@ const walk = async (pathname, walkFunc, _dirent) => { } }; -export default { +module.exports = { walk, skipDir, };