From 2f7e7700d5b6fa882745cb2bdf78ee1cf6463d98 Mon Sep 17 00:00:00 2001 From: Chris Thompson Date: Tue, 4 Apr 2017 13:54:28 -0600 Subject: [PATCH] [MAJOR] Provide significantly more control over processing and loading * filter - Used to determine which directories and files to include * fileTransform - Use as a custom loader or to ignore a file * dirTransform - Can prevent a dir from being added to the result --- README.md | 43 +++++++- index.js | 174 ++++++++++++------------------- lib/file.js | 105 +++++++++++++++++++ package.json | 7 +- test/async-test.js | 83 +++++++++++++++ test/index.js | 252 ++++++++++++++++++++++++++++++++++++++++----- 6 files changed, 525 insertions(+), 139 deletions(-) create mode 100644 lib/file.js create mode 100644 test/async-test.js diff --git a/README.md b/README.md index 77a91d8..d4905af 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,40 @@ dir-obj ======= -Create an object from a directory tree. Create a key with an object for each directory by name. All files within a directory that can be required (`.js` or `.json`) are added to the directory object, using the base filename as the key and `require('file.js')` as the value. +Create an object from a directory tree. + +`dir-obj` creates an object with keys for each directory (by name). +All files within a directory that can be required (`.js` or `.json`) are added to the directory object, using the base filename as the key and `require('file.js')` as the value. This is useful for things like loading test fixture data. +## API +You can also customize how `dir-obj` processes the directory tree + +```javascript +const fixtures = dirObj.readDirectory(path.join(__dirname, 'fixtures'), { + // Use the `filter` option to filter all directories and files + // Return `true` to include the directory or file (call `file.isDirectory` to check) + // Return `false` to exclude the file or directory from the result + // The default is to only allow `.js` and `.json` files + filter: file => (file.isDirectory || file.isRequirable) && file.name !== 'index.js', + // This function takes a File object and returns the value to be added to the object + // You can mutate `File.key` to change the object key used to reference the value + // You can return `undefined` to not include the file in the result + // The default `fileTransform` just calls `require(file.fullpath)` + // Example: Change whitespace in a filename to `_` then set the value using `require` + fileTransform: file => { + file.key = file.basename.replace(/\s/g, '_'); + return require(file.fullpath); + }, + // This function takes a File (which is a directory) and a value, which is the recursively + // calculated value of that directory + // Example: Do not include any directories that have no child keys + dirTransform: (file, value) => Object.getOwnPropertyNames(value).length > 0 ? value : undefined +}); + ## Example +Using the default options: ```javascript /* @@ -31,3 +60,15 @@ describe('test', function () { }); }); ``` + +Loading SQL files: + +```javascript +// Get an object where the value of each key is a string loaded from each SQL file + +const fixtures = dirObj.readDirectory(path.resolve('../../models/sql'), { + filter: file => file.isDirectory || file.ext === '.sql'. + fileTransform: file => file.readSync() // helper function equivilent to `fs.readFileSync(file.fullpath, 'utf8')` +}); + +``` diff --git a/index.js b/index.js index 248264d..27ac000 100644 --- a/index.js +++ b/index.js @@ -2,113 +2,63 @@ const fs = require('fs'); const nodeRequire = require; -const path = require('path'); -const jsonFile = /^\.js(?:on)?$/; - -/** - * Symbol to define file attributes cache - * - * This allows us to store a cache of file attributes but keep it hidden - * - * @type {Symbol} - * @private - */ -const _attributes = Symbol('attributes'); -const _key = Symbol('key'); - -/** - * File or directory - * @property {String} path The fully resolved path to the file (the parent directory) - * @property {String} fullpath The fully resolved path of the file - * @property {String} ext The file extension - * @property {String} name The full name of the file with no path - * @property {String} basename The name of the file without the extension - * @property {Boolean} isDirectory Returns `true` if the file is a directory - */ -class File { - constructor(dir, file) { - this.path = path.resolve(dir); - this.fullpath = path.join(this.path, file); - this.ext = path.extname(this.fullpath); - this.name = path.basename(this.fullpath); - this.basename = path.basename(this.name, this.ext); - }; - - /** - * Get a string value to be used as an object key - * - * @returns {String} - */ - get key() { - return this.attributes.key; - } - - set key(value) { - return (this.attributes.key = value); - } - - /** - * Get the file's attributes - * - * @see https://nodejs.org/api/fs.html#fs_fs_lstatsync_path - * @returns {Object} - */ - get attributes() { - if (!this[_attributes]) { - const a = this[_attributes] = fs.lstatSync(this.fullpath); - a.key = (a.isDirectory()) ? this.name : this.basename; - } - return this[_attributes]; - }; - - /** - * Check if file is a directory - * - * @returns {Boolean} - */ - get isDirectory() { - return this.attributes.isDirectory(); - } - - /** - * Check if file extension is either `.js` or `.json` - * - * @returns {Boolean} - */ - get isRequirable() { - return jsonFile.test(this.ext); - } -} +const File = require('./lib/file'); /** * Add a property to an object with a value * - * @param {Function} transform - * A transform function that takes the value and the file + * @param {Object} object + * @param {*} value + * @param {File} file + * @param {Function} transform A transform function that takes the value and the file * If `transform` returns a function, then the object will use it as a getter * */ -function defineProperty(object, value, file, transform) { +function defineProperty (object, file, transform, value) { const property = { enumerable: true }; - const result = transform(value, file); - // If dirTransform returns a function, then use it as a getter, otherwise just return the value - property[(typeof result === 'function') ? 'get' : 'value'] = result; - - return Object.defineProperty(object, file.key || file.basename, property); + const result = transform(file, value); + if (result !== undefined) { + // If dirTransform returns a function, then use it as a getter, otherwise just return the value + property[( result instanceof Function ) ? 'get' : 'value'] = result; + return Object.defineProperty(object, file.key || file.basename, property); + } } /** * Make sure transform functions are defined + * + * @param {Object} [options] + * @param {RegExp|Function} [options.filter] + * @param {Function} [options.dirTransform] + * @param {Function} [options.fileTransform] */ -function defaultOptions(options) { - const opts = options || {}; - - if (typeof opts.dirTransform !== 'function') { - opts.dirTransform = v => v; +function defaultOptions (options) { + const opts = {}; + + if (!(options instanceof Object)) { + options = {}; } - if (typeof opts.fileTransform !== 'function') { - opts.fileTransform = v => v; + if (options.filter instanceof RegExp) { + opts.filter = file => opts.filter.test(file.basename); + } else if (options.filter instanceof Function) { + opts.filter = options.filter; + } else { + opts.filter = file => ( + file.isDirectory || file.isRequirable + ); + } + + if (options.dirTransform instanceof Function) { + opts.dirTransform = options.dirTransform; + } else { + opts.dirTransform = (f, v) => v; + } + + if (options.fileTransform instanceof Function) { + opts.fileTransform = options.fileTransform; + } else { + opts.fileTransform = f => nodeRequire(f.fullpath); } return opts; @@ -119,33 +69,35 @@ function defaultOptions(options) { * * @param {String} dir The directory to read. Can be relative or absolute. * @param {Object} [options] - * @param {Boolean} [options.dirTransform=Object.freeze] A function called with the object result from scanning a directory - * @param {Function} [options.fileTransform=_.cloneDeep] A function called with one argument that is the result of require(file) + * @param {Boolean} [options.dirTransform] A function called with the object result from scanning a directory + * @param {Function} [options.fileTransform] A function called with one argument that is the result of require(file) * @returns {Promise.} */ -function readDirectoryAsync(dir, options) { +function readDirectoryAsync (dir, options) { const opts = defaultOptions(options); return new Promise((resolve, reject) => { fs.readdir(dir, (err, files) => { - if (err) { reject(err); } - if (!Array.isArray(files)) { resolve({}); } + if (err) { return reject(err); } + if (!Array.isArray(files)) { return resolve({}); } let promises = []; const result = files.reduce((accum, name) => { const file = new File(dir, name); - if (file.isDirectory) { - promises.push(readDirectory(file.fullpath, options) - .then(obj => defineProperty(accum, obj, file, opts.dirTransform))); - } else if (file.isRequirable) { - defineProperty(accum, require(file.fullpath), file, opts.fileTransform); + if (opts.filter(file)) { + if (file.isDirectory) { + promises.push(readDirectoryAsync(file.fullpath, options) + .then(obj => defineProperty(accum, file, opts.dirTransform, obj))); + } else { + defineProperty(accum, file, opts.fileTransform); + } } return accum; }, {}); Promise.all(promises).then(() => resolve(result)); }); - }) + }); } /** @@ -153,17 +105,15 @@ function readDirectoryAsync(dir, options) { * * @param {String} dir The directory to read. Can be relative or absolute. * @param {Object} [options] - * @param {Boolean} [options.dirTransform] A function called with the object result from scanning a directory - * Default is no transform + * @param {Function} [options.dirTransform] A function called with the object result from scanning a directory * @param {Function} [options.fileTransform] A function called with one argument that is the result of require(file) - * Default is a getter function that returns a clone of the file * @returns {Object} * * @example * readDirectory('./fixtures', { dirTransform: Object.freeze, fileTransform: v => () => _.cloneDeep(v) }) * .then(console.log) */ -function readDirectory(dir, options) { +function readDirectory (dir, options) { const opts = defaultOptions(options); const files = fs.readdirSync(dir); @@ -171,10 +121,12 @@ function readDirectory(dir, options) { return files.reduce((accum, name) => { const file = new File(dir, name); - if (file.isDirectory) { // Recurse into sub-directories - defineProperty(accum, readDirectory(file.fullpath, options), file, opts.dirTransform); - } else if (file.isRequirable) { - defineProperty(accum, nodeRequire(file.fullpath), file, opts.fileTransform); + if (opts.filter(file)) { + if (file.isDirectory) { // Recurse into sub-directories + defineProperty(accum, file, opts.dirTransform, readDirectory(file.fullpath, options)); + } else { + defineProperty(accum, file, opts.fileTransform); + } } return accum; }, {}); diff --git a/lib/file.js b/lib/file.js new file mode 100644 index 0000000..52e7819 --- /dev/null +++ b/lib/file.js @@ -0,0 +1,105 @@ +'use strict'; + +const path = require('path'); +const jsonFile = /^\.js(?:on)?$/; +const fs = require('fs'); + +/** + * Symbol to define file attributes cache + * + * This allows us to store a cache of file attributes but keep it hidden + * + * @type {Symbol} + * @private + */ +const _attributes = Symbol('attributes'); + +/** + * File or directory + * @property {String} path The fully resolved path to the file (the parent directory) + * @property {String} fullpath The fully resolved path of the file + * @property {String} ext The file extension + * @property {String} name The full name of the file with no path + * @property {String} basename The name of the file without the extension + * @property {Boolean} isDirectory Returns `true` if the file is a directory + */ +class File { + constructor (dir, file) { + this.path = path.resolve(dir); + this.fullpath = path.join(this.path, file); + this.ext = path.extname(this.fullpath); + this.name = path.basename(this.fullpath); + this.basename = path.basename(this.name, this.ext); + }; + + /** + * Load the file as a UTF8 string + * + * @param {{ encoding: string, flag: string }}options + */ + readSync (options) { + options = options || { encoding: 'utf8' }; + return fs.readFileSync(this.fullpath, options); + } + + /** + * Get a string value to be used as an object key + * + * @returns {string} + */ + get key () { + return this.attributes.key; + } + + /** + * Set a string to be used as an object key + * + * @param {string} value + */ + set key (value) { + this.attributes.key = value; + } + + /** + * Get the file's attributes + * + * @see https://nodejs.org/api/fs.html#fs_fs_lstatsync_path + * + * @returns {fs.Stats} + */ + get attributes () { + if (!this[_attributes]) { + const a = this[_attributes] = fs.lstatSync(this.fullpath); + a.key = (a.isDirectory()) ? this.name : this.basename; + } + return this[_attributes]; + }; + + /** + * Check if file is a directory + * + * @returns {Boolean} + */ + get isDirectory () { + return this.attributes.isDirectory(); + } + + /** + * Check if file extension is either `.js` or `.json` + * + * @returns {Boolean} + */ + get isRequirable () { + return jsonFile.test(this.ext); + } + + toString () { + return this.fullpath; + } + + get [Symbol.toStringTag] () { + return 'File'; + } +} + +module.exports = File; diff --git a/package.json b/package.json index 67c72b9..181de96 100644 --- a/package.json +++ b/package.json @@ -1,10 +1,13 @@ { "name": "dir-obj", - "version": "1.0.0", + "version": "2.0.0", "description": "Recusively load files from a directory and return an object with keys from the directory and file names", "main": "index.js", + "enginesStrict": { + "node": ">=4.0.0" + }, "scripts": { - "test": "node test" + "test": "node test/index.js && node test/async-test.js" }, "repository": { "type": "git", diff --git a/test/async-test.js b/test/async-test.js new file mode 100644 index 0000000..86c08ab --- /dev/null +++ b/test/async-test.js @@ -0,0 +1,83 @@ +'use strict'; + +const dirObj = require('..'); + +const assert = require('assert'); +const path = require('path'); +const fixturePath = path.join(__dirname, 'fixtures'); + +const test = (name, fn) => { + return fn() + .then(() => true) + .catch(e => { + console.log('Failed test: ' + name); + console.log(e.stack); + return false; + }); +}; + +test('dir-obj', () => { + const tests = [ + + test('no arguments requires all js and json files', () => { + const expected = { + fixtures1: { + 'name': 'fixture1.json' + }, + fixtures2: { + name: 'fixtures2.js' + } + }; + + return dirObj.readDirectoryAsync(path.join(fixturePath, 'dir1')) + .then(result => assert.deepEqual(result, expected)) + }), + + test('filter, fileTransform, and dirTransform', () => { + const expected = { + '3dir': { fixtures3: { name: 'dir2/fixtures3.sjon' } }, + '4.dir': { '_-_bad_%_file__name_': { name: ' - bad % file \\ name .json' } }, + 'dir.2': { + subdir1: { + subsubdir1: { + subfixture1: { name: 'subsubdir1/subfixture1.json' }, + subfixture2: true + } + } + }, + dir1: { + fixtures1: { name: 'fixture1.json' }, + fixtures2: { name: 'fixtures2.js' } + }, + root: 'This is the root' + }; + + return dirObj.readDirectoryAsync(fixturePath, { + filter: file => (file.isDirectory || file.isRequirable) && file.name !== 'index.js', + fileTransform: file => { + file.key = file.basename.replace(/\s/g, '_'); + return require(file.fullpath); + }, + dirTransform: (file, value) => Object.getOwnPropertyNames(value).length > 0 ? value : undefined + }) + .then(result => assert.deepEqual(result, expected)); + }) + + ]; + + return Promise.all(tests) + .then(testResults => { + const results = testResults.reduce((accum, t) => { + accum[t ? 'passed' : 'failed'] += 1; + return accum; + }, { + passed: 0, + failed: 0 + } + ); + + console.log(require('util').inspect(results, { colors: true })); + assert.equal(results.failed, 0); + }) + .catch(e => console.log(e.stack)); +}); diff --git a/test/index.js b/test/index.js index 0d8a237..4bd67f3 100644 --- a/test/index.js +++ b/test/index.js @@ -1,32 +1,234 @@ 'use strict'; -const assert = require('assert'); const dirObj = require('..'); -const expected = { - fixtures: - { '3dir': { fixtures3: { name: 'dir2/fixtures3.sjon' } }, - '4.dir': { '_-_bad_%_file__name_': { name: ' - bad % file \\ name .json' } }, - 'dir.2': - { subdir1: - { subsubdir1: - { subfixture1: { name: 'subsubdir1/subfixture1.json' }, - subfixture2: true }, - subsubdir2: { index: { name: 'subsubdir2/index.js' } } } }, - dir1: - { fixtures1: { name: 'fixture1.json' }, - fixtures2: { name: 'fixtures2.js' } }, - emptydir: {}, - root: 'This is the root' }, - index: {} -}; +const assert = require('assert'); +const path = require('path'); +const fixturePath = path.join(__dirname, 'fixtures'); -const result = dirObj.readDirectory(__dirname, { - fileTransform: (value, file) => { - file.key = file.basename.replace(/\s/g, '_'); - return value; +const test = (name, fn) => { + try { + fn(); + return true; + } catch (e) { + console.log('Failed test: ' + name); + console.log(e.stack); + return false; } -}); +}; + +test('dir-obj', () => { + const tests = [ + + test('no arguments requires all js and json files', () => { + const expected = { + fixtures1: { + 'name': 'fixture1.json' + }, + fixtures2: { + name: 'fixtures2.js' + } + }; + const result = dirObj.readDirectory(path.join(__dirname, '/fixtures/dir1')); + + assert.deepEqual(result, expected); + }), + + test('filter - only load JSON files', () => { + const expected = { + subdir1: { + subsubdir1: { + subfixture1: { + 'name': 'subsubdir1/subfixture1.json' + } + }, + subsubdir2: {} + } + }; + + const result = dirObj.readDirectory(path.join(fixturePath, 'dir.2'), { + filter: file => file.isDirectory || file.ext === '.json' + }); + + assert.deepEqual(result, expected); + }), + + test('filter - do not add keys for empty directories', () => { + const expected = { + 'dir.2': { + subdir1: { + subsubdir1: { subfixture2: true }, + subsubdir2: { + index: { name: 'subsubdir2/index.js' } + } + } + }, + dir1: { + fixtures2: { name: 'fixtures2.js' } + }, + root: 'This is the root' + }; + + const result = dirObj.readDirectory(fixturePath, { + filter: file => file.isDirectory || file.ext === '.js', + dirTransform: (file, value) => (Object.getOwnPropertyNames(value).length > 0) ? value : undefined + }); + + assert.deepEqual(result, expected); + }), + + test('fileTransform - change the object key', () => { + const expected = { + '3dir': { FIXTURES3: { name: 'dir2/fixtures3.sjon' } }, + '4.dir': { '_-_BAD_%_FILE__NAME_': { name: ' - bad % file \\ name .json' } }, + 'dir.2': { + subdir1: { + subsubdir1: { + SUBFIXTURE1: { name: 'subsubdir1/subfixture1.json' }, + SUBFIXTURE2: true + }, + subsubdir2: { INDEX: { name: 'subsubdir2/index.js' } } + } + }, + dir1: { + FIXTURES1: { name: 'fixture1.json' }, + FIXTURES2: { name: 'fixtures2.js' } + }, + ROOT: 'This is the root' + }; + + const result = dirObj.readDirectory(fixturePath, { + fileTransform: file => { + file.key = file.key.replace(/[. ]/g, '_').toUpperCase(); + return require(file.fullpath); + } + }); + + assert.deepEqual(result, expected); + }), + + test('fileTransform - use a custom file loader to get data from non-requireable files', () => { + const expected = { + dir1: { fixtures2: 'This will not get loaded\n' } + }; -assert.deepEqual(result, expected); -console.log('test passed'); + const result = dirObj.readDirectory(fixturePath, { + filter: file => file.isDirectory || file.ext === '.txt', + fileTransform: file => require('fs').readFileSync(file.fullpath, 'utf8'), + dirTransform: (file, value) => Object.getOwnPropertyNames(value).length > 0 ? value : undefined + }); + + assert.deepEqual(result, expected); + }), + + test('fileTransform - ozen object', () => { + const result = dirObj.readDirectory(path.join(fixturePath, 'dir1'), { + fileTransform: file => Object.freeze(require(file.fullpath)), + dirTransform: (file, value) => Object.getOwnPropertyNames(value).length > 0 ? value : undefined + }); + + assert.ok(Object.isFrozen(result.fixtures1)); + assert.throws(() => { result.fixtures1 = {}; }); + assert.throws(() => { result.fixtures1.test = 'test'; }); + }), + + test('fileTransform - ignore a file by returning undefined', () => { + const expected = { + subdir1: { + subsubdir1: {}, + subsubdir2: {} + } + }; + + const result = dirObj.readDirectory(path.join(fixturePath, 'dir.2'), { + fileTransform: file => undefined + }); + + assert.deepEqual(result, expected); + }), + + test('dirTransform - change directory key', () => { + const expected = { + _dir: { fixtures3: { name: 'dir2/fixtures3.sjon' } }, + __dir: { ' - bad % file name ': { name: ' - bad % file \\ name .json' } }, + dir__: { + subdir_: { + subsubdir_: { + subfixture1: { name: 'subsubdir1/subfixture1.json' }, + subfixture2: true + }, + subsubdir2: { index: { name: 'subsubdir2/index.js' } } + } + }, + dir_: { + fixtures1: { name: 'fixture1.json' }, + fixtures2: { name: 'fixtures2.js' } + }, + root: 'This is the root' + }; + + const result = dirObj.readDirectory(fixturePath, { + dirTransform: (file, value) => { + if (file.name !== 'subsubdir2') { + file.key = file.key.replace(/[\d.]/g, '_'); + } + return value; + } + }); + + assert.deepEqual(result, expected); + }), + + test('dirTransform - ignore a directory by returning undefined', () => { + const expected = { root: 'This is the root' }; + + const result = dirObj.readDirectory(fixturePath, { + dirTransform: file => undefined + }); + + assert.deepEqual(result, expected); + }), + + test('filter, fileTransform, and dirTransform', () => { + const expected = { + '3dir': { fixtures3: { name: 'dir2/fixtures3.sjon' } }, + '4.dir': { '_-_bad_%_file__name_': { name: ' - bad % file \\ name .json' } }, + 'dir.2': { + subdir1: { + subsubdir1: { + subfixture1: { name: 'subsubdir1/subfixture1.json' }, + subfixture2: true + } + } + }, + dir1: { + fixtures1: { name: 'fixture1.json' }, + fixtures2: { name: 'fixtures2.js' } + }, + root: 'This is the root' + }; + + const result = dirObj.readDirectory(fixturePath, { + filter: file => (file.isDirectory || file.isRequirable) && file.name !== 'index.js', + fileTransform: file => { + file.key = file.basename.replace(/\s/g, '_'); + return require(file.fullpath); + }, + dirTransform: (file, value) => Object.getOwnPropertyNames(value).length > 0 ? value : undefined + }); + + assert.deepEqual(result, expected); + }) + ]; + + const results = tests.reduce((accum, t) => { + accum[t ? 'passed' : 'failed'] += 1; + return accum; + }, { + passed: 0, + failed: 0 + }); + + console.log(require('util').inspect(results, { colors: true })); + assert.equal(results.failed, 0); +});