version 3 alpha1

better progress bars ability to inherit stdout/stderr: you will see npm activity now parallel tests
mutantcornholio · Aug 12, 2018 · 70ec133 · 70ec133
1 parent 891a757
commit 70ec133
Show file tree

Hide file tree

Showing 11 changed files with 145 additions and 69 deletions.
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "veendor",
-  "version": "2.1.0",
+  "version": "3.0.0-alpha.1",
   "description": "a tool for stroing your npm dependencies in arbitraty storage",
   "bin": {
     "veendor": "dist/bin/veendor.js"

diff --git a/src/lib/backends/s3.ts b/src/lib/backends/s3.ts
@@ -131,51 +131,59 @@ export async function pull(hash: string, options: S3Options, _cachedir: string,
     return Promise.all([downloadStreamPromise, extractPromise]);
 }
 
-export function push(hash: string, options: S3Options, _cachedir: string, toolsProvider: BackendToolsProvider) {
+export async function push(hash: string, options: S3Options, _cachedir: string, toolsProvider: BackendToolsProvider) {
     const filename = `${hash}.tar${tarWrapper.compression[options.compression]}`;
     const s3 = options.__s3;
 
     const controlToken: ControlToken = {};
 
-    const {stream, promise} = tarWrapper
-        .createStreamArchive([path.resolve(process.cwd(), 'node_modules')], options.compression, {controlToken});
+    let bundleExists = false;
+    try {
+        await s3.headObject({
+            Bucket: options.bucket,
+            Key: filename,
+        }).promise();
+        bundleExists = true;
+    } catch (error) {
+        if (error.statusCode !== 404) {
+            throw error;
+        }
+    }
+
+    if (bundleExists) {
+        throw new errors.BundleAlreadyExistsError();
+    }
 
     const progressStream = toolsProvider.getProgressStream('push');
+    progressStream.toggleVisibility(true);
 
-    setTimeout(() => progressStream.toggleVisibility(true), 500);
+    const {stream: tarWrapperStream, promise: tarWrapperPromise} = tarWrapper
+        .createStreamArchive([path.resolve(process.cwd(), 'node_modules')], options.compression, {controlToken});
 
-    stream.pipe(progressStream);
-    return s3.headObject({
+    tarWrapperStream.pipe(progressStream);
+
+    const s3Promise = s3.upload({
         Bucket: options.bucket,
         Key: filename,
-    }).promise()
-        .then(() => {
-            throw new errors.BundleAlreadyExistsError();
-        }, error => {
-            if (error.statusCode === 404) {
-                return s3.upload({
-                    Bucket: options.bucket,
-                    Key: filename,
-                    ACL: options.objectAcl,
-                    Body: progressStream,
-                }).promise().then(() => promise);
-            }
+        ACL: options.objectAcl,
+        Body: progressStream,
+    }).promise();
 
+    try {
+        await Promise.all([tarWrapperPromise, s3Promise]);
+    } catch (error) {
+        if (error instanceof errors.VeendorError) {
             throw error;
-        })
-        .catch(error => {
-            if (controlToken.terminate !== undefined) {
-                controlToken.terminate();
-            }
-
-            progressStream.die();
+        }
 
-            if (error instanceof errors.VeendorError) {
-                throw error;
-            }
+        throw new BundleUploadError(`${error.statusCode}: ${error.message}`);
+    } finally {
+        if (controlToken.terminate !== undefined) {
+            controlToken.terminate();
+        }
 
-            throw new BundleUploadError(`${error.statusCode}: ${error.message}`);
-        });
+        progressStream.die();
+    }
 }
 
 export class BundleDownloadError extends errors.VeendorError {}

diff --git a/src/lib/commandWrappers/gitWrapper.ts b/src/lib/commandWrappers/gitWrapper.ts
@@ -1,5 +1,6 @@
 import path from 'path';
 import * as helpers from './helpers';
+import {StdioPolicy} from './helpers';
 import * as errors from '../errors';
 import {getLogger} from '../util/logger';
 
@@ -75,14 +76,20 @@ export async function olderRevision(
 }
 
 export async function clone(repo: string, directory: string) {
-    return helpers.getOutput('git', ['clone', repo, directory], {pipeToParent: true});
+    return helpers.getOutput('git', ['clone', repo, directory], {
+        stdout: StdioPolicy.copy, stderr: StdioPolicy.inherit
+    });
 }
 export async function fetch(gitDirectory: string) {
-    return helpers.getOutput('git', ['fetch', '--tags'], {cwd: gitDirectory, pipeToParent: true});
+    return helpers.getOutput('git', ['fetch', '--tags'], {
+        cwd: gitDirectory, stdout: StdioPolicy.copy, stderr: StdioPolicy.inherit
+    });
 }
 
 export async function lfsPull(gitDirectory: string) {
-    return helpers.getOutput('git', ['lfs', 'pull'], {cwd: gitDirectory, pipeToParent: true});
+    return helpers.getOutput('git', ['lfs', 'pull'], {
+        cwd: gitDirectory, stdout: StdioPolicy.copy, stderr: StdioPolicy.inherit
+    });
 }
 
 export async function checkout(gitDirectory: string, gitId: string) {
@@ -106,7 +113,7 @@ export async function push(gitDirectory: string, gitId: string) {
             return helpers.getOutput(
                 'git',
                 ['push', remote.trim(), gitId],
-                {cwd: gitDirectory, pipeToParent: true}
+                {cwd: gitDirectory, stdout: StdioPolicy.copy, stderr: StdioPolicy.inherit}
             );
         }).catch(error => {
             if (!(error instanceof helpers.CommandReturnedNonZeroError)) {
@@ -141,7 +148,7 @@ export async function resetToRemote(gitDirectory: string, branch: string) {
             helpers.getOutput(
                 'git',
                 ['reset', '--hard', `${remote.trim()}/${branch}`],
-                {cwd: gitDirectory, pipeToParent: true}
+                {cwd: gitDirectory, stdout: StdioPolicy.copy, stderr: StdioPolicy.inherit}
             )
         );
 }
diff --git a/src/lib/commandWrappers/helpers.ts b/src/lib/commandWrappers/helpers.ts
@@ -20,6 +20,16 @@ export type ControlToken = {
     stdio?: [Writable, Readable, Readable];
 }
 
+export enum StdioPolicy {
+    inherit, // `process.stdout` or `process.stderr` is gonna be passed to child process.
+             // getOutput will not get data from corresponding stream
+    copy,    // each line sent stdout/stderr records, to `getOutput` result and sends to
+             // `process.stdout` / `process.stderr`
+    collect, // only record output to `getOutput` result
+    pipe,    // do not record output; corresponding stream will be available at controlToken's stdio
+    ignore,  // attach /dev/null to the stream
+}
+
 type GetOutputOptions = {
     controlToken?: ControlToken // You can pass an empty object here and it will populate
     // with useful stuff
@@ -28,15 +38,26 @@ type GetOutputOptions = {
 
     cwd?: string,
 
-    pipeToParent?: boolean,  // If true, every chunk of data will be pushed to stdout or stderr,
-    // like {stdio: 'inherit'}
+    stdout?: StdioPolicy,
+    stderr?: StdioPolicy,
+}
+
+function stdioPolicyToCpStdio(policy: StdioPolicy, fd: number): 'ignore' | 'pipe' | number {
+    if (policy === StdioPolicy.inherit) {
+        return fd;
+    } else if (policy === StdioPolicy.ignore) {
+        return 'ignore';
+    }
 
-    collectOutput?: boolean, // If false, getOutput will resolve into empty string,
-                             // but you can get actual output through `controlToken.stdio`
+    return 'pipe';
 }
 
 export function getOutput(executable: string, args: string[], {
-    timeoutDuration = 0, cwd = process.cwd(), pipeToParent = false, collectOutput = true, controlToken = {},
+    timeoutDuration = 0,
+    cwd = process.cwd(),
+    controlToken = {},
+    stdout = StdioPolicy.collect,
+    stderr = StdioPolicy.collect,
 }: GetOutputOptions = {}): Promise<string> {
     return new Promise((resolve, reject) => {
         const commandName = `[${executable} ${args.join(' ')}]`;
@@ -47,12 +68,19 @@ export function getOutput(executable: string, args: string[], {
         let timeout: NodeJS.Timer;
 
         logger.debug(`Running ${commandName}; cwd: ${cwd}`);
-        const proc = childProcess.spawn(executable, args, {stdio: 'pipe', cwd});
+        const proc = childProcess.spawn(executable, args, {
+            stdio: ['pipe', stdioPolicyToCpStdio(stdout, 1), stdioPolicyToCpStdio(stderr, 2)],
+            cwd,
+        });
         controlToken.terminate = () => {
             logger.debug(`Terminating ${commandName} using control token`);
-            proc.kill()
+            proc.kill();
         };
 
+        const deathHand = () => proc.kill();
+
+        process.on('exit', deathHand);
+
         controlToken.stdio = proc.stdio;
 
         if (timeoutDuration !== 0) {
@@ -67,25 +95,28 @@ export function getOutput(executable: string, args: string[], {
             }, timeoutDuration);
         }
 
-        if (collectOutput) {
+        if (stdout === StdioPolicy.collect || stdout === StdioPolicy.copy) {
             proc.stdout.on('data', data => {
                 result += data.toString();
 
-                if (pipeToParent) {
+                if (stdout === StdioPolicy.copy) {
                     process.stdout.write(data);
                 }
             });
+        }
 
+        if (stderr === StdioPolicy.collect || stderr === StdioPolicy.copy) {
             proc.stderr.on('data', data => {
                 result += data.toString();
 
-                if (pipeToParent) {
+                if (stdout === StdioPolicy.copy) {
                     process.stderr.write(data);
                 }
             });
         }
 
         proc.on('exit', (code, signal) => {
+            process.removeListener('exit', deathHand);
             if (!completed) {
                 if (code === 0) {
                     logger.debug(`Command ${commandName} exited with 0`);

diff --git a/src/lib/commandWrappers/npmWrapper.ts b/src/lib/commandWrappers/npmWrapper.ts
@@ -1,6 +1,7 @@
 import _ from 'lodash';
 import * as helpers from './helpers';
 import {StringMap} from '@/serviceTypes';
+import {StdioPolicy} from '@/lib/commandWrappers/helpers';
 
 export function install(packages: StringMap, timeoutDuration = 0) {
     const args = ['install'];
@@ -9,11 +10,15 @@ export function install(packages: StringMap, timeoutDuration = 0) {
         args.push(`${pkgname}@${version}`);
     });
 
-    return helpers.getOutput('npm', args, {timeoutDuration, pipeToParent: true});
+    return helpers.getOutput('npm', args, {
+        timeoutDuration, stdout: StdioPolicy.copy, stderr: StdioPolicy.inherit
+    });
 }
 
 export function installAll(timeoutDuration = 0) {
-    return helpers.getOutput('npm', ['install'], {timeoutDuration, pipeToParent: true});
+    return helpers.getOutput('npm', ['install'], {
+        timeoutDuration, stdout: StdioPolicy.copy, stderr: StdioPolicy.inherit
+    });
 }
 
 export function version() {
@@ -23,5 +28,7 @@ export function version() {
 export function uninstall(packages: string[], timeoutDuration = 0) {
     const args = ['uninstall'].concat(packages);
 
-    return helpers.getOutput('npm', args, {timeoutDuration, pipeToParent: true});
+    return helpers.getOutput('npm', args, {
+        timeoutDuration, stdout: StdioPolicy.copy, stderr: StdioPolicy.inherit
+    });
 }
diff --git a/src/lib/commandWrappers/tarWrapper.ts b/src/lib/commandWrappers/tarWrapper.ts
@@ -4,6 +4,7 @@ import {Readable} from 'stream';
 import * as errors from '../errors';
 import * as helpers from './helpers';
 import {ControlToken} from './helpers';
+import {StdioPolicy} from '@/lib/commandWrappers/helpers';
 
 
 export type Compression = 'gzip'| 'bzip2' | 'xz'
@@ -26,13 +27,13 @@ export function createArchive(outPath: string, inputPaths: string[], compression
         ...pathsToAdd
     ];
 
-    return helpers.getOutput('tar', args, {cwd: baseDir, pipeToParent: true});
+    return helpers.getOutput('tar', args, {cwd: baseDir, stdout: StdioPolicy.copy, stderr: StdioPolicy.inherit});
 }
 
 export function extractArchive(archive: string) {
     const args = ['--extract', '--file', archive];
 
-    return helpers.getOutput('tar', args, {pipeToParent: true});
+    return helpers.getOutput('tar', args, {stdout: StdioPolicy.copy, stderr: StdioPolicy.inherit});
 }
 
 class ControlTokenError extends errors.VeendorError {}
@@ -51,7 +52,7 @@ export function createStreamArchive(
     ];
 
     const procPromise = helpers.getOutput(
-        'tar', args, {pipeToParent: false, controlToken, collectOutput: false}
+        'tar', args, {stdout: StdioPolicy.pipe, stderr: StdioPolicy.pipe, controlToken}
     );
 
     if (!controlToken.stdio) {
@@ -67,7 +68,9 @@ export function createStreamArchive(
 export function extractArchiveFromStream(archiveStream: Readable, {controlToken = {}}: {controlToken: ControlToken}) {
     const args = ['--extract', '--file', '-'];
 
-    const procPromise = helpers.getOutput('tar', args, {pipeToParent: false, controlToken, collectOutput: false});
+    const procPromise = helpers.getOutput('tar', args, {
+        stdout: StdioPolicy.pipe, stderr: StdioPolicy.pipe, controlToken
+    });
     if (controlToken.stdio) {
         archiveStream.pipe(controlToken.stdio[0]);
         return procPromise;

diff --git a/src/lib/install/index.ts b/src/lib/install/index.ts
@@ -64,7 +64,11 @@ export default async function install(
         }
 
         if (!isRsyncModeEnabled) {
-            clearNodeModules().catch(() => {});
+            logger.trace('Started removing node_modules');
+            clearNodeModules().then(
+                () => {logger.trace('Successfully removed node_modules');},
+                err => {logger.debug(`Error during node_modules removal: ${err.stack}`);}
+            );
         }
     }
 
@@ -244,6 +248,9 @@ async function pullBackends(
         if (error instanceof errors.BundleNotFoundError) {
             return pullBackends(hash, config, lockfilePath, backendIndex + 1);
         } else {
+            logger.error(
+                `Backend '${backendConfig.alias}' failed on pull:`
+            );
             throw error;
         }
     }

diff --git a/src/lib/util/progress.ts b/src/lib/util/progress.ts
@@ -25,15 +25,20 @@ export class ProgressStream extends Transform {
 
         this.haveTotal = typeof this.total === 'number';
 
-        const progressWithTotal = `  ${colors.green(title)} [{bar}]  `
-            + `${colors.gray('{_value} / {_total} Mb')}   {percentage}%   {duration_formatted}`;
+        const progressWithTotal = rigthPad(2000, `  ${colors.green(title)} [{bar}]  `
+            + `${colors.gray('{_value} / {_total} Mb')}   {percentage}%   {duration_formatted}`);
 
-        const progressWithoutTotal = `  ${colors.green(title)} ${colors.gray('{_value} Mb')}   {duration_formatted}`;
+        const progressWithoutTotal = rigthPad(2000, `  ${colors.green(title)} ${colors.gray('{_value} Mb')}` +
+            `   {duration_formatted}`);
 
         this.progress = new cliProgress.Bar({
             format: this.haveTotal ? progressWithTotal : progressWithoutTotal,
             barsize: 40,
             etaBuffer: 50,
+            hideCursor: false,
+            clearOnComplete: true,
+            linewrap: false,
+            fps: 50,
         });
 
         this.completed = 0;
@@ -144,6 +149,12 @@ function leftPad(width: number, str: string): string {
     return Array(width).join(' ').substring(' ', width - str.length) + str;
 }
 
+function rigthPad(width: number, str: string): string {
+    // https://stackoverflow.com/questions/5366849/convert-1-to-0001-in-javascript
+    // @ts-ignore
+    return str + Array(width).join(' ').substring(' ', width - str.length);
+}
+
 const allTokens: ProgressContolToken[] = [];
 
 export function provideBackendCallTools(backendConfig: BackendConfig, callType: BackendCalls): BackendToolsProvider {