archive-gh-me/shelljs
1
0
Fork 0
mirror of https://github.com/shelljs/shelljs.git synced 2026-01-25 16:07:37 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #370 from shelljs/feat-pipes

Browse Source 
feat(pipe): add support for pipes between commands
This commit is contained in:
Ari Porad 2016-02-21 13:23:27 -08:00
commit 77b41d839e
9 changed files with 204 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
README.md
View File

@ -619,7 +619,7 @@ otherwise returns string explaining the error
Examples:
```
```javascript
var foo = ShellString('hello world');
```
@ -627,10 +627,25 @@ Turns a regular string into a string-like object similar to what each
command returns. This has special methods, like `.to()` and `.toEnd()`
### Pipes
Examples:
```javascript
grep('foo', 'file1.txt', 'file2.txt').sed(/o/g, 'a').to('output.txt');
echo('files with o\'s in the name:\n' + ls().grep('o'));
cat('test.js').exec('node'); // pipe to exec() call
```
Commands can send their output to another command in a pipe-like fashion.
`sed`, `grep`, `cat`, `exec`, `to`, and `toEnd` can appear on the right-hand
side of a pipe. Pipes can be chained.
## Configuration
### config.silent
Example:
```javascript
@ -645,6 +660,7 @@ Suppresses all command output if `true`, except for `echo()` calls.
Default is `false`.
### config.fatal
Example:
```javascript
@ -659,6 +675,7 @@ command encounters an error. Default is `false`. This is analogous to
Bash's `set -e`
### config.verbose
Example:
```javascript

17
shell.js
View File

@ -128,6 +128,20 @@ exports.error = _error;
//@include ./src/common
exports.ShellString = common.ShellString;
//@
//@ ### Pipes
//@
//@ Examples:
//@
//@ ```javascript
//@ grep('foo', 'file1.txt', 'file2.txt').sed(/o/g, 'a').to('output.txt');
//@ echo('files with o\'s in the name:\n' + ls().grep('o'));
//@ cat('test.js').exec('node'); // pipe to exec() call
//@ ```
//@
//@ Commands can send their output to another command in a pipe-like fashion.
//@ `sed`, `grep`, `cat`, `exec`, `to`, and `toEnd` can appear on the right-hand
//@ side of a pipe. Pipes can be chained.
//@
//@ ## Configuration
@ -137,6 +151,7 @@ exports.config = common.config;
//@
//@ ### config.silent
//@
//@ Example:
//@
//@ ```javascript
@ -152,6 +167,7 @@ exports.config = common.config;
//@
//@ ### config.fatal
//@
//@ Example:
//@
//@ ```javascript
@ -167,6 +183,7 @@ exports.config = common.config;
//@
//@ ### config.verbose
//@
//@ Example:
//@
//@ ```javascript

8
src/cat.js
View File

@ -17,14 +17,12 @@ var fs = require('fs');
//@ containing the files if more than one file is given (a new line character is
//@ introduced between each file). Wildcard `*` accepted.
function _cat(options, files) {
var cat = '';
var cat = common.readFromPipe(this);
if (!files)
if (!files && !cat)
common.error('no paths given');
if (typeof files === 'string')
files = [].slice.call(arguments, 1);
// if it's array leave it as it is
files = [].slice.call(arguments, 1);
files.forEach(function(file) {
if (!fs.existsSync(file))

27
src/common.js
View File

@ -5,6 +5,7 @@
var os = require('os');
var fs = require('fs');
var glob = require('glob');
var shell = require('..');
var _to = require('./to');
var _toEnd = require('./toEnd');
@ -57,18 +58,27 @@ exports.error = error;
//@
//@ Examples:
//@
//@ ```
//@ ```javascript
//@ var foo = ShellString('hello world');
//@ ```
//@
//@ Turns a regular string into a string-like object similar to what each
//@ command returns. This has special methods, like `.to()` and `.toEnd()`
var ShellString = function (str, stderr) {
var that = new String(str);
that.to = wrap('to', _to, {idx: 1});
that.toEnd = wrap('toEnd', _toEnd, {idx: 1});
that.stdout = str;
var ShellString = function (stdout, stderr) {
var that;
if (stdout instanceof Array) {
that = stdout;
that.stdout = stdout.join('\n')+'\n';
} else {
that = new String(stdout);
that.stdout = stdout;
}
that.stderr = stderr;
that.to = function() {wrap('to', _to, {idx: 1}).apply(that.stdout, arguments); return that;};
that.toEnd = function() {wrap('toEnd', _toEnd, {idx: 1}).apply(that.stdout, arguments); return that;};
['cat', 'sed', 'grep', 'exec'].forEach(function (cmd) {
that[cmd] = function() {return shell[cmd].apply(that.stdout, arguments);};
});
return that;
};
@ -281,3 +291,8 @@ function wrap(cmd, fn, options) {
};
} // wrap
exports.wrap = wrap;
function _readFromPipe(that) {
return that instanceof String ? that.toString() : '';
}
exports.readFromPipe = _readFromPipe;

67
src/exec.js
View File

@ -12,7 +12,7 @@ var DEFAULT_MAXBUFFER_SIZE = 20*1024*1024;
// (Can't do a wait loop that checks for internal Node variables/messages as
// Node is single-threaded; callbacks and other internal state changes are done in the
// event loop).
function execSync(cmd, opts) {
function execSync(cmd, opts, pipe) {
var tempDir = _tempDir();
var stdoutFile = path.resolve(tempDir+'/'+common.randomFileName()),
stderrFile = path.resolve(tempDir+'/'+common.randomFileName()),
@ -53,10 +53,6 @@ function execSync(cmd, opts) {
previousStreamContent = streamContent;
}
function escape(str) {
return (str+'').replace(/([\\"'])/g, "\\$1").replace(/\0/g, "\\0");
}
if (fs.existsSync(scriptFile)) common.unlinkSync(scriptFile);
if (fs.existsSync(stdoutFile)) common.unlinkSync(stdoutFile);
if (fs.existsSync(stderrFile)) common.unlinkSync(stderrFile);
@ -74,23 +70,26 @@ function execSync(cmd, opts) {
if (typeof child.execSync === 'function') {
script = [
"var child = require('child_process')",
" , fs = require('fs');",
"var childProcess = child.exec('"+escape(cmd)+"', "+optString+", function(err) {",
" fs.writeFileSync('"+escape(codeFile)+"', err ? err.code.toString() : '0');",
"});",
"var stdoutStream = fs.createWriteStream('"+escape(stdoutFile)+"');",
"var stderrStream = fs.createWriteStream('"+escape(stderrFile)+"');",
"childProcess.stdout.pipe(stdoutStream, {end: false});",
"childProcess.stderr.pipe(stderrStream, {end: false});",
"childProcess.stdout.pipe(process.stdout);",
"childProcess.stderr.pipe(process.stderr);",
"var stdoutEnded = false, stderrEnded = false;",
"function tryClosingStdout(){ if(stdoutEnded){ stdoutStream.end(); } }",
"function tryClosingStderr(){ if(stderrEnded){ stderrStream.end(); } }",
"childProcess.stdout.on('end', function(){ stdoutEnded = true; tryClosingStdout(); });",
"childProcess.stderr.on('end', function(){ stderrEnded = true; tryClosingStderr(); });"
].join('\n');
"var child = require('child_process')",
" , fs = require('fs');",
"var childProcess = child.exec("+JSON.stringify(cmd)+", "+optString+", function(err) {",
" fs.writeFileSync("+JSON.stringify(codeFile)+", err ? err.code.toString() : '0');",
"});",
"var stdoutStream = fs.createWriteStream("+JSON.stringify(stdoutFile)+");",
"var stderrStream = fs.createWriteStream("+JSON.stringify(stderrFile)+");",
"childProcess.stdout.pipe(stdoutStream, {end: false});",
"childProcess.stderr.pipe(stderrStream, {end: false});",
"childProcess.stdout.pipe(process.stdout);",
"childProcess.stderr.pipe(process.stderr);"
].join('\n') +
(pipe ? "\nchildProcess.stdin.end("+JSON.stringify(pipe)+");\n" : '\n') +
[
"var stdoutEnded = false, stderrEnded = false;",
"function tryClosingStdout(){ if(stdoutEnded){ stdoutStream.end(); } }",
"function tryClosingStderr(){ if(stderrEnded){ stderrStream.end(); } }",
"childProcess.stdout.on('end', function(){ stdoutEnded = true; tryClosingStdout(); });",
"childProcess.stderr.on('end', function(){ stderrEnded = true; tryClosingStderr(); });"
].join('\n');
fs.writeFileSync(scriptFile, script);
@ -115,12 +114,13 @@ function execSync(cmd, opts) {
cmd += ' > '+stdoutFile+' 2> '+stderrFile; // works on both win/unix
script = [
"var child = require('child_process')",
" , fs = require('fs');",
"var childProcess = child.exec('"+escape(cmd)+"', "+optString+", function(err) {",
" fs.writeFileSync('"+escape(codeFile)+"', err ? err.code.toString() : '0');",
"});"
].join('\n');
"var child = require('child_process')",
" , fs = require('fs');",
"var childProcess = child.exec("+JSON.stringify(cmd)+", "+optString+", function(err) {",
" fs.writeFileSync("+JSON.stringify(codeFile)+", err ? err.code.toString() : '0');",
"});"
].join('\n') +
(pipe ? "\nchildProcess.stdin.end("+JSON.stringify(pipe)+");\n" : '\n');
fs.writeFileSync(scriptFile, script);
@ -163,7 +163,7 @@ function execSync(cmd, opts) {
} // execSync()
// Wrapper around exec() to enable echoing output to console in real time
function execAsync(cmd, opts, callback) {
function execAsync(cmd, opts, pipe, callback) {
var stdout = '';
var stderr = '';
@ -179,6 +179,9 @@ function execAsync(cmd, opts, callback) {
callback(err ? err.code : 0, stdout, stderr);
});
if (pipe)
c.stdin.end(pipe);
c.stdout.on('data', function(data) {
stdout += data;
if (!opts.silent)
@ -233,6 +236,8 @@ function _exec(command, options, callback) {
if (!command)
common.error('must specify command');
var pipe = common.readFromPipe(this);
// Callback is defined instead of options.
if (typeof options === 'function') {
callback = options;
@ -251,9 +256,9 @@ function _exec(command, options, callback) {
try {
if (options.async)
return execAsync(command, options, callback);
return execAsync(command, options, pipe, callback);
else
return execSync(command, options);
return execSync(command, options, pipe);
} catch (e) {
common.error('internal error');
}

17
src/grep.js
View File

@ -24,21 +24,26 @@ function _grep(options, regex, files) {
'l': 'nameOnly'
});
if (!files)
// Check if this is coming from a pipe
var pipe = common.readFromPipe(this);
if (!files && !pipe)
common.error('no paths given');
if (typeof files === 'string')
files = [].slice.call(arguments, 2);
files = [].slice.call(arguments, 2);
if (pipe)
files.unshift('-');
var grep = [];
files.forEach(function(file) {
if (!fs.existsSync(file)) {
if (!fs.existsSync(file) && file !== '-') {
common.error('no such file or directory: ' + file, true);
return;
}
var contents = fs.readFileSync(file, 'utf8'),
lines = contents.split(/\r*\n/);
var contents = file === '-' ? pipe : fs.readFileSync(file, 'utf8');
var lines = contents.split(/\r*\n/);
if (options.nameOnly) {
if (contents.match(regex))
grep.push(file);

12
src/ls.js
View File

@ -3,8 +3,6 @@ var fs = require('fs');
var common = require('./common');
var _cd = require('./cd');
var _pwd = require('./pwd');
var _to = require('./to');
var _toEnd = require('./toEnd');
//@
//@ ### ls([options,] [path, ...])
@ -154,15 +152,7 @@ function _ls(options, paths) {
});
// Add methods, to make this more compatible with ShellStrings
list.stdout = list.join('\n') + '\n';
list.stderr = common.state.error;
list.to = function (file) {
common.wrap('to', _to, {idx: 1}).call(this.stdout, file);
};
list.toEnd = function(file) {
common.wrap('toEnd', _toEnd, {idx: 1}).call(this.stdout, file);
};
return list;
return new common.ShellString(list, common.state.error);
}
module.exports = _ls;

18
src/sed.js
View File

@ -22,6 +22,9 @@ function _sed(options, regex, replacement, files) {
'i': 'inplace'
});
// Check if this is coming from a pipe
var pipe = common.readFromPipe(this);
if (typeof replacement === 'string' || typeof replacement === 'function')
replacement = replacement; // no-op
else if (typeof replacement === 'number')
@ -33,21 +36,24 @@ function _sed(options, regex, replacement, files) {
if (typeof regex === 'string')
regex = RegExp(regex);
if (!files)
if (!files && !pipe)
common.error('no files given');
if (typeof files === 'string')
files = [].slice.call(arguments, 3);
// if it's array leave it as it is
files = [].slice.call(arguments, 3);
if (pipe)
files.unshift('-');
var sed = [];
files.forEach(function(file) {
if (!fs.existsSync(file)) {
if (!fs.existsSync(file) && file !== '-') {
common.error('no such file or directory: ' + file, true);
return;
}
var result = fs.readFileSync(file, 'utf8').split('\n').map(function (line) {
var contents = file === '-' ? pipe : fs.readFileSync(file, 'utf8');
var lines = contents.split(/\r*\n/);
var result = lines.map(function (line) {
return line.replace(regex, replacement);
}).join('\n');

85
test/pipe.js Normal file
View File

@ -0,0 +1,85 @@
var shell = require('..');
var assert = require('assert');
shell.config.silent = true;
shell.rm('-rf', 'tmp');
shell.mkdir('tmp');
//
// Invalids
//
// no piped methods for commands that don't return anything
assert.throws(function() {
shell.cp('resources/file1', 'tmp/');
assert.ok(!shell.error());
shell.cp('resources/file1', 'tmp/').cat();
});
// commands like `rm` can't be on the right side of pipes
assert.equal(typeof shell.ls('.').rm, 'undefined');
assert.equal(typeof shell.cat('resources/file1.txt').rm, 'undefined');
//
// Valids
//
// piping to cat() should return roughly the same thing
assert.strictEqual(shell.cat('resources/file1.txt').cat().toString(),
shell.cat('resources/file1.txt').toString());
// piping ls() into cat() converts to a string
assert.strictEqual(shell.ls('resources/').cat().toString(),
shell.ls('resources/').stdout);
var result;
result = shell.ls('resources/').grep('file1');
assert.equal(result + '', 'file1\nfile1.js\nfile1.txt\n');
result = shell.ls('resources/').cat().grep('file1');
assert.equal(result + '', 'file1\nfile1.js\nfile1.txt\n');
// Equivalent to a simple grep() test case
result = shell.cat('resources/grep/file').grep(/alpha*beta/);
assert.equal(shell.error(), null);
assert.equal(result.toString(), 'alphaaaaaaabeta\nalphbeta\n');
// Equivalent to a simple sed() test case
var result = shell.cat('resources/grep/file').sed(/l*\.js/, '');
assert.ok(!shell.error());
assert.equal(result.toString(), 'alphaaaaaaabeta\nhowareyou\nalphbeta\nthis line ends in\n\n');
// Synchronous exec
// TODO: add windows tests
if (process.platform !== 'win32') {
// unix-specific
if (shell.which('grep').stdout) {
result = shell.cat('resources/grep/file').exec("grep 'alpha*beta'");
assert.equal(shell.error(), null);
assert.equal(result, 'alphaaaaaaabeta\nalphbeta\n');
} else {
console.error('Warning: Cannot verify piped exec');
}
} else {
console.error('Warning: Cannot verify piped exec');
}
// Async exec
// TODO: add windows tests
if (process.platform !== 'win32') {
// unix-specific
if (shell.which('grep').stdout) {
shell.cat('resources/grep/file').exec("grep 'alpha*beta'", function(code, stdout) {
assert.equal(code, 0);
assert.equal(stdout, 'alphaaaaaaabeta\nalphbeta\n');
shell.exit(123);
});
} else {
console.error('Warning: Cannot verify piped exec');
}
} else {
console.error('Warning: Cannot verify piped exec');
shell.exit(123);
}