diff --git a/docs/providers/aws/cli-reference/invoke-local.md b/docs/providers/aws/cli-reference/invoke-local.md index 120aa51a4..6597cdf24 100644 --- a/docs/providers/aws/cli-reference/invoke-local.md +++ b/docs/providers/aws/cli-reference/invoke-local.md @@ -29,6 +29,8 @@ serverless invoke local --function functionName - `--contextPath` or `-x`, The path to a json file holding input context to be passed to the invoked function. This path is relative to the root directory of the service. - `--context` or `-c`, String data to be passed as a context to your function. Same like with `--data`, context included in `--contextPath` will overwrite the context you passed with `--context` flag. * `--env` or `-e` String representing an environment variable to set when invoking your function, in the form `=`. Can be repeated for more than one environment variable. +* `--docker` Enable docker support for NodeJS/Python/Ruby/Java. Enabled by default for other + runtimes. ## Environment @@ -107,7 +109,11 @@ serverless invoke local -f functionName -e VAR1=value1 -e VAR2=value2 ### Limitations -Currently, `invoke local` only supports the NodeJs, Python, Java, & Ruby runtimes. +Use of the `--docker` flag and runtimes other than NodeJs, Python, Java, & Ruby depend on having +[Docker](https://www.docker.com/) installed. On MacOS & Windows, install +[Docker Desktop](https://www.docker.com/products/docker-desktop); On Linux install +[Docker engine](https://www.docker.com/products/docker-engine) and ensure your user is in the +`docker` group so that you can invoke docker without `sudo`. **Note:** In order to get correct output when using Java runtime, your Response class must implement `toString()` method. diff --git a/lib/plugins/aws/invokeLocal/index.js b/lib/plugins/aws/invokeLocal/index.js index bf32955ca..26140da9a 100644 --- a/lib/plugins/aws/invokeLocal/index.js +++ b/lib/plugins/aws/invokeLocal/index.js @@ -4,12 +4,19 @@ const BbPromise = require('bluebird'); const _ = require('lodash'); const os = require('os'); const fs = BbPromise.promisifyAll(require('fs')); +const fse = require('fs-extra'); const path = require('path'); const validate = require('../lib/validate'); const chalk = require('chalk'); const stdin = require('get-stdin'); const spawn = require('child_process').spawn; const inspect = require('util').inspect; +const download = require('download'); +const mkdirp = require('mkdirp'); +const cachedir = require('cachedir'); +const jszip = require('jszip'); + +const cachePath = path.join(cachedir('serverless'), 'invokeLocal'); class AwsInvokeLocal { constructor(serverless, options) { @@ -28,6 +35,12 @@ class AwsInvokeLocal { }; } + getRuntime() { + return this.options.functionObj.runtime + || this.serverless.service.provider.runtime + || 'nodejs4.3'; + } + validateFile(filePath, key) { const absolutePath = path.isAbsolute(filePath) ? filePath : @@ -126,11 +139,13 @@ class AwsInvokeLocal { } invokeLocal() { - const runtime = this.options.functionObj.runtime - || this.serverless.service.provider.runtime - || 'nodejs4.3'; + const runtime = this.getRuntime(); const handler = this.options.functionObj.handler; + if (this.options.docker) { + return this.invokeLocalDocker(); + } + if (runtime.startsWith('nodejs')) { const handlerPath = handler.split('.')[0]; const handlerName = handler.split('.')[1]; @@ -177,8 +192,151 @@ class AwsInvokeLocal { this.options.context); } - throw new this.serverless.classes - .Error('You can only invoke Node.js, Python, Java & Ruby functions locally.'); + return this.invokeLocalDocker(); + } + + checkDockerDaemonStatus() { + return new BbPromise((resolve, reject) => { + const docker = spawn('docker', ['version']); + docker.on('exit', error => { + if (error) { + reject('Please start the Docker daemon to use the invoke local Docker integration.'); + } + resolve(); + }); + }); + } + + checkDockerImage() { + const runtime = this.getRuntime(); + + return new BbPromise((resolve, reject) => { + const docker = spawn('docker', ['images', '-q', `lambci/lambda:${runtime}`]); + let stdout = ''; + docker.stdout.on('data', (buf) => { stdout += buf.toString(); }); + docker.on('exit', error => (error ? reject(error) : resolve(Boolean(stdout.trim())))); + }); + } + + pullDockerImage() { + const runtime = this.getRuntime(); + + this.serverless.cli.log('Downloading base Docker image...'); + + return new BbPromise((resolve, reject) => { + const docker = spawn('docker', ['pull', `lambci/lambda:${runtime}`]); + docker.on('exit', error => (error ? reject(error) : resolve())); + }); + } + + getLayerPaths() { + const layers = _.mapKeys( + this.serverless.service.layers, + (value, key) => this.provider.naming.getLambdaLayerLogicalId(key) + ); + + return BbPromise.all( + (this.options.functionObj.layers || this.serverless.service.provider.layers || []) + .map(layer => { + if (layer.Ref) { + return layers[layer.Ref].path; + } + const arnParts = layer.split(':'); + const layerArn = arnParts.slice(0, -1).join(':'); + const layerVersion = Number(arnParts.slice(-1)[0]); + const layerContentsPath = path.join( + '.serverless', 'layers', arnParts[6], arnParts[7]); + const layerContentsCachePath = path.join( + cachePath, 'layers', arnParts[6], arnParts[7]); + if (fs.existsSync(layerContentsPath)) { + return layerContentsPath; + } + let downloadPromise = BbPromise.resolve(); + if (!fs.existsSync(layerContentsCachePath)) { + this.serverless.cli.log(`Downloading layer ${layer}...`); + mkdirp.sync(path.join(layerContentsCachePath)); + downloadPromise = this.provider.request( + 'Lambda', 'getLayerVersion', { LayerName: layerArn, VersionNumber: layerVersion }) + .then(layerInfo => download( + layerInfo.Content.Location, + layerContentsPath, + { extract: true })); + } + return downloadPromise + .then(() => fse.copySync(layerContentsCachePath, layerContentsPath)) + .then(() => layerContentsPath); + })); + } + + buildDockerImage(layerPaths) { + const runtime = this.getRuntime(); + + + const imageName = 'sls-docker'; + + return new BbPromise((resolve, reject) => { + let dockerfile = `FROM lambci/lambda:${runtime}`; + for (const layerPath of layerPaths) { + dockerfile += `\nADD --chown=sbx_user1051:495 ${layerPath} /opt`; + } + mkdirp.sync(path.join('.serverless', 'invokeLocal')); + const dockerfilePath = path.join('.serverless', 'invokeLocal', 'Dockerfile'); + fs.writeFileSync(dockerfilePath, dockerfile); + this.serverless.cli.log('Building Docker image...'); + const docker = spawn('docker', ['build', '-t', imageName, + `${this.serverless.config.servicePath}`, '-f', dockerfilePath]); + docker.on('exit', error => (error ? reject(error) : resolve(imageName))); + }); + } + + extractArtifact() { + const artifact = _.get(this.options.functionObj, 'package.artifact', _.get( + this.serverless.service, 'package.artifact' + )); + if (!artifact) { + return this.serverless.config.servicePath; + } + return fs.readFileAsync(artifact) + .then(jszip.loadAsync) + .then(zip => BbPromise.all( + Object.keys(zip.files) + .map(filename => zip.files[filename].async('nodebuffer').then(fileData => { + if (filename.endsWith(path.sep)) { + return BbPromise.resolve(); + } + mkdirp.sync(path.join( + '.serverless', 'invokeLocal', 'artifact')); + return fs.writeFileAsync(path.join( + '.serverless', 'invokeLocal', 'artifact', filename), fileData, { + mode: zip.files[filename].unixPermissions, + }); + })))) + .then(() => path.join( + this.serverless.config.servicePath, '.serverless', 'invokeLocal', 'artifact')); + } + + + invokeLocalDocker() { + const handler = this.options.functionObj.handler; + + return BbPromise.all([ + this.checkDockerDaemonStatus(), + this.checkDockerImage().then(exists => (exists ? {} : this.pullDockerImage())), + this.getLayerPaths().then(layerPaths => this.buildDockerImage(layerPaths)), + this.extractArtifact(), + ]) + .then((results) => new BbPromise((resolve, reject) => { + const imageName = results[2]; + const artifactPath = results[3]; + const dockerArgs = [ + 'run', '--rm', '-v', `${artifactPath}:/var/task`, imageName, + handler, JSON.stringify(this.options.data), + ]; + const docker = spawn('docker', dockerArgs); + docker.stdout.on('data', (buf) => this.serverless.cli.consoleLog(buf.toString())); + docker.stderr.on('data', (buf) => this.serverless.cli.consoleLog(buf.toString())); + docker.on('exit', error => (error ? reject(error) : resolve(imageName))); + })); } invokeLocalPython(runtime, handlerPath, handlerName, event, context) { @@ -360,7 +518,7 @@ class AwsInvokeLocal { this.serverless.cli.consoleLog(JSON.stringify(result, null, 4)); } - return new Promise((resolve) => { + return new BbPromise((resolve) => { const callback = (err, result) => { if (!hasResponded) { hasResponded = true; @@ -408,7 +566,7 @@ class AwsInvokeLocal { const maybeThennable = lambda(event, context, callback); if (!_.isUndefined(maybeThennable)) { - return Promise.resolve(maybeThennable) + return BbPromise.resolve(maybeThennable) .then( callback.bind(this, null), callback.bind(this) diff --git a/lib/plugins/aws/invokeLocal/index.test.js b/lib/plugins/aws/invokeLocal/index.test.js index c8bba83f4..c88c8ec31 100644 --- a/lib/plugins/aws/invokeLocal/index.test.js +++ b/lib/plugins/aws/invokeLocal/index.test.js @@ -32,6 +32,7 @@ describe('AwsInvokeLocal', () => { function: 'first', }; serverless = new Serverless(); + serverless.config.servicePath = 'servicePath'; serverless.cli = new CLI(serverless); provider = new AwsProvider(serverless, options); serverless.setProvider('aws', provider); @@ -334,6 +335,7 @@ describe('AwsInvokeLocal', () => { let invokeLocalPythonStub; let invokeLocalJavaStub; let invokeLocalRubyStub; + let invokeLocalDockerStub; beforeEach(() => { invokeLocalNodeJsStub = @@ -344,6 +346,8 @@ describe('AwsInvokeLocal', () => { sinon.stub(awsInvokeLocal, 'invokeLocalJava').resolves(); invokeLocalRubyStub = sinon.stub(awsInvokeLocal, 'invokeLocalRuby').resolves(); + invokeLocalDockerStub = + sinon.stub(awsInvokeLocal, 'invokeLocalDocker').resolves(); awsInvokeLocal.serverless.service.service = 'new-service'; awsInvokeLocal.provider.options.stage = 'dev'; @@ -468,10 +472,23 @@ describe('AwsInvokeLocal', () => { }); }); - it('throw error when using runtime other than Node.js, Python, Java or Ruby', () => { - awsInvokeLocal.options.functionObj.runtime = 'invalid-runtime'; - expect(() => awsInvokeLocal.invokeLocal()).to.throw(Error); - delete awsInvokeLocal.options.functionObj.runtime; + it('should call invokeLocalDocker if using runtime provided', () => { + awsInvokeLocal.options.functionObj.runtime = 'provided'; + awsInvokeLocal.options.functionObj.handler = 'handler.foobar'; + return awsInvokeLocal.invokeLocal().then(() => { + expect(invokeLocalDockerStub.calledOnce).to.be.equal(true); + expect(invokeLocalDockerStub.calledWithExactly()).to.be.equal(true); + }); + }); + + it('should call invokeLocalDocker if using --docker option with nodejs8.10', () => { + awsInvokeLocal.options.functionObj.runtime = 'nodejs8.10'; + awsInvokeLocal.options.functionObj.handler = 'handler.foobar'; + awsInvokeLocal.options.docker = true; + return awsInvokeLocal.invokeLocal().then(() => { + expect(invokeLocalDockerStub.calledOnce).to.be.equal(true); + expect(invokeLocalDockerStub.calledWithExactly()).to.be.equal(true); + }); }); }); @@ -1097,4 +1114,87 @@ describe('AwsInvokeLocal', () => { }); }); }); + + describe('#invokeLocalDocker()', () => { + let awsInvokeLocalMocked; + let spawnStub; + + beforeEach(() => { + awsInvokeLocal.provider.options.stage = 'dev'; + awsInvokeLocal.options = { + function: 'first', + functionObj: { + handler: 'handler.hello', + name: 'hello', + timeout: 4, + }, + data: {}, + }; + + spawnStub = sinon.stub().returns({ + stderr: new EventEmitter().on('data', () => {}), + stdout: new EventEmitter().on('data', () => {}), + stdin: { + write: () => {}, + end: () => {}, + }, + on: (key, callback) => callback(), + }); + mockRequire('child_process', { spawn: spawnStub }); + + // Remove Node.js internal "require cache" contents and re-require ./index.js + delete require.cache[require.resolve('./index')]; + delete require.cache[require.resolve('child_process')]; + + const AwsInvokeLocalMocked = require('./index'); // eslint-disable-line global-require + + serverless.setProvider('aws', new AwsProvider(serverless, options)); + awsInvokeLocalMocked = new AwsInvokeLocalMocked(serverless, options); + + awsInvokeLocalMocked.options = { + stage: 'dev', + function: 'first', + functionObj: { + handler: 'handler.hello', + name: 'hello', + timeout: 4, + runtime: 'nodejs8.10', + }, + data: {}, + }; + }); + + + afterEach(() => { + delete require.cache[require.resolve('./index')]; + delete require.cache[require.resolve('child_process')]; + }); + + it('calls docker', () => + awsInvokeLocalMocked.invokeLocalDocker().then(() => { + expect(spawnStub.getCall(0).args).to.deep.equal(['docker', ['version']]); + expect(spawnStub.getCall(1).args).to.deep.equal(['docker', + ['images', '-q', 'lambci/lambda:nodejs8.10']]); + expect(spawnStub.getCall(2).args).to.deep.equal(['docker', + ['pull', 'lambci/lambda:nodejs8.10']]); + expect(spawnStub.getCall(3).args).to.deep.equal(['docker', [ + 'build', + '-t', + 'sls-docker', + 'servicePath', + '-f', + '.serverless/invokeLocal/Dockerfile', + ]]); + expect(spawnStub.getCall(4).args).to.deep.equal(['docker', [ + 'run', + '--rm', + '-v', + 'servicePath:/var/task', + 'sls-docker', + 'handler.hello', + '{}', + ]]); + }) + ); + }); }); diff --git a/lib/plugins/aws/package/compile/functions/index.js b/lib/plugins/aws/package/compile/functions/index.js index 6eee5d75e..410480c74 100644 --- a/lib/plugins/aws/package/compile/functions/index.js +++ b/lib/plugins/aws/package/compile/functions/index.js @@ -361,10 +361,9 @@ class AwsCompileFunctions { if (functionObject.layers && _.isArray(functionObject.layers)) { newFunction.Properties.Layers = functionObject.layers; - /* TODO - is a DependsOn needed? - newLayer.DependsOn = [NEW LAYER??] - .concat(newLayer.DependsOn || []); - */ + } else if (this.serverless.service.provider.layers && _.isArray( + this.serverless.service.provider.layers)) { + newFunction.Properties.Layers = this.serverless.service.provider.layers; } const functionLogicalId = this.provider.naming diff --git a/lib/plugins/invoke/invoke.js b/lib/plugins/invoke/invoke.js index b01212b82..d07db0216 100644 --- a/lib/plugins/invoke/invoke.js +++ b/lib/plugins/invoke/invoke.js @@ -87,7 +87,9 @@ class Invoke { usage: 'Override environment variables. e.g. --env VAR1=val1 --env VAR2=val2', shortcut: 'e', }, + docker: { usage: 'Flag to turn on docker use for node/python/ruby/java' }, }, + }, }, }, diff --git a/package-lock.json b/package-lock.json index 0c506ee20..f4de13c7e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1166,6 +1166,11 @@ "unset-value": "^1.0.0" } }, + "cachedir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cachedir/-/cachedir-2.1.0.tgz", + "integrity": "sha512-xGBpPqoBvn3unBW7oxgb8aJn42K0m9m1/wyjmazah10Fq7bROGG3kRAE6OIyr3U3PIJUqGuebhCEdMk9OKJG0A==" + }, "caller-id": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/caller-id/-/caller-id-0.1.0.tgz", @@ -3735,8 +3740,7 @@ "immediate": { "version": "3.0.6", "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", - "integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=", - "dev": true + "integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=" }, "import-lazy": { "version": "2.1.0", @@ -4984,7 +4988,6 @@ "version": "3.1.5", "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.1.5.tgz", "integrity": "sha512-5W8NUaFRFRqTOL7ZDDrx5qWHJyBXy6velVudIzQUSoqAAYqzSh2Z7/m0Rf1QbmQJccegD0r+YZxBjzqoBiEeJQ==", - "dev": true, "requires": { "core-js": "~2.3.0", "es6-promise": "~3.0.2", @@ -4996,26 +4999,22 @@ "core-js": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.3.0.tgz", - "integrity": "sha1-+rg/uwstjchfpjbEudNMdUIMbWU=", - "dev": true + "integrity": "sha1-+rg/uwstjchfpjbEudNMdUIMbWU=" }, "es6-promise": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-3.0.2.tgz", - "integrity": "sha1-AQ1YWEI6XxGJeWZfRkhqlcbuK7Y=", - "dev": true + "integrity": "sha1-AQ1YWEI6XxGJeWZfRkhqlcbuK7Y=" }, "process-nextick-args": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz", - "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=", - "dev": true + "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=" }, "readable-stream": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz", "integrity": "sha1-j5A0HmilPMySh4jaz80Rs265t44=", - "dev": true, "requires": { "core-util-is": "~1.0.0", "inherits": "~2.0.1", @@ -5028,8 +5027,7 @@ "string_decoder": { "version": "0.10.31", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", - "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=", - "dev": true + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" } } }, @@ -5123,7 +5121,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/lie/-/lie-3.1.1.tgz", "integrity": "sha1-mkNrLMd0bKWd56QfpGmz77dr2H4=", - "dev": true, "requires": { "immediate": "~3.0.5" } @@ -6084,8 +6081,7 @@ "pako": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.8.tgz", - "integrity": "sha512-6i0HVbUfcKaTv+EG8ZTr75az7GFXcLYk9UyLEg7Notv/Ma+z/UG3TCoz6GiNeOrn1E/e63I0X/Hpw18jHOTUnA==", - "dev": true + "integrity": "sha512-6i0HVbUfcKaTv+EG8ZTr75az7GFXcLYk9UyLEg7Notv/Ma+z/UG3TCoz6GiNeOrn1E/e63I0X/Hpw18jHOTUnA==" }, "parse-github-url": { "version": "1.0.2", diff --git a/package.json b/package.json index 8113de4c4..68d51735b 100644 --- a/package.json +++ b/package.json @@ -77,7 +77,6 @@ "eslint-plugin-react": "^6.1.1", "istanbul": "^0.4.4", "jest-cli": "^23.1.0", - "jszip": "^3.1.2", "markdown-link": "^0.1.1", "markdown-magic": "^0.1.19", "markdown-table": "^1.1.1", @@ -91,10 +90,12 @@ "sinon-chai": "^2.9.0" }, "dependencies": { + "jszip": "^3.1.2", "archiver": "^1.1.0", "async": "^1.5.2", "aws-sdk": "^2.373.0", "bluebird": "^3.5.0", + "cachedir": "^2.1.0", "chalk": "^2.0.0", "ci-info": "^1.1.1", "download": "^5.0.2", @@ -112,6 +113,7 @@ "jwt-decode": "^2.2.0", "lodash": "^4.13.1", "minimist": "^1.2.0", + "mkdirp": "^0.5.1", "moment": "^2.13.0", "nanomatch": "^1.2.13", "node-fetch": "^1.6.0",