Merge pull request #5863 from serverless/invoke-local-docker

Invoke local docker

docs/providers/aws/cli-reference/invoke-local.md

@@ -29,6 +29,8 @@ serverless invoke local --function functionName
 - `--contextPath` or `-x`, The path to a json file holding input context to be passed to the invoked function. This path is relative to the root directory of the service.
 - `--context` or `-c`, String data to be passed as a context to your function. Same like with `--data`, context included in `--contextPath` will overwrite the context you passed with `--context` flag.
 * `--env` or `-e` String representing an environment variable to set when invoking your function, in the form `<name>=<value>`. Can be repeated for more than one environment variable.
+* `--docker` Enable docker support for NodeJS/Python/Ruby/Java. Enabled by default for other
+  runtimes.
 
 ## Environment
 
@@ -107,7 +109,11 @@ serverless invoke local -f functionName -e VAR1=value1 -e VAR2=value2
 
 ### Limitations
 
-Currently, `invoke local` only supports the NodeJs, Python, Java, & Ruby runtimes.
+Use of the `--docker` flag and runtimes other than NodeJs, Python, Java, & Ruby depend on having
+[Docker](https://www.docker.com/) installed. On MacOS & Windows, install
+[Docker Desktop](https://www.docker.com/products/docker-desktop); On Linux install
+[Docker engine](https://www.docker.com/products/docker-engine) and ensure your user is in the
+`docker` group so that you can invoke docker without `sudo`.
 
 **Note:** In order to get correct output when using Java runtime, your Response class must implement `toString()` method.
 

lib/plugins/aws/invokeLocal/index.js

@@ -4,12 +4,19 @@ const BbPromise = require('bluebird');
 const _ = require('lodash');
 const os = require('os');
 const fs = BbPromise.promisifyAll(require('fs'));
+const fse = require('fs-extra');
 const path = require('path');
 const validate = require('../lib/validate');
 const chalk = require('chalk');
 const stdin = require('get-stdin');
 const spawn = require('child_process').spawn;
 const inspect = require('util').inspect;
+const download = require('download');
+const mkdirp = require('mkdirp');
+const cachedir = require('cachedir');
+const jszip = require('jszip');
+
+const cachePath = path.join(cachedir('serverless'), 'invokeLocal');
 
 class AwsInvokeLocal {
   constructor(serverless, options) {
@@ -28,6 +35,12 @@ class AwsInvokeLocal {
     };
   }
 
+  getRuntime() {
+    return this.options.functionObj.runtime
+      || this.serverless.service.provider.runtime
+      || 'nodejs4.3';
+  }
+
   validateFile(filePath, key) {
     const absolutePath = path.isAbsolute(filePath) ?
       filePath :
@@ -126,11 +139,13 @@ class AwsInvokeLocal {
   }
 
   invokeLocal() {
-    const runtime = this.options.functionObj.runtime
-      || this.serverless.service.provider.runtime
-      || 'nodejs4.3';
+    const runtime = this.getRuntime();
     const handler = this.options.functionObj.handler;
 
+    if (this.options.docker) {
+      return this.invokeLocalDocker();
+    }
+
     if (runtime.startsWith('nodejs')) {
       const handlerPath = handler.split('.')[0];
       const handlerName = handler.split('.')[1];
@@ -177,8 +192,151 @@ class AwsInvokeLocal {
         this.options.context);
     }
 
-    throw new this.serverless.classes
-      .Error('You can only invoke Node.js, Python, Java & Ruby functions locally.');
+    return this.invokeLocalDocker();
+  }
+
+  checkDockerDaemonStatus() {
+    return new BbPromise((resolve, reject) => {
+      const docker = spawn('docker', ['version']);
+      docker.on('exit', error => {
+        if (error) {
+          reject('Please start the Docker daemon to use the invoke local Docker integration.');
+        }
+        resolve();
+      });
+    });
+  }
+
+  checkDockerImage() {
+    const runtime = this.getRuntime();
+
+    return new BbPromise((resolve, reject) => {
+      const docker = spawn('docker', ['images', '-q', `lambci/lambda:${runtime}`]);
+      let stdout = '';
+      docker.stdout.on('data', (buf) => { stdout += buf.toString(); });
+      docker.on('exit', error => (error ? reject(error) : resolve(Boolean(stdout.trim()))));
+    });
+  }
+
+  pullDockerImage() {
+    const runtime = this.getRuntime();
+
+    this.serverless.cli.log('Downloading base Docker image...');
+
+    return new BbPromise((resolve, reject) => {
+      const docker = spawn('docker', ['pull', `lambci/lambda:${runtime}`]);
+      docker.on('exit', error => (error ? reject(error) : resolve()));
+    });
+  }
+
+  getLayerPaths() {
+    const layers = _.mapKeys(
+      this.serverless.service.layers,
+      (value, key) => this.provider.naming.getLambdaLayerLogicalId(key)
+    );
+
+    return BbPromise.all(
+      (this.options.functionObj.layers || this.serverless.service.provider.layers || [])
+        .map(layer => {
+          if (layer.Ref) {
+            return layers[layer.Ref].path;
+          }
+          const arnParts = layer.split(':');
+          const layerArn = arnParts.slice(0, -1).join(':');
+          const layerVersion = Number(arnParts.slice(-1)[0]);
+          const layerContentsPath = path.join(
+            '.serverless', 'layers', arnParts[6], arnParts[7]);
+          const layerContentsCachePath = path.join(
+            cachePath, 'layers', arnParts[6], arnParts[7]);
+          if (fs.existsSync(layerContentsPath)) {
+            return layerContentsPath;
+          }
+          let downloadPromise = BbPromise.resolve();
+          if (!fs.existsSync(layerContentsCachePath)) {
+            this.serverless.cli.log(`Downloading layer ${layer}...`);
+            mkdirp.sync(path.join(layerContentsCachePath));
+            downloadPromise = this.provider.request(
+              'Lambda', 'getLayerVersion', { LayerName: layerArn, VersionNumber: layerVersion })
+              .then(layerInfo => download(
+                layerInfo.Content.Location,
+                layerContentsPath,
+                { extract: true }));
+          }
+          return downloadPromise
+            .then(() => fse.copySync(layerContentsCachePath, layerContentsPath))
+            .then(() => layerContentsPath);
+        }));
+  }
+
+  buildDockerImage(layerPaths) {
+    const runtime = this.getRuntime();
+
+
+    const imageName = 'sls-docker';
+
+    return new BbPromise((resolve, reject) => {
+      let dockerfile = `FROM lambci/lambda:${runtime}`;
+      for (const layerPath of layerPaths) {
+        dockerfile += `\nADD --chown=sbx_user1051:495 ${layerPath} /opt`;
+      }
+      mkdirp.sync(path.join('.serverless', 'invokeLocal'));
+      const dockerfilePath = path.join('.serverless', 'invokeLocal', 'Dockerfile');
+      fs.writeFileSync(dockerfilePath, dockerfile);
+      this.serverless.cli.log('Building Docker image...');
+      const docker = spawn('docker', ['build', '-t', imageName,
+        `${this.serverless.config.servicePath}`, '-f', dockerfilePath]);
+      docker.on('exit', error => (error ? reject(error) : resolve(imageName)));
+    });
+  }
+
+  extractArtifact() {
+    const artifact = _.get(this.options.functionObj, 'package.artifact', _.get(
+      this.serverless.service, 'package.artifact'
+    ));
+    if (!artifact) {
+      return this.serverless.config.servicePath;
+    }
+    return fs.readFileAsync(artifact)
+      .then(jszip.loadAsync)
+      .then(zip => BbPromise.all(
+        Object.keys(zip.files)
+          .map(filename => zip.files[filename].async('nodebuffer').then(fileData => {
+            if (filename.endsWith(path.sep)) {
+              return BbPromise.resolve();
+            }
+            mkdirp.sync(path.join(
+              '.serverless', 'invokeLocal', 'artifact'));
+            return fs.writeFileAsync(path.join(
+              '.serverless', 'invokeLocal', 'artifact', filename), fileData, {
+                mode: zip.files[filename].unixPermissions,
+              });
+          }))))
+      .then(() => path.join(
+        this.serverless.config.servicePath, '.serverless', 'invokeLocal', 'artifact'));
+  }
+
+
+  invokeLocalDocker() {
+    const handler = this.options.functionObj.handler;
+
+    return BbPromise.all([
+      this.checkDockerDaemonStatus(),
+      this.checkDockerImage().then(exists => (exists ? {} : this.pullDockerImage())),
+      this.getLayerPaths().then(layerPaths => this.buildDockerImage(layerPaths)),
+      this.extractArtifact(),
+    ])
+      .then((results) => new BbPromise((resolve, reject) => {
+        const imageName = results[2];
+        const artifactPath = results[3];
+        const dockerArgs = [
+          'run', '--rm', '-v', `${artifactPath}:/var/task`, imageName,
+          handler, JSON.stringify(this.options.data),
+        ];
+        const docker = spawn('docker', dockerArgs);
+        docker.stdout.on('data', (buf) => this.serverless.cli.consoleLog(buf.toString()));
+        docker.stderr.on('data', (buf) => this.serverless.cli.consoleLog(buf.toString()));
+        docker.on('exit', error => (error ? reject(error) : resolve(imageName)));
+      }));
   }
 
   invokeLocalPython(runtime, handlerPath, handlerName, event, context) {
@@ -360,7 +518,7 @@ class AwsInvokeLocal {
       this.serverless.cli.consoleLog(JSON.stringify(result, null, 4));
     }
 
-    return new Promise((resolve) => {
+    return new BbPromise((resolve) => {
       const callback = (err, result) => {
         if (!hasResponded) {
           hasResponded = true;
@@ -408,7 +566,7 @@ class AwsInvokeLocal {
 
       const maybeThennable = lambda(event, context, callback);
       if (!_.isUndefined(maybeThennable)) {
-        return Promise.resolve(maybeThennable)
+        return BbPromise.resolve(maybeThennable)
           .then(
             callback.bind(this, null),
             callback.bind(this)

lib/plugins/aws/invokeLocal/index.test.js

@@ -32,6 +32,7 @@ describe('AwsInvokeLocal', () => {
       function: 'first',
     };
     serverless = new Serverless();
+    serverless.config.servicePath = 'servicePath';
     serverless.cli = new CLI(serverless);
     provider = new AwsProvider(serverless, options);
     serverless.setProvider('aws', provider);
@@ -334,6 +335,7 @@ describe('AwsInvokeLocal', () => {
     let invokeLocalPythonStub;
     let invokeLocalJavaStub;
     let invokeLocalRubyStub;
+    let invokeLocalDockerStub;
 
     beforeEach(() => {
       invokeLocalNodeJsStub =
@@ -344,6 +346,8 @@ describe('AwsInvokeLocal', () => {
         sinon.stub(awsInvokeLocal, 'invokeLocalJava').resolves();
       invokeLocalRubyStub =
         sinon.stub(awsInvokeLocal, 'invokeLocalRuby').resolves();
+      invokeLocalDockerStub =
+        sinon.stub(awsInvokeLocal, 'invokeLocalDocker').resolves();
 
       awsInvokeLocal.serverless.service.service = 'new-service';
       awsInvokeLocal.provider.options.stage = 'dev';
@@ -468,10 +472,23 @@ describe('AwsInvokeLocal', () => {
       });
     });
 
-    it('throw error when using runtime other than Node.js, Python, Java or Ruby', () => {
-      awsInvokeLocal.options.functionObj.runtime = 'invalid-runtime';
-      expect(() => awsInvokeLocal.invokeLocal()).to.throw(Error);
-      delete awsInvokeLocal.options.functionObj.runtime;
+    it('should call invokeLocalDocker if using runtime provided', () => {
+      awsInvokeLocal.options.functionObj.runtime = 'provided';
+      awsInvokeLocal.options.functionObj.handler = 'handler.foobar';
+      return awsInvokeLocal.invokeLocal().then(() => {
+        expect(invokeLocalDockerStub.calledOnce).to.be.equal(true);
+        expect(invokeLocalDockerStub.calledWithExactly()).to.be.equal(true);
+      });
+    });
+
+    it('should call invokeLocalDocker if using --docker option with nodejs8.10', () => {
+      awsInvokeLocal.options.functionObj.runtime = 'nodejs8.10';
+      awsInvokeLocal.options.functionObj.handler = 'handler.foobar';
+      awsInvokeLocal.options.docker = true;
+      return awsInvokeLocal.invokeLocal().then(() => {
+        expect(invokeLocalDockerStub.calledOnce).to.be.equal(true);
+        expect(invokeLocalDockerStub.calledWithExactly()).to.be.equal(true);
+      });
     });
   });
 
@@ -1097,4 +1114,87 @@ describe('AwsInvokeLocal', () => {
       });
     });
   });
+
+  describe('#invokeLocalDocker()', () => {
+    let awsInvokeLocalMocked;
+    let spawnStub;
+
+    beforeEach(() => {
+      awsInvokeLocal.provider.options.stage = 'dev';
+      awsInvokeLocal.options = {
+        function: 'first',
+        functionObj: {
+          handler: 'handler.hello',
+          name: 'hello',
+          timeout: 4,
+        },
+        data: {},
+      };
+
+      spawnStub = sinon.stub().returns({
+        stderr: new EventEmitter().on('data', () => {}),
+        stdout: new EventEmitter().on('data', () => {}),
+        stdin: {
+          write: () => {},
+          end: () => {},
+        },
+        on: (key, callback) => callback(),
+      });
+      mockRequire('child_process', { spawn: spawnStub });
+
+      // Remove Node.js internal "require cache" contents and re-require ./index.js
+      delete require.cache[require.resolve('./index')];
+      delete require.cache[require.resolve('child_process')];
+
+      const AwsInvokeLocalMocked = require('./index'); // eslint-disable-line global-require
+
+      serverless.setProvider('aws', new AwsProvider(serverless, options));
+      awsInvokeLocalMocked = new AwsInvokeLocalMocked(serverless, options);
+
+      awsInvokeLocalMocked.options = {
+        stage: 'dev',
+        function: 'first',
+        functionObj: {
+          handler: 'handler.hello',
+          name: 'hello',
+          timeout: 4,
+          runtime: 'nodejs8.10',
+        },
+        data: {},
+      };
+    });
+
+
+    afterEach(() => {
+      delete require.cache[require.resolve('./index')];
+      delete require.cache[require.resolve('child_process')];
+    });
+
+    it('calls docker', () =>
+      awsInvokeLocalMocked.invokeLocalDocker().then(() => {
+        expect(spawnStub.getCall(0).args).to.deep.equal(['docker', ['version']]);
+        expect(spawnStub.getCall(1).args).to.deep.equal(['docker',
+          ['images', '-q', 'lambci/lambda:nodejs8.10']]);
+        expect(spawnStub.getCall(2).args).to.deep.equal(['docker',
+          ['pull', 'lambci/lambda:nodejs8.10']]);
+        expect(spawnStub.getCall(3).args).to.deep.equal(['docker', [
+          'build',
+          '-t',
+          'sls-docker',
+          'servicePath',
+          '-f',
+          '.serverless/invokeLocal/Dockerfile',
+        ]]);
+        expect(spawnStub.getCall(4).args).to.deep.equal(['docker', [
+          'run',
+          '--rm',
+          '-v',
+          'servicePath:/var/task',
+          'sls-docker',
+          'handler.hello',
+          '{}',
+        ]]);
+      })
+    );
+  });
 });

lib/plugins/aws/package/compile/functions/index.js

@@ -361,10 +361,9 @@ class AwsCompileFunctions {
 
     if (functionObject.layers && _.isArray(functionObject.layers)) {
       newFunction.Properties.Layers = functionObject.layers;
-      /* TODO - is a DependsOn needed?
-      newLayer.DependsOn = [NEW LAYER??]
-        .concat(newLayer.DependsOn || []);
-      */
+    } else if (this.serverless.service.provider.layers && _.isArray(
+      this.serverless.service.provider.layers)) {
+      newFunction.Properties.Layers = this.serverless.service.provider.layers;
     }
 
     const functionLogicalId = this.provider.naming

lib/plugins/invoke/invoke.js

@@ -87,7 +87,9 @@ class Invoke {
                 usage: 'Override environment variables. e.g. --env VAR1=val1 --env VAR2=val2',
                 shortcut: 'e',
               },
+              docker: { usage: 'Flag to turn on docker use for node/python/ruby/java' },
             },
+
           },
         },
       },

package-lock.json

@@ -1166,6 +1166,11 @@
         "unset-value": "^1.0.0"
       }
     },
+    "cachedir": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/cachedir/-/cachedir-2.1.0.tgz",
+      "integrity": "sha512-xGBpPqoBvn3unBW7oxgb8aJn42K0m9m1/wyjmazah10Fq7bROGG3kRAE6OIyr3U3PIJUqGuebhCEdMk9OKJG0A=="
+    },
     "caller-id": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/caller-id/-/caller-id-0.1.0.tgz",
@@ -3735,8 +3740,7 @@
     "immediate": {
       "version": "3.0.6",
       "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz",
-      "integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=",
-      "dev": true
+      "integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps="
     },
     "import-lazy": {
       "version": "2.1.0",
@@ -4984,7 +4988,6 @@
       "version": "3.1.5",
       "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.1.5.tgz",
       "integrity": "sha512-5W8NUaFRFRqTOL7ZDDrx5qWHJyBXy6velVudIzQUSoqAAYqzSh2Z7/m0Rf1QbmQJccegD0r+YZxBjzqoBiEeJQ==",
-      "dev": true,
       "requires": {
         "core-js": "~2.3.0",
         "es6-promise": "~3.0.2",
@@ -4996,26 +4999,22 @@
         "core-js": {
           "version": "2.3.0",
           "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.3.0.tgz",
-          "integrity": "sha1-+rg/uwstjchfpjbEudNMdUIMbWU=",
-          "dev": true
+          "integrity": "sha1-+rg/uwstjchfpjbEudNMdUIMbWU="
         },
         "es6-promise": {
           "version": "3.0.2",
           "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-3.0.2.tgz",
-          "integrity": "sha1-AQ1YWEI6XxGJeWZfRkhqlcbuK7Y=",
-          "dev": true
+          "integrity": "sha1-AQ1YWEI6XxGJeWZfRkhqlcbuK7Y="
         },
         "process-nextick-args": {
           "version": "1.0.7",
           "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz",
-          "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=",
-          "dev": true
+          "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M="
         },
         "readable-stream": {
           "version": "2.0.6",
           "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz",
           "integrity": "sha1-j5A0HmilPMySh4jaz80Rs265t44=",
-          "dev": true,
           "requires": {
             "core-util-is": "~1.0.0",
             "inherits": "~2.0.1",
@@ -5028,8 +5027,7 @@
         "string_decoder": {
           "version": "0.10.31",
           "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz",
-          "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=",
-          "dev": true
+          "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ="
         }
       }
     },
@@ -5123,7 +5121,6 @@
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/lie/-/lie-3.1.1.tgz",
       "integrity": "sha1-mkNrLMd0bKWd56QfpGmz77dr2H4=",
-      "dev": true,
       "requires": {
         "immediate": "~3.0.5"
       }
@@ -6084,8 +6081,7 @@
     "pako": {
       "version": "1.0.8",
       "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.8.tgz",
-      "integrity": "sha512-6i0HVbUfcKaTv+EG8ZTr75az7GFXcLYk9UyLEg7Notv/Ma+z/UG3TCoz6GiNeOrn1E/e63I0X/Hpw18jHOTUnA==",
-      "dev": true
+      "integrity": "sha512-6i0HVbUfcKaTv+EG8ZTr75az7GFXcLYk9UyLEg7Notv/Ma+z/UG3TCoz6GiNeOrn1E/e63I0X/Hpw18jHOTUnA=="
     },
     "parse-github-url": {
       "version": "1.0.2",

package.json

@@ -77,7 +77,6 @@
     "eslint-plugin-react": "^6.1.1",
     "istanbul": "^0.4.4",
     "jest-cli": "^23.1.0",
-    "jszip": "^3.1.2",
     "markdown-link": "^0.1.1",
     "markdown-magic": "^0.1.19",
     "markdown-table": "^1.1.1",
@@ -91,10 +90,12 @@
     "sinon-chai": "^2.9.0"
   },
   "dependencies": {
+    "jszip": "^3.1.2",
     "archiver": "^1.1.0",
     "async": "^1.5.2",
     "aws-sdk": "^2.373.0",
     "bluebird": "^3.5.0",
+    "cachedir": "^2.1.0",
     "chalk": "^2.0.0",
     "ci-info": "^1.1.1",
     "download": "^5.0.2",
@@ -112,6 +113,7 @@
     "jwt-decode": "^2.2.0",
     "lodash": "^4.13.1",
     "minimist": "^1.2.0",
+    "mkdirp": "^0.5.1",
     "moment": "^2.13.0",
     "nanomatch": "^1.2.13",
     "node-fetch": "^1.6.0",