[WIP] Support for publishing Lambda Layers

lib/classes/Service.js

@@ -156,6 +156,10 @@ class Service {
       that.package.excludeDevDependencies = serverlessFile.package.excludeDevDependencies;
     }
 
+    if (that.provider.name === 'aws') {
+      that.layers = serverlessFile.layers || {};
+    }
+
     return this;
   }
 
@@ -223,6 +227,10 @@ class Service {
     return Object.keys(this.functions);
   }
 
+  getAllLayers() {
+    return this.layers ? Object.keys(this.layers) : [];
+  }
+
   getAllFunctionsNames() {
     return this.getAllFunctions().map((func) => this.getFunction(func).name);
   }
@@ -234,6 +242,13 @@ class Service {
     throw new ServerlessError(`Function "${functionName}" doesn't exist in this Service`);
   }
 
+  getLayer(layerName) {
+    if (layerName in this.layers) {
+      return this.layers[layerName];
+    }
+    throw new ServerlessError(`Layer "${layerName}" doesn't exist in this Service`);
+  }
+
   getEventInFunction(eventName, functionName) {
     const event = this.getFunction(functionName).events
       .find(e => Object.keys(e)[0] === eventName);

lib/plugins/Plugins.json

@@ -33,6 +33,7 @@
     "./aws/rollback/index.js",
     "./aws/rollbackFunction/index.js",
     "./aws/package/compile/functions/index.js",
+    "./aws/package/compile/layers/index.js",
     "./aws/package/compile/events/schedule/index.js",
     "./aws/package/compile/events/s3/index.js",
     "./aws/package/compile/events/apiGateway/index.js",

lib/plugins/aws/deploy/lib/uploadArtifacts.js

@@ -16,7 +16,7 @@ module.exports = {
   uploadArtifacts() {
     return BbPromise.bind(this)
       .then(this.uploadCloudFormationFile)
-      .then(this.uploadFunctions);
+      .then(this.uploadFunctionsAndLayers);
   },
 
   uploadCloudFormationFile() {
@@ -78,7 +78,7 @@ module.exports = {
       params);
   },
 
-  uploadFunctions() {
+  uploadFunctionsAndLayers() {
     this.serverless.cli.log('Uploading artifacts...');
 
     const functionNames = this.serverless.service.getAllFunctions();
@@ -103,6 +103,17 @@ module.exports = {
       })
     );
 
+    const layerNames = this.serverless.service.getAllLayers();
+    artifactFilePaths.push(..._.map(layerNames, (name) => {
+      const layerObject = this.serverless.service.getLayer(name);
+
+      if (layerObject.package && layerObject.package.artifact) {
+        return layerObject.package.artifact;
+      }
+
+      return path.join(this.packagePath, this.provider.naming.getLayerArtifactName(name));
+    }));
+
     return BbPromise.map(artifactFilePaths, (artifactFilePath) => {
       const stats = fs.statSync(artifactFilePath);
       this.serverless.cli.log(`Uploading service .zip file to S3 (${filesize(stats.size)})...`);

lib/plugins/aws/deploy/lib/uploadArtifacts.test.js

@@ -61,17 +61,18 @@ describe('uploadArtifacts', () => {
     it('should run promise chain in order', () => {
       const uploadCloudFormationFileStub = sinon
         .stub(awsDeploy, 'uploadCloudFormationFile').resolves();
-      const uploadFunctionsStub = sinon
-        .stub(awsDeploy, 'uploadFunctions').resolves();
+      const uploadFunctionsAndLayersStub = sinon
+        .stub(awsDeploy, 'uploadFunctionsAndLayers').resolves();
 
       return awsDeploy.uploadArtifacts().then(() => {
         expect(uploadCloudFormationFileStub.calledOnce)
           .to.be.equal(true);
 
-        expect(uploadFunctionsStub.calledAfter(uploadCloudFormationFileStub)).to.be.equal(true);
+        expect(uploadFunctionsAndLayersStub.calledAfter(uploadCloudFormationFileStub)).to.be.equal(
+          true);
 
         awsDeploy.uploadCloudFormationFile.restore();
-        awsDeploy.uploadFunctions.restore();
+        awsDeploy.uploadFunctionsAndLayers.restore();
       });
     });
   });
@@ -224,7 +225,7 @@ describe('uploadArtifacts', () => {
     });
   });
 
-  describe('#uploadFunctions()', () => {
+  describe('#uploadFunctionsAndLayers()', () => {
     let uploadZipFileStub;
 
     beforeEach(() => {
@@ -241,7 +242,7 @@ describe('uploadArtifacts', () => {
       awsDeploy.serverless.config.servicePath = 'some/path';
       awsDeploy.serverless.service.service = 'new-service';
 
-      return awsDeploy.uploadFunctions().then(() => {
+      return awsDeploy.uploadFunctionsAndLayers().then(() => {
         expect(uploadZipFileStub.calledOnce).to.be.equal(true);
         const expectedPath = path.join('foo', '.serverless', 'new-service.zip');
         expect(uploadZipFileStub.args[0][0]).to.be.equal(expectedPath);
@@ -262,7 +263,7 @@ describe('uploadArtifacts', () => {
         },
       };
 
-      return awsDeploy.uploadFunctions().then(() => {
+      return awsDeploy.uploadFunctionsAndLayers().then(() => {
         expect(uploadZipFileStub.calledOnce).to.be.equal(true);
         expect(uploadZipFileStub.args[0][0]).to.be.equal('artifact.zip');
       });
@@ -283,7 +284,7 @@ describe('uploadArtifacts', () => {
         },
       };
 
-      return awsDeploy.uploadFunctions().then(() => {
+      return awsDeploy.uploadFunctionsAndLayers().then(() => {
         expect(uploadZipFileStub.calledTwice).to.be.equal(true);
         expect(uploadZipFileStub.args[0][0])
           .to.be.equal(awsDeploy.serverless.service.functions.first.package.artifact);
@@ -307,7 +308,7 @@ describe('uploadArtifacts', () => {
         },
       };
 
-      return awsDeploy.uploadFunctions().then(() => {
+      return awsDeploy.uploadFunctionsAndLayers().then(() => {
         expect(uploadZipFileStub.calledTwice).to.be.equal(true);
         expect(uploadZipFileStub.args[0][0])
           .to.be.equal(awsDeploy.serverless.service.functions.first.package.artifact);
@@ -322,7 +323,7 @@ describe('uploadArtifacts', () => {
 
       sinon.spy(awsDeploy.serverless.cli, 'log');
 
-      return awsDeploy.uploadFunctions().then(() => {
+      return awsDeploy.uploadFunctionsAndLayers().then(() => {
         const expected = 'Uploading service .zip file to S3 (1 KB)...';
         expect(awsDeploy.serverless.cli.log.calledWithExactly(expected)).to.be.equal(true);
       });

lib/plugins/aws/lib/naming.js

@@ -61,6 +61,10 @@ module.exports = {
     return `${functionName}.zip`;
   },
 
+  getLayerArtifactName(layerName) {
+    return `${layerName}.zip`;
+  },
+
   getServiceStateFileName() {
     return 'serverless-state.json';
   },
@@ -136,6 +140,9 @@ module.exports = {
   getLambdaLogicalId(functionName) {
     return `${this.getNormalizedFunctionName(functionName)}LambdaFunction`;
   },
+  getLambdaLayerLogicalId(functionName) {
+    return `${this.getNormalizedFunctionName(functionName)}LambdaLayer`;
+  },
   getLambdaLogicalIdRegex() {
     return /LambdaFunction$/;
   },

lib/plugins/aws/package/compile/functions/index.js

@@ -316,6 +316,10 @@ class AwsCompileFunctions {
 
     if (functionObject.layers && _.isArray(functionObject.layers)) {
       newFunction.Properties.Layers = functionObject.layers;
+      /* TODO - is a DependsOn needed?
+      newLayer.DependsOn = [NEW LAYER??]
+        .concat(newLayer.DependsOn || []);
+      */
     }
 
     const functionLogicalId = this.provider.naming

lib/plugins/aws/package/compile/layers/index.js

@@ -0,0 +1,85 @@
+'use strict';
+
+const BbPromise = require('bluebird');
+const _ = require('lodash');
+const path = require('path');
+
+class AwsCompileLayers {
+  constructor(serverless, options) {
+    this.serverless = serverless;
+    this.options = options;
+    const servicePath = this.serverless.config.servicePath || '';
+    this.packagePath = this.serverless.service.package.path ||
+      path.join(servicePath || '.', '.serverless');
+
+    this.provider = this.serverless.getProvider('aws');
+
+    this.hooks = {
+      'package:compileLayers': () => BbPromise.bind(this)
+        .then(this.compileLayers),
+    };
+  }
+
+  compileLayer(layerName) {
+    const newLayer = this.cfLambdaLayerTemplate();
+    const layerObject = this.serverless.service.getLayer(layerName);
+    layerObject.package = layerObject.package || {};
+
+    const artifactFileName = this.provider.naming.getLayerArtifactName(layerName);
+    const artifactFilePath = layerObject.package && layerObject.package.artifact
+      ? layerObject.package.artifact
+      : path.join(this.serverless.config.servicePath, '.serverless', artifactFileName);
+
+    if (this.serverless.service.package.deploymentBucket) {
+      newLayer.Properties.Content.S3Bucket = this.serverless.service.package.deploymentBucket;
+    }
+
+    const s3Folder = this.serverless.service.package.artifactDirectoryName;
+    const s3FileName = artifactFilePath.split(path.sep).pop();
+    newLayer.Properties.Content.S3Key = `${s3Folder}/${s3FileName}`;
+
+    newLayer.Properties.LayerName = layerObject.name || layerName;
+    if (layerObject.description) {
+      newLayer.Properties.Description = layerObject.description;
+    }
+    if (layerObject.licenseInfo) {
+      newLayer.Properties.LicenseInfo = layerObject.licenseInfo;
+    }
+    if (layerObject.compatibleRuntimes) {
+      newLayer.Properties.CompatibleRuntimes = layerObject.compatibleRuntimes;
+    }
+
+    const layerLogicalId = this.provider.naming.getLambdaLayerLogicalId(layerName);
+    const newLayerObject = {
+      [layerLogicalId]: newLayer,
+    };
+
+    _.merge(this.serverless.service.provider.compiledCloudFormationTemplate.Resources,
+      newLayerObject);
+  }
+
+  compileLayers() {
+    const allLayers = this.serverless.service.getAllLayers();
+    return BbPromise.each(
+      allLayers,
+      layerName => this.compileLayer(layerName)
+    );
+  }
+
+  cfLambdaLayerTemplate() {
+    return {
+      Type: 'AWS::Lambda::LayerVersion',
+      Properties: {
+        Content: {
+          S3Bucket: {
+            Ref: 'ServerlessDeploymentBucket',
+          },
+          S3Key: 'S3Key',
+        },
+        LayerName: 'LayerName',
+      },
+    };
+  }
+}
+
+module.exports = AwsCompileLayers;

lib/plugins/aws/package/index.js

@@ -65,6 +65,9 @@ class AwsPackage {
       'before:package:compileFunctions': () => BbPromise.bind(this)
         .then(this.generateArtifactDirectoryName),
 
+      'before:package:compileLayers': () => BbPromise.bind(this)
+        .then(this.generateArtifactDirectoryName),
+
       'package:finalize': () => BbPromise.bind(this)
         .then(() => this.serverless.pluginManager.spawn('aws:package:finalize')),
 

lib/plugins/aws/package/lib/generateArtifactDirectoryName.js

@@ -4,12 +4,15 @@ const BbPromise = require('bluebird');
 
 module.exports = {
   generateArtifactDirectoryName() {
-    const date = new Date();
-    const serviceStage = `${this.serverless.service.service}/${this.provider.getStage()}`;
-    const dateString = `${date.getTime().toString()}-${date.toISOString()}`;
-    const prefix = this.provider.getDeploymentPrefix();
-    this.serverless.service.package
-      .artifactDirectoryName = `${prefix}/${serviceStage}/${dateString}`;
+    // Don't regenerate name if it's already set
+    if (!this.serverless.service.package.artifactDirectoryName) {
+      const date = new Date();
+      const serviceStage = `${this.serverless.service.service}/${this.provider.getStage()}`;
+      const dateString = `${date.getTime().toString()}-${date.toISOString()}`;
+      const prefix = this.provider.getDeploymentPrefix();
+      this.serverless.service.package
+        .artifactDirectoryName = `${prefix}/${serviceStage}/${dateString}`;
+    }
 
     return BbPromise.resolve();
   },

lib/plugins/package/lib/packageService.js

@@ -24,15 +24,18 @@ module.exports = {
     return _.union(packageIncludes, include);
   },
 
-  getExcludes(exclude) {
+  getExcludes(exclude, excludeLayers) {
     const packageExcludes = this.serverless.service.package.exclude || [];
     // add local service plugins Path
     const pluginsLocalPath = this.serverless.pluginManager
       .parsePluginsObject(this.serverless.service.plugins).localPath;
-
     const localPathExcludes = pluginsLocalPath ? [pluginsLocalPath] : [];
+    // add layer paths
+    const layerExcludes = excludeLayers ? this.serverless.service.getAllLayers().map(
+      (layer) => `${this.serverless.service.getLayer(layer).path}/**`) : [];
     // add defaults for exclude
-    return _.union(this.defaultExcludes, localPathExcludes, packageExcludes, exclude);
+    return _.union(
+      this.defaultExcludes, localPathExcludes, packageExcludes, layerExcludes, exclude);
   },
 
   packageService() {
@@ -56,6 +59,15 @@ module.exports = {
       shouldPackageService = true;
       return BbPromise.resolve();
     });
+    const allLayers = this.serverless.service.getAllLayers();
+    packagePromises.push(..._.map(allLayers, layerName => {
+      const layerObject = this.serverless.service.getLayer(layerName);
+      layerObject.package = layerObject.package || {};
+      if (layerObject.package.artifact) {
+        return BbPromise.resolve();
+      }
+      return this.packageLayer(layerName);
+    }));
 
     return BbPromise.all(packagePromises).then(() => {
       if (shouldPackageService && !this.serverless.service.package.artifact) {
@@ -113,8 +125,24 @@ module.exports = {
     );
   },
 
+  packageLayer(layerName) {
+    const layerObject = this.serverless.service.getLayer(layerName);
+
+    const zipFileName = `${layerName}.zip`;
+
+    return this.resolveFilePathsLayer(layerName)
+      .then(filePaths =>
+        this.zipFiles(filePaths, zipFileName, layerObject.path).then(artifactPath => {
+          layerObject.package = {
+            artifact: artifactPath,
+          };
+          return artifactPath;
+        })
+      );
+  },
+
   resolveFilePathsAll() {
-    const params = { exclude: this.getExcludes(), include: this.getIncludes() };
+    const params = { exclude: this.getExcludes([], true), include: this.getIncludes() };
     return this.excludeDevDependencies(params).then(() =>
       this.resolveFilePathsFromPatterns(params));
   },
@@ -124,14 +152,26 @@ module.exports = {
     const funcPackageConfig = functionObject.package || {};
 
     const params = {
-      exclude: this.getExcludes(funcPackageConfig.exclude),
+      exclude: this.getExcludes(funcPackageConfig.exclude, true),
       include: this.getIncludes(funcPackageConfig.include),
     };
     return this.excludeDevDependencies(params).then(() =>
       this.resolveFilePathsFromPatterns(params));
   },
 
-  resolveFilePathsFromPatterns(params) {
+  resolveFilePathsLayer(layerName) {
+    const layerObject = this.serverless.service.getLayer(layerName);
+    const layerPackageConfig = layerObject.package || {};
+
+    const params = {
+      exclude: this.getExcludes(layerPackageConfig.exclude),
+      include: this.getIncludes(layerPackageConfig.include),
+    };
+    return this.excludeDevDependencies(params).then(() => this.resolveFilePathsFromPatterns(
+      params, layerObject.path));
+  },
+
+  resolveFilePathsFromPatterns(params, prefix) {
     const patterns = ['**'];
 
     params.exclude.forEach((pattern) => {
@@ -149,7 +189,7 @@ module.exports = {
     });
 
     return globby(patterns, {
-      cwd: this.serverless.config.servicePath,
+      cwd: path.join(this.serverless.config.servicePath, prefix || ''),
       dot: true,
       silent: true,
       follow: true,

lib/plugins/package/lib/zipService.js

@@ -55,7 +55,7 @@ module.exports = {
       this.zipFiles(filePaths, params.zipFileName));
   },
 
-  zipFiles(files, zipFileName) {
+  zipFiles(files, zipFileName, prefix) {
     if (files.length === 0) {
       const error = new this.serverless.classes.Error('No files to package');
       return BbPromise.reject(error);
@@ -80,10 +80,12 @@ module.exports = {
       output.on('open', () => {
         zip.pipe(output);
 
-        BbPromise.all(files.map(this.getFileContentAndStat.bind(this))).then((contents) => {
+        const filePaths = files.map(file => (prefix ? path.join(prefix, file) : file));
+        BbPromise.all(filePaths.map(this.getFileContentAndStat.bind(this))).then((contents) => {
           _.forEach(_.sortBy(contents, ['filePath']), (file) => {
+            const name = file.filePath.slice(prefix ? `${prefix}${path.sep}`.length : 0);
             zip.append(file.data, {
-              name: file.filePath,
+              name,
               mode: file.stat.mode,
               date: new Date(0), // necessary to get the same hash when zipping the same content
             });

lib/plugins/package/package.js

@@ -29,6 +29,7 @@ class Package {
           'initialize',
           'setupProviderConfiguration',
           'createDeploymentArtifacts',
+          'compileLayers',
           'compileFunctions',
           'compileEvents',
           'finalize',