From 6a5029bfdc100b59d166e00a088f977b9fe4f80e Mon Sep 17 00:00:00 2001 From: Austen Collins Date: Mon, 14 Mar 2016 14:25:27 -0700 Subject: [PATCH] Runtimes: clean up --- lib/Function.js | 29 ++-- lib/Project.js | 4 + lib/RuntimeBase.js | 220 +++++++++++++++++++------------ lib/RuntimeNode.js | 210 ++++++++++++++++++----------- lib/RuntimePython27.js | 2 +- lib/actions/CodeDeployLambda.js | 9 +- lib/actions/CodePackageLambda.js | 25 +--- lib/utils/index.js | 10 +- 8 files changed, 303 insertions(+), 206 deletions(-) diff --git a/lib/Function.js b/lib/Function.js index 117bb09f0..d5b49ddc0 100644 --- a/lib/Function.js +++ b/lib/Function.js @@ -1,12 +1,12 @@ 'use strict'; const SError = require('./Error'), - SerializerFileSystem = require('./SerializerFileSystem'), - BbPromise = require('bluebird'), - async = require('async'), - path = require('path'), - fs = BbPromise.promisifyAll(require('fs')), - _ = require('lodash'); + SerializerFileSystem = require('./SerializerFileSystem'), + BbPromise = require('bluebird'), + async = require('async'), + path = require('path'), + fs = BbPromise.promisifyAll(require('fs')), + _ = require('lodash'); let SUtils; @@ -37,7 +37,11 @@ class Function extends SerializerFileSystem { }; this.endpoints = []; this.events = []; - this.environment = {}; + this.environment = { + SERVERLESS_PROJECT: this.getProject().getName(), + SERVERLESS_STAGE: "${stage}", + SERVERLESS_REGION: "${region}" + }; this.vpc = { securityGroupIds: [], subnetIds: [] @@ -67,16 +71,13 @@ class Function extends SerializerFileSystem { toObjectPopulated(options) { options = options || {}; - // Validate: Check Stage & Region - if (!options.stage || !options.region) throw new SError('Both "stage" and "region" params are required'); - // Validate: Check project path is set if (!this._S.hasProject()) throw new SError('Function could not be populated because no project path has been set on Serverless instance'); // Merge templates let templates = _.merge( - this.getProject().getTemplates().toObject(), - this.getTemplates().toObject()); + this.getProject().getTemplates().toObject(), + this.getTemplates().toObject()); // Populate return SUtils.populate(this.getProject(), templates, this.toObject(), options.stage, options.region); @@ -116,7 +117,7 @@ class Function extends SerializerFileSystem { setEndpoint(endpoint) { let _this = this, - added = false; + added = false; for (let i = 0; i < _this.endpoints.length; i++){ let e = _this.endpoints[i]; @@ -135,7 +136,7 @@ class Function extends SerializerFileSystem { setEvent(event) { let _this = this, - added = false; + added = false; for (let i = 0; i < _this.events.length; i++){ let e = _this.events[i]; diff --git a/lib/Project.js b/lib/Project.js index 4f42039e5..6f414e5c3 100644 --- a/lib/Project.js +++ b/lib/Project.js @@ -141,6 +141,10 @@ class Project extends SerializerFileSystem { return path.join.apply( path, args ); } + getTempPath() { + return this.getRootPath('_meta', '_tmp'); + } + getName() { return this.name; } diff --git a/lib/RuntimeBase.js b/lib/RuntimeBase.js index f80b838e5..ece1a244b 100644 --- a/lib/RuntimeBase.js +++ b/lib/RuntimeBase.js @@ -1,11 +1,12 @@ 'use strict'; const SError = require('./Error'), - BbPromise = require('bluebird'), - fs = BbPromise.promisifyAll(require('fs')), - path = require('path'), - wrench = require('wrench'), - _ = require('lodash'); + BbPromise = require('bluebird'), + fs = require('fs'), + fse = BbPromise.promisifyAll(require('fs-extra')), + path = require('path'), + wrench = require('wrench'), + _ = require('lodash'); /** * This is the base class that all Serverless Runtimes should extend. @@ -15,39 +16,92 @@ const SError = require('./Error'), let SUtils; class ServerlessRuntimeBase { + constructor(S, name) { - - SUtils = S.utils; - + SUtils = S.utils; this.S = S; this.name = name; } - installDepedencies( dir ) { - return BbPromise.reject(new SError(`Runtime "${this.getName()}" should implement "installDepedencies()" method`)); - } - - // Helper methods for derived classes - - getName() { - return this.name; - } + /** + * Scaffold + * - Scaffold the function in this runtime + */ scaffold(func) { return BbPromise.resolve(); } + /** + * Run + * - Run the function in this runtime + */ + run(func) { return BbPromise.reject(new SError(`Runtime "${this.getName()}" should implement "run()" method`)); } - build(func, pathDist, stage, region) { - return this._copyDir(func, pathDist, stage, region) - .then(() => this._afterCopyDir(func, pathDist, stage, region)) - .then(() => this._generateIncludePaths(func, pathDist)); + /** + * Build + * - Build the function in this runtime + */ + + build(func, stage, region) { + return BbPromise.reject(new SError(`Runtime "${this.getName()}" should implement "build()" method`)); } - _copyDir(func, pathDist, stage, region) { + getName() { + return this.name; + } + + /** + * Get ENV Vars + * - Gets ENV vars for this function and sets some defaults + */ + + getEnvVars(func, stage, region) { + + const envVars = func.toObjectPopulated({stage, region}).environment, + project = this.S.getProject(); + + const defaultVars = { + SERVERLESS_PROJECT: project.getName(), + SERVERLESS_STAGE: stage, + SERVERLESS_REGION: region, + SERVERLESS_DATA_MODEL_STAGE: stage ? project.getStage(stage).getName() : stage + }; + + return BbPromise.resolve(_.defaults(defaultVars, envVars)); + } + + /** + * Create Dist Dir + * - Creates a distribution folder for this function in _meta/_tmp + */ + + createDistDir(funcName) { + + let d = new Date(), + pathDist = this.S.getProject().getRootPath('_meta', '_tmp', funcName + '@' + d.getTime()); + + return new BbPromise(function(resolve, reject) { + try { + fse.mkdirsSync(path.dirname(pathDist)); + } catch (e) { + reject(new SError(`Error creating parent folders when writing this file: ${pathDist} + ${e.message}`)); + } + + resolve(pathDist); + }); + } + + /** + * Copy Function + * - Copies function to dist dir + */ + + copyFunction(func, pathDist, stage, region) { return BbPromise.try(() => { // Status SUtils.sDebug(`"${stage} - ${region} - ${func.getName()}": Copying in dist dir ${pathDist}`); @@ -63,63 +117,33 @@ class ServerlessRuntimeBase { let packageRoot = handlerFullPath.replace(func.handler, ''); return wrench.copyDirSyncRecursive(packageRoot, pathDist, { - exclude: this._exclude(func, pathDist, stage, region) + exclude: this._processExcludePatterns(func, pathDist, stage, region) }); }); } - _exclude(func, pathDist, stage, region) { - // Copy entire test project to temp folder, don't include anything in excludePatterns - let excludePatterns = func.custom.excludePatterns || []; + /** + * Install Dependencies + */ - return function(name, prefix) { - - if (!excludePatterns.length) { return false;} - - let relPath = path.join(prefix.replace(pathDist, ''), name); - - return excludePatterns.some(sRegex => { - relPath = (relPath.charAt(0) == path.sep) ? relPath.substr(1) : relPath; - - let re = new RegExp(sRegex), - matches = re.exec(relPath), - willExclude = (matches && matches.length > 0); - - if (willExclude) { - SUtils.sDebug(`"${stage} - ${region} - ${func.name}": Excluding - ${relPath}`); - } - - return willExclude; - }); - } + installDependencies(dir ) { + return BbPromise.reject(new SError(`Runtime "${this.getName()}" should implement "installDependencies()" method`)); } - _getEnvVars(func, stage, region) { - const envVars = func.toObjectPopulated({stage, region}).environment, - project = this.S.getProject(); + /** + * Generate Paths + * - Generate and return an array of paths of the function + */ - const defaultVars = { - SERVERLESS_STAGE: stage, - SERVERLESS_REGION: region, - SERVERLESS_DATA_MODEL_STAGE: project.getStage(stage).getName(), - SERVERLESS_PROJECT_NAME: project.getName() - }; + generatePaths(func, pathDist) { - return BbPromise.resolve(_.defaults(envVars, defaultVars)); - } - - _afterCopyDir(func, pathDist, stage, region) { - return BbPromise.resolve(); - } - - _generateIncludePaths(func, pathDist) { let compressPaths = [], - ignore = ['.DS_Store'], - stats, - fullPath; + ignore = ['.DS_Store'], + stats, + fullPath; // Zip up whatever is in back - let includePaths = func.custom.includePaths || ['.']; + let includePaths = ['.']; includePaths.forEach(p => { @@ -143,35 +167,61 @@ class ServerlessRuntimeBase { let dirname = path.basename(p); wrench - .readdirSyncRecursive(fullPath) - .forEach(file => { + .readdirSyncRecursive(fullPath) + .forEach(file => { - // Ignore certain files - for (let i = 0; i < ignore.length; i++) { - if (file.toLowerCase().indexOf(ignore[i]) > -1) return; - } + // Ignore certain files + for (let i = 0; i < ignore.length; i++) { + if (file.toLowerCase().indexOf(ignore[i]) > -1) return; + } - let filePath = path.join(fullPath, file); - if (fs.lstatSync(filePath).isFile()) { + let filePath = path.join(fullPath, file); + if (fs.lstatSync(filePath).isFile()) { - let pathInZip = path.join(dirname, file); + let pathInZip = path.join(dirname, file); - compressPaths.push({ - name: pathInZip, - path: filePath - }); - } - }); + compressPaths.push({ + name: pathInZip, + path: filePath + }); + } + }); } }); return BbPromise.resolve(compressPaths); } - getHandler(func) { - return func.handler; - } + /** + * Process Exclude Patterns + * - Process exclude patterns in function.custom.excludePatterns + */ + _processExcludePatterns(func, pathDist, stage, region) { + // Copy entire test project to temp folder, don't include anything in excludePatterns + let excludePatterns = func.custom.excludePatterns || []; + + return function(name, prefix) { + + if (!excludePatterns.length) { return false;} + + let relPath = path.join(prefix.replace(pathDist, ''), name); + + return excludePatterns.some(sRegex => { + relPath = (relPath.charAt(0) == path.sep) ? relPath.substr(1) : relPath; + + let re = new RegExp(sRegex), + matches = re.exec(relPath), + willExclude = (matches && matches.length > 0); + + if (willExclude) { + SUtils.sDebug(`"${stage} - ${region} - ${func.name}": Excluding - ${relPath}`); + } + + return willExclude; + }); + } + } } diff --git a/lib/RuntimeNode.js b/lib/RuntimeNode.js index 5bfd694fb..6d6efc20d 100644 --- a/lib/RuntimeNode.js +++ b/lib/RuntimeNode.js @@ -1,125 +1,179 @@ 'use strict'; -const SError = require('./Error'), - RuntimeBase = require('./RuntimeBase'), - SCli = require('./utils/cli'), - _ = require('lodash'), - BbPromise = require('bluebird'), - chalk = require('chalk'), - context = require('./utils/context'), - path = require('path'), - fs = BbPromise.promisifyAll(require('fs')); - +const SError = require('./Error'), + RuntimeBase = require('./RuntimeBase'), + SCli = require('./utils/cli'), + _ = require('lodash'), + BbPromise = require('bluebird'), + chalk = require('chalk'), + context = require('./utils/context'), + path = require('path'), + fs = BbPromise.promisifyAll(require('fs')); let SUtils; class ServerlessRuntimeNode extends RuntimeBase { + constructor(S) { super( S, 'nodejs' ); - SUtils = S.utils; } - installDepedencies( dir ) { - SCli.log('Installing "serverless-helpers" for this component via NPM...'); - SCli.log(`-----------------`); - SUtils.npmInstall(this.S.getProject().getRootPath(dir)); - SCli.log(`-----------------`); - } - - _loadFunctionHandler(func) { - return BbPromise.try(() => { - const handlerArr = func.handler.split('/').pop().split('.'), - functionFile = func.getRootPath(handlerArr[0] + '.js'), - functionHandler = handlerArr[1]; - - return require(functionFile)[functionHandler]; - }); - } + /** + * Scaffold + * - Create scaffolding for new Node.js function + */ scaffold(func) { const handlerPath = path.join(this.S.getServerlessPath(), 'templates', 'nodejs', 'handler.js'); return fs.readFileAsync(handlerPath) - .then(handlerJs => BbPromise.all([ - SUtils.writeFile(func.getRootPath('handler.js'), handlerJs), - SUtils.writeFile(func.getRootPath('event.json'), {}) - ])); + .then(handlerJs => BbPromise.all([ + SUtils.writeFile(func.getRootPath('handler.js'), handlerJs), + SUtils.writeFile(func.getRootPath('event.json'), {}) + ])); } + /** + * Run + * - Run this function locally + */ + run(func) { - return BbPromise - .all([this._loadFunctionHandler(func), SUtils.readFile(func.getRootPath('event.json'))]) - .spread((functionHandler, functionEvent) => { - return new BbPromise(resolve => { - functionHandler(functionEvent, context(func, (err, result) => { - SCli.log(`-----------------`); + let _this = this, + functionEvent, + functionCall; - // Show error - if (err) { - SCli.log(chalk.bold('Failed - This Error Was Returned:')); - SCli.log(err.message); - SCli.log(err.stack); + return BbPromise.try(function() { - return resolve({ - status: 'error', - response: err.message, - error: err - }); - } + // Load Event + functionEvent = SUtils.readFileSync(func.getRootPath('event.json')); - // Show success response - SCli.log(chalk.bold('Success! - This Response Was Returned:')); - SCli.log(JSON.stringify(result, null, 4)); - return resolve({ - status: 'success', - response: result - }); - })); + // Load Function + let handlerArr = func.handler.split('/').pop().split('.'), + functionFile = func.getRootPath(handlerArr[0] + '.js'), + functionHandler = handlerArr[1]; + + functionCall = require(functionFile)[functionHandler]; }) - }) - .catch((err) => { - SCli.log(`-----------------`); + .then(() => _this.getEnvVars(func)) + .then(function(envVars) { - SCli.log(chalk.bold('Failed - This Error Was Thrown:')); - SCli.log(err.stack || err); + // Add ENV vars (from no stage/region) to environment + for (var key in envVars) { + process.env[key] = envVars[key]; + } + }) + .then(() => { - return { - status: 'error', - response: err.message, - error: err - }; - }) + return new BbPromise(resolve => { + + // Call Function + functionCall(functionEvent, context(func, (err, result) => { + + SCli.log(`-----------------`); + + // Show error + if (err) { + SCli.log(chalk.bold('Failed - This Error Was Returned:')); + SCli.log(err.message); + SCli.log(err.stack); + + return resolve({ + status: 'error', + response: err.message, + error: err + }); + } + + // Show success response + SCli.log(chalk.bold('Success! - This Response Was Returned:')); + SCli.log(JSON.stringify(result, null, 4)); + return resolve({ + status: 'success', + response: result + }); + })); + }) + }) + .catch((err) => { + SCli.log(`-----------------`); + + SCli.log(chalk.bold('Failed - This Error Was Thrown:')); + SCli.log(err.stack || err); + + return { + status: 'error', + response: err.message, + error: err + }; + }); } + /** + * Build + * - Build the function in this runtime + */ + + build(func, stage, region) { + + // Validate + if (!func._class || func._class !== 'Function') return BbPromise.reject(new SError('A function instance is required')); + + let pathDist; + + return this.createDistDir(func.name) + .then(function(distDir) { pathDist = distDir }) + .then(() => this.copyFunction(func, pathDist, stage, region)) + .then(() => this._addEnvVarsInline(func, pathDist, stage, region)) + .then(() => this.generatePaths(func, pathDist)); + } + + /** + * Get Handler + */ + getHandler(func) { return path.join(path.dirname(func.handler), "_serverless_handler.handler").replace(/\\/g, '/'); } - _afterCopyDir(func, pathDist, stage, region) { - return this._getEnvVars(func, stage, region) - .then(envVars => { + /** + * Install NPM Dependencies + */ - const handlerArr = func.handler.split('.'), + installDependencies(dir) { + SCli.log(`Installing NPM dependencies in dir: ${dir}`); + SCli.log(`-----------------`); + SUtils.npmInstall(this.S.getProject().getRootPath(dir)); + SCli.log(`-----------------`); + } + + /** + * Add ENV Vars In-line + * - Adds a new handler that loads in ENV vars before running the main handler + */ + + _addEnvVarsInline(func, pathDist, stage, region) { + + return this.getEnvVars(func, stage, region) + .then(envVars => { + + const handlerArr = func.handler.split('.'), handlerDir = path.dirname(func.handler), handlerFile = handlerArr[0].split('/').pop(), handlerMethod = handlerArr[1]; - const loader = ` + const loader = ` var envVars = ${JSON.stringify(envVars, null, 2)}; - for (var key in envVars) { process.env[key] = envVars[key]; } - exports.handler = require("./${handlerFile}")["${handlerMethod}"]; `; - return fs.writeFileAsync(path.join(pathDist, handlerDir, '_serverless_handler.js'), loader); - }); + return fs.writeFileAsync(path.join(pathDist, handlerDir, '_serverless_handler.js'), loader); + }); } - - } module.exports = ServerlessRuntimeNode; diff --git a/lib/RuntimePython27.js b/lib/RuntimePython27.js index 92e4a1a2c..7b500cf6d 100644 --- a/lib/RuntimePython27.js +++ b/lib/RuntimePython27.js @@ -101,7 +101,7 @@ class ServerlessRuntimePython27 extends RuntimeBase { } _afterCopyDir(func, pathDist, stage, region) { - return this._getEnvVars(func, stage, region) + return this.getEnvVars(func, stage, region) .then(envVars => { const handlerArr = func.handler.split('.'), diff --git a/lib/actions/CodeDeployLambda.js b/lib/actions/CodeDeployLambda.js index 0b9bd3805..56b9d5543 100644 --- a/lib/actions/CodeDeployLambda.js +++ b/lib/actions/CodeDeployLambda.js @@ -122,6 +122,10 @@ module.exports = function(SPlugin, serverlessPath) { zip.file(nc.name, fs.readFileSync(nc.path)); }); + // Set zipfile name + this.zipName = `${this.function.getName()}_${this.evt.options.stage}_${this.evt.options.region}`; + + // Compress this.zipBuffer = zip.generate({ type: 'nodebuffer', compression: 'DEFLATE' @@ -134,11 +138,8 @@ module.exports = function(SPlugin, serverlessPath) { ); } - // Set path of compressed package - this.pathCompressed = path.join(this.evt.options.pathDist, 'package.zip'); - // Create compressed package - fs.writeFileSync(this.pathCompressed, this.zipBuffer); + fs.writeFileSync(path.join(this.project.getTempPath(), this.zipName), this.zipBuffer); SUtils.sDebug(`"${this.evt.options.stage} - ${this.evt.options.region} - ${this.functionName}": Compressed file created - ${this.pathCompressed}`); diff --git a/lib/actions/CodePackageLambda.js b/lib/actions/CodePackageLambda.js index b81e21774..77ffc5b1c 100644 --- a/lib/actions/CodePackageLambda.js +++ b/lib/actions/CodePackageLambda.js @@ -66,7 +66,6 @@ module.exports = function(SPlugin, serverlessPath) { // Flow return _this._validateAndPrepare() .bind(_this) - .then(_this._createDistFolder) .then(_this._package) .then(function() { @@ -75,7 +74,6 @@ module.exports = function(SPlugin, serverlessPath) { */ _this.evt.data.pathsPackaged = _this.pathsPackaged; - _this.evt.data.pathDist = _this.pathDist; return _this.evt; }); @@ -94,7 +92,7 @@ module.exports = function(SPlugin, serverlessPath) { if (!_this.function) BbPromise.reject(new SError(`Function could not be found: ${_this.evt.options.name}`)); - //TODO: Use Function.validate() + //TODO: Use Function.validate()? // Validate if (!_this.function.name) { @@ -115,33 +113,18 @@ module.exports = function(SPlugin, serverlessPath) { return BbPromise.resolve(); } - /** - * Create Distribution Folder - */ - - _createDistFolder() { - - let _this = this; - - // Set Dist Dir - let d = new Date(); - _this.pathDist = _this.S.getProject().getRootPath('_meta', '_tmp', _this.function.name + '@' + d.getTime()); - - return BbPromise.resolve(); - } - /** * Package + * - Build lambda package */ _package() { // Create pathsPackaged for each file ready to compress - return this.function.build(this.pathDist, this.evt.options.stage, this.evt.options.region) + return this.function.getRuntime().build(this.function, this.evt.options.stage, this.evt.options.region) .then(paths => this.pathsPackaged = paths); } - } return( CodePackageLambda ); -}; +}; \ No newline at end of file diff --git a/lib/utils/index.js b/lib/utils/index.js index 4bd36f657..8f6201e6c 100644 --- a/lib/utils/index.js +++ b/lib/utils/index.js @@ -290,13 +290,17 @@ module.exports = { populate: function(project, templates, data, stage, region) { // Validate required params - if (!project || !templates || !data || !stage || !region) throw new SError(`Missing required params: Serverless, project, stage, region`); + if (!project || !templates || !data) throw new SError(`Missing required params: Serverless, project, stage, region`); // Validate: Check stage exists - if (!project.validateStageExists(stage)) throw new SError(`Stage doesn't exist`); + if (stage) { + if (!project.validateStageExists(stage)) throw new SError(`Stage doesn't exist`); + } // Validate: Check region exists in stage - if (!project.validateRegionExists(stage, region)) throw new SError(`Region "${region}" doesn't exist in provided stage "${stage}"`); + if (stage && region) { + if (!project.validateRegionExists(stage, region)) throw new SError(`Region "${region}" doesn't exist in provided stage "${stage}"`); + } let varTemplateSyntax = /\${([\s\S]+?)}/g, templateTemplateSyntax = /\$\${([\s\S]+?)}/g;