Merge pull request #4713 from erikerikson/fix-4687-cyclic-variable-dependencies

Eliminate/Report Hung Promises (#4687), Prepopulate Stage and Region (#4311), Handle Quoted Strings (#4734)
This commit is contained in:
Frank Schmid 2018-02-20 13:56:27 +01:00 committed by GitHub
commit 39caddc404
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 1405 additions and 1171 deletions

View File

@ -0,0 +1,53 @@
'use strict';
const logWarning = require('./Error').logWarning;
class PromiseTracker {
constructor() {
this.promiseList = [];
this.promiseMap = {};
this.startTime = Date.now();
}
start() {
this.interval = setInterval(this.report.bind(this), 2500);
}
report() {
const delta = Date.now() - this.startTime;
logWarning('################################################################################');
logWarning(`# ${delta}: ${this.getSettled().length} of ${
this.getAll().length} promises have settled`);
const pending = this.getPending();
logWarning(`# ${delta}: ${pending.length} unsettled promises:`);
pending.forEach((promise) => {
logWarning(`# ${delta}: ${promise.waitList}`);
});
logWarning('################################################################################');
}
stop() {
clearInterval(this.interval);
}
add(variable, prms, specifier) {
const promise = prms;
promise.waitList = `${variable} waited on by: ${specifier}`;
promise.state = 'pending';
promise.then( // creates a promise with the following effects but that we otherwise ignore
() => { promise.state = 'resolved'; },
() => { promise.state = 'rejected'; });
this.promiseList.push(promise);
this.promiseMap[variable] = promise;
return promise;
}
contains(variable) {
return variable in this.promiseMap;
}
get(variable, specifier) {
const promise = this.promiseMap[variable];
promise.waitList += ` ${specifier}`;
return promise;
}
getPending() { return this.promiseList.filter(p => (p.state === 'pending')); }
getSettled() { return this.promiseList.filter(p => (p.state !== 'pending')); }
getAll() { return this.promiseList; }
}
module.exports = PromiseTracker;

View File

@ -0,0 +1,64 @@
'use strict';
/* eslint-disable no-unused-expressions */
const BbPromise = require('bluebird');
const chai = require('chai');
const PromiseTracker = require('../../lib/classes/PromiseTracker');
chai.use(require('chai-as-promised'));
const expect = chai.expect;
/**
* Mostly this class is tested by its use in peer ~/lib/classes/Variables.js
*
* Mostly, I'm creating coverage but if errors are discovered, coverage for the specific cases
* can be created here.
*/
describe('PromiseTracker', () => {
let promiseTracker;
beforeEach(() => {
promiseTracker = new PromiseTracker();
});
it('logs a warning without throwing', () => {
promiseTracker.add('foo', BbPromise.resolve(), '${foo:}');
promiseTracker.add('foo', BbPromise.delay(10), '${foo:}');
promiseTracker.report(); // shouldn't throw
});
it('reports no pending promises when none have been added', () => {
const promises = promiseTracker.getPending();
expect(promises).to.be.an.instanceof(Array);
expect(promises.length).to.equal(0);
});
it('reports one pending promise when one has been added', () => {
let resolve;
const promise = new BbPromise((rslv) => { resolve = rslv; });
promiseTracker.add('foo', promise, '${foo:}');
return BbPromise.delay(1).then(() => {
const promises = promiseTracker.getPending();
expect(promises).to.be.an.instanceof(Array);
expect(promises.length).to.equal(1);
expect(promises[0]).to.equal(promise);
}).then(() => { resolve(); });
});
it('reports no settled promises when none have been added', () => {
const promises = promiseTracker.getSettled();
expect(promises).to.be.an.instanceof(Array);
expect(promises.length).to.equal(0);
});
it('reports one settled promise when one has been added', () => {
const promise = BbPromise.resolve();
promiseTracker.add('foo', promise, '${foo:}');
promise.state = 'resolved';
const promises = promiseTracker.getSettled();
expect(promises).to.be.an.instanceof(Array);
expect(promises.length).to.equal(1);
expect(promises[0]).to.equal(promise);
});
it('reports no promises when none have been added', () => {
const promises = promiseTracker.getAll();
expect(promises).to.be.an('array').that.is.empty;
});
});

View File

@ -1,34 +1,102 @@
'use strict';
const _ = require('lodash');
const path = require('path');
const replaceall = require('replaceall');
const logWarning = require('./Error').logWarning;
const BbPromise = require('bluebird');
const os = require('os');
const path = require('path');
const replaceall = require('replaceall');
const fse = require('../utils/fs/fse');
const logWarning = require('./Error').logWarning;
const PromiseTracker = require('./PromiseTracker');
class Variables {
constructor(serverless) {
this.serverless = serverless;
this.service = this.serverless.service;
this.cache = {};
this.tracker = new PromiseTracker();
this.overwriteSyntax = RegExp(/,/g);
this.deep = [];
this.deepRefSyntax = RegExp(/^(\${)?deep:\d+(\.[^}]+)*()}?$/);
this.overwriteSyntax = RegExp(/\s*(?:,\s*)+/g);
this.fileRefSyntax = RegExp(/^file\((~?[a-zA-Z0-9._\-/]+?)\)/g);
this.envRefSyntax = RegExp(/^env:/g);
this.optRefSyntax = RegExp(/^opt:/g);
this.selfRefSyntax = RegExp(/^self:/g);
this.cfRefSyntax = RegExp(/^cf:/g);
this.s3RefSyntax = RegExp(/^s3:(.+?)\/(.+)$/);
this.stringRefSyntax = RegExp(/('.*')|(".*")/g);
this.stringRefSyntax = RegExp(/(?:('|").*?\1)/g);
this.ssmRefSyntax = RegExp(/^ssm:([a-zA-Z0-9_.\-/]+)[~]?(true|false)?/);
}
loadVariableSyntax() {
this.variableSyntax = RegExp(this.service.provider.variableSyntax, 'g');
}
// #############
// ## SERVICE ##
// #############
prepopulateService() {
const dependentServices = [
{ name: 'CloudFormation', regex: this.cfRefSyntax },
{ name: 'S3', regex: this.s3RefSyntax },
{ name: 'SSM', regex: this.ssmRefSyntax },
];
const dependencyMessage = (configName, configValue, serviceName) =>
`Variable Failure: value ${
configName
} set to '${
configValue
}' references ${
serviceName
} which requires a ${
configName
} value for use.`;
const getVariableParts = (variableString) => {
const matches = this.getMatches(variableString);
return matches.reduce(
(accumulation, current) =>
accumulation.concat(this.splitByComma(current.variable)),
[]);
};
const serviceMatch = variablePart =>
dependentServices.find((service) => {
let variable = variablePart;
if (variable.match(this.deepRefSyntax)) {
variable = this.getVariableFromDeep(variablePart);
}
return variable.match(service.regex);
});
const getUntilValid = (config) => {
const parts = getVariableParts(config.value);
const service = parts.reduce(
(accumulation, part) => accumulation || serviceMatch(part),
undefined);
if (service) {
const msg = dependencyMessage(config.name, config.value, service.name);
return BbPromise.reject(new this.serverless.classes.Error(msg));
}
return this.populateValue(config.value, false)
.then(populated => (
populated.match(this.variableSyntax) ?
getUntilValid(_.assign(config, { value: populated })) :
_.assign({}, config, { populated })
));
};
const provider = this.serverless.getProvider('aws');
if (provider) {
const requiredConfig = [
_.assign({ name: 'region' }, provider.getRegionSourceValue()),
_.assign({ name: 'stage' }, provider.getStageSourceValue()),
];
const configToPopulate = requiredConfig.filter(config =>
!_.isUndefined(config.value) &&
(_.isString(config.value) && config.value.match(this.variableSyntax)));
const configPromises = configToPopulate.map(getUntilValid);
return this.assignProperties(provider, configPromises);
}
return BbPromise.resolve();
}
/**
* Populate all variables in the service, conviently remove and restore the service attributes
* that confuse the population methods.
@ -42,14 +110,116 @@ class Variables {
// store
const variableSyntaxProperty = this.service.provider.variableSyntax;
// remove
this.service.provider.variableSyntax = true; // matches itself
this.serverless.service.serverless = null;
return this.populateObject(this.service).then(() => {
// restore
this.service.provider.variableSyntax = variableSyntaxProperty;
this.serverless.service.serverless = this.serverless;
return BbPromise.resolve(this.service);
});
this.service.provider.variableSyntax = undefined; // otherwise matches itself
this.service.serverless = undefined;
this.tracker.start();
return this.prepopulateService()
.then(() => this.populateObject(this.service))
.finally(() => {
// restore
this.tracker.stop();
this.service.serverless = this.serverless;
this.service.provider.variableSyntax = variableSyntaxProperty;
})
.then(() => this.service);
}
// ############
// ## OBJECT ##
// ############
/**
* The declaration of a terminal property. This declaration includes the path and value of the
* property.
* Example Input:
* {
* foo: {
* bar: 'baz'
* }
* }
* Example Result:
* [
* {
* path: ['foo', 'bar']
* value: 'baz
* }
* ]
* @typedef {Object} TerminalProperty
* @property {String[]} path The path to the terminal property
* @property {Date|RegEx|String} The value of the terminal property
*/
/**
* Generate an array of objects noting the terminal properties of the given root object and their
* paths
* @param root The object to generate a terminal property path/value set for
* @param current The current part of the given root that terminal properties are being sought
* within
* @param [context] An array containing the path to the current object root (intended for internal
* use)
* @param [results] An array of current results (intended for internal use)
* @returns {TerminalProperty[]} The terminal properties of the given root object, with the path
* and value of each
*/
getProperties(root, atRoot, current, cntxt, rslts) {
let context = cntxt;
if (!context) {
context = [];
}
let results = rslts;
if (!results) {
results = [];
}
const addContext = (value, key) =>
this.getProperties(root, false, value, context.concat(key), results);
if (
_.isArray(current)
) {
_.map(current, addContext);
} else if (
_.isObject(current) &&
!_.isDate(current) &&
!_.isRegExp(current) &&
!_.isFunction(current)
) {
if (atRoot || current !== root) {
_.mapValues(current, addContext);
}
} else {
results.push({ path: context, value: current });
}
return results;
}
/**
* @typedef {TerminalProperty} TerminalPropertyPopulated
* @property {Object} populated The populated value of the value at the path
*/
/**
* Populate the given terminal properties, returning promises to do so
* @param properties The terminal properties to populate
* @returns {Promise<TerminalPropertyPopulated[]>[]} The promises that will resolve to the
* populated values of the given terminal properties
*/
populateVariables(properties) {
const variables = properties.filter(property =>
_.isString(property.value) &&
property.value.match(this.variableSyntax));
return _.map(variables,
variable => this.populateValue(variable.value, false)
.then(populated => _.assign({}, variable, { populated })));
}
/**
* Assign the populated values back to the target object
* @param target The object to which the given populated terminal properties should be applied
* @param populations The fully populated terminal properties
* @returns {Promise<number>} resolving with the number of changes that were applied to the given
* target
*/
assignProperties(target, populations) { // eslint-disable-line class-methods-use-this
return BbPromise.all(populations)
.then((results) => results.forEach((result) => {
if (result.value !== result.populated) {
_.set(target, result.path, result.populated);
}
}));
}
/**
* Populate the variables in the given object.
@ -57,78 +227,113 @@ class Variables {
* @returns {Promise.<TResult>|*} A promise resolving to the in-place populated object.
*/
populateObject(objectToPopulate) {
// Map terminal values of given root (i.e. for every leaf value...)
const forEachLeaf = (root, context, callback) => {
const addContext = (value, key) => forEachLeaf(value, context.concat(key), callback);
if (
_.isArray(root)
) {
return _.map(root, addContext);
} else if (
_.isObject(root) &&
!_.isDate(root) &&
!_.isRegExp(root) &&
!_.isFunction(root)
) {
return _.extend({}, root, _.mapValues(root, addContext));
}
return callback(root, context);
};
// For every leaf value...
const pendingLeaves = [];
forEachLeaf(
objectToPopulate,
[],
(leafValue, leafPath) => {
if (typeof leafValue === 'string') {
pendingLeaves.push(this
.populateProperty(leafValue, true)
.then(leafValuePopulated => _.set(objectToPopulate, leafPath, leafValuePopulated))
);
}
}
);
return BbPromise.all(pendingLeaves).then(() => objectToPopulate);
const leaves = this.getProperties(objectToPopulate, true, objectToPopulate);
const populations = this.populateVariables(leaves);
return this.assignProperties(objectToPopulate, populations)
.then(() => (populations.length ?
this.populateObject(objectToPopulate) :
objectToPopulate));
}
// ##############
// ## PROPERTY ##
// ##############
/**
* Standard logic for cleaning a variable
* Example: cleanVariable('${opt:foo}') => 'opt:foo'
* @param match The variable match instance variable part
* @returns {string} The cleaned variable match
*/
cleanVariable(match) {
return match.replace(
this.variableSyntax,
(context, contents) => contents.trim()
).replace(/\s/g, '');
}
/**
* Populate variables, in-place if specified, in the given property value.
* @param propertyToPopulate The property to populate (only strings with variables are altered).
* @param populateInPlace Whether to deeply clone the given property prior to population.
* @returns {Promise.<TResult>|*} A promise resolving to the populated result.
* @typedef {Object} MatchResult
* @property {String} match The original property value that matched the variable syntax
* @property {String} variable The cleaned variable string that specifies the origin for the
* property value
*/
populateProperty(propertyToPopulate, populateInPlace) {
let property = propertyToPopulate;
if (!populateInPlace) {
property = _.cloneDeep(propertyToPopulate);
/**
* Get matches against the configured variable syntax
* @param property The property value to attempt extracting matches from
* @returns {Object|String|MatchResult[]} The given property or the identified matches
*/
getMatches(property) {
if (typeof property !== 'string') {
return property;
}
if (
typeof property !== 'string' ||
!property.match(this.variableSyntax)
) {
const matches = property.match(this.variableSyntax);
if (!matches || !matches.length) {
return property;
}
return _.map(matches, match => ({
match,
variable: this.cleanVariable(match),
}));
}
/**
* Populate the given matches, returning an array of Promises which will resolve to the populated
* values of the given matches
* @param {MatchResult[]} matches The matches to populate
* @returns {Promise[]} Promises for the eventual populated values of the given matches
*/
populateMatches(matches) {
return _.map(matches, (match) => {
const parts = this.splitByComma(match.variable);
if (parts.length > 1) {
return this.overwrite(parts, match.match);
}
return this.getValueFromSource(parts[0], match.match);
});
}
/**
* Render the given matches and their associated results to the given value
* @param value The value into which to render the given results
* @param matches The matches on the given value where the results are to be rendered
* @param results The results that are to be rendered to the given value
* @returns {*} The populated value with the given results rendered according to the given matches
*/
renderMatches(value, matches, results) {
let result = value;
for (let i = 0; i < matches.length; i += 1) {
this.warnIfNotFound(matches[i].variable, results[i]);
result = this.populateVariable(result, matches[i].match, results[i]);
}
return result;
}
/**
* Populate the given value, recursively if root is true
* @param valueToPopulate The value to populate variables within
* @param root Whether the caller is the root populator and thereby whether to recursively
* populate
* @returns {PromiseLike<T>} A promise that resolves to the populated value, recursively if root
* is true
*/
populateValue(valueToPopulate, root) {
const property = _.cloneDeep(valueToPopulate);
const matches = this.getMatches(property);
if (!_.isArray(matches)) {
return BbPromise.resolve(property);
}
const pendingMatches = [];
property.match(this.variableSyntax).forEach((matchedString) => {
const variableString = matchedString
.replace(this.variableSyntax, (match, varName) => varName.trim())
.replace(/\s/g, '');
let pendingMatch;
if (variableString.match(this.overwriteSyntax)) {
pendingMatch = this.overwrite(variableString);
} else {
pendingMatch = this.getValueFromSource(variableString);
}
pendingMatches.push(pendingMatch.then(matchedValue => {
this.warnIfNotFound(variableString, matchedValue);
return this.populateVariable(property, matchedString, matchedValue)
.then((populatedProperty) => {
property = populatedProperty;
});
}));
});
return BbPromise.all(pendingMatches)
.then(() => this.populateProperty(property, true));
const populations = this.populateMatches(matches);
return BbPromise.all(populations)
.then(results => this.renderMatches(property, matches, results))
.then((result) => {
if (root && matches.length) {
return this.populateValue(result);
}
return result;
});
}
/**
* Populate variables in the given property.
* @param propertyToPopulate The property to populate (replace variables with their values).
* @returns {Promise.<TResult>|*} A promise resolving to the populated result.
*/
populateProperty(propertyToPopulate) {
return this.populateValue(propertyToPopulate, true);
}
/**
* Populate a given property, given the matched string to replace and the value to replace the
@ -155,20 +360,63 @@ class Variables {
].join('');
throw new this.serverless.classes.Error(errorMessage);
}
return BbPromise.resolve(property);
return property;
}
// ###############
// ## VARIABLES ##
// ###############
/**
* Split a given string by whitespace padded commas excluding those within single or double quoted
* strings.
* @param string The string to split by comma.
*/
splitByComma(string) {
const input = string.trim();
const stringMatches = [];
let match = this.stringRefSyntax.exec(input);
while (match) {
stringMatches.push({
start: match.index,
end: this.stringRefSyntax.lastIndex,
});
match = this.stringRefSyntax.exec(input);
}
const commaReplacements = [];
const contained = commaMatch => // curry the current commaMatch
stringMatch => // check whether stringMatch containing the commaMatch
stringMatch.start < commaMatch.index &&
this.overwriteSyntax.lastIndex < stringMatch.end;
match = this.overwriteSyntax.exec(input);
while (match) {
const matchContained = contained(match);
const containedBy = stringMatches.find(matchContained);
if (!containedBy) { // if uncontained, this comma respresents a splitting location
commaReplacements.push({
start: match.index,
end: this.overwriteSyntax.lastIndex,
});
}
match = this.overwriteSyntax.exec(input);
}
let prior = 0;
const results = [];
commaReplacements.forEach((replacement) => {
results.push(input.slice(prior, replacement.start));
prior = replacement.end;
});
results.push(input.slice(prior));
return results;
}
/**
* Overwrite the given variable string, resolve each variable and resolve to the first valid
* value.
* Resolve the given variable string that expresses a series of fallback values in case the
* initial values are not valid, resolving each variable and resolving to the first valid value.
* @param variableStringsString The overwrite string of variables to populate and choose from.
* @returns {Promise.<TResult>|*} A promise resolving to the first validly populating variable
* in the given variable strings string.
*/
overwrite(variableStringsString) {
const variableStrings = variableStringsString.split(',');
overwrite(variableStrings, propertyString) {
const variableValues = variableStrings.map(variableString =>
this.getValueFromSource(variableString)
);
this.getValueFromSource(variableString, propertyString));
const validValue = value => (
value !== null &&
typeof value !== 'undefined' &&
@ -176,53 +424,50 @@ class Variables {
);
return BbPromise.all(variableValues)
.then(values => // find and resolve first valid value, undefined if none
BbPromise.resolve(values.find(validValue))
);
BbPromise.resolve(values.find(validValue)));
}
/**
* Given any variable string, return the value it should be populated with.
* @param variableString The variable string to retrieve a value for.
* @returns {Promise.<TResult>|*} A promise resolving to the given variables value.
*/
getValueFromSource(variableString) {
if (!(variableString in this.cache)) {
let value;
getValueFromSource(variableString, propertyString) {
let ret;
if (this.tracker.contains(variableString)) {
ret = this.tracker.get(variableString, propertyString);
} else {
if (variableString.match(this.envRefSyntax)) {
value = this.getValueFromEnv(variableString);
ret = this.getValueFromEnv(variableString);
} else if (variableString.match(this.optRefSyntax)) {
value = this.getValueFromOptions(variableString);
ret = this.getValueFromOptions(variableString);
} else if (variableString.match(this.selfRefSyntax)) {
value = this.getValueFromSelf(variableString);
ret = this.getValueFromSelf(variableString);
} else if (variableString.match(this.fileRefSyntax)) {
value = this.getValueFromFile(variableString);
ret = this.getValueFromFile(variableString);
} else if (variableString.match(this.cfRefSyntax)) {
value = this.getValueFromCf(variableString);
ret = this.getValueFromCf(variableString);
} else if (variableString.match(this.s3RefSyntax)) {
value = this.getValueFromS3(variableString);
ret = this.getValueFromS3(variableString);
} else if (variableString.match(this.stringRefSyntax)) {
value = this.getValueFromString(variableString);
ret = this.getValueFromString(variableString);
} else if (variableString.match(this.ssmRefSyntax)) {
value = this.getValueFromSsm(variableString);
ret = this.getValueFromSsm(variableString);
} else if (variableString.match(this.deepRefSyntax)) {
ret = this.getValueFromDeep(variableString);
} else {
const errorMessage = [
`Invalid variable reference syntax for variable ${variableString}.`,
' You can only reference env vars, options, & files.',
' You can check our docs for more info.',
].join('');
throw new this.serverless.classes.Error(errorMessage);
ret = BbPromise.reject(new this.serverless.classes.Error(errorMessage));
}
this.cache[variableString] = BbPromise.resolve(value)
.then(variableValue => {
if (_.isObject(variableValue) && variableValue !== this.service) {
return this.populateObject(variableValue);
}
return variableValue;
});
ret = this.tracker.add(variableString, ret, propertyString);
}
return this.cache[variableString];
return ret;
}
getValueFromEnv(variableString) {
getValueFromEnv(variableString) { // eslint-disable-line class-methods-use-this
const requestedEnvVar = variableString.split(':')[1];
let valueToPopulate;
if (requestedEnvVar !== '' || '' in process.env) {
@ -233,7 +478,7 @@ class Variables {
return BbPromise.resolve(valueToPopulate);
}
getValueFromString(variableString) {
getValueFromString(variableString) { // eslint-disable-line class-methods-use-this
const valueToPopulate = variableString.replace(/^['"]|['"]$/g, '');
return BbPromise.resolve(valueToPopulate);
}
@ -262,13 +507,13 @@ class Variables {
.replace('~', os.homedir());
let referencedFileFullPath = (path.isAbsolute(referencedFileRelativePath) ?
referencedFileRelativePath :
path.join(this.serverless.config.servicePath, referencedFileRelativePath));
referencedFileRelativePath :
path.join(this.serverless.config.servicePath, referencedFileRelativePath));
// Get real path to handle potential symlinks (but don't fatal error)
referencedFileFullPath = fse.existsSync(referencedFileFullPath) ?
fse.realpathSync(referencedFileFullPath) :
referencedFileFullPath;
fse.realpathSync(referencedFileFullPath) :
referencedFileFullPath;
let fileExtension = referencedFileRelativePath.split('.');
fileExtension = fileExtension[fileExtension.length - 1];
@ -281,7 +526,8 @@ class Variables {
// Process JS files
if (fileExtension === 'js') {
const jsFile = require(referencedFileFullPath); // eslint-disable-line global-require
// eslint-disable-next-line global-require, import/no-dynamic-require
const jsFile = require(referencedFileFullPath);
const variableArray = variableString.split(':');
let returnValueFunction;
if (variableArray[1]) {
@ -293,29 +539,28 @@ class Variables {
}
if (typeof returnValueFunction !== 'function') {
throw new this.serverless.classes
.Error([
'Invalid variable syntax when referencing',
` file "${referencedFileRelativePath}".`,
' Check if your javascript is exporting a function that returns a value.',
].join(''));
const errorMessage = [
'Invalid variable syntax when referencing',
` file "${referencedFileRelativePath}".`,
' Check if your javascript is exporting a function that returns a value.',
].join('');
return BbPromise.reject(new this.serverless.classes.Error(errorMessage));
}
valueToPopulate = returnValueFunction.call(jsFile);
return BbPromise.resolve(valueToPopulate).then(valueToPopulateResolved => {
return BbPromise.resolve(valueToPopulate).then((valueToPopulateResolved) => {
let deepProperties = variableString.replace(matchedFileRefString, '');
deepProperties = deepProperties.slice(1).split('.');
deepProperties.splice(0, 1);
return this.getDeepValue(deepProperties, valueToPopulateResolved)
.then(deepValueToPopulateResolved => {
.then((deepValueToPopulateResolved) => {
if (typeof deepValueToPopulateResolved === 'undefined') {
const errorMessage = [
'Invalid variable syntax when referencing',
` file "${referencedFileRelativePath}".`,
' Check if your javascript is returning the correct data.',
].join('');
throw new this.serverless.classes
.Error(errorMessage);
return BbPromise.reject(new this.serverless.classes.Error(errorMessage));
}
return BbPromise.resolve(deepValueToPopulateResolved);
});
@ -334,8 +579,7 @@ class Variables {
` file "${referencedFileRelativePath}" sub properties`,
' Please use ":" to reference sub properties.',
].join('');
throw new this.serverless.classes
.Error(errorMessage);
return BbPromise.reject(new this.serverless.classes.Error(errorMessage));
}
deepProperties = deepProperties.slice(1).split('.');
return this.getDeepValue(deepProperties, valueToPopulate);
@ -352,9 +596,8 @@ class Variables {
.request('CloudFormation',
'describeStacks',
{ StackName: stackName },
{ useCache: true } // Use request cache
)
.then(result => {
{ useCache: true })// Use request cache
.then((result) => {
const outputs = result.Stacks[0].Outputs;
const output = outputs.find(x => x.OutputKey === outputLogicalId);
@ -364,11 +607,9 @@ class Variables {
` Stack name: "${stackName}"`,
` Requested variable: "${outputLogicalId}".`,
].join('');
throw new this.serverless.classes
.Error(errorMessage);
return BbPromise.reject(new this.serverless.classes.Error(errorMessage));
}
return output.OutputValue;
return BbPromise.resolve(output.OutputValue);
});
}
@ -376,60 +617,89 @@ class Variables {
const groups = variableString.match(this.s3RefSyntax);
const bucket = groups[1];
const key = groups[2];
return this.serverless.getProvider('aws')
.request('S3',
return this.serverless.getProvider('aws').request(
'S3',
'getObject',
{
Bucket: bucket,
Key: key,
},
{ useCache: true } // Use request cache
)
.then(
response => response.Body.toString(),
err => {
{ useCache: true }) // Use request cache
.then(response => BbPromise.resolve(response.Body.toString()))
.catch((err) => {
const errorMessage = `Error getting value for ${variableString}. ${err.message}`;
throw new this.serverless.classes.Error(errorMessage);
}
);
return BbPromise.reject(new this.serverless.classes.Error(errorMessage));
});
}
getValueFromSsm(variableString) {
const groups = variableString.match(this.ssmRefSyntax);
const param = groups[1];
const decrypt = (groups[2] === 'true');
return this.serverless.getProvider('aws')
.request('SSM',
return this.serverless.getProvider('aws').request(
'SSM',
'getParameter',
{
Name: param,
WithDecryption: decrypt,
},
{ useCache: true } // Use request cache
)
.then(
response => BbPromise.resolve(response.Parameter.Value),
err => {
{ useCache: true }) // Use request cache
.then(response => BbPromise.resolve(response.Parameter.Value))
.catch((err) => {
const expectedErrorMessage = `Parameter ${param} not found.`;
if (err.message !== expectedErrorMessage) {
throw new this.serverless.classes.Error(err.message);
return BbPromise.reject(new this.serverless.classes.Error(err.message));
}
return BbPromise.resolve(undefined);
}
);
});
}
getDeepIndex(variableString) {
const deepIndexReplace = RegExp(/^deep:|(\.[^}]+)*$/g);
return variableString.replace(deepIndexReplace, '');
}
getVariableFromDeep(variableString) {
const index = this.getDeepIndex(variableString);
return this.deep[index];
}
getValueFromDeep(variableString) {
const deepPrefixReplace = RegExp(/(?:^deep:)\d+\.?/g);
const variable = this.getVariableFromDeep(variableString);
const deepRef = variableString.replace(deepPrefixReplace, '');
const sourceString = `\${deep:\${${variable}}${deepRef.length ? `.${deepRef}` : ''}}`;
return this.getValueFromSource(variable, sourceString);
}
getDeepValue(deepProperties, valueToPopulate) {
return BbPromise.reduce(deepProperties, (computedValueToPopulateParam, subProperty) => {
let computedValueToPopulate = computedValueToPopulateParam;
if (typeof computedValueToPopulate === 'undefined') {
if ( // in build deep variable mode
_.isString(computedValueToPopulate) &&
computedValueToPopulate.match(this.deepRefSyntax)
) {
if (subProperty !== '') {
computedValueToPopulate = `${
computedValueToPopulate.slice(0, computedValueToPopulate.length - 1)
}.${
subProperty
}}`;
}
return BbPromise.resolve(computedValueToPopulate);
} else if (typeof computedValueToPopulate === 'undefined') { // in get deep value mode
computedValueToPopulate = {};
} else if (subProperty !== '' || '' in computedValueToPopulate) {
computedValueToPopulate = computedValueToPopulate[subProperty];
}
if (typeof computedValueToPopulate === 'string' &&
computedValueToPopulate.match(this.variableSyntax)) {
return this.populateProperty(computedValueToPopulate, true);
if (
typeof computedValueToPopulate === 'string' &&
computedValueToPopulate.match(this.variableSyntax)
) {
const computedVariable = this.cleanVariable(computedValueToPopulate);
let index = this.deep.findIndex((item) => computedVariable === item);
if (index < 0) {
index = this.deep.push(computedVariable) - 1;
}
return BbPromise.resolve(`\${deep:${index}}`);
}
return BbPromise.resolve(computedValueToPopulate);
}, valueToPopulate);
@ -453,10 +723,10 @@ class Variables {
} else if (variableString.match(this.ssmRefSyntax)) {
varType = 'SSM parameter';
}
logWarning(
`A valid ${varType} to satisfy the declaration '${variableString}' could not be found.`
);
logWarning(`A valid ${varType} to satisfy the declaration '${
variableString}' could not be found.`);
}
return valueToPopulate;
}
}

File diff suppressed because it is too large Load Diff

View File

@ -330,13 +330,28 @@ class AwsProvider {
credentials.useAccelerateEndpoint = true; // eslint-disable-line no-param-reassign
}
getValues(source, paths) {
return paths.map(path => ({
path,
value: _.get(source, path.join('.')),
}));
}
firstValue(values) {
return values.reduce((result, current) => (result.value ? result : current), {});
}
getRegionSourceValue() {
const values = this.getValues(this, [
['options', 'region'],
['serverless', 'config', 'region'],
['serverless', 'service', 'provider', 'region'],
]);
return this.firstValue(values);
}
getRegion() {
const defaultRegion = 'us-east-1';
return _.get(this, 'options.region')
|| _.get(this, 'serverless.config.region')
|| _.get(this, 'serverless.service.provider.region')
|| defaultRegion;
const regionSourceValue = this.getRegionSourceValue();
return regionSourceValue.value || defaultRegion;
}
getServerlessDeploymentBucketName() {
@ -352,13 +367,18 @@ class AwsProvider {
).then((result) => result.StackResourceDetail.PhysicalResourceId);
}
getStageSourceValue() {
const values = this.getValues(this, [
['options', 'stage'],
['serverless', 'config', 'stage'],
['serverless', 'service', 'provider', 'stage'],
]);
return this.firstValue(values);
}
getStage() {
const defaultStage = 'dev';
return _.get(this, 'options.stage')
|| _.get(this, 'serverless.config.stage')
|| _.get(this, 'serverless.service.provider.stage')
|| defaultStage;
const stageSourceValue = this.getStageSourceValue();
return stageSourceValue.value || defaultStage;
}
getAccountId() {

View File

@ -72,118 +72,118 @@ describe('AwsProvider', () => {
// clear env
delete process.env.AWS_CLIENT_TIMEOUT;
});
});
describe('#constructor() certificate authority - environment variable', () => {
afterEach('Environment Variable Cleanup', () => {
// clear env
delete process.env.ca;
});
it('should set AWS ca single', () => {
process.env.ca = '-----BEGIN CERTIFICATE-----\n-----END CERTIFICATE-----';
const newAwsProvider = new AwsProvider(serverless, options);
describe('certificate authority - environment variable', () => {
afterEach('Environment Variable Cleanup', () => {
// clear env
delete process.env.ca;
});
it('should set AWS ca single', () => {
process.env.ca = '-----BEGIN CERTIFICATE-----\n-----END CERTIFICATE-----';
const newAwsProvider = new AwsProvider(serverless, options);
expect(typeof newAwsProvider.sdk.config.httpOptions.agent).to.not.equal('undefined');
expect(typeof newAwsProvider.sdk.config.httpOptions.agent).to.not.equal('undefined');
});
it('should set AWS ca multiple', () => {
const certContents = '-----BEGIN CERTIFICATE-----\n-----END CERTIFICATE-----';
process.env.ca = `${certContents},${certContents}`;
const newAwsProvider = new AwsProvider(serverless, options);
expect(typeof newAwsProvider.sdk.config.httpOptions.agent).to.not.equal('undefined');
});
});
it('should set AWS ca multiple', () => {
describe('certificate authority - file', () => {
const certContents = '-----BEGIN CERTIFICATE-----\n-----END CERTIFICATE-----';
process.env.ca = `${certContents},${certContents}`;
const newAwsProvider = new AwsProvider(serverless, options);
const tmpdir = os.tmpdir();
let file1 = null;
let file2 = null;
beforeEach('Create CA Files and env vars', () => {
file1 = path.join(tmpdir, 'ca1.txt');
file2 = path.join(tmpdir, 'ca2.txt');
fs.writeFileSync(file1, certContents);
fs.writeFileSync(file2, certContents);
});
expect(typeof newAwsProvider.sdk.config.httpOptions.agent).to.not.equal('undefined');
});
});
afterEach('CA File Cleanup', () => {
// delete files
fs.unlinkSync(file1);
fs.unlinkSync(file2);
// clear env
delete process.env.ca;
delete process.env.cafile;
});
describe('#constructor() certificate authority - file', () => {
const certContents = '-----BEGIN CERTIFICATE-----\n-----END CERTIFICATE-----';
const tmpdir = os.tmpdir();
let file1 = null;
let file2 = null;
beforeEach('Create CA Files and env vars', () => {
file1 = path.join(tmpdir, 'ca1.txt');
file2 = path.join(tmpdir, 'ca2.txt');
fs.writeFileSync(file1, certContents);
fs.writeFileSync(file2, certContents);
it('should set AWS cafile single', () => {
process.env.cafile = file1;
const newAwsProvider = new AwsProvider(serverless, options);
expect(typeof newAwsProvider.sdk.config.httpOptions.agent).to.not.equal('undefined');
});
it('should set AWS cafile multiple', () => {
process.env.cafile = `${file1},${file2}`;
const newAwsProvider = new AwsProvider(serverless, options);
expect(typeof newAwsProvider.sdk.config.httpOptions.agent).to.not.equal('undefined');
});
it('should set AWS ca and cafile', () => {
process.env.ca = certContents;
process.env.cafile = file1;
const newAwsProvider = new AwsProvider(serverless, options);
expect(typeof newAwsProvider.sdk.config.httpOptions.agent).to.not.equal('undefined');
});
});
afterEach('CA File Cleanup', () => {
// delete files
fs.unlinkSync(file1);
fs.unlinkSync(file2);
// clear env
delete process.env.ca;
delete process.env.cafile;
});
describe('deploymentBucket configuration', () => {
it('should do nothing if not defined', () => {
serverless.service.provider.deploymentBucket = undefined;
it('should set AWS cafile single', () => {
process.env.cafile = file1;
const newAwsProvider = new AwsProvider(serverless, options);
const newAwsProvider = new AwsProvider(serverless, options);
expect(typeof newAwsProvider.sdk.config.httpOptions.agent).to.not.equal('undefined');
});
expect(newAwsProvider.serverless.service.provider.deploymentBucket).to.equal(undefined);
});
it('should set AWS cafile multiple', () => {
process.env.cafile = `${file1},${file2}`;
const newAwsProvider = new AwsProvider(serverless, options);
it('should do nothing if the value is a string', () => {
serverless.service.provider.deploymentBucket = 'my.deployment.bucket';
expect(typeof newAwsProvider.sdk.config.httpOptions.agent).to.not.equal('undefined');
});
const newAwsProvider = new AwsProvider(serverless, options);
it('should set AWS ca and cafile', () => {
process.env.ca = certContents;
process.env.cafile = file1;
const newAwsProvider = new AwsProvider(serverless, options);
expect(newAwsProvider.serverless.service.provider.deploymentBucket)
.to.equal('my.deployment.bucket');
});
expect(typeof newAwsProvider.sdk.config.httpOptions.agent).to.not.equal('undefined');
});
});
it('should save a given object and use name from it', () => {
const deploymentBucketObject = {
name: 'my.deployment.bucket',
serverSideEncryption: 'AES256',
};
serverless.service.provider.deploymentBucket = deploymentBucketObject;
describe('when checking for the deploymentBucket config', () => {
it('should do nothing if the deploymentBucket config is not used', () => {
serverless.service.provider.deploymentBucket = undefined;
const newAwsProvider = new AwsProvider(serverless, options);
const newAwsProvider = new AwsProvider(serverless, options);
expect(newAwsProvider.serverless.service.provider.deploymentBucket)
.to.equal('my.deployment.bucket');
expect(newAwsProvider.serverless.service.provider.deploymentBucketObject)
.to.deep.equal(deploymentBucketObject);
});
expect(newAwsProvider.serverless.service.provider.deploymentBucket).to.equal(undefined);
});
it('should save a given object and nullify the name if one is not provided', () => {
const deploymentBucketObject = {
serverSideEncryption: 'AES256',
};
serverless.service.provider.deploymentBucket = deploymentBucketObject;
it('should do nothing if the deploymentBucket config is a string', () => {
serverless.service.provider.deploymentBucket = 'my.deployment.bucket';
const newAwsProvider = new AwsProvider(serverless, options);
const newAwsProvider = new AwsProvider(serverless, options);
expect(newAwsProvider.serverless.service.provider.deploymentBucket)
.to.equal('my.deployment.bucket');
});
it('should save the object and use the name for the deploymentBucket if provided', () => {
const deploymentBucketObject = {
name: 'my.deployment.bucket',
serverSideEncryption: 'AES256',
};
serverless.service.provider.deploymentBucket = deploymentBucketObject;
const newAwsProvider = new AwsProvider(serverless, options);
expect(newAwsProvider.serverless.service.provider.deploymentBucket)
.to.equal('my.deployment.bucket');
expect(newAwsProvider.serverless.service.provider.deploymentBucketObject)
.to.deep.equal(deploymentBucketObject);
});
it('should save the object and nullify the name if it is not provided', () => {
const deploymentBucketObject = {
serverSideEncryption: 'AES256',
};
serverless.service.provider.deploymentBucket = deploymentBucketObject;
const newAwsProvider = new AwsProvider(serverless, options);
expect(newAwsProvider.serverless.service.provider.deploymentBucket)
.to.equal(null);
expect(newAwsProvider.serverless.service.provider.deploymentBucketObject)
.to.deep.equal(deploymentBucketObject);
expect(newAwsProvider.serverless.service.provider.deploymentBucket)
.to.equal(null);
expect(newAwsProvider.serverless.service.provider.deploymentBucketObject)
.to.deep.equal(deploymentBucketObject);
});
});
});
@ -693,6 +693,69 @@ describe('AwsProvider', () => {
});
});
describe('values', () => {
const obj = {
a: 'b',
c: {
d: 'e',
f: {
g: 'h',
},
},
};
const paths = [
['a'],
['c', 'd'],
['c', 'f', 'g'],
];
const getExpected = [
{ path: paths[0], value: obj.a },
{ path: paths[1], value: obj.c.d },
{ path: paths[2], value: obj.c.f.g },
];
describe('#getValues', () => {
it('should return an array of values given paths to them', () => {
expect(awsProvider.getValues(obj, paths)).to.eql(getExpected);
});
});
describe('#firstValue', () => {
it('should ignore entries without a \'value\' attribute', () => {
const input = _.cloneDeep(getExpected);
delete input[0].value;
delete input[2].value;
expect(awsProvider.firstValue(input)).to.eql(getExpected[1]);
});
it('should ignore entries with an undefined \'value\' attribute', () => {
const input = _.cloneDeep(getExpected);
input[0].value = undefined;
input[2].value = undefined;
expect(awsProvider.firstValue(input)).to.eql(getExpected[1]);
});
it('should return the first value', () => {
expect(awsProvider.firstValue(getExpected)).to.equal(getExpected[0]);
});
it('should return the middle value', () => {
const input = _.cloneDeep(getExpected);
delete input[0].value;
delete input[2].value;
expect(awsProvider.firstValue(input)).to.equal(input[1]);
});
it('should return the last value', () => {
const input = _.cloneDeep(getExpected);
delete input[0].value;
delete input[1].value;
expect(awsProvider.firstValue(input)).to.equal(input[2]);
});
it('should return the last object if none have valid values', () => {
const input = _.cloneDeep(getExpected);
delete input[0].value;
delete input[1].value;
delete input[2].value;
expect(awsProvider.firstValue(input)).to.equal(input[2]);
});
});
});
describe('#getRegion()', () => {
let newAwsProvider;