diff --git a/CHANGELOG.md b/CHANGELOG.md index 73414341f..8d238b713 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,40 @@ +# 1.55.0 (2019-10-23) + +- [Allow empty arrays in overrides](https://github.com/serverless/serverless/pull/6813) +- [Make question mark available as variables fallback](https://github.com/serverless/serverless/pull/6808) +- [Improve plugins resolution and initialization flow](https://github.com/serverless/serverless/pull/6814) +- [Azure Python template](https://github.com/serverless/serverless/pull/6822) +- [Chore - stop using deprecated 'new Buffer()' method.](https://github.com/serverless/serverless/pull/6829) +- [AWS - adding naming function for S3 compiled template file name.](https://github.com/serverless/serverless/pull/6828) +- [Span docs! and full `serverless_sdk` docs](https://github.com/serverless/serverless/pull/6809) +- [Fix perms with several CloudWatch log subscriptions](https://github.com/serverless/serverless/pull/6827) +- [Fixing an Azure docs broken link](https://github.com/serverless/serverless/pull/6838) +- [Adding note to Azure nodejs template](https://github.com/serverless/serverless/pull/6839) +- [Updated Azure Functions documentation](https://github.com/serverless/serverless/pull/6840) +- [Support for NotAction and NotResource in IAM role statements](https://github.com/serverless/serverless/pull/6842) +- [added frontmatter to sdk docs](https://github.com/serverless/serverless/pull/6845) +- [Setup completion via CLI command and interactive CLI step](https://github.com/serverless/serverless/pull/6835) +- [Upgrade gradle version](https://github.com/serverless/serverless/pull/6855) +- [Update Google provider documentation for functions](https://github.com/serverless/serverless/pull/6854) +- [SNS integration tests](https://github.com/serverless/serverless/pull/6846) +- [SQS integration tests](https://github.com/serverless/serverless/pull/6847) +- [Streams integration tests](https://github.com/serverless/serverless/pull/6848) +- [Improvements on SQS docs as suggested on #6516](https://github.com/serverless/serverless/pull/6853) +- [Schedule integration tests](https://github.com/serverless/serverless/pull/6851) +- [Update event documentation](https://github.com/serverless/serverless/pull/6857) +- [Upgrade groovy/gradle/plugin versions and dependencies (aws-groovy-gradle)](https://github.com/serverless/serverless/pull/6862) +- [Upgrade gradle/plugins version and dependencies (aws-clojure-gradle)](https://github.com/serverless/serverless/pull/6861) +- [IoT integration tests](https://github.com/serverless/serverless/pull/6837) +- [Update https-proxy-agent dependency](https://github.com/serverless/serverless/pull/6866) +- [Allow to use Ref in stream arn property](https://github.com/serverless/serverless/pull/6856) +- [Add Tests for resolveFilePathsFromPatterns()](https://github.com/serverless/serverless/pull/6825) +- [Integration tests improvements and fixes](https://github.com/serverless/serverless/pull/6867) +- [Honor cfnRole in custom resources](https://github.com/serverless/serverless/pull/6871) + +## Meta + +- [Comparison since last release](https://github.com/serverless/serverless/compare/v1.54.0...v1.55.0) + # 1.54.0 (2019-10-09) - [Fixing typos in variable names](https://github.com/serverless/serverless/pull/6746) diff --git a/docs/providers/aws/events/streams.md b/docs/providers/aws/events/streams.md index 1c06e3f48..a37c659b3 100644 --- a/docs/providers/aws/events/streams.md +++ b/docs/providers/aws/events/streams.md @@ -47,6 +47,10 @@ functions: type: kinesis arn: Fn::ImportValue: MyExportedKinesisStreamArnId + - stream: + type: dynamodb + arn: + Ref: MyDynamoDbTableStreamArn - stream: type: kinesis arn: diff --git a/lib/classes/CLI.test.js b/lib/classes/CLI.test.js index 2bfa8bd84..0527f296a 100644 --- a/lib/classes/CLI.test.js +++ b/lib/classes/CLI.test.js @@ -1,14 +1,11 @@ 'use strict'; -/* eslint-disable no-unused-expressions */ - const chai = require('chai'); const sinon = require('sinon'); const CLI = require('../../lib/classes/CLI'); -const os = require('os'); const fse = require('fs-extra'); -const exec = require('child_process').exec; -const path = require('path'); +const spawn = require('child-process-ext/spawn'); +const resolveAwsEnv = require('@serverless/test/resolve-env'); const stripAnsi = require('strip-ansi'); const Serverless = require('../../lib/Serverless'); const { getTmpDirPath } = require('../../tests/utils/fs'); @@ -624,8 +621,10 @@ describe('CLI', () => { }); describe('Integration tests', function() { - this.timeout(0); + this.timeout(1000 * 60 * 10); const that = this; + const serverlessExec = require('../../tests/serverless-binary'); + const env = resolveAwsEnv(); before(() => { const tmpDir = getTmpDirPath(); @@ -634,62 +633,27 @@ describe('CLI', () => { fse.mkdirsSync(tmpDir); process.chdir(tmpDir); - - serverless = new Serverless(); - return serverless.init().then(() => { - // Cannot rely on shebang in severless.js to invoke script using NodeJS on Windows. - const execPrefix = os.platform() === 'win32' ? 'node ' : ''; - - that.serverlessExec = - execPrefix + path.join(serverless.config.serverlessPath, '..', 'bin', 'serverless'); - }); }); after(() => { process.chdir(that.cwd); }); - it('should print general --help to stdout', done => { - exec(`${this.serverlessExec} --help`, (err, stdout, stderr) => { - if (err) { - process.stdout.write(stdout); - process.stderr.write(stderr); - done(err); - return; - } - - expect(stdout).to.contain('contextual help'); - done(); - }); - }); - - it('should print command --help to stdout', done => { - exec(`${this.serverlessExec} deploy --help`, (err, stdout, stderr) => { - if (err) { - process.stdout.write(stdout); - process.stderr.write(stderr); - done(err); - return; - } + it('should print general --help to stdout', () => + spawn(serverlessExec, ['--help'], { env }).then(({ stdoutBuffer }) => + expect(String(stdoutBuffer)).to.contain('contextual help') + )); + it('should print command --help to stdout', () => + spawn(serverlessExec, ['deploy', '--help'], { env }).then(({ stdoutBuffer }) => { + const stdout = String(stdoutBuffer); expect(stdout).to.contain('deploy'); expect(stdout).to.contain('--stage'); - done(); - }); - }); + })); - it('should print help --verbose to stdout', done => { - exec(`${this.serverlessExec} help --verbose`, (err, stdout, stderr) => { - if (err) { - process.stdout.write(stdout); - process.stderr.write(stderr); - done(err); - return; - } - - expect(stdout).to.contain('Commands by plugin'); - done(); - }); - }); + it('should print help --verbose to stdout', () => + spawn(serverlessExec, ['help', '--verbose'], { env }).then(({ stdoutBuffer }) => + expect(String(stdoutBuffer)).to.contain('Commands by plugin') + )); }); }); diff --git a/lib/classes/PluginManager.test.js b/lib/classes/PluginManager.test.js index 9f04a22ad..5c2a2377c 100644 --- a/lib/classes/PluginManager.test.js +++ b/lib/classes/PluginManager.test.js @@ -5,6 +5,8 @@ const chai = require('chai'); const overrideEnv = require('process-utils/override-env'); const cjsResolve = require('ncjsm/resolve/sync'); +const spawn = require('child-process-ext/spawn'); +const resolveAwsEnv = require('@serverless/test/resolve-env'); const Serverless = require('../../lib/Serverless'); const CLI = require('../../lib/classes/CLI'); const Create = require('../../lib/plugins/create/create'); @@ -14,12 +16,10 @@ const path = require('path'); const fs = require('fs'); const fse = require('fs-extra'); const mockRequire = require('mock-require'); -const os = require('os'); const sinon = require('sinon'); const proxyquire = require('proxyquire'); const BbPromise = require('bluebird'); const getCacheFilePath = require('../utils/getCacheFilePath'); -const { execSync } = require('child_process'); const { installPlugin } = require('../../tests/utils/plugins'); const { getTmpDirPath } = require('../../tests/utils/fs'); @@ -29,6 +29,7 @@ chai.use(require('sinon-chai')); const expect = chai.expect; describe('PluginManager', () => { + const env = resolveAwsEnv(); let pluginManager; let serverless; @@ -2005,35 +2006,25 @@ describe('PluginManager', () => { }); describe('Plugin / CLI integration', function() { - this.timeout(0); + this.timeout(1000 * 60 * 10); - const cwd = process.cwd(); let serverlessInstance; let serviceDir; - let serverlessExec; + const serverlessExec = require('../../tests/serverless-binary'); beforeEach(() => { // eslint-disable-line prefer-arrow-callback serverlessInstance = new Serverless(); return serverlessInstance.init().then(() => { // Cannot rely on shebang in severless.js to invoke script using NodeJS on Windows. - const execPrefix = os.platform() === 'win32' ? 'node ' : ''; - serverlessExec = - execPrefix + - path.join(serverlessInstance.config.serverlessPath, '..', 'bin', 'serverless'); const tmpDir = getTmpDirPath(); serviceDir = path.join(tmpDir, 'service'); fse.mkdirsSync(serviceDir); - process.chdir(serviceDir); - try { - execSync(`${serverlessExec} create --template aws-nodejs`); - } catch (error) { - // Expose process output in case of crash - process.stdout.write(error.stdout); - process.stderr.write(error.stderr); - throw error; - } + return spawn(serverlessExec, ['create', '--template', 'aws-nodejs'], { + env, + cwd: serviceDir, + }); }); }); @@ -2057,22 +2048,14 @@ describe('PluginManager', () => { 'plugins:\n - local-plugin\n - parent-plugin' ); - let output; - try { - output = execSync(serverlessExec); - } catch (error) { - process.stdout.write(error.stdout); - process.stdout.write(error.stderr); - throw error; - } - const stringifiedOutput = Buffer.from(output, 'base64').toString(); - expect(stringifiedOutput).to.contain('SynchronousPluginMock'); - expect(stringifiedOutput).to.contain('PromisePluginMock'); + return spawn(serverlessExec, [], { env, cwd: serviceDir }).then(({ stdoutBuffer }) => { + const stringifiedOutput = String(stdoutBuffer); + expect(stringifiedOutput).to.contain('SynchronousPluginMock'); + expect(stringifiedOutput).to.contain('PromisePluginMock'); + }); }); afterEach(() => { - // eslint-disable-line prefer-arrow-callback - process.chdir(cwd); try { fse.removeSync(serviceDir); } catch (e) { diff --git a/lib/plugins/aws/customResources/index.js b/lib/plugins/aws/customResources/index.js index b3b9b4f99..3b84d4663 100644 --- a/lib/plugins/aws/customResources/index.js +++ b/lib/plugins/aws/customResources/index.js @@ -86,78 +86,83 @@ function addCustomResourceToService(awsProvider, resourceName, iamRoleStatements const s3FileName = outputFilePath.split(path.sep).pop(); const S3Key = `${s3Folder}/${s3FileName}`; - let customResourceRole = Resources[customResourcesRoleLogicalId]; - if (!customResourceRole) { - customResourceRole = { - Type: 'AWS::IAM::Role', - Properties: { - AssumeRolePolicyDocument: { - Version: '2012-10-17', - Statement: [ - { - Effect: 'Allow', - Principal: { - Service: ['lambda.amazonaws.com'], - }, - Action: ['sts:AssumeRole'], - }, - ], - }, - Policies: [ - { - PolicyName: { - 'Fn::Join': [ - '-', - [ - awsProvider.getStage(), - awsProvider.serverless.service.service, - 'custom-resources-lambda', - ], - ], - }, - PolicyDocument: { - Version: '2012-10-17', - Statement: [], - }, - }, - ], - }, - }; + const cfnRoleArn = serverless.service.provider.cfnRole; - if (shouldWriteLogs) { - const logGroupsPrefix = awsProvider.naming.getLogGroupName(funcPrefix); - customResourceRole.Properties.Policies[0].PolicyDocument.Statement.push( - { - Effect: 'Allow', - Action: ['logs:CreateLogStream'], - Resource: [ + if (!cfnRoleArn) { + let customResourceRole = Resources[customResourcesRoleLogicalId]; + if (!customResourceRole) { + customResourceRole = { + Type: 'AWS::IAM::Role', + Properties: { + AssumeRolePolicyDocument: { + Version: '2012-10-17', + Statement: [ + { + Effect: 'Allow', + Principal: { + Service: ['lambda.amazonaws.com'], + }, + Action: ['sts:AssumeRole'], + }, + ], + }, + Policies: [ { - 'Fn::Sub': - 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}' + - `:log-group:${logGroupsPrefix}*:*`, + PolicyName: { + 'Fn::Join': [ + '-', + [ + awsProvider.getStage(), + awsProvider.serverless.service.service, + 'custom-resources-lambda', + ], + ], + }, + PolicyDocument: { + Version: '2012-10-17', + Statement: [], + }, }, ], }, - { - Effect: 'Allow', - Action: ['logs:PutLogEvents'], - Resource: [ - { - 'Fn::Sub': - 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}' + - `:log-group:${logGroupsPrefix}*:*:*`, - }, - ], - } - ); + }; + Resources[customResourcesRoleLogicalId] = customResourceRole; + + if (shouldWriteLogs) { + const logGroupsPrefix = awsProvider.naming.getLogGroupName(funcPrefix); + customResourceRole.Properties.Policies[0].PolicyDocument.Statement.push( + { + Effect: 'Allow', + Action: ['logs:CreateLogStream'], + Resource: [ + { + 'Fn::Sub': + 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}' + + `:log-group:${logGroupsPrefix}*:*`, + }, + ], + }, + { + Effect: 'Allow', + Action: ['logs:PutLogEvents'], + Resource: [ + { + 'Fn::Sub': + 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}' + + `:log-group:${logGroupsPrefix}*:*:*`, + }, + ], + } + ); + } } + const { Statement } = customResourceRole.Properties.Policies[0].PolicyDocument; + iamRoleStatements.forEach(newStmt => { + if (!Statement.find(existingStmt => existingStmt.Resource === newStmt.Resource)) { + Statement.push(newStmt); + } + }); } - const { Statement } = customResourceRole.Properties.Policies[0].PolicyDocument; - iamRoleStatements.forEach(newStmt => { - if (!Statement.find(existingStmt => existingStmt.Resource === newStmt.Resource)) { - Statement.push(newStmt); - } - }); const customResourceFunction = { Type: 'AWS::Lambda::Function', @@ -169,19 +174,21 @@ function addCustomResourceToService(awsProvider, resourceName, iamRoleStatements FunctionName: absoluteFunctionName, Handler, MemorySize: 1024, - Role: { - 'Fn::GetAtt': [customResourcesRoleLogicalId, 'Arn'], - }, Runtime: 'nodejs10.x', Timeout: 180, }, - DependsOn: [customResourcesRoleLogicalId], + DependsOn: [], }; + Resources[customResourceFunctionLogicalId] = customResourceFunction; - Object.assign(Resources, { - [customResourceFunctionLogicalId]: customResourceFunction, - [customResourcesRoleLogicalId]: customResourceRole, - }); + if (cfnRoleArn) { + customResourceFunction.Properties.Role = cfnRoleArn; + } else { + customResourceFunction.Properties.Role = { + 'Fn::GetAtt': [customResourcesRoleLogicalId, 'Arn'], + }; + customResourceFunction.DependsOn.push(customResourcesRoleLogicalId); + } if (shouldWriteLogs) { const customResourceLogGroupLogicalId = awsProvider.naming.getLogGroupLogicalId( diff --git a/lib/plugins/aws/customResources/index.test.js b/lib/plugins/aws/customResources/index.test.js index ceb5a328d..639f25cd8 100644 --- a/lib/plugins/aws/customResources/index.test.js +++ b/lib/plugins/aws/customResources/index.test.js @@ -221,6 +221,119 @@ describe('#addCustomResourceToService()', () => { }); }); + it('Should not setup new IAM role, when cfnRole is provided', () => { + const cfnRoleArn = (serverless.service.provider.cfnRole = + 'arn:aws:iam::999999999999:role/some-role'); + return expect( + BbPromise.all([ + // add the custom S3 resource + addCustomResourceToService(provider, 's3', [ + ...iamRoleStatements, + { + Effect: 'Allow', + Resource: 'arn:aws:s3:::some-bucket', + Action: ['s3:PutBucketNotification', 's3:GetBucketNotification'], + }, + ]), + // add the custom Cognito User Pool resource + addCustomResourceToService(provider, 'cognitoUserPool', [ + ...iamRoleStatements, + { + Effect: 'Allow', + Resource: '*', + Action: [ + 'cognito-idp:ListUserPools', + 'cognito-idp:DescribeUserPool', + 'cognito-idp:UpdateUserPool', + ], + }, + ]), + // add the custom Event Bridge resource + addCustomResourceToService(provider, 'eventBridge', [ + ...iamRoleStatements, + { + Effect: 'Allow', + Resource: 'arn:aws:events:*:*:rule/some-rule', + Action: [ + 'events:PutRule', + 'events:RemoveTargets', + 'events:PutTargets', + 'events:DeleteRule', + ], + }, + { + Action: ['events:CreateEventBus', 'events:DeleteEventBus'], + Effect: 'Allow', + Resource: 'arn:aws:events:*:*:event-bus/some-event-bus', + }, + ]), + ]) + ).to.be.fulfilled.then(() => { + const { Resources } = serverless.service.provider.compiledCloudFormationTemplate; + const customResourcesZipFilePath = path.join( + tmpDirPath, + '.serverless', + 'custom-resources.zip' + ); + + expect(execAsyncStub).to.have.callCount(3); + expect(fs.existsSync(customResourcesZipFilePath)).to.equal(true); + // S3 Lambda Function + expect(Resources.CustomDashresourceDashexistingDashs3LambdaFunction).to.deep.equal({ + Type: 'AWS::Lambda::Function', + Properties: { + Code: { + S3Bucket: { Ref: 'ServerlessDeploymentBucket' }, + S3Key: 'artifact-dir-name/custom-resources.zip', + }, + FunctionName: `${serviceName}-dev-custom-resource-existing-s3`, + Handler: 's3/handler.handler', + MemorySize: 1024, + Role: cfnRoleArn, + Runtime: 'nodejs10.x', + Timeout: 180, + }, + DependsOn: [], + }); + // Cognito User Pool Lambda Function + expect(Resources.CustomDashresourceDashexistingDashcupLambdaFunction).to.deep.equal({ + Type: 'AWS::Lambda::Function', + Properties: { + Code: { + S3Bucket: { Ref: 'ServerlessDeploymentBucket' }, + S3Key: 'artifact-dir-name/custom-resources.zip', + }, + FunctionName: `${serviceName}-dev-custom-resource-existing-cup`, + Handler: 'cognitoUserPool/handler.handler', + MemorySize: 1024, + Role: cfnRoleArn, + Runtime: 'nodejs10.x', + Timeout: 180, + }, + DependsOn: [], + }); + // Event Bridge Lambda Function + expect(Resources.CustomDashresourceDasheventDashbridgeLambdaFunction).to.deep.equal({ + Type: 'AWS::Lambda::Function', + Properties: { + Code: { + S3Bucket: { Ref: 'ServerlessDeploymentBucket' }, + S3Key: 'artifact-dir-name/custom-resources.zip', + }, + FunctionName: `${serviceName}-dev-custom-resource-event-bridge`, + Handler: 'eventBridge/handler.handler', + MemorySize: 1024, + Role: cfnRoleArn, + Runtime: 'nodejs10.x', + Timeout: 180, + }, + DependsOn: [], + }); + // Iam Role + expect(Resources.IamRoleCustomResourcesLambdaExecution).to.be.undefined; + }); + }); + it('should setup CloudWatch Logs when logs.frameworkLambda is true', () => { serverless.service.provider.logs = { frameworkLambda: true }; return BbPromise.all([ diff --git a/lib/plugins/aws/package/compile/events/stream/index.js b/lib/plugins/aws/package/compile/events/stream/index.js index b977b1dda..8ef01ecf7 100644 --- a/lib/plugins/aws/package/compile/events/stream/index.js +++ b/lib/plugins/aws/package/compile/events/stream/index.js @@ -72,13 +72,15 @@ class AwsCompileStreamEvents { !( _.has(event.stream.arn, 'Fn::ImportValue') || _.has(event.stream.arn, 'Fn::GetAtt') || + (_.has(event.stream.arn, 'Ref') && + _.has(this.serverless.service.resources.Parameters, event.stream.arn.Ref)) || _.has(event.stream.arn, 'Fn::Join') ) ) { const errorMessage = [ `Bad dynamic ARN property on stream event in function "${functionName}"`, - ' If you use a dynamic "arn" (such as with Fn::GetAtt, Fn::Join', - ' or Fn::ImportValue) there must only be one key (either Fn::GetAtt, Fn::Join', + ' If you use a dynamic "arn" (such as with Fn::GetAtt, Fn::Join, Ref', + ' or Fn::ImportValue) there must only be one key (either Fn::GetAtt, Fn::Join, Ref', ' or Fn::ImportValue) in the arn object. Please check the docs for more info.', ].join(''); throw new this.serverless.classes.Error(errorMessage); @@ -108,6 +110,8 @@ class AwsCompileStreamEvents { return EventSourceArn['Fn::GetAtt'][0]; } else if (EventSourceArn['Fn::ImportValue']) { return EventSourceArn['Fn::ImportValue']; + } else if (EventSourceArn.Ref) { + return EventSourceArn.Ref; } else if (EventSourceArn['Fn::Join']) { // [0] is the used delimiter, [1] is the array with values const name = EventSourceArn['Fn::Join'][1].slice(-1).pop(); @@ -147,9 +151,9 @@ class AwsCompileStreamEvents { ) { dependsOn = `"${funcRole['Fn::GetAtt'][0]}"`; } else if ( - // otherwise, check if we have an import + // otherwise, check if we have an import or parameters ref typeof funcRole === 'object' && - 'Fn::ImportValue' in funcRole + ('Fn::ImportValue' in funcRole || 'Ref' in funcRole) ) { dependsOn = '[]'; } else if (typeof funcRole === 'string') { diff --git a/lib/plugins/aws/package/compile/events/stream/index.test.js b/lib/plugins/aws/package/compile/events/stream/index.test.js index f4feddb64..7701870c1 100644 --- a/lib/plugins/aws/package/compile/events/stream/index.test.js +++ b/lib/plugins/aws/package/compile/events/stream/index.test.js @@ -234,6 +234,35 @@ describe('AwsCompileStreamEvents', () => { ).to.equal(null); }); + it('should not throw error if IAM role is referenced from cloudformation parameters', () => { + awsCompileStreamEvents.serverless.service.functions = { + first: { + role: { Ref: 'MyStreamRoleArn' }, + events: [ + { + // doesn't matter if DynamoDB or Kinesis stream + stream: 'arn:aws:dynamodb:region:account:table/foo/stream/1', + }, + ], + }, + }; + + // pretend that the default IamRoleLambdaExecution is not in place + awsCompileStreamEvents.serverless.service.provider.compiledCloudFormationTemplate.Resources.IamRoleLambdaExecution = null; + + expect(() => { + awsCompileStreamEvents.compileStreamEvents(); + }).to.not.throw(Error); + expect( + awsCompileStreamEvents.serverless.service.provider.compiledCloudFormationTemplate.Resources + .FirstEventSourceMappingDynamodbFoo.DependsOn.length + ).to.equal(0); + expect( + awsCompileStreamEvents.serverless.service.provider.compiledCloudFormationTemplate.Resources + .IamRoleLambdaExecution + ).to.equal(null); + }); + it('should not throw error if IAM role is imported', () => { awsCompileStreamEvents.serverless.service.functions = { first: { @@ -444,6 +473,14 @@ describe('AwsCompileStreamEvents', () => { }); it('should allow specifying DynamoDB and Kinesis streams as CFN reference types', () => { + awsCompileStreamEvents.serverless.service.resources.Parameters = { + SomeDdbTableStreamArn: { + Type: 'String', + }, + ForeignKinesisStreamArn: { + Type: 'String', + }, + }; awsCompileStreamEvents.serverless.service.functions = { first: { events: [ @@ -481,6 +518,18 @@ describe('AwsCompileStreamEvents', () => { type: 'kinesis', }, }, + { + stream: { + arn: { Ref: 'SomeDdbTableStreamArn' }, + type: 'dynamodb', + }, + }, + { + stream: { + arn: { Ref: 'ForeignKinesisStreamArn' }, + type: 'kinesis', + }, + }, ], }, }; @@ -507,6 +556,9 @@ describe('AwsCompileStreamEvents', () => { { 'Fn::GetAtt': ['SomeDdbTable', 'StreamArn'], }, + { + Ref: 'SomeDdbTableStreamArn', + }, ], }); @@ -537,6 +589,60 @@ describe('AwsCompileStreamEvents', () => { ], ], }); + + expect( + awsCompileStreamEvents.serverless.service.provider.compiledCloudFormationTemplate + .Resources.IamRoleLambdaExecution.Properties.Policies[0].PolicyDocument.Statement[1] + ).to.deep.equal({ + Effect: 'Allow', + Action: [ + 'kinesis:GetRecords', + 'kinesis:GetShardIterator', + 'kinesis:DescribeStream', + 'kinesis:ListStreams', + ], + Resource: [ + { + 'Fn::ImportValue': 'ForeignKinesis', + }, + { + 'Fn::Join': [ + ':', + [ + 'arn', + 'aws', + 'kinesis', + { + Ref: 'AWS::Region', + }, + { + Ref: 'AWS::AccountId', + }, + 'stream/MyStream', + ], + ], + }, + { + Ref: 'ForeignKinesisStreamArn', + }, + ], + }); + }); + + it('fails if Ref/dynamic stream ARN is used without defining it to the CF parameters', () => { + awsCompileStreamEvents.serverless.service.functions = { + first: { + events: [ + { + stream: { + arn: { Ref: 'SomeDdbTableStreamArn' }, + }, + }, + ], + }, + }; + + expect(() => awsCompileStreamEvents.compileStreamEvents()).to.throw(Error); }); it('fails if Fn::GetAtt/dynamic stream ARN is used without a type', () => { diff --git a/lib/plugins/package/lib/packageService.test.js b/lib/plugins/package/lib/packageService.test.js index eb47cfc0b..9c591075e 100644 --- a/lib/plugins/package/lib/packageService.test.js +++ b/lib/plugins/package/lib/packageService.test.js @@ -3,11 +3,13 @@ const _ = require('lodash'); const BbPromise = require('bluebird'); const path = require('path'); +const fse = require('fs-extra'); const chai = require('chai'); const sinon = require('sinon'); const Package = require('../package'); const Serverless = require('../../../Serverless'); const serverlessConfigFileUtils = require('../../../../lib/utils/getServerlessConfigFile'); +const { createTmpDir } = require('../../../../tests/utils/fs'); // Configure chai chai.use(require('chai-as-promised')); @@ -604,4 +606,55 @@ describe('#packageService()', () => { ); }); }); + + describe('#resolveFilePathsFromPatterns()', () => { + // NOTE: the path.join in `beforeEach` will take care of OS + // independent file paths + const handlerFile = 'src/function/handler.js'; + const utilsFile = 'src/utils/utils.js'; + let servicePath; + + beforeEach(() => { + servicePath = createTmpDir(); + fse.ensureFileSync(path.join(servicePath, handlerFile)); + fse.ensureFileSync(path.join(servicePath, utilsFile)); + }); + + it('should exclude all and include function/handler.js', () => { + const params = { + exclude: ['**'], + include: [handlerFile], + }; + serverless.config.servicePath = servicePath; + + return expect(packagePlugin.resolveFilePathsFromPatterns(params)).to.be.fulfilled.then( + actual => expect(actual).to.deep.equal([handlerFile]) + ); + }); + + it('should include file specified with `!` in exclude params', () => { + const params = { + exclude: ['**', `!${utilsFile}`], + include: [handlerFile], + }; + serverless.config.servicePath = servicePath; + + return expect(packagePlugin.resolveFilePathsFromPatterns(params)).to.be.fulfilled.then( + actual => expect(actual).to.deep.equal([handlerFile, utilsFile]) + ); + }); + + it('should exclude file specified with `!` in include params', () => { + const params = { + exclude: [], + include: [`!${utilsFile}`], + }; + const expected = [handlerFile]; + serverless.config.servicePath = servicePath; + + return expect(packagePlugin.resolveFilePathsFromPatterns(params)).to.be.fulfilled.then( + actual => expect(actual).to.deep.equal(expected) + ); + }); + }); }); diff --git a/lib/utils/isTrackingDisabled.js b/lib/utils/isTrackingDisabled.js index 8095d3011..a0262443e 100644 --- a/lib/utils/isTrackingDisabled.js +++ b/lib/utils/isTrackingDisabled.js @@ -2,4 +2,4 @@ const configUtils = require('./config'); -module.exports = configUtils.get('trackingDisabled'); +module.exports = Boolean(process.env.SLS_TRACKING_DISABLED || configUtils.get('trackingDisabled')); diff --git a/lib/utils/tabCompletion/isSupported.test.js b/lib/utils/tabCompletion/isSupported.test.js index 6516f28dc..d7bc3f2ea 100644 --- a/lib/utils/tabCompletion/isSupported.test.js +++ b/lib/utils/tabCompletion/isSupported.test.js @@ -2,8 +2,8 @@ const { expect } = require('chai'); -const isTabtabCompletionSuported = require('../isTrackingDisabled'); +const isSuported = require('./isSupported'); describe('isTabtabCompletionSuported', () => { - it('Should resolve boolean', () => expect(typeof isTabtabCompletionSuported).to.equal('boolean')); + it('Should resolve boolean', () => expect(typeof isSuported).to.equal('boolean')); }); diff --git a/package.json b/package.json index 813cb8516..a23069b4b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "serverless", - "version": "1.54.0", + "version": "1.55.0", "engines": { "node": ">=6.0" }, @@ -73,11 +73,13 @@ "root": true }, "eslintIgnore": [ - "lib/plugins/create/templates/**" + "lib/plugins/create/templates/**", + "lib/plugins/aws/customResources/node_modules/**" ], "mocha": { "reporter": "./tests/mocha-reporter", "require": [ + "@serverless/test/setup/log", "@serverless/test/setup/async-leaks-detector", "@serverless/test/setup/async-leaks-detector/bluebird-patch", "@serverless/test/setup/mock-homedir", @@ -87,34 +89,35 @@ }, "devDependencies": { "@serverless/eslint-config": "^1.2.0", - "@serverless/test": "^2.1.0", + "@serverless/test": "^2.4.0", "chai": "^4.2.0", "chai-as-promised": "^7.1.1", "child-process-ext": "^2.1.0", "cli-progress-footer": "^1.1.1", - "coveralls": "^3.0.6", + "coveralls": "^3.0.7", "eslint": "^6.5.1", "eslint-plugin-import": "^2.18.2", "git-list-updated": "^1.2.1", - "mocha": "^6.2.1", + "log": "^6.0.0", + "mocha": "^6.2.2", "mocha-lcov-reporter": "^1.3.0", "mock-require": "^3.0.3", "nyc": "^14.1.1", "prettier": "^1.18.2", - "process-utils": "^2.5.0", + "process-utils": "^3.0.1", "proxyquire": "^2.1.3", "sinon": "^7.5.0", "sinon-chai": "^3.3.0", "strip-ansi": "^5.2.0", - "ws": "^7.1.2" + "ws": "^7.2.0" }, "dependencies": { - "@serverless/cli": "^1.2.3", + "@serverless/cli": "^1.4.0", "@serverless/enterprise-plugin": "^3.1.2", "archiver": "^1.3.0", "async": "^1.5.2", - "aws-sdk": "^2.545.0", - "bluebird": "^3.7.0", + "aws-sdk": "^2.554.0", + "bluebird": "^3.7.1", "cachedir": "^2.2.0", "chalk": "^2.4.2", "ci-info": "^1.6.0", diff --git a/tests/.eslintrc.js b/tests/.eslintrc.js index 9938cda39..d26dc4ead 100644 --- a/tests/.eslintrc.js +++ b/tests/.eslintrc.js @@ -6,4 +6,18 @@ module.exports = { // console.info allowed to report on long going tasks or valuable debug information 'no-console': ['error', { allow: ['info'] }], }, + overrides: [ + { + files: ['utils/**.js'], + parserOptions: { + ecmaVersion: 2015, + }, + }, + { + files: ['utils/aws-cleanup.js', 'utils/integration.js'], + parserOptions: { + ecmaVersion: 2017, + }, + }, + ], }; diff --git a/tests/integration-all/api-gateway/tests.js b/tests/integration-all/api-gateway/tests.js index 79e16db46..c51982321 100644 --- a/tests/integration-all/api-gateway/tests.js +++ b/tests/integration-all/api-gateway/tests.js @@ -3,17 +3,16 @@ const path = require('path'); const AWS = require('aws-sdk'); const _ = require('lodash'); -const fetch = require('node-fetch'); const { expect } = require('chai'); const { getTmpDirPath, readYamlFile, writeYamlFile } = require('../../utils/fs'); +const { region, confirmCloudWatchLogs } = require('../../utils/misc'); const { - region, - confirmCloudWatchLogs, createTestService, deployService, removeService, -} = require('../../utils/misc'); + fetch, +} = require('../../utils/integration'); const { createRestApi, deleteRestApi, getResources } = require('../../utils/api-gateway'); const CF = new AWS.CloudFormation({ region }); @@ -30,11 +29,11 @@ describe('AWS - API Gateway Integration Test', function() { let apiKey; const stage = 'dev'; - before(() => { + before(async () => { tmpDirPath = getTmpDirPath(); console.info(`Temporary path: ${tmpDirPath}`); serverlessFilePath = path.join(tmpDirPath, 'serverless.yml'); - const serverlessConfig = createTestService(tmpDirPath, { + const serverlessConfig = await createTestService(tmpDirPath, { templateDir: path.join(__dirname, 'service'), serverlessConfigHook: // Ensure unique API key for each test (to avoid collision among concurrent CI runs) @@ -46,36 +45,12 @@ describe('AWS - API Gateway Integration Test', function() { serviceName = serverlessConfig.service; stackName = `${serviceName}-${stage}`; console.info(`Deploying "${stackName}" service...`); - deployService(tmpDirPath); - // create an external REST API - const externalRestApiName = `${stage}-${serviceName}-ext-api`; - return createRestApi(externalRestApiName) - .then(restApiMeta => { - restApiId = restApiMeta.id; - return getResources(restApiId); - }) - .then(resources => { - restApiRootResourceId = resources[0].id; - console.info( - 'Created external rest API ' + - `(id: ${restApiId}, root resource id: ${restApiRootResourceId})` - ); - }); + await deployService(tmpDirPath); }); - after(() => { - // NOTE: deleting the references to the old, external REST API - const serverless = readYamlFile(serverlessFilePath); - delete serverless.provider.apiGateway.restApiId; - delete serverless.provider.apiGateway.restApiRootResourceId; - writeYamlFile(serverlessFilePath, serverless); - // NOTE: deploying once again to get the stack into the original state - console.info('Redeploying service...'); - deployService(tmpDirPath); + after(async () => { console.info('Removing service...'); - removeService(tmpDirPath); - console.info('Deleting external rest API...'); - return deleteRestApi(restApiId); + await removeService(tmpDirPath); }); beforeEach(() => { @@ -224,7 +199,7 @@ describe('AWS - API Gateway Integration Test', function() { }); describe('Using stage specific configuration', () => { - before(() => { + before(async () => { const serverless = readYamlFile(serverlessFilePath); // enable Logs, Tags and Tracing _.merge(serverless.provider, { @@ -240,7 +215,7 @@ describe('AWS - API Gateway Integration Test', function() { }, }); writeYamlFile(serverlessFilePath, serverless); - deployService(tmpDirPath); + await deployService(tmpDirPath); }); it('should update the stage without service interruptions', () => { @@ -261,7 +236,22 @@ describe('AWS - API Gateway Integration Test', function() { // NOTE: this test should be at the very end because we're using an external REST API here describe('when using an existing REST API with stage specific configuration', () => { - before(() => { + before(async () => { + // create an external REST API + const externalRestApiName = `${stage}-${serviceName}-ext-api`; + await createRestApi(externalRestApiName) + .then(restApiMeta => { + restApiId = restApiMeta.id; + return getResources(restApiId); + }) + .then(resources => { + restApiRootResourceId = resources[0].id; + console.info( + 'Created external rest API ' + + `(id: ${restApiId}, root resource id: ${restApiRootResourceId})` + ); + }); + const serverless = readYamlFile(serverlessFilePath); // enable Logs, Tags and Tracing _.merge(serverless.provider, { @@ -281,7 +271,21 @@ describe('AWS - API Gateway Integration Test', function() { }, }); writeYamlFile(serverlessFilePath, serverless); - deployService(tmpDirPath); + console.info('Redeploying service (with external Rest API ID)...'); + await deployService(tmpDirPath); + }); + + after(async () => { + // NOTE: deleting the references to the old, external REST API + const serverless = readYamlFile(serverlessFilePath); + delete serverless.provider.apiGateway.restApiId; + delete serverless.provider.apiGateway.restApiRootResourceId; + writeYamlFile(serverlessFilePath, serverless); + // NOTE: deploying once again to get the stack into the original state + console.info('Redeploying service (without external Rest API ID)...'); + await deployService(tmpDirPath); + console.info('Deleting external rest API...'); + return deleteRestApi(restApiId); }); it('should update the stage without service interruptions', () => { diff --git a/tests/integration-all/cognito-user-pool/tests.js b/tests/integration-all/cognito-user-pool/tests.js index 28b1c5215..e431fd98b 100644 --- a/tests/integration-all/cognito-user-pool/tests.js +++ b/tests/integration-all/cognito-user-pool/tests.js @@ -20,7 +20,7 @@ const { deployService, removeService, waitForFunctionLogs, -} = require('../../utils/misc'); +} = require('../../utils/integration'); const { getMarkers } = require('../shared/utils'); describe('AWS - Cognito User Pool Integration Test', function() { @@ -34,10 +34,10 @@ describe('AWS - Cognito User Pool Integration Test', function() { let poolExistingSimpleSetupConfig; const stage = 'dev'; - before(() => { + before(async () => { tmpDirPath = getTmpDirPath(); console.info(`Temporary path: ${tmpDirPath}`); - const serverlessConfig = createTestService(tmpDirPath, { + const serverlessConfig = await createTestService(tmpDirPath, { templateDir: path.join(__dirname, 'service'), filesToAdd: [path.join(__dirname, '..', 'shared')], serverlessConfigHook: @@ -67,13 +67,13 @@ describe('AWS - Cognito User Pool Integration Test', function() { createUserPool(poolExistingMultiSetup), ]).then(() => { console.info(`Deploying "${stackName}" service...`); - deployService(tmpDirPath); + return deployService(tmpDirPath); }); }); - after(() => { + after(async () => { console.info('Removing service...'); - removeService(tmpDirPath); + await removeService(tmpDirPath); console.info('Deleting Cognito User Pools'); return BbPromise.all([ deleteUserPool(poolExistingSimpleSetup), diff --git a/tests/integration-all/event-bridge/tests.js b/tests/integration-all/event-bridge/tests.js index f8060739e..a1e43fcd4 100644 --- a/tests/integration-all/event-bridge/tests.js +++ b/tests/integration-all/event-bridge/tests.js @@ -5,12 +5,13 @@ const { expect } = require('chai'); const { getTmpDirPath, readYamlFile, writeYamlFile } = require('../../utils/fs'); const { createEventBus, putEvents, deleteEventBus } = require('../../utils/eventBridge'); + const { createTestService, deployService, removeService, waitForFunctionLogs, -} = require('../../utils/misc'); +} = require('../../utils/integration'); const { getMarkers } = require('../shared/utils'); describe('AWS - Event Bridge Integration Test', function() { @@ -31,10 +32,10 @@ describe('AWS - Event Bridge Integration Test', function() { }, ]; - before(() => { + before(async () => { tmpDirPath = getTmpDirPath(); console.info(`Temporary path: ${tmpDirPath}`); - const serverlessConfig = createTestService(tmpDirPath, { + const serverlessConfig = await createTestService(tmpDirPath, { templateDir: path.join(__dirname, 'service'), filesToAdd: [path.join(__dirname, '..', 'shared')], serverlessConfigHook: @@ -61,13 +62,13 @@ describe('AWS - Event Bridge Integration Test', function() { writeYamlFile(serverlessFilePath, config); // deploy the service console.info(`Deploying "${stackName}" service...`); - deployService(tmpDirPath); + return deployService(tmpDirPath); }); }); - after(() => { + after(async () => { console.info('Removing service...'); - removeService(tmpDirPath); + await removeService(tmpDirPath); console.info(`Deleting Event Bus "${arnEventBusName}"...`); return deleteEventBus(arnEventBusName); }); diff --git a/tests/integration-all/iot/tests.js b/tests/integration-all/iot/tests.js index 9349bb790..24e64d350 100644 --- a/tests/integration-all/iot/tests.js +++ b/tests/integration-all/iot/tests.js @@ -10,7 +10,7 @@ const { deployService, removeService, waitForFunctionLogs, -} = require('../../utils/misc'); +} = require('../../utils/integration'); const { getMarkers } = require('../shared/utils'); describe('AWS - IoT Integration Test', function() { @@ -21,10 +21,10 @@ describe('AWS - IoT Integration Test', function() { let tmpDirPath; const stage = 'dev'; - before(() => { + before(async () => { tmpDirPath = getTmpDirPath(); console.info(`Temporary path: ${tmpDirPath}`); - const serverlessConfig = createTestService(tmpDirPath, { + const serverlessConfig = await createTestService(tmpDirPath, { templateDir: path.join(__dirname, 'service'), filesToAdd: [path.join(__dirname, '..', 'shared')], serverlessConfigHook: @@ -43,7 +43,7 @@ describe('AWS - IoT Integration Test', function() { after(() => { // Topics are ephemeral and IoT endpoint is part of the account console.info('Removing service...'); - removeService(tmpDirPath); + return removeService(tmpDirPath); }); describe('Basic Setup', () => { diff --git a/tests/integration-all/s3/tests.js b/tests/integration-all/s3/tests.js index d7a4f1df4..ae4e3331f 100644 --- a/tests/integration-all/s3/tests.js +++ b/tests/integration-all/s3/tests.js @@ -11,7 +11,7 @@ const { deployService, removeService, waitForFunctionLogs, -} = require('../../utils/misc'); +} = require('../../utils/integration'); const { getMarkers } = require('../shared/utils'); describe('AWS - S3 Integration Test', function() { @@ -25,10 +25,10 @@ describe('AWS - S3 Integration Test', function() { let bucketExistingComplexSetup; const stage = 'dev'; - before(() => { + before(async () => { tmpDirPath = getTmpDirPath(); console.info(`Temporary path: ${tmpDirPath}`); - const serverlessConfig = createTestService(tmpDirPath, { + const serverlessConfig = await createTestService(tmpDirPath, { templateDir: path.join(__dirname, 'service'), filesToAdd: [path.join(__dirname, '..', 'shared')], serverlessConfigHook: @@ -57,13 +57,13 @@ describe('AWS - S3 Integration Test', function() { createBucket(bucketExistingComplexSetup), ]).then(() => { console.info(`Deploying "${stackName}" service...`); - deployService(tmpDirPath); + return deployService(tmpDirPath); }); }); - after(() => { + after(async () => { console.info('Removing service...'); - removeService(tmpDirPath); + await removeService(tmpDirPath); console.info('Deleting S3 buckets'); return BbPromise.all([ deleteBucket(bucketExistingSimpleSetup), diff --git a/tests/integration-all/schedule/tests.js b/tests/integration-all/schedule/tests.js index 004e8f055..0dda32f71 100644 --- a/tests/integration-all/schedule/tests.js +++ b/tests/integration-all/schedule/tests.js @@ -9,7 +9,7 @@ const { deployService, removeService, waitForFunctionLogs, -} = require('../../utils/misc'); +} = require('../../utils/integration'); const { getMarkers } = require('../shared/utils'); describe('AWS - Schedule Integration Test', function() { @@ -19,22 +19,22 @@ describe('AWS - Schedule Integration Test', function() { let tmpDirPath; const stage = 'dev'; - before(() => { + before(async () => { tmpDirPath = getTmpDirPath(); console.info(`Temporary path: ${tmpDirPath}`); - const serverlessConfig = createTestService(tmpDirPath, { + const serverlessConfig = await createTestService(tmpDirPath, { templateDir: path.join(__dirname, 'service'), filesToAdd: [path.join(__dirname, '..', 'shared')], }); serviceName = serverlessConfig.service; stackName = `${serviceName}-${stage}`; console.info(`Deploying "${stackName}" service...`); - deployService(tmpDirPath); + return deployService(tmpDirPath); }); - after(() => { + after(async () => { console.info('Removing service...'); - removeService(tmpDirPath); + return removeService(tmpDirPath); }); describe('Minimal Setup', () => { diff --git a/tests/integration-all/sns/tests.js b/tests/integration-all/sns/tests.js index e64b7bf81..41ab45e5d 100644 --- a/tests/integration-all/sns/tests.js +++ b/tests/integration-all/sns/tests.js @@ -11,7 +11,7 @@ const { deployService, removeService, waitForFunctionLogs, -} = require('../../utils/misc'); +} = require('../../utils/integration'); const { getMarkers } = require('../shared/utils'); describe('AWS - SNS Integration Test', function() { @@ -24,10 +24,10 @@ describe('AWS - SNS Integration Test', function() { let existingTopicName; const stage = 'dev'; - before(() => { + before(async () => { tmpDirPath = getTmpDirPath(); console.info(`Temporary path: ${tmpDirPath}`); - const serverlessConfig = createTestService(tmpDirPath, { + const serverlessConfig = await createTestService(tmpDirPath, { templateDir: path.join(__dirname, 'service'), filesToAdd: [path.join(__dirname, '..', 'shared')], serverlessConfigHook: @@ -53,13 +53,13 @@ describe('AWS - SNS Integration Test', function() { console.info(`Creating SNS topic "${existingTopicName}"...`); return createSnsTopic(existingTopicName).then(() => { console.info(`Deploying "${stackName}" service...`); - deployService(tmpDirPath); + return deployService(tmpDirPath); }); }); - after(() => { + after(async () => { console.info('Removing service...'); - removeService(tmpDirPath); + await removeService(tmpDirPath); console.info('Deleting SNS topics'); return removeSnsTopic(existingTopicName); }); diff --git a/tests/integration-all/sqs/tests.js b/tests/integration-all/sqs/tests.js index 6939b90de..e1d6f5f2b 100644 --- a/tests/integration-all/sqs/tests.js +++ b/tests/integration-all/sqs/tests.js @@ -10,7 +10,7 @@ const { deployService, removeService, waitForFunctionLogs, -} = require('../../utils/misc'); +} = require('../../utils/integration'); const { getMarkers } = require('../shared/utils'); describe('AWS - SQS Integration Test', function() { @@ -21,10 +21,10 @@ describe('AWS - SQS Integration Test', function() { let queueName; const stage = 'dev'; - before(() => { + before(async () => { tmpDirPath = getTmpDirPath(); console.info(`Temporary path: ${tmpDirPath}`); - const serverlessConfig = createTestService(tmpDirPath, { + const serverlessConfig = await createTestService(tmpDirPath, { templateDir: path.join(__dirname, 'service'), filesToAdd: [path.join(__dirname, '..', 'shared')], serverlessConfigHook: @@ -41,13 +41,13 @@ describe('AWS - SQS Integration Test', function() { console.info(`Creating SQS queue "${queueName}"...`); return createSqsQueue(queueName).then(() => { console.info(`Deploying "${stackName}" service...`); - deployService(tmpDirPath); + return deployService(tmpDirPath); }); }); - after(() => { + after(async () => { console.info('Removing service...'); - removeService(tmpDirPath); + await removeService(tmpDirPath); console.info('Deleting SQS queue'); return deleteSqsQueue(queueName); }); diff --git a/tests/integration-all/stream/tests.js b/tests/integration-all/stream/tests.js index e28df2da5..c03fd7f8b 100644 --- a/tests/integration-all/stream/tests.js +++ b/tests/integration-all/stream/tests.js @@ -15,7 +15,7 @@ const { deployService, removeService, waitForFunctionLogs, -} = require('../../utils/misc'); +} = require('../../utils/integration'); const { getMarkers } = require('../shared/utils'); describe('AWS - Stream Integration Test', function() { @@ -28,10 +28,10 @@ describe('AWS - Stream Integration Test', function() { const historicStreamMessage = 'Hello from the Kinesis horizon!'; const stage = 'dev'; - before(() => { + before(async () => { tmpDirPath = getTmpDirPath(); console.info(`Temporary path: ${tmpDirPath}`); - const serverlessConfig = createTestService(tmpDirPath, { + const serverlessConfig = await createTestService(tmpDirPath, { templateDir: path.join(__dirname, 'service'), filesToAdd: [path.join(__dirname, '..', 'shared')], serverlessConfigHook: @@ -56,13 +56,13 @@ describe('AWS - Stream Integration Test', function() { console.info( `Deploying "${stackName}" service with DynamoDB table resource "${tableName}"...` ); - deployService(tmpDirPath); + return deployService(tmpDirPath); }); }); - after(() => { + after(async () => { console.info(`Removing service (and DynamoDB table resource "${tableName}")...`); - removeService(tmpDirPath); + await removeService(tmpDirPath); console.info('Deleting Kinesis stream'); return deleteKinesisStream(streamName); }); diff --git a/tests/integration-all/websocket/tests.js b/tests/integration-all/websocket/tests.js index 5aceecc4e..efc3faa13 100644 --- a/tests/integration-all/websocket/tests.js +++ b/tests/integration-all/websocket/tests.js @@ -7,14 +7,8 @@ const _ = require('lodash'); const { expect } = require('chai'); const { getTmpDirPath, readYamlFile, writeYamlFile } = require('../../utils/fs'); -const { - region, - confirmCloudWatchLogs, - createTestService, - deployService, - removeService, - wait, -} = require('../../utils/misc'); +const { region, confirmCloudWatchLogs, wait } = require('../../utils/misc'); +const { createTestService, deployService, removeService } = require('../../utils/integration'); const { createApi, deleteApi, @@ -33,22 +27,22 @@ describe('AWS - API Gateway Websocket Integration Test', function() { let serverlessFilePath; const stage = 'dev'; - before(() => { + before(async () => { tmpDirPath = getTmpDirPath(); console.info(`Temporary path: ${tmpDirPath}`); serverlessFilePath = path.join(tmpDirPath, 'serverless.yml'); - const serverlessConfig = createTestService(tmpDirPath, { + const serverlessConfig = await createTestService(tmpDirPath, { templateDir: path.join(__dirname, 'service'), }); serviceName = serverlessConfig.service; stackName = `${serviceName}-${stage}`; console.info(`Deploying "${stackName}" service...`); - deployService(tmpDirPath); + return deployService(tmpDirPath); }); after(() => { console.info('Removing service...'); - removeService(tmpDirPath); + return removeService(tmpDirPath); }); describe('Minimal Setup', () => { @@ -108,7 +102,7 @@ describe('AWS - API Gateway Websocket Integration Test', function() { }, }); writeYamlFile(serverlessFilePath, serverless); - deployService(tmpDirPath); + return deployService(tmpDirPath); }); after(async () => { @@ -121,7 +115,7 @@ describe('AWS - API Gateway Websocket Integration Test', function() { await deleteStage(websocketApiId, 'dev'); // NOTE: deploying once again to get the stack into the original state console.info('Redeploying service...'); - deployService(tmpDirPath); + await deployService(tmpDirPath); console.info('Deleting external websocket API...'); await deleteApi(websocketApiId); }); diff --git a/tests/integration-basic/tests.js b/tests/integration-basic/tests.js index 9ea327139..a28870550 100644 --- a/tests/integration-basic/tests.js +++ b/tests/integration-basic/tests.js @@ -3,15 +3,15 @@ const path = require('path'); const fs = require('fs'); const fse = require('fs-extra'); -const BbPromise = require('bluebird'); const AWS = require('aws-sdk'); const stripAnsi = require('strip-ansi'); const { expect } = require('chai'); -const { execSync } = require('../utils/child-process'); +const spawn = require('child-process-ext/spawn'); +const resolveAwsEnv = require('@serverless/test/resolve-aws-env'); const { getTmpDirPath } = require('../utils/fs'); const { region, getServiceName } = require('../utils/misc'); -const serverlessExec = path.join(__dirname, '..', '..', 'bin', 'serverless'); +const serverlessExec = require('../serverless-binary'); const CF = new AWS.CloudFormation({ region }); @@ -19,36 +19,58 @@ describe('Service Lifecyle Integration Test', function() { this.timeout(1000 * 60 * 10); // Involves time-taking deploys const templateName = 'aws-nodejs'; const tmpDir = getTmpDirPath(); + const env = resolveAwsEnv(); + const spawnOptions = { + cwd: tmpDir, + env, + // As in invoke we optionally read stdin, we need to ensure it's closed + // See https://github.com/sindresorhus/get-stdin/issues/13#issuecomment-279234249 + shouldCloseStdin: true, + }; let serviceName; let StackName; before(() => { serviceName = getServiceName(); StackName = `${serviceName}-dev`; + console.info(`Temporary path: ${tmpDir}`); fse.mkdirsSync(tmpDir); }); - it('should create service in tmp directory', () => { - execSync(`${serverlessExec} create --template ${templateName} --name ${serviceName}`, { - cwd: tmpDir, - }); + after(async () => { + try { + await CF.describeStacks({ StackName }).promise(); + } catch (error) { + if (error.message.indexOf('does not exist') > -1) return; + throw error; + } + await spawn(serverlessExec, ['remove'], { cwd: tmpDir, env }); + }); + + it('should create service in tmp directory', async () => { + await spawn( + serverlessExec, + ['create', '--template', templateName, '--name', serviceName], + spawnOptions + ); expect(fs.existsSync(path.join(tmpDir, 'serverless.yml'))).to.be.equal(true); expect(fs.existsSync(path.join(tmpDir, 'handler.js'))).to.be.equal(true); }); - it('should deploy service to aws', () => { - execSync(`${serverlessExec} deploy`, { cwd: tmpDir }); + it('should deploy service to aws', async () => { + await spawn(serverlessExec, ['deploy'], { cwd: tmpDir, env }); - return CF.describeStacks({ StackName }) - .promise() - .then(d => expect(d.Stacks[0].StackStatus).to.be.equal('UPDATE_COMPLETE')); + const d = await CF.describeStacks({ StackName }).promise(); + expect(d.Stacks[0].StackStatus).to.be.equal('UPDATE_COMPLETE'); }); - it('should invoke function from aws', () => { - const invoked = execSync(`${serverlessExec} invoke --function hello --noGreeting true`, { - cwd: tmpDir, - }); - const result = JSON.parse(Buffer.from(invoked, 'base64').toString()); + it('should invoke function from aws', async () => { + const { stdoutBuffer: invoked } = await spawn( + serverlessExec, + ['invoke', '--function', 'hello', '--noGreeting', 'true'], + spawnOptions + ); + const result = JSON.parse(invoked); // parse it once again because the body is stringified to be LAMBDA-PROXY ready const message = JSON.parse(result.body).message; expect(message).to.be.equal('Go Serverless v1.0! Your function executed successfully!'); @@ -64,20 +86,26 @@ describe('Service Lifecyle Integration Test', function() { `; fs.writeFileSync(path.join(tmpDir, 'handler.js'), newHandler); - execSync(`${serverlessExec} deploy`, { cwd: tmpDir }); + return spawn(serverlessExec, ['deploy'], spawnOptions); }); - it('should invoke updated function from aws', () => { - const invoked = execSync(`${serverlessExec} invoke --function hello --noGreeting true`, { - cwd: tmpDir, - }); - const result = JSON.parse(Buffer.from(invoked, 'base64').toString()); + it('should invoke updated function from aws', async () => { + const { stdoutBuffer: invoked } = await spawn( + serverlessExec, + ['invoke', '--function', 'hello', '--noGreeting', 'true'], + spawnOptions + ); + const result = JSON.parse(invoked); expect(result.message).to.be.equal('Service Update Succeeded'); }); - it('should list existing deployments and roll back to first deployment', () => { + it('should list existing deployments and roll back to first deployment', async () => { let timestamp; - const listDeploys = execSync(`${serverlessExec} deploy list`, { cwd: tmpDir }); + const { stdoutBuffer: listDeploys } = await spawn( + serverlessExec, + ['deploy', 'list'], + spawnOptions + ); const output = stripAnsi(listDeploys.toString()); const match = output.match(new RegExp('Datetime: (.+)')); if (match) { @@ -86,26 +114,31 @@ describe('Service Lifecyle Integration Test', function() { // eslint-disable-next-line no-unused-expressions expect(timestamp).to.not.undefined; - execSync(`${serverlessExec} rollback -t ${timestamp}`, { cwd: tmpDir }); + await spawn(serverlessExec, ['rollback', '-t', timestamp], { cwd: tmpDir, env }); - const invoked = execSync(`${serverlessExec} invoke --function hello --noGreeting true`, { - cwd: tmpDir, - }); - const result = JSON.parse(Buffer.from(invoked, 'base64').toString()); + const { stdoutBuffer: invoked } = await spawn( + serverlessExec, + ['invoke', '--function', 'hello', '--noGreeting', 'true'], + spawnOptions + ); + const result = JSON.parse(invoked); // parse it once again because the body is stringified to be LAMBDA-PROXY ready const message = JSON.parse(result.body).message; expect(message).to.be.equal('Go Serverless v1.0! Your function executed successfully!'); }); - it('should remove service from aws', () => { - execSync(`${serverlessExec} remove`, { cwd: tmpDir }); + it('should remove service from aws', async () => { + await spawn(serverlessExec, ['remove'], { cwd: tmpDir, env }); - return CF.describeStacks({ StackName }) - .promise() - .then(d => expect(d.Stacks[0].StackStatus).to.be.equal('DELETE_COMPLETE')) - .catch(error => { - if (error.message.indexOf('does not exist') > -1) return BbPromise.resolve(); - throw new Error(error); - }); + const d = await (async () => { + try { + return await CF.describeStacks({ StackName }).promise(); + } catch (error) { + if (error.message.indexOf('does not exist') > -1) return null; + throw error; + } + })(); + if (!d) return; + expect(d.Stacks[0].StackStatus).to.be.equal('DELETE_COMPLETE'); }); }); diff --git a/tests/integration-package/cloudformation.tests.js b/tests/integration-package/cloudformation.tests.js index 420149a84..0adc13fdc 100644 --- a/tests/integration-package/cloudformation.tests.js +++ b/tests/integration-package/cloudformation.tests.js @@ -5,7 +5,7 @@ const path = require('path'); const { expect } = require('chai'); const fse = require('fs-extra'); const { execSync } = require('../utils/child-process'); -const { serverlessExec } = require('../utils/misc'); +const serverlessExec = require('../serverless-binary'); const { getTmpDirPath } = require('../utils/fs'); const fixturePaths = { diff --git a/tests/integration-package/lambda-files.tests.js b/tests/integration-package/lambda-files.tests.js index b05107604..0f12d1cd8 100644 --- a/tests/integration-package/lambda-files.tests.js +++ b/tests/integration-package/lambda-files.tests.js @@ -4,7 +4,7 @@ const path = require('path'); const { expect } = require('chai'); const fse = require('fs-extra'); const { execSync } = require('../utils/child-process'); -const { serverlessExec } = require('../utils/misc'); +const serverlessExec = require('../serverless-binary'); const { getTmpDirPath, listZipFiles } = require('../utils/fs'); const fixturePaths = { diff --git a/tests/serverless-binary.js b/tests/serverless-binary.js new file mode 100644 index 000000000..180765a6a --- /dev/null +++ b/tests/serverless-binary.js @@ -0,0 +1,8 @@ +'use strict'; + +const path = require('path'); + +module.exports = (() => { + if (process.env.SERVERLESS_BINARY_PATH) return path.resolve(process.env.SERVERLESS_BINARY_PATH); + return path.join(__dirname, '../bin/serverless.js'); +})(); diff --git a/tests/utils/cognito/index.js b/tests/utils/cognito/index.js index 594594c8c..b25efd46a 100644 --- a/tests/utils/cognito/index.js +++ b/tests/utils/cognito/index.js @@ -1,6 +1,7 @@ 'use strict'; const AWS = require('aws-sdk'); +const log = require('log').get('aws'); const { region, persistentRequest } = require('../misc'); function createUserPool(name, config = {}) { @@ -30,6 +31,7 @@ function deleteUserPool(name) { } function findUserPoolByName(name) { + log.debug('find cognito user pool by name %s', name); const cognito = new AWS.CognitoIdentityServiceProvider({ region }); const params = { @@ -42,6 +44,7 @@ function findUserPoolByName(name) { .listUserPools(params) .promise() .then(result => { + log.debug('cognito.listUserPools %j', result); const matches = result.UserPools.filter(pool => pool.Name === name); if (matches.length) { return matches.shift(); @@ -57,7 +60,13 @@ function findUserPoolByName(name) { function describeUserPool(userPoolId) { const cognito = new AWS.CognitoIdentityServiceProvider({ region }); - return cognito.describeUserPool({ UserPoolId: userPoolId }).promise(); + return cognito + .describeUserPool({ UserPoolId: userPoolId }) + .promise() + .then(result => { + log.debug('cognito.describeUserPool %s %j', userPoolId, result); + return result; + }); } function createUser(userPoolId, username, password) { diff --git a/tests/utils/integration.js b/tests/utils/integration.js new file mode 100644 index 000000000..9cdbdee9b --- /dev/null +++ b/tests/utils/integration.js @@ -0,0 +1,132 @@ +// Integration tests related utils + +'use strict'; + +const path = require('path'); +const fse = require('fs-extra'); +const spawn = require('child-process-ext/spawn'); +const nodeFetch = require('node-fetch'); +const logFetch = require('log').get('fetch'); +const resolveAwsEnv = require('@serverless/test/resolve-aws-env'); +const { getServiceName, wait } = require('./misc'); +const { readYamlFile, writeYamlFile } = require('./fs'); + +const serverlessExec = require('../serverless-binary'); + +const env = resolveAwsEnv(); + +async function createTestService( + tmpDir, + options = { + // Either templateName or templateDir have to be provided + templateName: null, // Generic template to use (e.g. 'aws-nodejs') + templateDir: null, // Path to custom pre-prepared service template + filesToAdd: [], // Array of additional files to add to the service directory + serverlessConfigHook: null, // Eventual hook that allows to customize serverless config + } +) { + const serviceName = getServiceName(); + + fse.mkdirsSync(tmpDir); + + if (options.templateName) { + // create a new Serverless service + await spawn(serverlessExec, ['create', '--template', options.templateName], { + cwd: tmpDir, + env, + }); + } else if (options.templateDir) { + fse.copySync(options.templateDir, tmpDir, { clobber: true, preserveTimestamps: true }); + } else { + throw new Error("Either 'templateName' or 'templateDir' options have to be provided"); + } + + if (options.filesToAdd && options.filesToAdd.length) { + options.filesToAdd.forEach(filePath => { + fse.copySync(filePath, tmpDir, { preserveTimestamps: true }); + }); + } + + const serverlessFilePath = path.join(tmpDir, 'serverless.yml'); + const serverlessConfig = readYamlFile(serverlessFilePath); + // Ensure unique service name + serverlessConfig.service = serviceName; + if (options.serverlessConfigHook) options.serverlessConfigHook(serverlessConfig); + writeYamlFile(serverlessFilePath, serverlessConfig); + + process.env.TOPIC_1 = `${serviceName}-1`; + process.env.TOPIC_2 = `${serviceName}-1`; + process.env.BUCKET_1 = `${serviceName}-1`; + process.env.BUCKET_2 = `${serviceName}-2`; + process.env.COGNITO_USER_POOL_1 = `${serviceName}-1`; + process.env.COGNITO_USER_POOL_2 = `${serviceName}-2`; + + return serverlessConfig; +} + +async function deployService(cwd) { + return spawn(serverlessExec, ['deploy'], { cwd, env }); +} + +async function removeService(cwd) { + return spawn(serverlessExec, ['remove'], { cwd, env }); +} + +async function getFunctionLogs(cwd, functionName) { + let logs; + try { + ({ stdoutBuffer: logs } = await spawn( + serverlessExec, + ['logs', '--function', functionName, '--noGreeting', 'true'], + { + cwd, + env, + } + )); + } catch (_) { + // Attempting to read logs before first invocation will will result in a "No existing streams for the function" error + return null; + } + return String(logs); +} + +async function waitForFunctionLogs(cwd, functionName, startMarker, endMarker) { + await wait(2000); + const logs = await getFunctionLogs(cwd, functionName); + if (logs && logs.includes(startMarker) && logs.includes(endMarker)) return logs; + return waitForFunctionLogs(cwd, functionName, startMarker, endMarker); +} + +let lastRequestId = 0; +async function fetch(url, options) { + const requestId = ++lastRequestId; + logFetch.debug('[%d] %s %o', requestId, url, options); + + let response; + try { + response = await nodeFetch(url, options); + } catch (error) { + logFetch.error('[%d] request error: %o', requestId, error); + throw error; + } + + /* eslint-disable no-underscore-dangle */ + logFetch.debug('[%d] %d %j', requestId, response.status, response.headers._headers); + const responseDecodeResult = response._decode(); + response._decode = () => responseDecodeResult; + /* eslint-enable */ + responseDecodeResult.then( + buffer => logFetch.debug('[%d] %s', requestId, String(buffer)), + error => logFetch.error('[%d] response resolution error: %o', requestId, error) + ); + return response; +} + +module.exports = { + createTestService, + deployService, + env, + fetch, + removeService, + waitForFunctionLogs, +}; diff --git a/tests/utils/misc/index.js b/tests/utils/misc/index.js index bd4fdfbac..85e42956b 100644 --- a/tests/utils/misc/index.js +++ b/tests/utils/misc/index.js @@ -1,11 +1,7 @@ 'use strict'; -const path = require('path'); -const fse = require('fs-extra'); const BbPromise = require('bluebird'); const CloudWatchLogsSdk = require('aws-sdk/clients/cloudwatchlogs'); -const { execSync } = require('../child-process'); -const { readYamlFile, writeYamlFile } = require('../fs'); const logger = console; @@ -14,8 +10,6 @@ const cloudWatchLogsSdk = new CloudWatchLogsSdk({ region }); const testServiceIdentifier = 'integ-test'; -const serverlessExec = path.resolve(__dirname, '..', '..', '..', 'bin', 'serverless'); - const serviceNameRegex = new RegExp(`${testServiceIdentifier}-d+`); function getServiceName() { @@ -23,14 +17,6 @@ function getServiceName() { return `${testServiceIdentifier}-${hrtime[1]}`; } -function deployService(cwd) { - execSync(`${serverlessExec} deploy`, { cwd }); -} - -function removeService(cwd) { - execSync(`${serverlessExec} remove`, { cwd }); -} - function replaceEnv(values) { const originals = {}; for (const key of Object.keys(values)) { @@ -48,80 +34,6 @@ function replaceEnv(values) { return originals; } -function createTestService( - tmpDir, - options = { - // Either templateName or templateDir have to be provided - templateName: null, // Generic template to use (e.g. 'aws-nodejs') - templateDir: null, // Path to custom pre-prepared service template - filesToAdd: [], // Array of additional files to add to the service directory - serverlessConfigHook: null, // Eventual hook that allows to customize serverless config - } -) { - const serviceName = getServiceName(); - - fse.mkdirsSync(tmpDir); - - if (options.templateName) { - // create a new Serverless service - execSync(`${serverlessExec} create --template ${options.templateName}`, { cwd: tmpDir }); - } else if (options.templateDir) { - fse.copySync(options.templateDir, tmpDir, { clobber: true, preserveTimestamps: true }); - } else { - throw new Error("Either 'templateName' or 'templateDir' options have to be provided"); - } - - if (options.filesToAdd && options.filesToAdd.length) { - options.filesToAdd.forEach(filePath => { - fse.copySync(filePath, tmpDir, { preserveTimestamps: true }); - }); - } - - const serverlessFilePath = path.join(tmpDir, 'serverless.yml'); - const serverlessConfig = readYamlFile(serverlessFilePath); - // Ensure unique service name - serverlessConfig.service = serviceName; - if (options.serverlessConfigHook) options.serverlessConfigHook(serverlessConfig); - writeYamlFile(serverlessFilePath, serverlessConfig); - - process.env.TOPIC_1 = `${serviceName}-1`; - process.env.TOPIC_2 = `${serviceName}-1`; - process.env.BUCKET_1 = `${serviceName}-1`; - process.env.BUCKET_2 = `${serviceName}-2`; - process.env.COGNITO_USER_POOL_1 = `${serviceName}-1`; - process.env.COGNITO_USER_POOL_2 = `${serviceName}-2`; - - return serverlessConfig; -} - -function getFunctionLogs(cwd, functionName) { - try { - const logs = execSync(`${serverlessExec} logs --function ${functionName} --noGreeting true`, { - cwd, - }); - const logsString = Buffer.from(logs, 'base64').toString(); - process.stdout.write(logsString); - return logsString; - } catch (_) { - // Attempting to read logs before first invocation will will result in a "No existing streams for the function" error - return null; - } -} - -function waitForFunctionLogs(cwd, functionName, startMarker, endMarker) { - let logs; - return new BbPromise(resolve => { - const interval = setInterval(() => { - logs = getFunctionLogs(cwd, functionName); - if (logs && logs.includes(startMarker) && logs.includes(endMarker)) { - clearInterval(interval); - return resolve(logs); - } - return null; - }, 2000); - }); -} - /** * Cloudwatch logs when turned on, are usually take some time for being effective * This function allows to confirm that new setting (turned on cloudwatch logs) @@ -175,15 +87,9 @@ module.exports = { region, confirmCloudWatchLogs, testServiceIdentifier, - serverlessExec, serviceNameRegex, getServiceName, - deployService, - removeService, replaceEnv, - createTestService, - getFunctionLogs, - waitForFunctionLogs, persistentRequest, wait, };