From 2945880e680d07b571e8490dc3ea4e8dc99f7c32 Mon Sep 17 00:00:00 2001 From: Alex Oskotsky Date: Wed, 10 May 2017 22:55:59 -0400 Subject: [PATCH] Add support for S3 variables --- docs/providers/aws/guide/variables.md | 12 +++++ lib/classes/Variables.js | 25 +++++++++++ lib/classes/Variables.test.js | 63 +++++++++++++++++++++++++++ 3 files changed, 100 insertions(+) diff --git a/docs/providers/aws/guide/variables.md b/docs/providers/aws/guide/variables.md index 05d2c4538..a26bb31d7 100644 --- a/docs/providers/aws/guide/variables.md +++ b/docs/providers/aws/guide/variables.md @@ -23,6 +23,7 @@ The Serverless framework provides a powerful variable system which allows you to - Recursively nest variable references within each other for ultimate flexibility - Combine multiple variable references to overwrite each other - Define your own variable syntax if it conflicts with CF syntax +- Reference & load variables from S3 **Note:** You can only use variables in `serverless.yml` property **values**, not property keys. So you can't use variables to generate dynamic logical IDs in the custom resources section for example. @@ -110,6 +111,17 @@ functions: ``` In that case, the framework will fetch the values of those `functionPrefix` outputs from the provided stack names and populate your variables. There are many use cases for this functionality and it allows your service to communicate with other services/stacks. +## Referencing S3 Options +You can reference S3 values as the source of your variables to use in your service with the `s3:bucketName/key` syntax. For example: +```yml +service: new-service +provider: aws +functions: + hello: + name: ${s3:myBucket/myKey}-hello + handler: handler.hello +``` +In the above example, the value for `myKey` in the `myBucket` S3 bucket will be looked up and used to populate the variable. ## Reference Variables in Other Files To reference variables in other YAML or JSON files, use the `${file(./myFile.yml):someProperty}` syntax in your `serverless.yml` configuration file. This functionality is recursive, so you can go as deep in that file as you want. Here's an example: diff --git a/lib/classes/Variables.js b/lib/classes/Variables.js index 724981983..04bd6f141 100644 --- a/lib/classes/Variables.js +++ b/lib/classes/Variables.js @@ -20,6 +20,7 @@ class Variables { this.optRefSyntax = RegExp(/^opt:/g); this.selfRefSyntax = RegExp(/^self:/g); this.cfRefSyntax = RegExp(/^cf:/g); + this.s3RefSynax = RegExp(/^s3:(.+?)\/(.+)$/); } loadVariableSyntax() { @@ -153,6 +154,8 @@ class Variables { return this.getValueFromFile(variableString); } else if (variableString.match(this.cfRefSyntax)) { return this.getValueFromCf(variableString); + } else if (variableString.match(this.s3RefSynax)) { + return this.getValueFromS3(variableString); } const errorMessage = [ `Invalid variable reference syntax for variable ${variableString}.`, @@ -272,6 +275,28 @@ class Variables { }); } + getValueFromS3(variableString) { + const groups = variableString.match(this.s3RefSynax); + const bucket = groups[1]; + const key = groups[2]; + return this.serverless.getProvider('aws') + .request('S3', + 'getObject', + { + Bucket: bucket, + Key: key, + }, + this.options.stage, + this.options.region) + .then( + response => response.Body.toString(), + err => { + const errorMessage = `Error getting value for ${variableString}. ${err.message}`; + throw new this.serverless.classes.Error(errorMessage); + } + ); + } + getDeepValue(deepProperties, valueToPopulate) { return BbPromise.reduce(deepProperties, (computedValueToPopulateParam, subProperty) => { let computedValueToPopulate = computedValueToPopulateParam; diff --git a/lib/classes/Variables.test.js b/lib/classes/Variables.test.js index f96a33b2d..c2ff60c5a 100644 --- a/lib/classes/Variables.test.js +++ b/lib/classes/Variables.test.js @@ -349,6 +349,19 @@ describe('Variables', () => { }); }); + it('should call getValueFromS3 if referencing variable in S3', () => { + const serverless = new Serverless(); + const getValueFromS3Stub = sinon + .stub(serverless.variables, 'getValueFromS3').resolves('variableValue'); + return serverless.variables.getValueFromSource('s3:test-bucket/path/to/key') + .then(valueToPopulate => { + expect(valueToPopulate).to.equal('variableValue'); + expect(getValueFromS3Stub.called).to.equal(true); + expect(getValueFromS3Stub.calledWith('s3:test-bucket/path/to/key')).to.equal(true); + serverless.variables.getValueFromS3.restore(); + }); + }); + it('should throw error if referencing an invalid source', () => { const serverless = new Serverless(); expect(() => serverless.variables.getValueFromSource('weird:source')) @@ -613,6 +626,56 @@ describe('Variables', () => { }); }); + describe('#getValueFromS3()', () => { + let serverless; + let awsProvider; + + beforeEach(() => { + serverless = new Serverless(); + const options = { + stage: 'prod', + region: 'us-west-2', + }; + awsProvider = new AwsProvider(serverless, options); + serverless.setProvider('aws', awsProvider); + serverless.variables.options = options; + }); + + it('should get variable from S3', () => { + const awsResponseMock = { + Body: 'MockValue', + }; + const s3Stub = sinon.stub(awsProvider, 'request').resolves(awsResponseMock); + + return serverless.variables.getValueFromS3('s3:some.bucket/path/to/key').then(value => { + expect(value).to.be.equal('MockValue'); + expect(s3Stub.calledOnce).to.be.equal(true); + expect(s3Stub.calledWithExactly( + 'S3', + 'getObject', + { + Bucket: 'some.bucket', + Key: 'path/to/key', + }, + serverless.variables.options.stage, + serverless.variables.options.region + )).to.be.equal(true); + }); + }); + + it('should throw error if error getting value from S3', () => { + const error = new Error('The specified bucket is not valid'); + sinon.stub(awsProvider, 'request').rejects(error); + + return serverless.variables.getValueFromS3('s3:some.bucket/path/to/key').then(() => { + throw new Error('S3 value was populated for invalid S3 bucket'); + }, (err) => { + expect(err.message).to.be.equal('Error getting value for s3:some.bucket/path/to/key. ' + + 'The specified bucket is not valid'); + }); + }); + }); + describe('#getDeepValue()', () => { it('should get deep values', () => { const serverless = new Serverless();