From 360925d2e0cddb6fbbbb72ca47495aa71a43d1fc Mon Sep 17 00:00:00 2001 From: Stephen Date: Fri, 30 Apr 2021 07:33:32 +0000 Subject: [PATCH] fix(AWS S3): Fix parsing of the artifact S3 url (#9380) --- docs/providers/aws/guide/packaging.md | 8 +++-- lib/plugins/aws/package/compile/functions.js | 17 +++++----- lib/plugins/aws/utils/parse-s3-uri.js | 26 ++++++++++++++ .../plugins/aws/utils/parse-s3-uri.test.js | 34 +++++++++++++++++++ 4 files changed, 73 insertions(+), 12 deletions(-) create mode 100644 lib/plugins/aws/utils/parse-s3-uri.js create mode 100644 test/unit/lib/plugins/aws/utils/parse-s3-uri.test.js diff --git a/docs/providers/aws/guide/packaging.md b/docs/providers/aws/guide/packaging.md index 4c7953504..5d4cbde1e 100644 --- a/docs/providers/aws/guide/packaging.md +++ b/docs/providers/aws/guide/packaging.md @@ -119,7 +119,9 @@ functions: #### Artifacts hosted on S3 -Artifacts can also be fetched from a remote S3 bucket. In this case you just need to provide the S3 object URL as the artifact value. This applies to both, service-wide and function-level artifact setups. +Artifacts can also be fetched from a remote S3 bucket. In this case you just need to provide the S3 object URI (old style or new) as the artifact value. This applies to both, service-wide and function-level artifact setups. + +**Note:** At this time, only S3 URIs are supported. Serverless does not yet support fetching artifacts from non-S3 remote locations. ##### Service package @@ -127,7 +129,7 @@ Artifacts can also be fetched from a remote S3 bucket. In this case you just nee service: my-service package: - artifact: https://s3.amazonaws.com/some-bucket/service-artifact.zip + artifact: s3://some-bucket/path/to/service-artifact.zip ``` ##### Individual function packages @@ -142,7 +144,7 @@ functions: hello: handler: com.serverless.Handler package: - artifact: https://s3.amazonaws.com/some-bucket/function-artifact.zip + artifact: s3://some-bucket/path/to/service-artifact.zip ``` ### Packaging functions separately diff --git a/lib/plugins/aws/package/compile/functions.js b/lib/plugins/aws/package/compile/functions.js index e259a986e..39c11a4ee 100644 --- a/lib/plugins/aws/package/compile/functions.js +++ b/lib/plugins/aws/package/compile/functions.js @@ -9,6 +9,7 @@ const path = require('path'); const ServerlessError = require('../../../../serverless-error'); const deepSortObjectByKey = require('../../../../utils/deepSortObjectByKey'); const getHashForFilePath = require('../lib/getHashForFilePath'); +const parseS3URI = require('../../utils/parse-s3-uri'); class AwsCompileFunctions { constructor(serverless, options) { @@ -102,18 +103,16 @@ class AwsCompileFunctions { _.get(functionObject, 'package.artifact') || _.get(this, 'serverless.service.package.artifact'); - const regex = new RegExp('s3\\.amazonaws\\.com/(.+)/(.+)'); - const match = artifactFilePath.match(regex); - - if (!match) return; + const s3Object = parseS3URI(artifactFilePath); + if (!s3Object) return; + if (process.env.SLS_DEBUG) { + this.serverless.cli.log(`Downloading ${s3Object.Key} from bucket ${s3Object.Bucket}`); + } await new Promise((resolve, reject) => { const tmpDir = this.serverless.utils.getTmpDirPath(); - const filePath = path.join(tmpDir, match[2]); + const filePath = path.join(tmpDir, path.basename(s3Object.Key)); - const readStream = S3.getObject({ - Bucket: match[1], - Key: match[2], - }).createReadStream(); + const readStream = S3.getObject(s3Object).createReadStream(); const writeStream = fs.createWriteStream(filePath); readStream diff --git a/lib/plugins/aws/utils/parse-s3-uri.js b/lib/plugins/aws/utils/parse-s3-uri.js new file mode 100644 index 000000000..ecb843c48 --- /dev/null +++ b/lib/plugins/aws/utils/parse-s3-uri.js @@ -0,0 +1,26 @@ +'use strict'; + +const patterns = [ + // S3 URI. Ex: s3://bucket/path/to/artifact.zip + new RegExp('^s3://([^/]+)/(.+)'), + + // New style S3 URL. Ex: https://bucket.s3.amazonaws.com/path/to/artifact.zip + new RegExp('([^/]+)\\.s3\\.amazonaws\\.com/(.+)'), + + // Old style S3 URL. Ex: https://s3.amazonaws.com/bucket/path/to/artifact.zip + new RegExp('s3\\.amazonaws\\.com/([^/]+)/(.+)'), +]; + +module.exports = (url) => { + for (const regex of patterns) { + const match = url.match(regex); + if (match) { + return { + Bucket: match[1], + Key: match[2], + }; + } + } + + return null; +}; diff --git a/test/unit/lib/plugins/aws/utils/parse-s3-uri.test.js b/test/unit/lib/plugins/aws/utils/parse-s3-uri.test.js new file mode 100644 index 000000000..99aad0e3e --- /dev/null +++ b/test/unit/lib/plugins/aws/utils/parse-s3-uri.test.js @@ -0,0 +1,34 @@ +'use strict'; +const expect = require('chai').expect; +const parseS3URI = require('../../../../../../lib/plugins/aws/utils/parse-s3-uri'); + +describe('test/unit/lib/plugins/aws/utils/parse-s3-uri.test.js', () => { + it('should parse an S3 URI', () => { + const expected = { + Bucket: 'test-bucket', + Key: 'path/to/artifact.zip', + }; + const actual = parseS3URI('s3://test-bucket/path/to/artifact.zip'); + expect(actual).to.deep.equal(expected); + }); + it('should parse an old style S3 URL', () => { + const expected = { + Bucket: 'test-bucket', + Key: 'path/to/artifact.zip', + }; + const actual = parseS3URI('https://s3.amazonaws.com/test-bucket/path/to/artifact.zip'); + expect(actual).to.deep.equal(expected); + }); + it('should parse a new style S3 URL', () => { + const expected = { + Bucket: 'test-bucket', + Key: 'path/to/artifact.zip', + }; + const actual = parseS3URI('https://test-bucket.s3.amazonaws.com/path/to/artifact.zip'); + expect(actual).to.deep.equal(expected); + }); + it('should reject non S3 URLs', () => { + const actual = parseS3URI('https://example.com/path/to/artifact.zip'); + expect(actual).to.be.null; + }); +});