diff --git a/docs/providers/aws/guide/functions.md b/docs/providers/aws/guide/functions.md index 7eaf62319..2f5d02d63 100644 --- a/docs/providers/aws/guide/functions.md +++ b/docs/providers/aws/guide/functions.md @@ -501,3 +501,25 @@ functions: maximumEventAge: 7200 maximumRetryAttempts: 1 ``` + +## EFS Configuration + +You can use [Amazon EFS with Lambda](https://docs.aws.amazon.com/lambda/latest/dg/services-efs.html) by adding a `fileSystemConfig` property in the function configuration in `serverless.yml`. `fileSystemConfig` should be an object that contains the `arn` and `localMountPath` properties. The `arn` property should reference an existing EFS Access Point, where the `localMountPath` should specifiy the absolute path under which the file system will be mounted. Here's an example configuration: + +```yml +# serverless.yml +service: service-name +provider: aws + +functions: + hello: + handler: handler.hello + fileSystemConfig: + localMountPath: /mnt/example + arn: arn:aws:elasticfilesystem:us-east-1:111111111111:access-point/fsap-0d0d0d0d0d0d0d0d0 + vpc: + securityGroupIds: + - securityGroupId1 + subnetIds: + - subnetId1 +``` diff --git a/lib/plugins/aws/package/compile/functions/index.js b/lib/plugins/aws/package/compile/functions/index.js index 8a7048ca9..dcebf88cf 100644 --- a/lib/plugins/aws/package/compile/functions/index.js +++ b/lib/plugins/aws/package/compile/functions/index.js @@ -397,6 +397,42 @@ class AwsCompileFunctions { delete functionResource.Properties.VpcConfig; } + const fileSystemConfig = functionObject.fileSystemConfig; + + if (fileSystemConfig) { + if (!functionResource.Properties.VpcConfig) { + const errorMessage = [ + `Function "${functionName}": when using fileSystemConfig, `, + 'ensure that function has vpc configured ', + 'on function or provider level', + ].join(''); + throw new this.serverless.classes.Error( + errorMessage, + 'LAMBDA_FILE_SYSTEM_CONFIG_MISSING_VPC' + ); + } + + const iamRoleLambdaExecution = cfTemplate.Resources.IamRoleLambdaExecution; + + const stmt = { + Effect: 'Allow', + Action: ['elasticfilesystem:ClientMount', 'elasticfilesystem:ClientWrite'], + Resource: [fileSystemConfig.arn], + }; + + // update the PolicyDocument statements (if default policy is used) + if (iamRoleLambdaExecution) { + iamRoleLambdaExecution.Properties.Policies[0].PolicyDocument.Statement.push(stmt); + } + + const cfFileSystemConfig = { + Arn: fileSystemConfig.arn, + LocalMountPath: fileSystemConfig.localMountPath, + }; + + functionResource.Properties.FileSystemConfigs = [cfFileSystemConfig]; + } + if (functionObject.reservedConcurrency || functionObject.reservedConcurrency === 0) { // Try convert reservedConcurrency to integer const reservedConcurrency = Number(functionObject.reservedConcurrency); diff --git a/lib/plugins/aws/package/compile/functions/index.test.js b/lib/plugins/aws/package/compile/functions/index.test.js index 122df45fe..bb170e8a0 100644 --- a/lib/plugins/aws/package/compile/functions/index.test.js +++ b/lib/plugins/aws/package/compile/functions/index.test.js @@ -2842,4 +2842,118 @@ describe('AwsCompileFunctions #2', () => { ); }); }); + + describe('when using fileSystemConfig', () => { + const arn = + 'arn:aws:elasticfilesystem:us-east-1:111111111111:access-point/fsap-a1a1a1a1a1a1a1a1a'; + const localMountPath = '/mnt/path'; + const securityGroupIds = ['sg-0a0a0a0a']; + const subnetIds = ['subnet-01010101']; + + let functionConfig; + let defaultIamRole; + + before(() => + fixtures + .extend('function', { + functions: { + foo: { + vpc: { + subnetIds, + securityGroupIds, + }, + fileSystemConfig: { + localMountPath, + arn, + }, + }, + }, + }) + .then(fixturePath => + runServerless({ + cwd: fixturePath, + cliArgs: ['package'], + }).then(({ awsNaming, cfTemplate }) => { + functionConfig = cfTemplate.Resources[awsNaming.getLambdaLogicalId('foo')].Properties; + defaultIamRole = cfTemplate.Resources.IamRoleLambdaExecution; + }) + ) + ); + + it('should correctly set Arn and LocalMountPath', () => { + expect(functionConfig.FileSystemConfigs).to.deep.equal([ + { + Arn: arn, + LocalMountPath: localMountPath, + }, + ]); + }); + + it('should update default IAM role', () => { + expect(defaultIamRole.Properties.Policies[0].PolicyDocument.Statement).to.deep.include({ + Effect: 'Allow', + Action: ['elasticfilesystem:ClientMount', 'elasticfilesystem:ClientWrite'], + Resource: [arn], + }); + }); + + it('should support vpc defined on provider level', () => { + return fixtures + .extend('function', { + provider: { + vpc: { + subnetIds, + securityGroupIds, + }, + }, + functions: { + foo: { + fileSystemConfig: { + localMountPath, + arn, + }, + }, + }, + }) + .then(fixturePath => + runServerless({ + cwd: fixturePath, + cliArgs: ['package'], + }).then(({ cfTemplate, awsNaming }) => { + const cfResources = cfTemplate.Resources; + const naming = awsNaming; + const fnConfig = cfResources[naming.getLambdaLogicalId('foo')].Properties; + + expect(fnConfig.FileSystemConfigs).to.deep.equal([ + { + Arn: arn, + LocalMountPath: localMountPath, + }, + ]); + }) + ); + }); + + it('should throw error when function has no vpc configured', () => { + return fixtures + .extend('function', { + functions: { + foo: { + fileSystemConfig: { + localMountPath, + arn, + }, + }, + }, + }) + .then(fixturePath => + runServerless({ + cwd: fixturePath, + cliArgs: ['package'], + }).catch(error => { + expect(error).to.have.property('code', 'LAMBDA_FILE_SYSTEM_CONFIG_MISSING_VPC'); + }) + ); + }); + }); }); diff --git a/lib/plugins/aws/provider/awsProvider.js b/lib/plugins/aws/provider/awsProvider.js index 08a1eda3c..8aa62b392 100644 --- a/lib/plugins/aws/provider/awsProvider.js +++ b/lib/plugins/aws/provider/awsProvider.js @@ -220,7 +220,22 @@ class AwsProvider { }, function: { // TODO: Complete schema, see https://github.com/serverless/serverless/issues/8017 - properties: { handler: { type: 'string' } }, + properties: { + handler: { type: 'string' }, + fileSystemConfig: { + type: 'object', + properties: { + localMountPath: { type: 'string', pattern: '^/mnt/[a-zA-Z0-9-_.]+$' }, + arn: { + type: 'string', + pattern: + '^arn:aws[a-zA-Z-]*:elasticfilesystem:[a-z]{2}((-gov)|(-iso(b?)))?-[a-z]+-[1-9]{1}:[0-9]{12}:access-point/fsap-[a-f0-9]{17}$', + }, + }, + additionalProperties: false, + required: ['localMountPath', 'arn'], + }, + }, }, }); } diff --git a/tests/integration-all/file-system-config/cloudformation.yml b/tests/integration-all/file-system-config/cloudformation.yml new file mode 100644 index 000000000..0660a89c0 --- /dev/null +++ b/tests/integration-all/file-system-config/cloudformation.yml @@ -0,0 +1,60 @@ +Description: This template deploys a minimal stack for testing EFS + +Resources: + VPC: + Type: AWS::EC2::VPC + Properties: + CidrBlock: 10.192.0.0/16 + EnableDnsSupport: true + EnableDnsHostnames: true + + Subnet: + Type: AWS::EC2::Subnet + Properties: + VpcId: !Ref VPC + AvailabilityZone: !Select [0, !GetAZs ''] + CidrBlock: 10.192.21.0/24 + MapPublicIpOnLaunch: false + + FileSystem: + Type: AWS::EFS::FileSystem + Properties: + PerformanceMode: generalPurpose + FileSystemTags: + - Key: Name + Value: ServerlessFrameworkTestsVolume + + MountTarget: + Type: AWS::EFS::MountTarget + Properties: + FileSystemId: !Ref FileSystem + SubnetId: !Ref Subnet + SecurityGroups: + - !GetAtt VPC.DefaultSecurityGroup + + AccessPointResource: + Type: AWS::EFS::AccessPoint + Properties: + FileSystemId: !Ref FileSystem + PosixUser: + Uid: 1001 + Gid: 1001 + RootDirectory: + CreationInfo: + OwnerGid: 1001 + OwnerUid: 1001 + Permissions: 770 + Path: /efs + +Outputs: + Subnet: + Description: A reference to the subnet + Value: !Ref Subnet + + SecurityGroup: + Description: Security group + Value: !GetAtt VPC.DefaultSecurityGroup + + AccessPointARN: + Description: Access Point ARN + Value: !GetAtt AccessPointResource.Arn diff --git a/tests/integration-all/file-system-config/service/core.js b/tests/integration-all/file-system-config/service/core.js new file mode 100644 index 000000000..4662ecd2e --- /dev/null +++ b/tests/integration-all/file-system-config/service/core.js @@ -0,0 +1,17 @@ +'use strict'; + +const fs = require('fs'); + +const filename = '/mnt/testing/file.txt'; + +function writer(event, context, callback) { + fs.writeFileSync(filename, 'fromlambda', 'utf8'); + return callback(null, event); +} + +function reader(event, context, callback) { + const result = fs.readFileSync(filename, 'utf8'); + return callback(null, { result }); +} + +module.exports = { writer, reader }; diff --git a/tests/integration-all/file-system-config/service/serverless.yml b/tests/integration-all/file-system-config/service/serverless.yml new file mode 100644 index 000000000..c22e0a510 --- /dev/null +++ b/tests/integration-all/file-system-config/service/serverless.yml @@ -0,0 +1,17 @@ +service: CHANGE_TO_UNIQUE_PER_RUN + +configValidationMode: error + +# VPC and EFS configuration is added dynamically during test run +# Because it has to be provisioned separately via CloudFormation stack + +provider: + name: aws + runtime: nodejs12.x + versionFunctions: false + +functions: + writer: + handler: core.writer + reader: + handler: core.reader diff --git a/tests/integration-all/file-system-config/tests.js b/tests/integration-all/file-system-config/tests.js new file mode 100644 index 000000000..ee23d04fa --- /dev/null +++ b/tests/integration-all/file-system-config/tests.js @@ -0,0 +1,105 @@ +'use strict'; + +const path = require('path'); +const { expect } = require('chai'); + +const awsRequest = require('@serverless/test/aws-request'); +const fs = require('fs'); +const { getTmpDirPath } = require('../../utils/fs'); +const crypto = require('crypto'); +const { createTestService, deployService, removeService } = require('../../utils/integration'); + +describe('AWS - FileSystemConfig Integration Test', function() { + this.timeout(1000 * 60 * 100); // Involves time-taking deploys + let serviceName; + let stackName; + let tmpDirPath; + const stage = 'dev'; + const resourcesStackName = `efs-integration-tests-deps-stack-${crypto + .randomBytes(8) + .toString('hex')}`; + + before(async () => { + tmpDirPath = getTmpDirPath(); + console.info(`Temporary path: ${tmpDirPath}`); + const cfnTemplate = fs.readFileSync(path.join(__dirname, 'cloudformation.yml'), 'utf8'); + + console.info('Deploying CloudFormation stack with required resources...'); + await awsRequest('CloudFormation', 'createStack', { + StackName: resourcesStackName, + TemplateBody: cfnTemplate, + }); + const waitForResult = await awsRequest('CloudFormation', 'waitFor', 'stackCreateComplete', { + StackName: resourcesStackName, + }); + + const outputMap = waitForResult.Stacks[0].Outputs.reduce((map, output) => { + map[output.OutputKey] = output.OutputValue; + return map; + }, {}); + + const serverlessConfig = await createTestService(tmpDirPath, { + templateDir: path.join(__dirname, 'service'), + serverlessConfigHook: config => { + const fileSystemConfig = { + localMountPath: '/mnt/testing', + arn: outputMap.AccessPointARN, + }; + config.provider.vpc = { + subnetIds: [outputMap.Subnet], + securityGroupIds: [outputMap.SecurityGroup], + }; + config.functions.writer.fileSystemConfig = fileSystemConfig; + config.functions.reader.fileSystemConfig = fileSystemConfig; + }, + }); + serviceName = serverlessConfig.service; + stackName = `${serviceName}-${stage}`; + console.info(`Deploying "${stackName}" service...`); + return deployService(tmpDirPath); + }); + + after(async () => { + console.info('Removing service...'); + await removeService(tmpDirPath); + console.info('Removing CloudFormation stack with required resources...'); + await awsRequest('CloudFormation', 'deleteStack', { StackName: resourcesStackName }); + return awsRequest('CloudFormation', 'waitFor', 'stackDeleteComplete', { + StackName: resourcesStackName, + }); + }); + + describe('Basic Setup', () => { + let startTime; + + before(() => { + startTime = Date.now(); + }); + + it('should be able to write to efs and read from it in a separate function', async function self() { + try { + await awsRequest('Lambda', 'invoke', { + FunctionName: `${stackName}-writer`, + InvocationType: 'RequestResponse', + }); + } catch (e) { + // Sometimes EFS is not available right away which causes invoke to fail, + // here we retry it to avoid that issue + if (e.code === 'EFSMountFailureException' && startTime > Date.now() - 1000 * 60) { + console.info('Failed to invoke, retry'); + return self(); + } + throw e; + } + + const readerResult = await awsRequest('Lambda', 'invoke', { + FunctionName: `${stackName}-reader`, + InvocationType: 'RequestResponse', + }); + const payload = JSON.parse(readerResult.Payload); + + expect(payload).to.deep.equal({ result: 'fromlambda' }); + return null; + }); + }); +}); diff --git a/tests/integration-all/fileSystemConfig/service/serverless.yml b/tests/integration-all/fileSystemConfig/service/serverless.yml new file mode 100644 index 000000000..1be4e124a --- /dev/null +++ b/tests/integration-all/fileSystemConfig/service/serverless.yml @@ -0,0 +1,12 @@ +service: CHANGE_TO_UNIQUE_PER_RUN + +provider: + name: aws + runtime: nodejs12.x + versionFunctions: false + +functions: + writer: + handler: core.writer + reader: + handler: core.reader