feat(AWS Lambda): Support EFS attachment (#8042)

docs/providers/aws/guide/functions.md

@@ -501,3 +501,25 @@ functions:
     maximumEventAge: 7200
     maximumRetryAttempts: 1
 ```
+
+## EFS Configuration
+
+You can use [Amazon EFS with Lambda](https://docs.aws.amazon.com/lambda/latest/dg/services-efs.html) by adding a `fileSystemConfig` property in the function configuration in `serverless.yml`. `fileSystemConfig` should be an object that contains the `arn` and `localMountPath` properties. The `arn` property should reference an existing EFS Access Point, where the `localMountPath` should specifiy the absolute path under which the file system will be mounted. Here's an example configuration:
+
+```yml
+# serverless.yml
+service: service-name
+provider: aws
+
+functions:
+  hello:
+    handler: handler.hello
+    fileSystemConfig:
+      localMountPath: /mnt/example
+      arn: arn:aws:elasticfilesystem:us-east-1:111111111111:access-point/fsap-0d0d0d0d0d0d0d0d0
+    vpc:
+      securityGroupIds:
+        - securityGroupId1
+      subnetIds:
+        - subnetId1
+```

lib/plugins/aws/package/compile/functions/index.js

@@ -397,6 +397,42 @@ class AwsCompileFunctions {
         delete functionResource.Properties.VpcConfig;
       }
 
+      const fileSystemConfig = functionObject.fileSystemConfig;
+
+      if (fileSystemConfig) {
+        if (!functionResource.Properties.VpcConfig) {
+          const errorMessage = [
+            `Function "${functionName}": when using fileSystemConfig, `,
+            'ensure that function has vpc configured ',
+            'on function or provider level',
+          ].join('');
+          throw new this.serverless.classes.Error(
+            errorMessage,
+            'LAMBDA_FILE_SYSTEM_CONFIG_MISSING_VPC'
+          );
+        }
+
+        const iamRoleLambdaExecution = cfTemplate.Resources.IamRoleLambdaExecution;
+
+        const stmt = {
+          Effect: 'Allow',
+          Action: ['elasticfilesystem:ClientMount', 'elasticfilesystem:ClientWrite'],
+          Resource: [fileSystemConfig.arn],
+        };
+
+        // update the PolicyDocument statements (if default policy is used)
+        if (iamRoleLambdaExecution) {
+          iamRoleLambdaExecution.Properties.Policies[0].PolicyDocument.Statement.push(stmt);
+        }
+
+        const cfFileSystemConfig = {
+          Arn: fileSystemConfig.arn,
+          LocalMountPath: fileSystemConfig.localMountPath,
+        };
+
+        functionResource.Properties.FileSystemConfigs = [cfFileSystemConfig];
+      }
+
       if (functionObject.reservedConcurrency || functionObject.reservedConcurrency === 0) {
         // Try convert reservedConcurrency to integer
         const reservedConcurrency = Number(functionObject.reservedConcurrency);

lib/plugins/aws/package/compile/functions/index.test.js

@@ -2842,4 +2842,118 @@ describe('AwsCompileFunctions #2', () => {
         );
     });
   });
+
+  describe('when using fileSystemConfig', () => {
+    const arn =
+      'arn:aws:elasticfilesystem:us-east-1:111111111111:access-point/fsap-a1a1a1a1a1a1a1a1a';
+    const localMountPath = '/mnt/path';
+    const securityGroupIds = ['sg-0a0a0a0a'];
+    const subnetIds = ['subnet-01010101'];
+
+    let functionConfig;
+    let defaultIamRole;
+
+    before(() =>
+      fixtures
+        .extend('function', {
+          functions: {
+            foo: {
+              vpc: {
+                subnetIds,
+                securityGroupIds,
+              },
+              fileSystemConfig: {
+                localMountPath,
+                arn,
+              },
+            },
+          },
+        })
+        .then(fixturePath =>
+          runServerless({
+            cwd: fixturePath,
+            cliArgs: ['package'],
+          }).then(({ awsNaming, cfTemplate }) => {
+            functionConfig = cfTemplate.Resources[awsNaming.getLambdaLogicalId('foo')].Properties;
+            defaultIamRole = cfTemplate.Resources.IamRoleLambdaExecution;
+          })
+        )
+    );
+
+    it('should correctly set Arn and LocalMountPath', () => {
+      expect(functionConfig.FileSystemConfigs).to.deep.equal([
+        {
+          Arn: arn,
+          LocalMountPath: localMountPath,
+        },
+      ]);
+    });
+
+    it('should update default IAM role', () => {
+      expect(defaultIamRole.Properties.Policies[0].PolicyDocument.Statement).to.deep.include({
+        Effect: 'Allow',
+        Action: ['elasticfilesystem:ClientMount', 'elasticfilesystem:ClientWrite'],
+        Resource: [arn],
+      });
+    });
+
+    it('should support vpc defined on provider level', () => {
+      return fixtures
+        .extend('function', {
+          provider: {
+            vpc: {
+              subnetIds,
+              securityGroupIds,
+            },
+          },
+          functions: {
+            foo: {
+              fileSystemConfig: {
+                localMountPath,
+                arn,
+              },
+            },
+          },
+        })
+        .then(fixturePath =>
+          runServerless({
+            cwd: fixturePath,
+            cliArgs: ['package'],
+          }).then(({ cfTemplate, awsNaming }) => {
+            const cfResources = cfTemplate.Resources;
+            const naming = awsNaming;
+            const fnConfig = cfResources[naming.getLambdaLogicalId('foo')].Properties;
+
+            expect(fnConfig.FileSystemConfigs).to.deep.equal([
+              {
+                Arn: arn,
+                LocalMountPath: localMountPath,
+              },
+            ]);
+          })
+        );
+    });
+
+    it('should throw error when function has no vpc configured', () => {
+      return fixtures
+        .extend('function', {
+          functions: {
+            foo: {
+              fileSystemConfig: {
+                localMountPath,
+                arn,
+              },
+            },
+          },
+        })
+        .then(fixturePath =>
+          runServerless({
+            cwd: fixturePath,
+            cliArgs: ['package'],
+          }).catch(error => {
+            expect(error).to.have.property('code', 'LAMBDA_FILE_SYSTEM_CONFIG_MISSING_VPC');
+          })
+        );
+    });
+  });
 });

lib/plugins/aws/provider/awsProvider.js

@@ -220,7 +220,22 @@ class AwsProvider {
         },
         function: {
           // TODO: Complete schema, see https://github.com/serverless/serverless/issues/8017
-          properties: { handler: { type: 'string' } },
+          properties: {
+            handler: { type: 'string' },
+            fileSystemConfig: {
+              type: 'object',
+              properties: {
+                localMountPath: { type: 'string', pattern: '^/mnt/[a-zA-Z0-9-_.]+$' },
+                arn: {
+                  type: 'string',
+                  pattern:
+                    '^arn:aws[a-zA-Z-]*:elasticfilesystem:[a-z]{2}((-gov)|(-iso(b?)))?-[a-z]+-[1-9]{1}:[0-9]{12}:access-point/fsap-[a-f0-9]{17}$',
+                },
+              },
+              additionalProperties: false,
+              required: ['localMountPath', 'arn'],
+            },
+          },
         },
       });
     }

tests/integration-all/file-system-config/cloudformation.yml

@@ -0,0 +1,60 @@
+Description: This template deploys a minimal stack for testing EFS
+
+Resources:
+  VPC:
+    Type: AWS::EC2::VPC
+    Properties:
+      CidrBlock: 10.192.0.0/16
+      EnableDnsSupport: true
+      EnableDnsHostnames: true
+
+  Subnet:
+    Type: AWS::EC2::Subnet
+    Properties:
+      VpcId: !Ref VPC
+      AvailabilityZone: !Select [0, !GetAZs '']
+      CidrBlock: 10.192.21.0/24
+      MapPublicIpOnLaunch: false
+
+  FileSystem:
+    Type: AWS::EFS::FileSystem
+    Properties:
+      PerformanceMode: generalPurpose
+      FileSystemTags:
+        - Key: Name
+          Value: ServerlessFrameworkTestsVolume
+
+  MountTarget:
+    Type: AWS::EFS::MountTarget
+    Properties:
+      FileSystemId: !Ref FileSystem
+      SubnetId: !Ref Subnet
+      SecurityGroups:
+        - !GetAtt VPC.DefaultSecurityGroup
+
+  AccessPointResource:
+    Type: AWS::EFS::AccessPoint
+    Properties:
+      FileSystemId: !Ref FileSystem
+      PosixUser:
+        Uid: 1001
+        Gid: 1001
+      RootDirectory:
+        CreationInfo:
+          OwnerGid: 1001
+          OwnerUid: 1001
+          Permissions: 770
+        Path: /efs
+
+Outputs:
+  Subnet:
+    Description: A reference to the subnet
+    Value: !Ref Subnet
+
+  SecurityGroup:
+    Description: Security group
+    Value: !GetAtt VPC.DefaultSecurityGroup
+
+  AccessPointARN:
+    Description: Access Point ARN
+    Value: !GetAtt AccessPointResource.Arn

tests/integration-all/file-system-config/service/core.js

@@ -0,0 +1,17 @@
+'use strict';
+
+const fs = require('fs');
+
+const filename = '/mnt/testing/file.txt';
+
+function writer(event, context, callback) {
+  fs.writeFileSync(filename, 'fromlambda', 'utf8');
+  return callback(null, event);
+}
+
+function reader(event, context, callback) {
+  const result = fs.readFileSync(filename, 'utf8');
+  return callback(null, { result });
+}
+
+module.exports = { writer, reader };

tests/integration-all/file-system-config/service/serverless.yml

@@ -0,0 +1,17 @@
+service: CHANGE_TO_UNIQUE_PER_RUN
+
+configValidationMode: error
+
+# VPC and EFS configuration is added dynamically during test run
+# Because it has to be provisioned separately via CloudFormation stack
+
+provider:
+  name: aws
+  runtime: nodejs12.x
+  versionFunctions: false
+
+functions:
+  writer:
+    handler: core.writer
+  reader:
+    handler: core.reader

tests/integration-all/file-system-config/tests.js

@@ -0,0 +1,105 @@
+'use strict';
+
+const path = require('path');
+const { expect } = require('chai');
+
+const awsRequest = require('@serverless/test/aws-request');
+const fs = require('fs');
+const { getTmpDirPath } = require('../../utils/fs');
+const crypto = require('crypto');
+const { createTestService, deployService, removeService } = require('../../utils/integration');
+
+describe('AWS - FileSystemConfig Integration Test', function() {
+  this.timeout(1000 * 60 * 100); // Involves time-taking deploys
+  let serviceName;
+  let stackName;
+  let tmpDirPath;
+  const stage = 'dev';
+  const resourcesStackName = `efs-integration-tests-deps-stack-${crypto
+    .randomBytes(8)
+    .toString('hex')}`;
+
+  before(async () => {
+    tmpDirPath = getTmpDirPath();
+    console.info(`Temporary path: ${tmpDirPath}`);
+    const cfnTemplate = fs.readFileSync(path.join(__dirname, 'cloudformation.yml'), 'utf8');
+
+    console.info('Deploying CloudFormation stack with required resources...');
+    await awsRequest('CloudFormation', 'createStack', {
+      StackName: resourcesStackName,
+      TemplateBody: cfnTemplate,
+    });
+    const waitForResult = await awsRequest('CloudFormation', 'waitFor', 'stackCreateComplete', {
+      StackName: resourcesStackName,
+    });
+
+    const outputMap = waitForResult.Stacks[0].Outputs.reduce((map, output) => {
+      map[output.OutputKey] = output.OutputValue;
+      return map;
+    }, {});
+
+    const serverlessConfig = await createTestService(tmpDirPath, {
+      templateDir: path.join(__dirname, 'service'),
+      serverlessConfigHook: config => {
+        const fileSystemConfig = {
+          localMountPath: '/mnt/testing',
+          arn: outputMap.AccessPointARN,
+        };
+        config.provider.vpc = {
+          subnetIds: [outputMap.Subnet],
+          securityGroupIds: [outputMap.SecurityGroup],
+        };
+        config.functions.writer.fileSystemConfig = fileSystemConfig;
+        config.functions.reader.fileSystemConfig = fileSystemConfig;
+      },
+    });
+    serviceName = serverlessConfig.service;
+    stackName = `${serviceName}-${stage}`;
+    console.info(`Deploying "${stackName}" service...`);
+    return deployService(tmpDirPath);
+  });
+
+  after(async () => {
+    console.info('Removing service...');
+    await removeService(tmpDirPath);
+    console.info('Removing CloudFormation stack with required resources...');
+    await awsRequest('CloudFormation', 'deleteStack', { StackName: resourcesStackName });
+    return awsRequest('CloudFormation', 'waitFor', 'stackDeleteComplete', {
+      StackName: resourcesStackName,
+    });
+  });
+
+  describe('Basic Setup', () => {
+    let startTime;
+
+    before(() => {
+      startTime = Date.now();
+    });
+
+    it('should be able to write to efs and read from it in a separate function', async function self() {
+      try {
+        await awsRequest('Lambda', 'invoke', {
+          FunctionName: `${stackName}-writer`,
+          InvocationType: 'RequestResponse',
+        });
+      } catch (e) {
+        // Sometimes EFS is not available right away which causes invoke to fail,
+        // here we retry it to avoid that issue
+        if (e.code === 'EFSMountFailureException' && startTime > Date.now() - 1000 * 60) {
+          console.info('Failed to invoke, retry');
+          return self();
+        }
+        throw e;
+      }
+
+      const readerResult = await awsRequest('Lambda', 'invoke', {
+        FunctionName: `${stackName}-reader`,
+        InvocationType: 'RequestResponse',
+      });
+      const payload = JSON.parse(readerResult.Payload);
+
+      expect(payload).to.deep.equal({ result: 'fromlambda' });
+      return null;
+    });
+  });
+});

tests/integration-all/fileSystemConfig/service/serverless.yml

@@ -0,0 +1,12 @@
+service: CHANGE_TO_UNIQUE_PER_RUN
+
+provider:
+  name: aws
+  runtime: nodejs12.x
+  versionFunctions: false
+
+functions:
+  writer:
+    handler: core.writer
+  reader:
+    handler: core.reader