Skip to content

Commit

Permalink
[New Template] vpc/vpc-flow-logs-s3 - Deliver VPC Flow Logs to S3 (wi…
Browse files Browse the repository at this point in the history
  • Loading branch information
michaelwittig authored Aug 13, 2019
1 parent e6d3049 commit bf594b6
Show file tree
Hide file tree
Showing 5 changed files with 224 additions and 35 deletions.
1 change: 1 addition & 0 deletions docs/state.md
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,7 @@ S3 bucket with different access requirements:
| ConfigWrite | Allow Config to store data in this bucket. |
| CloudTrailWrite | Allow CloudTrail to store data in this bucket. |
| VpcEndpointRead | Allow reads from requests coming over a specific VPC endpoint (see `ParentVpcEndpointStack` parameter) |
| FlowLogWrite | Allow VPC Flow Logs to store data in this bucket. |

## Installation Guide
1. [![Launch Stack](./img/launch-stack.png)](https://console.aws.amazon.com/cloudformation/home#/stacks/create/review?templateURL=https://s3-eu-west-1.amazonaws.com/widdix-aws-cf-templates-releases-eu-west-1/__VERSION__/state/s3.yaml&stackName=s3)
Expand Down
89 changes: 70 additions & 19 deletions state/s3.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ Parameters:
Description: 'Access policy of the bucket.'
Type: String
Default: Private
AllowedValues: [Private, PublicRead, CloudFrontRead, CloudFrontAccessLogWrite, ElbAccessLogWrite, ConfigWrite, CloudTrailWrite, VpcEndpointRead]
AllowedValues: [Private, PublicRead, CloudFrontRead, CloudFrontAccessLogWrite, ElbAccessLogWrite, ConfigWrite, CloudTrailWrite, VpcEndpointRead, FlowLogWrite]
Versioning:
Description: 'Enable versioning to keep a backup if objects change.'
Type: String
Expand All @@ -78,12 +78,16 @@ Conditions:
HasConfigWriteAccess: !Equals [!Ref Access, ConfigWrite]
HasCloudTrailWriteAccess: !Equals [!Ref Access, CloudTrailWrite]
HasVpcEndpointReadAccess: !Equals [!Ref Access, VpcEndpointRead]
HasFlowLogWriteAccess: !Equals [!Ref Access, FlowLogWrite]
HasBucketName: !Not [!Equals [!Ref BucketName, '']]
HasVersioning: !Equals [!Ref Versioning, true]
HadVersioning: !Equals [!Ref Versioning, 'false-but-was-true']
HasNoncurrentVersionExpirationInDays: !Not [!Equals [!Ref NoncurrentVersionExpirationInDays, 0]]
HasExpirationInDays: !Not [!Equals [!Ref ExpirationInDays, 0]]
HasPrivateAccessAndKmsKey: !And [!Condition HasPrivateAccess, !Condition HasKmsKey]
HasPartitionPublic: !Equals [!Ref 'AWS::Partition', 'aws']
HasPartitionChina: !Equals [!Ref 'AWS::Partition', 'aws-cn']
HasPartitionUsGov: !Equals [!Ref 'AWS::Partition', 'aws-us-gov']
Resources:
Bucket: # cannot be deleted with data
Type: 'AWS::S3::Bucket'
Expand Down Expand Up @@ -112,6 +116,7 @@ Resources:
Properties:
Bucket: !Ref Bucket
PolicyDocument:
Version: '2012-10-17'
Statement:
- Principal: '*'
Action: 's3:PutObject*'
Expand All @@ -127,6 +132,7 @@ Resources:
Properties:
Bucket: !Ref Bucket
PolicyDocument:
Version: '2012-10-17'
Statement:
- Principal: '*'
Action: 's3:GetObject'
Expand All @@ -138,6 +144,7 @@ Resources:
Properties:
Bucket: !Ref Bucket
PolicyDocument:
Version: '2012-10-17'
Statement:
- Principal:
CanonicalUser: !GetAtt CloudFrontOriginAccessIdentity.S3CanonicalUserId
Expand All @@ -150,35 +157,54 @@ Resources:
Properties:
Bucket: !Ref Bucket
PolicyDocument:
Version: '2012-10-17'
Statement:
- Principal:
AWS: # https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-access-logs.html#access-logging-bucket-permissions
- 'arn:aws:iam::127311923021:root' # us-east-1
- 'arn:aws:iam::033677994240:root' # us-east-2
- 'arn:aws:iam::027434742980:root' # us-west-1
- 'arn:aws:iam::797873946194:root' # us-west-2
- 'arn:aws:iam::985666609251:root' # ca-central-1
- 'arn:aws:iam::054676820928:root' # eu-central-1
- 'arn:aws:iam::156460612806:root' # eu-west-1
- 'arn:aws:iam::652711504416:root' # eu-west-2
- 'arn:aws:iam::009996457667:root' # eu-west-3
- 'arn:aws:iam::897822967062:root' # eu-north-1
- 'arn:aws:iam::582318560864:root' # ap-northeast-1
- 'arn:aws:iam::600734575887:root' # ap-northeast-2
- 'arn:aws:iam::383597477331:root' # ap-northeast-3
- 'arn:aws:iam::114774131450:root' # ap-southeast-1
- 'arn:aws:iam::783225319266:root' # ap-southeast-2
- 'arn:aws:iam::718504428378:root' # ap-south-1
- 'arn:aws:iam::507241528517:root' # sa-east-1
- !If [HasPartitionPublic, 'arn:aws:iam::127311923021:root', !Ref 'AWS::NoValue'] # us-east-1
- !If [HasPartitionPublic, 'arn:aws:iam::033677994240:root', !Ref 'AWS::NoValue'] # us-east-2
- !If [HasPartitionPublic, 'arn:aws:iam::027434742980:root', !Ref 'AWS::NoValue'] # us-west-1
- !If [HasPartitionPublic, 'arn:aws:iam::797873946194:root', !Ref 'AWS::NoValue'] # us-west-2
- !If [HasPartitionPublic, 'arn:aws:iam::985666609251:root', !Ref 'AWS::NoValue'] # ca-central-1
- !If [HasPartitionPublic, 'arn:aws:iam::054676820928:root', !Ref 'AWS::NoValue'] # eu-central-1
- !If [HasPartitionPublic, 'arn:aws:iam::156460612806:root', !Ref 'AWS::NoValue'] # eu-west-1
- !If [HasPartitionPublic, 'arn:aws:iam::652711504416:root', !Ref 'AWS::NoValue'] # eu-west-2
- !If [HasPartitionPublic, 'arn:aws:iam::009996457667:root', !Ref 'AWS::NoValue'] # eu-west-3
- !If [HasPartitionPublic, 'arn:aws:iam::897822967062:root', !Ref 'AWS::NoValue'] # eu-north-1
- !If [HasPartitionPublic, 'arn:aws:iam::582318560864:root', !Ref 'AWS::NoValue'] # ap-northeast-1
- !If [HasPartitionPublic, 'arn:aws:iam::600734575887:root', !Ref 'AWS::NoValue'] # ap-northeast-2
- !If [HasPartitionPublic, 'arn:aws:iam::383597477331:root', !Ref 'AWS::NoValue'] # ap-northeast-3
- !If [HasPartitionPublic, 'arn:aws:iam::114774131450:root', !Ref 'AWS::NoValue'] # ap-southeast-1
- !If [HasPartitionPublic, 'arn:aws:iam::783225319266:root', !Ref 'AWS::NoValue'] # ap-southeast-2
- !If [HasPartitionPublic, 'arn:aws:iam::718504428378:root', !Ref 'AWS::NoValue'] # ap-south-1
- !If [HasPartitionPublic, 'arn:aws:iam::507241528517:root', !Ref 'AWS::NoValue'] # sa-east-1
- !If [HasPartitionUsGov, 'arn:aws-us-gov:iam::048591011584:root', !Ref 'AWS::NoValue'] # us-gov-west-1*
- !If [HasPartitionUsGov, 'arn:aws-us-gov:iam::190560391635:root', !Ref 'AWS::NoValue'] # us-gov-east-1*
- !If [HasPartitionChina, 'arn:aws-cn:iam::638102146993:root', !Ref 'AWS::NoValue'] # cn-north-1*
- !If [HasPartitionChina, 'arn:aws-cn:iam::037604701340:root', !Ref 'AWS::NoValue'] # cn-northwest-1*
Action: 's3:PutObject'
Effect: Allow
Resource: !Sub '${Bucket.Arn}/*AWSLogs/${AWS::AccountId}/*'
- Principal:
Service: 'delivery.logs.amazonaws.com' # https://docs.aws.amazon.com/elasticloadbalancing/latest/network/load-balancer-access-logs.html
Action: 's3:PutObject'
Effect: Allow
Resource: !Sub '${Bucket.Arn}/*AWSLogs/${AWS::AccountId}/*'
Condition:
StringEquals:
's3:x-amz-acl': 'bucket-owner-full-control'
- Principal:
Service: 'delivery.logs.amazonaws.com' # https://docs.aws.amazon.com/elasticloadbalancing/latest/network/load-balancer-access-logs.html
Action: 's3:GetBucketAcl'
Effect: Allow
Resource: !GetAtt 'Bucket.Arn'
BucketPolicyConfigWrite:
Condition: HasConfigWriteAccess
Type: 'AWS::S3::BucketPolicy'
Properties:
Bucket: !Ref Bucket
PolicyDocument:
Version: '2012-10-17'
Statement:
- Effect: Allow
Principal:
Expand All @@ -189,7 +215,7 @@ Resources:
Principal:
Service: 'config.amazonaws.com'
Action: 's3:PutObject'
Resource: !Sub '${Bucket.Arn}/*AWSLogs/${AWS::AccountId}/Config/*'
Resource: !Sub '${Bucket.Arn}/*AWSLogs/${AWS::AccountId}/*'
Condition:
StringEquals:
's3:x-amz-acl': 'bucket-owner-full-control'
Expand All @@ -199,6 +225,7 @@ Resources:
Properties:
Bucket: !Ref Bucket
PolicyDocument:
Version: '2012-10-17'
Statement:
- Effect: Allow
Principal:
Expand All @@ -219,6 +246,7 @@ Resources:
Properties:
Bucket: !Ref Bucket
PolicyDocument:
Version: '2012-10-17'
Statement:
- Principal: '*'
Action: 's3:GetObject'
Expand All @@ -227,6 +255,29 @@ Resources:
Condition:
StringEquals:
'aws:sourceVpce': {'Fn::ImportValue': !Sub '${ParentVpcEndpointStack}-EndpointS3'}
BucketPolicyFlowLogWrite:
Condition: HasFlowLogWriteAccess
Type: 'AWS::S3::BucketPolicy'
Properties:
Bucket: !Ref Bucket
PolicyDocument:
Version: '2012-10-17'
Statement: # https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs-s3.html#flow-logs-s3-permissions
- Sid: AWSLogDeliveryWrite
Effect: Allow
Principal:
Service: 'delivery.logs.amazonaws.com'
Action: 's3:PutObject'
Resource: !Sub '${Bucket.Arn}/*AWSLogs/${AWS::AccountId}/*'
Condition:
StringEquals:
's3:x-amz-acl': 'bucket-owner-full-control'
- Sid: AWSLogDeliveryAclCheck
Effect: Allow
Principal:
Service: 'delivery.logs.amazonaws.com'
Action: 's3:GetBucketAcl'
Resource: !GetAtt 'Bucket.Arn'
CloudFrontOriginAccessIdentity:
Condition: HasCloudFrontReadAccess
Type: 'AWS::CloudFront::CloudFrontOriginAccessIdentity'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,28 +11,22 @@ public void test() {
final String vpcStackName = "vpc-2azs-" + this.random8String();
final String flowLogsStackName = "vpc-flow-logs-" + this.random8String();
final String classB = "10";
final String keyName = "key-" + this.random8String();
try {
this.createKey(keyName);
this.createStack(vpcStackName,
"vpc/vpc-2azs.yaml",
new Parameter().withParameterKey("ClassB").withParameterValue(classB)
);
try {
this.createStack(vpcStackName,
"vpc/vpc-2azs.yaml",
new Parameter().withParameterKey("ClassB").withParameterValue(classB)
this.createStack(flowLogsStackName,
"vpc/vpc-flow-logs.yaml",
new Parameter().withParameterKey("ParentVPCStack").withParameterValue(vpcStackName)
);
try {
this.createStack(flowLogsStackName,
"vpc/vpc-flow-logs.yaml",
new Parameter().withParameterKey("ParentVPCStack").withParameterValue(vpcStackName)
);
// TODO how can we check if this stack works?
} finally {
this.deleteStack(flowLogsStackName);
}
// TODO how can we check if this stack works?
} finally {
this.deleteStack(vpcStackName);
this.deleteStack(flowLogsStackName);
}
} finally {
this.deleteKey(keyName);
this.deleteStack(vpcStackName);
}
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package de.widdix.awscftemplates.vpc;

import com.amazonaws.services.cloudformation.model.Parameter;
import de.widdix.awscftemplates.ACloudFormationTest;
import org.junit.Test;

public class TestVPCFlowLogsS3 extends ACloudFormationTest {

@Test
public void test() {
final String vpcStackName = "vpc-2azs-" + this.random8String();
final String flowLogsStackName = "vpc-flow-logs-s3-" + this.random8String();
final String classB = "10";
try {
this.createStack(vpcStackName,
"vpc/vpc-2azs.yaml",
new Parameter().withParameterKey("ClassB").withParameterValue(classB)
);
try {
this.createStack(flowLogsStackName,
"vpc/vpc-flow-logs-s3.yaml",
new Parameter().withParameterKey("ParentVPCStack").withParameterValue(vpcStackName)
);
// TODO how can we check if this stack works?
} finally {
this.deleteStack(flowLogsStackName);
}
} finally {
this.deleteStack(vpcStackName);
}
}

}
110 changes: 110 additions & 0 deletions vpc/vpc-flow-logs-s3.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
---
# Copyright 2018 widdix GmbH
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
AWSTemplateFormatVersion: '2010-09-09'
Description: 'VPC: Publish flow logs to S3, a cloudonaut.io template'
Metadata:
'AWS::CloudFormation::Interface':
ParameterGroups:
- Label:
default: 'Parent Stacks'
Parameters:
- ParentVPCStack
- Label:
default: 'Flow Logs Parameters'
Parameters:
- ExternalLogBucket
- LogFilePrefix
- TrafficType
Parameters:
ParentVPCStack:
Description: 'Stack name of parent VPC stack based on vpc/vpc-*azs.yaml template.'
Type: String
ExternalLogBucket:
Description: 'Optional The name of an S3 bucket where you want to store flow logs. If you leave this empty, the Amazon S3 bucket is created for you.'
Type: String
Default: ''
LogFilePrefix:
Description: 'Optional The log file prefix.'
Type: String
Default: ''
TrafficType:
Description: 'The type of traffic to log.'
Type: String
Default: REJECT
AllowedValues:
- ACCEPT
- REJECT
- ALL
Conditions:
InternalBucket: !Equals [!Ref ExternalLogBucket, '']
ExternalBucket: !Not [!Equals [!Ref ExternalLogBucket, '']]
HasLogFilePrefix: !Not [!Equals [!Ref LogFilePrefix, '']]
Resources:
LogBucket:
Condition: InternalBucket
Type: 'AWS::S3::Bucket'
Properties: {}
LogBucketPolicy:
Condition: InternalBucket
Type: 'AWS::S3::BucketPolicy'
Properties:
Bucket: !Ref LogBucket
PolicyDocument:
Version: '2012-10-17'
Statement: # https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs-s3.html#flow-logs-s3-permissions
- Sid: AWSLogDeliveryWrite
Effect: Allow
Principal:
Service: 'delivery.logs.amazonaws.com'
Action: 's3:PutObject'
Resource: !If [HasLogFilePrefix, !Sub '${LogBucket.Arn}/${LogFilePrefix}/AWSLogs/${AWS::AccountId}/*', !Sub '${LogBucket.Arn}/AWSLogs/${AWS::AccountId}/*']
Condition:
StringEquals:
's3:x-amz-acl': 'bucket-owner-full-control'
- Sid: AWSLogDeliveryAclCheck
Effect: Allow
Principal:
Service: 'delivery.logs.amazonaws.com'
Action: 's3:GetBucketAcl'
Resource: !GetAtt 'LogBucket.Arn'
FlowLogInternalBucket:
Condition: InternalBucket
DependsOn: LogBucketPolicy
Type: 'AWS::EC2::FlowLog'
Properties:
LogDestination: !If [HasLogFilePrefix, !Sub '${LogBucket.Arn}/${LogFilePrefix}/', !GetAtt 'LogBucket.Arn']
LogDestinationType: s3
ResourceId: {'Fn::ImportValue': !Sub '${ParentVPCStack}-VPC'}
ResourceType: 'VPC'
TrafficType: !Ref TrafficType
FlowLogExternalBucket:
Condition: ExternalBucket
Type: 'AWS::EC2::FlowLog'
Properties:
LogDestination: !If [HasLogFilePrefix, !Sub 'arn:aws:s3:::${ExternalLogBucket}/${LogFilePrefix}/', !Sub 'arn:aws:s3:::${ExternalLogBucket}']
LogDestinationType: s3
ResourceId: {'Fn::ImportValue': !Sub '${ParentVPCStack}-VPC'}
ResourceType: 'VPC'
TrafficType: !Ref TrafficType
Outputs:
TemplateID:
Description: 'cloudonaut.io template id.'
Value: 'vpc/vpc-flow-logs-s3'
TemplateVersion:
Description: 'cloudonaut.io template version.'
Value: '__VERSION__'
StackName:
Description: 'Stack name.'
Value: !Sub '${AWS::StackName}'

0 comments on commit bf594b6

Please sign in to comment.