[Improvement] state/* - support multiple engine versions for data sto…

…res (widdix#318)
jalawala · May 22, 2019 · f059fd4 · f059fd4
1 parent e05aeef
commit f059fd4
Show file tree

Hide file tree

Showing 11 changed files with 138 additions and 68 deletions.
diff --git a/fargate/service-cloudmap.yaml b/fargate/service-cloudmap.yaml
@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# FIXME add to cfn-modules
 AWSTemplateFormatVersion: '2010-09-09'
 Description: 'Fargate: service that runs on a Fargate cluster based on fargate/cluster.yaml with service discovery via Cloud Map, a cloudonaut.io template'
 Metadata:

diff --git a/fargate/service-cluster-alb.yaml b/fargate/service-cluster-alb.yaml
@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# FIXME add to cfn-modules
 AWSTemplateFormatVersion: '2010-09-09'
 Description: 'Fargate: service that runs on a Fargate cluster based on fargate/cluster.yaml and uses the cluster ALB, a cloudonaut.io template'
 Metadata:

diff --git a/fargate/service-dedicated-alb.yaml b/fargate/service-dedicated-alb.yaml
@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# FIXME add to cfn-modules
 AWSTemplateFormatVersion: '2010-09-09'
 Description: 'Fargate: service that runs on a Fargate cluster based on fargate/cluster.yaml and uses a dedicated ALB, a cloudonaut.io template'
 Metadata:

diff --git a/state/elasticache-memcached.yaml b/state/elasticache-memcached.yaml
@@ -28,6 +28,7 @@ Metadata:
     - Label:
         default: 'ElastiCache Parameters'
       Parameters:
+      - EngineVersion
       - CacheNodeType
       - NumCacheNodes
       - SubDomainNameWithDot
@@ -54,6 +55,14 @@ Parameters:
     Description: 'Optional but recommended stack name of parent alert stack based on operations/alert.yaml template.'
     Type: String
     Default: ''
+  EngineVersion:
+    Description: 'memcached version'
+    Type: String
+    Default: '1.4.34'
+    AllowedValues: # aws elasticache describe-cache-engine-versions --engine memcached --query "CacheEngineVersions[].EngineVersion"
+    - '1.5.10'
+    - '1.4.5'
+    - '1.4.34'
   CacheNodeType:
     Description: 'The compute and memory capacity of the nodes in the node group (shard).'
     Type: 'String'
@@ -71,6 +80,14 @@ Parameters:
     Type: Number
     Default: 14
     AllowedValues: [1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, 3653]
+Mappings:
+  EngineVersionMap:
+    '1.4.34':
+      CacheParameterGroupFamily: 'memcached1.4'
+    '1.4.5':
+      CacheParameterGroupFamily: 'memcached1.4'
+    '1.5.10':
+      CacheParameterGroupFamily: 'memcached1.5'
 Conditions:
   HasZone: !Not [!Equals [!Ref ParentZoneStack, '']]
   HasSSHBastionSecurityGroup: !Not [!Equals [!Ref ParentSSHBastionStack, '']]
@@ -93,7 +110,7 @@ Resources:
   CacheParameterGroupName:
     Type: 'AWS::ElastiCache::ParameterGroup'
     Properties:
-      CacheParameterGroupFamily: 'memcached1.4'
+      CacheParameterGroupFamily: !FindInMap [EngineVersionMap, !Ref EngineVersion, CacheParameterGroupFamily]
       Description: !Ref 'AWS::StackName'
       Properties: {}
   CacheSubnetGroupName:
@@ -131,7 +148,7 @@ Resources:
       CacheParameterGroupName: !Ref CacheParameterGroupName
       CacheSubnetGroupName: !Ref CacheSubnetGroupName
       Engine: memcached
-      EngineVersion: '1.4.34'
+      EngineVersion: !Ref EngineVersion
       NotificationTopicArn: !If [HasAlertTopic, {'Fn::ImportValue': !Sub '${ParentAlertStack}-TopicARN'}, !Ref 'AWS::NoValue']
       NumCacheNodes: !Ref NumCacheNodes
       PreferredAvailabilityZones: !Split [',', !GetAtt 'PreferredAvailabilityZonesSelector.PreferredAvailabilityZones']

diff --git a/state/elasticache-redis.yaml b/state/elasticache-redis.yaml
@@ -28,8 +28,8 @@ Metadata:
     - Label:
         default: 'ElastiCache Parameters'
       Parameters:
-      - CacheNodeType
       - EngineVersion
+      - CacheNodeType
       - TransitEncryption
       - AuthToken
       - SubDomainName
@@ -53,41 +53,18 @@ Parameters:
     Type: String
     Default: ''
   EngineVersion:
-    Description: 'Version of the redis engine to be used'
+    Description: 'Redis version'
     Type: String
     Default: '5.0.0'
-    AllowedValues:
+    AllowedValues: # aws elasticache describe-cache-engine-versions --engine redis --query "CacheEngineVersions[].EngineVersion"
+    - '5.0.4'
     - '5.0.0'
     - '4.0.10'
     - '3.2.6' # 3.2.4 and 3.2.10 do not support encryption
   CacheNodeType:
     Description: 'The compute and memory capacity of the nodes in the node group (shard).'
     Type: 'String'
     Default: 'cache.t2.micro'
-    AllowedValues:
-    - 'cache.t2.micro'
-    - 'cache.t2.small'
-    - 'cache.t2.medium'
-    - 'cache.m3.medium'
-    - 'cache.m3.large'
-    - 'cache.m3.xlarge'
-    - 'cache.m3.2xlarge'
-    - 'cache.m4.large'
-    - 'cache.m4.xlarge'
-    - 'cache.m4.2xlarge'
-    - 'cache.m4.4xlarge'
-    - 'cache.m4.10xlarge'
-    - 'cache.r3.large'
-    - 'cache.r3.xlarge'
-    - 'cache.r3.2xlarge'
-    - 'cache.r3.4xlarge'
-    - 'cache.r3.8xlarge'
-    - 'cache.r4.large'
-    - 'cache.r4.xlarge'
-    - 'cache.r4.2xlarge'
-    - 'cache.r4.4xlarge'
-    - 'cache.r4.8xlarge'
-    - 'cache.r4.16xlarge'
   TransitEncryption:
     Description: 'Enable encryption for data in transit? When transit encryption is enabled also specify an auth token.'
     Type: 'String'
@@ -105,13 +82,15 @@ Parameters:
     Type: String
     Default: redis
 Mappings:
-  CacheParameterGroupFamilyMap:
-    '5.0.0':
-      redis: 'redis5.0'
-    '4.0.10':
-      redis: 'redis4.0'
+  EngineVersionMap:
     '3.2.6':
-      redis: 'redis3.2'
+      CacheParameterGroupFamily: 'redis3.2'
+    '4.0.10':
+      CacheParameterGroupFamily: 'redis4.0'
+    '5.0.0':
+      CacheParameterGroupFamily: 'redis5.0'
+    '5.0.4':
+      CacheParameterGroupFamily: 'redis5.0'
 Conditions:
   HasZone: !Not [!Equals [!Ref ParentZoneStack, '']]
   HasSSHBastionSecurityGroup: !Not [!Equals [!Ref ParentSSHBastionStack, '']]
@@ -136,7 +115,7 @@ Resources:
   CacheParameterGroup:
     Type: 'AWS::ElastiCache::ParameterGroup'
     Properties:
-      CacheParameterGroupFamily: !FindInMap [CacheParameterGroupFamilyMap, !Ref EngineVersion, redis]
+      CacheParameterGroupFamily: !FindInMap [EngineVersionMap, !Ref EngineVersion, CacheParameterGroupFamily]
       Description: !Ref 'AWS::StackName'
       Properties: {}
   CacheSubnetGroupName:
@@ -180,7 +159,7 @@ Resources:
       CacheNodeType: !Ref CacheNodeType
       CacheParameterGroupName: !Ref CacheParameterGroup
       CacheSubnetGroupName: !Ref CacheSubnetGroupName
-      Engine: 'redis'
+      Engine: redis
       EngineVersion: !Ref EngineVersion
       NotificationTopicArn: !If [HasAlertTopic, {'Fn::ImportValue': !Sub '${ParentAlertStack}-TopicARN'}, !Ref 'AWS::NoValue']
       NumNodeGroups: 1

diff --git a/state/elasticsearch.yaml b/state/elasticsearch.yaml
@@ -30,6 +30,7 @@ Metadata:
         default: 'Elasticsearch Parameters'
       Parameters:
       - DomainName
+      - ElasticsearchVersion
       - EBSEnabled
       - EBSVolumeSize
       - ClusterInstanceCount
@@ -72,6 +73,11 @@ Parameters:
     Description: 'The size of the EBS volume for each data node. The minimum and maximum size of an EBS volume depends on the EBS volume type and the instance type to which it is attached.'
     Type: Number
     Default: 10
+  ElasticsearchVersion:
+    Description: 'Elasticsearch version'
+    Type: String
+    Default: '5.5'
+    AllowedValues: ['6.5', '6.4', '6.3', '6.2', '6.0', '5.6', '5.5'] # aws es list-elasticsearch-versions --query "ElasticsearchVersions[]"
   ClusterInstanceCount:
     Description: 'The number of data nodes (instances) to use in the Amazon ES domain.'
     Type: Number
@@ -157,7 +163,7 @@ Resources:
         InstanceCount: !Ref ClusterInstanceCount
         InstanceType: !Ref ClusterInstanceType
         ZoneAwarenessEnabled: !If [HasSingleClusterInstance, false, true]
-      ElasticsearchVersion: '5.5'
+      ElasticsearchVersion: !Ref ElasticsearchVersion
       EncryptionAtRestOptions: !If [HasKmsKey, {Enabled: true, KmsKeyId: {'Fn::ImportValue': !Sub '${ParentKmsKeyStack}-KeyId'}}, !Ref 'AWS::NoValue']
       SnapshotOptions:
         AutomatedSnapshotStartHour: 10

diff --git a/state/rds-aurora-serverless.yaml b/state/rds-aurora-serverless.yaml
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 AWSTemplateFormatVersion: '2010-09-09'
-Description: 'State: RDS Aurora Serverless, a cloudonaut.io template, sponsored by https://github.com/ngault'
+Description: 'State: RDS Aurora Serverless MySQL, a cloudonaut.io template, sponsored by https://github.com/ngault'
 Metadata:
   'AWS::CloudFormation::Interface':
     ParameterGroups:
@@ -29,7 +29,7 @@ Metadata:
     - Label:
         default: 'RDS Parameters'
       Parameters:
-      - Engine
+      - EngineVersion
       - DBSnapshotIdentifier
       - DBName
       - DBBackupRetentionPeriod
@@ -69,10 +69,9 @@ Parameters:
     Type: String
     Default: ''
   Engine:
-    Description: 'The name of the database engine that you want to use for this DB cluster.'
+    Description: 'DEPRECATED since v9, removed in v11, value is ignored!'
     Type: String
-    AllowedValues:
-    - aurora # MySQL 5.6-compatible Aurora
+    AllowedValues: [aurora]
     Default: aurora
   DBSnapshotIdentifier:
     Description: 'Optional identifier for the DB cluster snapshot from which you want to restore (leave blank to create an empty cluster).'
@@ -130,11 +129,14 @@ Parameters:
     MinValue: 1
     MaxValue: 86400
     Default: 300
+  EngineVersion:
+    Description: 'Aurora Serverless MySQL version.'
+    Type: String
+    Default: '5.6.10a'
+    AllowedValues: ['5.6.10a'] # aws rds describe-db-engine-versions --engine aurora --query 'DBEngineVersions[?contains(SupportedEngineModes,`serverless`)]'
 Mappings:
-  EngineMap:
-    aurora:
-      EngineVersion: '5.6.10a'
-      Port: 3306
+  EngineVersionMap:
+    '5.6.10a':
       ClusterParameterGroupFamily: 'aurora5.6'
 Conditions:
   HasZone: !Not [!Equals [!Ref ParentZoneStack, '']]
@@ -162,8 +164,8 @@ Resources:
       GroupDescription: !Ref 'AWS::StackName'
       SecurityGroupIngress:
       - IpProtocol: tcp
-        FromPort: !FindInMap [EngineMap, !Ref Engine, Port]
-        ToPort: !FindInMap [EngineMap, !Ref Engine, Port]
+        FromPort: 3306
+        ToPort: 3306
         SourceSecurityGroupId: {'Fn::ImportValue': !Sub '${ParentClientStack}-ClientSecurityGroup'}
       VpcId: {'Fn::ImportValue': !Sub '${ParentVPCStack}-VPC'}
   ClusterSecurityGroupInSSHBastion:
@@ -172,8 +174,8 @@ Resources:
     Properties:
       GroupId: !Ref ClusterSecurityGroup
       IpProtocol: tcp
-      FromPort: !FindInMap [EngineMap, !Ref Engine, Port]
-      ToPort: !FindInMap [EngineMap, !Ref Engine, Port]
+      FromPort: 3306
+      ToPort: 3306
       SourceSecurityGroupId: {'Fn::ImportValue': !Sub '${ParentSSHBastionStack}-SecurityGroup'}
   DBSubnetGroup:
     Type: 'AWS::RDS::DBSubnetGroup'
@@ -184,7 +186,7 @@ Resources:
     Type: 'AWS::RDS::DBClusterParameterGroup'
     Properties:
       Description: !Ref 'AWS::StackName'
-      Family: !FindInMap [EngineMap, !Ref Engine, ClusterParameterGroupFamily]
+      Family: !FindInMap [EngineVersionMap, !Ref EngineVersion, ClusterParameterGroupFamily]
       Parameters:
         character_set_client: utf8
         character_set_connection: utf8
@@ -203,13 +205,12 @@ Resources:
       DatabaseName: !If [HasDBSnapshotIdentifier, !Ref 'AWS::NoValue', !Ref DBName]
       DBClusterParameterGroupName: !Ref DBClusterParameterGroup
       DBSubnetGroupName: !Ref DBSubnetGroup
-      Engine: !Ref Engine
+      Engine: aurora
       EngineMode: serverless
-      EngineVersion: !FindInMap [EngineMap, !Ref Engine, EngineVersion]
+      EngineVersion: !Ref EngineVersion
       KmsKeyId: !If [HasDBSnapshotIdentifier, !Ref 'AWS::NoValue', {'Fn::ImportValue': !Sub '${ParentKmsKeyStack}-KeyArn'}]
       MasterUsername: !If [HasDBSnapshotIdentifier, !Ref 'AWS::NoValue', !Ref DBMasterUsername]
       MasterUserPassword: !If [HasDBSnapshotIdentifier, !Ref 'AWS::NoValue', !Ref DBMasterUserPassword]
-      # Port: !FindInMap [EngineMap, !Ref Engine, Port] TODO re-enable as soon as CloudFormation bug ix fixed
       # PreferredBackupWindow: !Ref PreferredBackupWindow TODO re-enable as soon as CloudFormation bug ix fixed
       # PreferredMaintenanceWindow: !Ref PreferredMaintenanceWindow TODO re-enable as soon as CloudFormation bug ix fixed
       ScalingConfiguration: