Issue 311: now supporting ec2 cluster instance sizes and placement gr…

…oups
francescomapelli · Jul 18, 2010 · d626a98 · d626a98
1 parent cd5fddf
commit d626a98
Show file tree

Hide file tree

Showing 82 changed files with 2,866 additions and 1,623 deletions.
diff --git a/aws/core/src/main/java/org/jclouds/aws/ec2/EC2AsyncClient.java b/aws/core/src/main/java/org/jclouds/aws/ec2/EC2AsyncClient.java
@@ -25,6 +25,7 @@
 import org.jclouds.aws.ec2.services.InstanceAsyncClient;
 import org.jclouds.aws.ec2.services.KeyPairAsyncClient;
 import org.jclouds.aws.ec2.services.MonitoringAsyncClient;
+import org.jclouds.aws.ec2.services.PlacementGroupAsyncClient;
 import org.jclouds.aws.ec2.services.SecurityGroupAsyncClient;
 import org.jclouds.aws.ec2.services.WindowsAsyncClient;
 import org.jclouds.rest.annotations.Delegate;
@@ -67,6 +68,12 @@ public interface EC2AsyncClient {
    @Delegate
    SecurityGroupAsyncClient getSecurityGroupServices();
 
+   /**
+    * Provides asynchronous access to PlacementGroup services.
+    */
+   @Delegate
+   PlacementGroupAsyncClient getPlacementGroupServices();
+
    /**
     * Provides asynchronous access to Monitoring services.
     */

diff --git a/aws/core/src/main/java/org/jclouds/aws/ec2/EC2Client.java b/aws/core/src/main/java/org/jclouds/aws/ec2/EC2Client.java
@@ -27,6 +27,7 @@
 import org.jclouds.aws.ec2.services.InstanceClient;
 import org.jclouds.aws.ec2.services.KeyPairClient;
 import org.jclouds.aws.ec2.services.MonitoringClient;
+import org.jclouds.aws.ec2.services.PlacementGroupClient;
 import org.jclouds.aws.ec2.services.SecurityGroupClient;
 import org.jclouds.aws.ec2.services.WindowsClient;
 import org.jclouds.concurrent.Timeout;
@@ -69,6 +70,12 @@ public interface EC2Client {
    @Delegate
    SecurityGroupClient getSecurityGroupServices();
 
+   /**
+    * Provides synchronous access to PlacementGroup services.
+    */
+   @Delegate
+   PlacementGroupClient getPlacementGroupServices();
+
    /**
     * Provides synchronous access to Monitoring services.
     */

diff --git a/aws/core/src/main/java/org/jclouds/aws/ec2/EC2PropertiesBuilder.java b/aws/core/src/main/java/org/jclouds/aws/ec2/EC2PropertiesBuilder.java
@@ -43,18 +43,16 @@ protected Properties defaultProperties() {
       properties.setProperty(PROPERTY_HEADER_TAG, "amz");
       properties.setProperty(PROPERTY_ENDPOINT, "https://ec2.us-east-1.amazonaws.com");
       properties.setProperty(PROPERTY_API_VERSION, EC2AsyncClient.VERSION);
-      properties.setProperty(PROPERTY_ELB_ENDPOINT,
-               "https://elasticloadbalancing.us-east-1.amazonaws.com");
+      properties.setProperty(PROPERTY_ELB_ENDPOINT, "https://elasticloadbalancing.us-east-1.amazonaws.com");
       // alestic, canonical, and rightscale
       properties.setProperty(PROPERTY_EC2_AMI_OWNERS, "063491364108,099720109477,411009282317");
       // amis that work with the cluster instances
       properties.setProperty(PROPERTY_EC2_CC_AMIs, "us-east-1/ami-7ea24a17");
       // auth fail sometimes happens in EC2, as the rc.local script that injects the
       // authorized key executes after ssh has started
-      properties.setProperty("jclouds.ssh.max_retries", "6");
-      properties
-               .setProperty("jclouds.ssh.retryable_messages",
-                        "Auth fail,invalid data,End of IO Stream Read,Connection reset,socket is not established");
+      properties.setProperty("jclouds.ssh.max_retries", "7");
+      properties.setProperty("jclouds.ssh.retryable_messages",
+               "Auth fail,invalid data,End of IO Stream Read,Connection reset,socket is not established");
       return properties;
    }
 

diff --git a/aws/core/src/main/java/org/jclouds/aws/ec2/compute/EC2ComputeService.java b/aws/core/src/main/java/org/jclouds/aws/ec2/compute/EC2ComputeService.java
@@ -18,6 +18,7 @@
  */
 package org.jclouds.aws.ec2.compute;
 
+import static com.google.common.base.Preconditions.checkState;
 import static org.jclouds.aws.ec2.util.EC2Utils.parseHandle;
 import static org.jclouds.util.Utils.checkNotEmpty;
 
@@ -32,11 +33,14 @@
 import javax.inject.Singleton;
 
 import org.jclouds.Constants;
+import org.jclouds.aws.AWSResponseException;
 import org.jclouds.aws.ec2.EC2Client;
 import org.jclouds.aws.ec2.compute.domain.RegionAndName;
 import org.jclouds.aws.ec2.compute.domain.RegionNameAndIngressRules;
 import org.jclouds.aws.ec2.compute.options.EC2TemplateOptions;
 import org.jclouds.aws.ec2.domain.KeyPair;
+import org.jclouds.aws.ec2.domain.PlacementGroup;
+import org.jclouds.aws.ec2.domain.PlacementGroup.State;
 import org.jclouds.compute.ComputeServiceContext;
 import org.jclouds.compute.domain.Image;
 import org.jclouds.compute.domain.NodeMetadata;
@@ -52,6 +56,7 @@
 import org.jclouds.compute.util.ComputeUtils;
 import org.jclouds.domain.Location;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Predicate;
 import com.google.common.collect.Maps;
 
@@ -63,82 +68,96 @@ public class EC2ComputeService extends BaseComputeService {
    private final EC2Client ec2Client;
    private final Map<RegionAndName, KeyPair> credentialsMap;
    private final Map<RegionAndName, String> securityGroupMap;
+   private final Map<RegionAndName, String> placementGroupMap;
+   private final Predicate<PlacementGroup> placementGroupDeleted;
 
    @Inject
-   protected EC2ComputeService(ComputeServiceContext context,
-         Provider<Set<? extends Image>> images,
-         Provider<Set<? extends Size>> sizes,
-         Provider<Set<? extends Location>> locations,
-         ListNodesStrategy listNodesStrategy,
-         GetNodeMetadataStrategy getNodeMetadataStrategy,
-         RunNodesAndAddToSetStrategy runNodesAndAddToSetStrategy,
-         RebootNodeStrategy rebootNodeStrategy,
-         DestroyNodeStrategy destroyNodeStrategy,
-         Provider<TemplateBuilder> templateBuilderProvider,
-         Provider<TemplateOptions> templateOptionsProvider,
-         @Named("NODE_RUNNING") Predicate<NodeMetadata> nodeRunning,
-         @Named("NODE_TERMINATED") Predicate<NodeMetadata> nodeTerminated,
-         ComputeUtils utils,
-         @Named(Constants.PROPERTY_USER_THREADS) ExecutorService executor,
-         EC2Client ec2Client, Map<RegionAndName, KeyPair> credentialsMap,
-         Map<RegionAndName, String> securityGroupMap) {
-      super(context, images, sizes, locations, listNodesStrategy,
-            getNodeMetadataStrategy, runNodesAndAddToSetStrategy,
-            rebootNodeStrategy, destroyNodeStrategy, templateBuilderProvider,
-            templateOptionsProvider, nodeRunning, nodeTerminated, utils,
-            executor);
+   protected EC2ComputeService(ComputeServiceContext context, Provider<Set<? extends Image>> images,
+            Provider<Set<? extends Size>> sizes, Provider<Set<? extends Location>> locations,
+            ListNodesStrategy listNodesStrategy, GetNodeMetadataStrategy getNodeMetadataStrategy,
+            RunNodesAndAddToSetStrategy runNodesAndAddToSetStrategy, RebootNodeStrategy rebootNodeStrategy,
+            DestroyNodeStrategy destroyNodeStrategy, Provider<TemplateBuilder> templateBuilderProvider,
+            Provider<TemplateOptions> templateOptionsProvider,
+            @Named("NODE_RUNNING") Predicate<NodeMetadata> nodeRunning,
+            @Named("NODE_TERMINATED") Predicate<NodeMetadata> nodeTerminated, ComputeUtils utils,
+            @Named(Constants.PROPERTY_USER_THREADS) ExecutorService executor, EC2Client ec2Client,
+            Map<RegionAndName, KeyPair> credentialsMap, @Named("SECURITY") Map<RegionAndName, String> securityGroupMap,
+            @Named("PLACEMENT") Map<RegionAndName, String> placementGroupMap,
+            @Named("DELETED") Predicate<PlacementGroup> placementGroupDeleted) {
+      super(context, images, sizes, locations, listNodesStrategy, getNodeMetadataStrategy, runNodesAndAddToSetStrategy,
+               rebootNodeStrategy, destroyNodeStrategy, templateBuilderProvider, templateOptionsProvider, nodeRunning,
+               nodeTerminated, utils, executor);
       this.ec2Client = ec2Client;
       this.credentialsMap = credentialsMap;
       this.securityGroupMap = securityGroupMap;
+      this.placementGroupMap = placementGroupMap;
+      this.placementGroupDeleted = placementGroupDeleted;
    }
 
-   private void deleteSecurityGroup(String region, String tag) {
+   @VisibleForTesting
+   void deletePlacementGroup(String region, String tag) {
       checkNotEmpty(tag, "tag");
-      String group = "jclouds#" + tag;
-      if (ec2Client.getSecurityGroupServices().describeSecurityGroupsInRegion(
-            region, group).size() > 0) {
+      String group = String.format("jclouds#%s#%s", tag, region);
+      if (ec2Client.getPlacementGroupServices().describePlacementGroupsInRegion(region, group).size() > 0) {
+         logger.debug(">> deleting placementGroup(%s)", group);
+         try {
+            ec2Client.getPlacementGroupServices().deletePlacementGroupInRegion(region, group);
+            checkState(placementGroupDeleted.apply(new PlacementGroup(region, group, "cluster", State.PENDING)), String
+                     .format("placementGroup region(%s) name(%s) failed to delete", region, group));
+            placementGroupMap.remove(new RegionAndName(region, tag));
+            logger.debug("<< deleted placementGroup(%s)", group);
+         } catch (AWSResponseException e) {
+            if (e.getError().getCode().equals("InvalidPlacementGroup.InUse")) {
+               logger.debug("<< inUse placementGroup(%s)", group);
+            } else {
+               throw e;
+            }
+         }
+      }
+   }
+
+   @VisibleForTesting
+   void deleteSecurityGroup(String region, String tag) {
+      checkNotEmpty(tag, "tag");
+      String group = String.format("jclouds#%s#%s", tag, region);
+      if (ec2Client.getSecurityGroupServices().describeSecurityGroupsInRegion(region, group).size() > 0) {
          logger.debug(">> deleting securityGroup(%s)", group);
-         ec2Client.getSecurityGroupServices().deleteSecurityGroupInRegion(
-               region, group);
+         ec2Client.getSecurityGroupServices().deleteSecurityGroupInRegion(region, group);
          // TODO: test this clear happens
-         securityGroupMap.remove(new RegionNameAndIngressRules(region, tag,
-               null, false));
+         securityGroupMap.remove(new RegionNameAndIngressRules(region, tag, null, false));
          logger.debug("<< deleted securityGroup(%s)", group);
       }
    }
 
-   private void deleteKeyPair(String region, String tag) {
-      for (KeyPair keyPair : ec2Client.getKeyPairServices()
-            .describeKeyPairsInRegion(region)) {
-         if (keyPair.getKeyName().matches("jclouds#" + tag + "-[0-9]+")) {
+   @VisibleForTesting
+   void deleteKeyPair(String region, String tag) {
+      for (KeyPair keyPair : ec2Client.getKeyPairServices().describeKeyPairsInRegion(region)) {
+         if (keyPair.getKeyName().matches(String.format("jclouds#%s#%s#%s", tag, region, "[0-9a-f]+"))) {
             logger.debug(">> deleting keyPair(%s)", keyPair.getKeyName());
-            ec2Client.getKeyPairServices().deleteKeyPairInRegion(region,
-                  keyPair.getKeyName());
+            ec2Client.getKeyPairServices().deleteKeyPairInRegion(region, keyPair.getKeyName());
             // TODO: test this clear happens
-            credentialsMap.remove(new RegionAndName(region, keyPair
-                  .getKeyName()));
+            credentialsMap.remove(new RegionAndName(region, keyPair.getKeyName()));
             logger.debug("<< deleted keyPair(%s)", keyPair.getKeyName());
          }
       }
    }
 
    /**
-    * like {@link BaseComputeService#destroyNodesMatching} except that this will
-    * clean implicit keypairs and security groups.
+    * like {@link BaseComputeService#destroyNodesMatching} except that this will clean implicit
+    * keypairs and security groups.
     */
    @Override
-   public Set<? extends NodeMetadata> destroyNodesMatching(
-         Predicate<NodeMetadata> filter) {
+   public Set<? extends NodeMetadata> destroyNodesMatching(Predicate<NodeMetadata> filter) {
       Set<? extends NodeMetadata> deadOnes = super.destroyNodesMatching(filter);
       Map<String, String> regionTags = Maps.newHashMap();
       for (NodeMetadata nodeMetadata : deadOnes) {
          if (nodeMetadata.getTag() != null)
-            regionTags.put(parseHandle(nodeMetadata.getId())[0], nodeMetadata
-                  .getTag());
+            regionTags.put(parseHandle(nodeMetadata.getId())[0], nodeMetadata.getTag());
       }
       for (Entry<String, String> regionTag : regionTags.entrySet()) {
          deleteKeyPair(regionTag.getKey(), regionTag.getValue());
          deleteSecurityGroup(regionTag.getKey(), regionTag.getValue());
+         deletePlacementGroup(regionTag.getKey(), regionTag.getValue());
       }
       return deadOnes;
    }