Skip to content

Commit

Permalink
Better tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@aboitreaud
aboitreaud committed Dec 20, 2024
1 parent 5ff77e9 commit ad9ff10
Show file tree
Hide file tree
Showing 4 changed files with 273 additions and 21 deletions.
33 changes: 12 additions & 21 deletions pkg/fleet/installer/setup/djm/emr_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ package djm

import (
"context"
_ "embed"
"fmt"
"os"
"path/filepath"
Expand All @@ -20,10 +21,14 @@ import (
"github.com/DataDog/datadog-agent/pkg/fleet/telemetry"
)

// Unless explicitly stated otherwise all files in this repository are licensed
// under the Apache License Version 2.0.
// This product includes software developed at Datadog (https://www.datadoghq.com/).
// Copyright 2016-present Datadog, Inc.
//go:embed testdata/instance.json
var instanceJSON string

//go:embed testdata/extraInstanceData.json
var extraInstanceJSON string

//go:embed testdata/emrDescribeClusterResponse.json
var emrDescribeClusterResponse string

func TestSetupCommonEmrHostTags(t *testing.T) {

Expand All @@ -33,34 +38,24 @@ func TestSetupCommonEmrHostTags(t *testing.T) {

executeCommandWithTimeout = func(command string, args ...string) ([]byte, error) {
if command == "aws" && args[0] == "emr" && args[1] == "describe-cluster" {
return []byte(`{"Cluster": {"Name": "TestCluster"}}`), nil
return []byte(emrDescribeClusterResponse), nil
}
return nil, fmt.Errorf("unexpected command: %s", command)
}

// Write info files in temp dir
emrInfoPath = t.TempDir()
instanceJSON := `{"InstanceGroupID": "ig-123", "IsMaster": true}`
extraInstanceJSON := `{"JobFlowID": "j-456", "ReleaseLabel": "emr-7.2.0"}`
err := os.WriteFile(filepath.Join(emrInfoPath, "instance.json"), []byte(instanceJSON), 0644)

if err != nil {
t.Fatalf("failed to write instance.json: %v", err)
}
require.NoError(t, os.WriteFile(filepath.Join(emrInfoPath, "instance.json"), []byte(instanceJSON), 0644))

err = os.WriteFile(filepath.Join(emrInfoPath, "extraInstanceData.json"), []byte(extraInstanceJSON), 0644)
if err != nil {
t.Fatalf("failed to write extraInstanceData.json: %v", err)
}
require.NoError(t, os.WriteFile(filepath.Join(emrInfoPath, "extraInstanceData.json"), []byte(extraInstanceJSON), 0644))

tests := []struct {
name string
env map[string]string
wantTags []string
}{
{
name: "basic fields json",

wantTags: []string{
"instance_group_id:ig-123",
"is_master_node:true",
Expand All @@ -73,10 +68,6 @@ func TestSetupCommonEmrHostTags(t *testing.T) {

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
os.Clearenv()
for k, v := range tt.env {
require.NoError(t, os.Setenv(k, v))
}
span, _ := telemetry.StartSpanFromContext(context.Background(), "test")
s := &common.Setup{Span: span}

Expand Down
167 changes: 167 additions & 0 deletions pkg/fleet/installer/setup/djm/testdata/emrDescribeClusterResponse.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
{
"Cluster": {
"Id": "j-456",
"Name": "TestCluster",
"Status": {
"State": "WAITING",
"StateChangeReason": {
"Message": "Cluster ready to run steps."
},
"Timeline": {
"CreationDateTime": "2024-12-19T12:33:52.519000+00:00",
"ReadyDateTime": "2024-12-19T12:39:40.738000+00:00"
}
},
"Ec2InstanceAttributes": {
"Ec2KeyName": "name-key",
"Ec2SubnetId": "subnet-123",
"RequestedEc2SubnetIds": [
"subnet-a75bb2c3"
],
"Ec2AvailabilityZone": "us-east-1a",
"RequestedEc2AvailabilityZones": [],
"IamInstanceProfile": "IamInstanceProfile",
"EmrManagedMasterSecurityGroup": "sg-EmrManagedMasterSecurityGroup",
"EmrManagedSlaveSecurityGroup": "sg-EmrManagedSlaveSecurityGroup",
"AdditionalMasterSecurityGroups": [
"sg-0348948a9f025a14e",
"sg-0cca41352bd4bf322"
],
"AdditionalSlaveSecurityGroups": [
"sg-1",
"sg-2"
]
},
"InstanceCollectionType": "INSTANCE_GROUP",
"LogUri": "s3n://bucket/emr_logs/",
"ReleaseLabel": "emr-7.2.0",
"AutoTerminate": false,
"TerminationProtected": false,
"UnhealthyNodeReplacement": true,
"VisibleToAllUsers": true,
"Applications": [
{
"Name": "Hadoop",
"Version": "3.3.6"
},
{
"Name": "Hive",
"Version": "3.1.3"
},
{
"Name": "JupyterEnterpriseGateway",
"Version": "2.6.0"
},
{
"Name": "Livy",
"Version": "0.8.0"
},
{
"Name": "Spark",
"Version": "3.5.1"
}
],
"Tags": [
{
"Key": "emr_env",
"Value": "sandbox"
}
],
"ServiceRole": "arn:aws:iam::123456:role/EMR_DefaultRole",
"NormalizedInstanceHours": 16,
"NormalizedInstanceHours": 16,
"MasterPublicDnsName": "ec2-1-2-3-4.compute-1.amazonaws.com",
"Configurations": [],
"ScaleDownBehavior": "TERMINATE_AT_TASK_COMPLETION",
"KerberosAttributes": {},
"ClusterArn": "arn:aws:elasticmapreduce:us-east-1:601427279990:cluster/j-456",
"StepConcurrencyLevel": 2,
"PlacementGroups": [],
"OSReleaseLabel": "2023.6.20241121.0",
"BootstrapActions": [],
"InstanceGroups": [
{
"Id": "ig-123",
"Name": "Primary",
"Market": "ON_DEMAND",
"InstanceGroupType": "MASTER",
"InstanceType": "m4.xlarge",
"RequestedInstanceCount": 1,
"RunningInstanceCount": 1,
"Status": {
"State": "RUNNING",
"StateChangeReason": {
"Message": ""
},
"Timeline": {
"CreationDateTime": "2024-12-19T12:33:52.539000+00:00",
"ReadyDateTime": "2024-12-19T12:39:11.301000+00:00"
}
},
"Configurations": [],
"ConfigurationsVersion": 0,
"LastSuccessfullyAppliedConfigurations": [],
"LastSuccessfullyAppliedConfigurationsVersion": 0,
"EbsBlockDevices": [
{
"VolumeSpecification": {
"VolumeType": "gp2",
"SizeInGB": 32
},
"Device": "/dev/sdb"
},
{
"VolumeSpecification": {
"VolumeType": "gp2",
"SizeInGB": 32
},
"Device": "/dev/sdc"
}
],
"EbsOptimized": true,
"ShrinkPolicy": {}
},
{
"Id": "ig-124",
"Name": "Core",
"Market": "ON_DEMAND",
"InstanceGroupType": "CORE",
"InstanceType": "m4.xlarge",
"RequestedInstanceCount": 1,
"RunningInstanceCount": 1,
"Status": {
"State": "RUNNING",
"StateChangeReason": {
"Message": ""
},
"Timeline": {
"CreationDateTime": "2024-12-19T12:33:52.539000+00:00",
"ReadyDateTime": "2024-12-19T12:39:40.738000+00:00"
}
},
"Configurations": [],
"ConfigurationsVersion": 0,
"LastSuccessfullyAppliedConfigurations": [],
"LastSuccessfullyAppliedConfigurationsVersion": 0,
"EbsBlockDevices": [
{
"VolumeSpecification": {
"VolumeType": "gp2",
"SizeInGB": 32
},
"Device": "/dev/sdb"
},
{
"VolumeSpecification": {
"VolumeType": "gp2",
"SizeInGB": 32
},
"Device": "/dev/sdc"
}
],
"EbsOptimized": true,
"ShrinkPolicy": {}
}
]
}
}
90 changes: 90 additions & 0 deletions pkg/fleet/installer/setup/djm/testdata/extraInstanceData.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
{
"masterHost": "localhost",
"systemActions": [],
"startupScripts": [],
"hadoopEnv": {},
"jobFlowId": "j-456",
"jobFlowCreationInstant": 1734611632519,
"instanceCount": 2,
"masterInstanceType": "m4.xlarge",
"instanceGroups": {
"CORE": [
{
"instanceType": "m4.xlarge",
"instanceCount": 1
}
],
"MASTER": [
{
"instanceType": "m4.xlarge",
"instanceCount": 1
}
]
},
"slaveInstanceType": "m4.xlarge",
"supportedProducts": [],
"runJobTracker": false,
"runTaskTracker": false,
"runNameNode": true,
"runDataNode": false,
"runResourceManager": true,
"runNodeManager": false,
"instanceGroupId": "ig-123",
"instanceRole": "master",
"marketType": "on_demand",
"instanceGroupInstanceType": "m4.xlarge",
"hasIAMInstanceProfile": true,
"hadoopVersion": "Hadoop_3_3_6",
"region": "us-east-1",
"regionSet": "aws",
"deleteSteps": false,
"stepConcurrencyLevel": 2,
"emrNotebookProfilerIntegrationEnabled": true,
"debuggingInfo": {
"sdbDisabled": false,
"sqsEnabled": true,
"sqsQueueName": "AWS-ElasticMapReduce-j-456",
"stepFailureAnalysisEnabled": true,
"emrLogAnalyticsEnabled": false,
"appViewerForSparkEnabled": true,
"frequencyOfUpload": 120,
"appViewerDelayBetweenCyclesInSeconds": 120,
"appViewerFileSizeThresholdToDelayUploadInGb": 2.0,
"appViewerLargeFileCycleDelay": 2,
"onClusterUiProxyEnabled": true
},
"accountId": "123456",
"releaseLabel": "emr-7.2.0",
"serviceEndpoints": {
"S3_DEFAULT": "s3.amazonaws.com",
"S3_LOCAL": "s3.amazonaws.com",
"DYNAMODB": "dynamodb.us-east-1.amazonaws.com",
"MONITORING": "http://monitor-api.amazon.com",
"DNS": "dns-manage.amazon.com",
"TIMBER": "https://timber-iad.amazon.com",
"CLOUDWATCH": "monitoring.us-east-1.amazonaws.com",
"EDP": "datapipeline.us-east-1.amazonaws.com",
"STS": "sts.us-east-1.amazonaws.com",
"EC2": "ec2.us-east-1.amazonaws.com",
"SNS": "sns.us-east-1.amazonaws.com",
"IAMADMIN": "iamadmin.amazonaws.com",
"RDS": "rds.us-east-1.amazonaws.com",
"SWF": "swf.us-east-1.amazonaws.com",
"MONITORING_READ": "http://monitor-api.amazon.com",
"SIMPLEDB": "sdb.amazonaws.com",
"EMR": "elasticmapreduce.us-east-1.amazonaws.com",
"KMS": "kms.us-east-1.amazonaws.com",
"IAM": "iam.amazonaws.com",
"SQS": "sqs.us-east-1.amazonaws.com",
"SIM": "https://maxis-service-prod-iad.amazon.com",
"FLUXO": "ticket-api.amazon.com",
"KINESIS": "kinesis.us-east-1.amazonaws.com",
"AEGIR": "https://aegir-us-east-1.amazon.com",
"REDSHIFT": "redshift.us-east-1.amazonaws.com",
"ANYSCALE": "autoscaling.us-east-1.amazonaws.com"
},
"numCandidates": 1,
"haEnabled": false,
"candidates": []

}
4 changes: 4 additions & 0 deletions pkg/fleet/installer/setup/djm/testdata/instance.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"instanceGroupId": "ig-123",
"isMaster": true
}

0 comments on commit ad9ff10

Please sign in to comment.