Skip to content

Commit

Permalink
Merge pull request prometheus#1085 from mjibson/elb
Browse files Browse the repository at this point in the history
Add SD for Amazon EC2 instances
  • Loading branch information
fabxc committed Oct 8, 2015
2 parents 5ebef5c + 5ba5a31 commit be8b83f
Show file tree
Hide file tree
Showing 55 changed files with 30,426 additions and 2 deletions.
35 changes: 34 additions & 1 deletion config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ var (
patJobName = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_-]*$`)
patFileSDName = regexp.MustCompile(`^[^*]*(\*[^/]*)?\.(json|yml|yaml|JSON|YML|YAML)$`)
patRulePath = regexp.MustCompile(`^[^*]*(\*[^/]*)?$`)
patAuthLine = regexp.MustCompile(`((?:username|password|bearer_token):\s+)(".+"|'.+'|[^\s]+)`)
patAuthLine = regexp.MustCompile(`((?:username|password|bearer_token|secret_key):\s+)(".+"|'.+'|[^\s]+)`)
)

// Load parses the YAML input s into a Config.
Expand Down Expand Up @@ -128,6 +128,12 @@ var (
RequestTimeout: Duration(10 * time.Second),
RetryInterval: Duration(1 * time.Second),
}

// DefaultEC2SDConfig is the default EC2 SD configuration.
DefaultEC2SDConfig = EC2SDConfig{
Port: 80,
RefreshInterval: Duration(60 * time.Second),
}
)

// URL is a custom URL type that allows validation at configuration load time.
Expand Down Expand Up @@ -351,6 +357,8 @@ type ScrapeConfig struct {
MarathonSDConfigs []*MarathonSDConfig `yaml:"marathon_sd_configs,omitempty"`
// List of Kubernetes service discovery configurations.
KubernetesSDConfigs []*KubernetesSDConfig `yaml:"kubernetes_sd_configs,omitempty"`
// List of EC2 service discovery configurations.
EC2SDConfigs []*EC2SDConfig `yaml:"ec2_sd_configs,omitempty"`

// List of target relabel configurations.
RelabelConfigs []*RelabelConfig `yaml:"relabel_configs,omitempty"`
Expand Down Expand Up @@ -664,6 +672,31 @@ func (c *KubernetesSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) er
return checkOverflow(c.XXX, "kubernetes_sd_config")
}

// EC2SDConfig is the configuration for EC2 based service discovery.
type EC2SDConfig struct {
Region string `yaml:"region"`
AccessKey string `yaml:"access_key,omitempty"`
SecretKey string `yaml:"secret_key,omitempty"`
RefreshInterval Duration `yaml:"refresh_interval,omitempty"`
Port int `yaml:"port"`
// Catches all undefined fields and must be empty after parsing.
XXX map[string]interface{} `yaml:",inline"`
}

// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (c *EC2SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
*c = DefaultEC2SDConfig
type plain EC2SDConfig
err := unmarshal((*plain)(c))
if err != nil {
return err
}
if c.Region == "" {
return fmt.Errorf("EC2 SD configuration requires a region")
}
return checkOverflow(c.XXX, "ec2_sd_config")
}

// RelabelAction is the action to be performed on relabeling.
type RelabelAction string

Expand Down
19 changes: 19 additions & 0 deletions config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,25 @@ var expectedConf = &Config{
},
},
},
{
JobName: "service-ec2",

ScrapeInterval: Duration(15 * time.Second),
ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout,

MetricsPath: DefaultScrapeConfig.MetricsPath,
Scheme: DefaultScrapeConfig.Scheme,

EC2SDConfigs: []*EC2SDConfig{
{
Region: "us-east-1",
AccessKey: "access",
SecretKey: "secret",
RefreshInterval: Duration(60 * time.Second),
Port: 80,
},
},
},
},
original: "",
}
Expand Down
6 changes: 6 additions & 0 deletions config/testdata/conf.good.yml
Original file line number Diff line number Diff line change
Expand Up @@ -114,3 +114,9 @@ scrape_configs:
marathon_sd_configs:
- servers:
- 'http://marathon.example.com:8080'

- job_name: service-ec2
ec2_sd_configs:
- region: us-east-1
access_key: access
secret_key: secret
131 changes: 131 additions & 0 deletions retrieval/discovery/ec2.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package discovery

import (
"fmt"
"time"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/prometheus/common/log"
"github.com/prometheus/common/model"

"github.com/aws/aws-sdk-go/service/ec2"
"github.com/prometheus/prometheus/config"
"github.com/prometheus/prometheus/util/strutil"
)

const (
ec2Label = model.MetaLabelPrefix + "ec2_"
ec2LabelInstanceID = ec2Label + "instance_id"
ec2LabelPublicIP = ec2Label + "public_ip"
ec2LabelPrivateIP = ec2Label + "private_ip"
ec2LabelTag = ec2Label + "tag_"
)

// EC2Discovery periodically performs EC2-SD requests. It implements
// the TargetProvider interface.
type EC2Discovery struct {
aws *aws.Config
done chan struct{}
interval time.Duration
port int
}

// NewEC2Discovery returns a new EC2Discovery which periodically refreshes its targets.
func NewEC2Discovery(conf *config.EC2SDConfig) *EC2Discovery {
creds := credentials.NewStaticCredentials(conf.AccessKey, conf.SecretKey, "")
if conf.AccessKey == "" && conf.SecretKey == "" {
creds = credentials.NewEnvCredentials()
}
return &EC2Discovery{
aws: &aws.Config{
Region: &conf.Region,
Credentials: creds,
},
done: make(chan struct{}),
interval: time.Duration(conf.RefreshInterval),
port: conf.Port,
}
}

// Run implements the TargetProvider interface.
func (ed *EC2Discovery) Run(ch chan<- *config.TargetGroup, done <-chan struct{}) {
defer close(ch)

ticker := time.NewTicker(ed.interval)
defer ticker.Stop()

// Get an initial set right away.
tg, err := ed.refresh()
if err != nil {
log.Error(err)
} else {
ch <- tg
}

for {
select {
case <-ticker.C:
tg, err := ed.refresh()
if err != nil {
log.Error(err)
} else {
ch <- tg
}
case <-done:
return
}
}
}

// Sources implements the TargetProvider interface.
func (ed *EC2Discovery) Sources() []string {
return []string{*ed.aws.Region}
}

func (ed *EC2Discovery) refresh() (*config.TargetGroup, error) {
ec2s := ec2.New(ed.aws)
tg := &config.TargetGroup{
Source: *ed.aws.Region,
}
if err := ec2s.DescribeInstancesPages(nil, func(p *ec2.DescribeInstancesOutput, lastPage bool) bool {
for _, r := range p.Reservations {
for _, inst := range r.Instances {
if inst.PrivateIpAddress == nil {
continue
}
labels := model.LabelSet{
ec2LabelInstanceID: model.LabelValue(*inst.InstanceId),
}
if inst.PublicIpAddress != nil {
labels[ec2LabelPublicIP] = model.LabelValue(*inst.PublicIpAddress)
}
labels[ec2LabelPrivateIP] = model.LabelValue(*inst.PrivateIpAddress)
addr := fmt.Sprintf("%s:%d", *inst.PrivateIpAddress, ed.port)
labels[model.AddressLabel] = model.LabelValue(addr)
for _, t := range inst.Tags {
name := strutil.SanitizeLabelName(*t.Key)
labels[ec2LabelTag+model.LabelName(name)] = model.LabelValue(*t.Value)
}
tg.Targets = append(tg.Targets, labels)
}
}
return true
}); err != nil {
return nil, fmt.Errorf("could not describe instances: %s", err)
}
return tg, nil
}
3 changes: 3 additions & 0 deletions retrieval/targetmanager.go
Original file line number Diff line number Diff line change
Expand Up @@ -438,6 +438,9 @@ func providersFromConfig(cfg *config.ScrapeConfig) []TargetProvider {
for i, c := range cfg.ServersetSDConfigs {
app("serverset", i, discovery.NewServersetDiscovery(c))
}
for i, c := range cfg.EC2SDConfigs {
app("ec2", i, discovery.NewEC2Discovery(c))
}
if len(cfg.TargetGroups) > 0 {
app("static", 0, NewStaticProvider(cfg.TargetGroups))
}
Expand Down
105 changes: 105 additions & 0 deletions vendor/github.com/aws/aws-sdk-go/aws/awserr/error.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit be8b83f

Please sign in to comment.