Skip to content

Commit

Permalink
WIP - Snapshot of Moving to Client Model.
Browse files Browse the repository at this point in the history
  • Loading branch information
@matttproud
matttproud committed Jun 25, 2013
1 parent 42198c1 commit 30b1cf8
Show file tree
Hide file tree
Showing 94 changed files with 1,969 additions and 3,758 deletions.
0 build/.gitignore → .build/.gitignore
View file
File renamed without changes.
0 build/Makefile → .build/Makefile
View file
File renamed without changes.
0 build/cache/.gitignore → .build/cache/.gitignore
View file
File renamed without changes.
0 build/cache/Makefile → .build/cache/Makefile
View file
File renamed without changes.
0 build/dirty/.gitignore → .build/dirty/.gitignore
View file
File renamed without changes.
0 build/dirty/Makefile → .build/dirty/Makefile
View file
File renamed without changes.
0 build/package/.gitignore → .build/package/.gitignore
View file
File renamed without changes.
0 build/package/Makefile → .build/package/Makefile
View file
File renamed without changes.
0 build/package/lib/.gitignore → .build/package/lib/.gitignore
View file
File renamed without changes.
0 build/package/run_prometheus.sh → .build/package/run_prometheus.sh
View file
File renamed without changes.
0 build/root/Makefile → .build/root/Makefile
View file
File renamed without changes.
0 build/root/bin/.gitignore → .build/root/bin/.gitignore
View file
File renamed without changes.
0 build/root/include/.gitignore → .build/root/include/.gitignore
View file
File renamed without changes.
0 build/root/lib/.gitignore → .build/root/lib/.gitignore
View file
File renamed without changes.
0 build/root/share/.gitignore → .build/root/share/.gitignore
View file
File renamed without changes.
25 changes: 14 additions & 11 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ include Makefile.INCLUDE

all: binary test

$(GOCC): build/cache/$(GOPKG)
tar -C build/root -xzf $<
$(GOCC): $(BUILD_PATH)/cache/$(GOPKG) source_path
tar -C $(BUILD_PATH)/root -xzf $<
touch $@

advice:
Expand All @@ -28,14 +28,14 @@ binary: build

build: config dependencies model preparation tools web
$(GO) build -o prometheus $(BUILDFLAGS) .
cp prometheus build/package/prometheus
rsync -av build/root/lib/ build/package/lib/
cp prometheus $(BUILD_PATH)/package/prometheus
rsync -av --delete $(BUILD_PATH)/root/lib/ $(BUILD_PATH)/package/lib/

build/cache/$(GOPKG):
$(BUILD_PATH)/cache/$(GOPKG):
curl -o $@ http://go.googlecode.com/files/$(GOPKG)

clean:
$(MAKE) -C build clean
$(MAKE) -C $(BUILD_PATH) clean
$(MAKE) -C tools clean
$(MAKE) -C web clean
rm -rf $(TEST_ARTIFACTS)
Expand All @@ -53,16 +53,16 @@ documentation: search_index
godoc -http=:6060 -index -index_files='search_index'

format:
find . -iname '*.go' | egrep -v "^./build/|generated|\.(l|y)\.go" | xargs -n1 $(GOFMT) -w -s=true
find . -iname '*.go' | egrep -v "^\./\.build|./generated|\.(l|y)\.go" | xargs -n1 $(GOFMT) -w -s=true

model: dependencies preparation
$(MAKE) -C model

preparation: $(GOCC) source_path
$(MAKE) -C build
$(MAKE) -C $(BUILD_PATH)

race_condition_binary: build
CGO_CFLAGS="-I$(PWD)/build/root/include" CGO_LDFLAGS="-L$(PWD)/build/root/lib" $(GO) build -race -o prometheus.race $(BUILDFLAGS) .
CGO_CFLAGS="-I$(BUILD_PATH)/root/include" CGO_LDFLAGS="-L$(BUILD_PATH)/root/lib" $(GO) build -race -o prometheus.race $(BUILDFLAGS) .

race_condition_run: race_condition_binary
./prometheus.race $(ARGUMENTS)
Expand All @@ -83,13 +83,16 @@ source_path:
[ -d "$(FULL_GOPATH)" ]

test: build
$(GOENV) find . -maxdepth 1 -mindepth 1 -type d -and -not -path ./build -exec $(GOCC) test {}/... $(GO_TEST_FLAGS) \;
$(GOENV) find . -maxdepth 1 -mindepth 1 -type d -and -not -path $(BUILD_PATH) -exec $(GOCC) test {}/... $(GO_TEST_FLAGS) \;
$(GO) test $(GO_TEST_FLAGS)

tools: dependencies preparation
$(MAKE) -C tools

update:
$(GO) get -d

web: config dependencies model preparation
$(MAKE) -C web

.PHONY: advice binary build clean config dependencies documentation format model package preparation race_condition_binary race_condition_run run search_index source_path test tools
.PHONY: advice binary build clean config dependencies documentation format model package preparation race_condition_binary race_condition_run run search_index source_path test tools update
24 changes: 13 additions & 11 deletions Makefile.INCLUDE
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,17 +28,19 @@ endif
OS=$(shell uname)
ARCH=$(shell uname -m)

BUILD_PATH = $(PWD)/.build

GO_VERSION := 1.1
GOOS = $(subst Darwin,darwin,$(subst Linux,linux,$(OS)))
GOARCH = $(subst x86_64,amd64,$(ARCH))
GOPKG = go$(GO_VERSION).$(GOOS)-$(GOARCH).tar.gz
GOROOT = $(PWD)/build/root/go
GOPATH = $(PWD)/build/root/gopath
GOCC = $(GOROOT)/bin/go
TMPDIR = /tmp
GOENV = TMPDIR=$(TMPDIR) GOROOT=$(GOROOT) GOPATH=$(GOPATH)
GO = $(GOENV) $(GOCC)
GOFMT = $(GOROOT)/bin/gofmt
GOOS = $(subst Darwin,darwin,$(subst Linux,linux,$(OS)))
GOARCH = $(subst x86_64,amd64,$(ARCH))
GOPKG = go$(GO_VERSION).$(GOOS)-$(GOARCH).tar.gz
GOROOT = $(BUILD_PATH)/root/go
GOPATH = $(BUILD_PATH)/root/gopath
GOCC = $(GOROOT)/bin/go
TMPDIR = /tmp
GOENV = TMPDIR=$(TMPDIR) GOROOT=$(GOROOT) GOPATH=$(GOPATH)
GO = $(GOENV) $(GOCC)
GOFMT = $(GOROOT)/bin/gofmt

LEVELDB_VERSION := 1.12.0
PROTOCOL_BUFFERS_VERSION := 2.5.0
Expand All @@ -48,7 +50,7 @@ UNAME := $(shell uname)
FULL_GOPATH := $(GOPATH)/src/github.com/prometheus/prometheus
FULL_GOPATH_BASE := $(GOPATH)/src/github.com/prometheus

export PREFIX=$(PWD)/build/root
export PREFIX=$(BUILD_PATH)/root

export LOCAL_BINARIES=$(PREFIX)/bin

Expand Down
50 changes: 19 additions & 31 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,32 +15,31 @@ package main

import (
"flag"
"log"
"os"
"os/signal"
"sync"
"time"

"github.com/prometheus/client_golang/extraction"

"github.com/prometheus/prometheus/config"
"github.com/prometheus/prometheus/retrieval"
"github.com/prometheus/prometheus/retrieval/format"
"github.com/prometheus/prometheus/rules"
"github.com/prometheus/prometheus/storage/metric"
"github.com/prometheus/prometheus/storage/raw/leveldb"
"github.com/prometheus/prometheus/web"
"github.com/prometheus/prometheus/web/api"
"log"
"os"
"os/signal"
"sync"
"time"
)

const (
deletionBatchSize = 100
)
const deletionBatchSize = 100

// Commandline flags.
var (
printVersion = flag.Bool("version", false, "print version information")
configFile = flag.String("configFile", "prometheus.conf", "Prometheus configuration file name.")
metricsStoragePath = flag.String("metricsStoragePath", "/tmp/metrics", "Base path for metrics storage.")
scrapeResultsQueueCapacity = flag.Int("scrapeResultsQueueCapacity", 4096, "The size of the scrape results queue.")
ruleResultsQueueCapacity = flag.Int("ruleResultsQueueCapacity", 4096, "The size of the rule results queue.")
samplesQueueCapacity = flag.Int("samplesQueueCapacity", 4096, "The size of the unwritten samples queue.")
concurrentRetrievalAllowance = flag.Int("concurrentRetrievalAllowance", 15, "The number of concurrent metrics retrieval requests allowed.")
diskAppendQueueCapacity = flag.Int("queue.diskAppendCapacity", 1000000, "The size of the queue for items that are pending writing to disk.")
memoryAppendQueueCapacity = flag.Int("queue.memoryAppendCapacity", 10000, "The size of the queue for items that are pending writing to memory.")
Expand Down Expand Up @@ -76,8 +75,7 @@ type prometheus struct {
databaseStates chan []leveldb.DatabaseState
stopBackgroundOperations chan bool

ruleResults chan *rules.Result
scrapeResults chan format.Result
unwrittenSamples chan *extraction.Result

storage *metric.TieredStorage
}
Expand Down Expand Up @@ -198,8 +196,7 @@ func main() {
log.Fatalln("Nil tiered storage.")
}

scrapeResults := make(chan format.Result, *scrapeResultsQueueCapacity)
ruleResults := make(chan *rules.Result, *ruleResultsQueueCapacity)
unwrittenSamples := make(chan *extraction.Result, *samplesQueueCapacity)
curationState := make(chan metric.CurationState, 1)
databaseStates := make(chan []leveldb.DatabaseState, 1)
// Coprime numbers, fool!
Expand All @@ -209,11 +206,11 @@ func main() {
deletionTimer := time.NewTicker(*deleteInterval)

// Queue depth will need to be exposed
targetManager := retrieval.NewTargetManager(scrapeResults, *concurrentRetrievalAllowance)
targetManager := retrieval.NewTargetManager(unwrittenSamples, *concurrentRetrievalAllowance)
targetManager.AddTargetsFromConfig(conf)

// Queue depth will need to be exposed
ruleManager := rules.NewRuleManager(ruleResults, conf.EvaluationInterval(), ts)
ruleManager := rules.NewRuleManager(unwrittenSamples, conf.EvaluationInterval(), ts)
err = ruleManager.AddRulesFromConfig(conf)
if err != nil {
log.Fatalf("Error loading rule files: %v", err)
Expand Down Expand Up @@ -259,7 +256,7 @@ func main() {
AlertsHandler: alertsHandler,
}

prometheus := prometheus{
prometheus := &prometheus{
bodyCompactionTimer: bodyCompactionTimer,
headCompactionTimer: headCompactionTimer,
tailCompactionTimer: tailCompactionTimer,
Expand All @@ -271,8 +268,7 @@ func main() {
curationState: curationState,
databaseStates: databaseStates,

ruleResults: ruleResults,
scrapeResults: scrapeResults,
unwrittenSamples: unwrittenSamples,

stopBackgroundOperations: make(chan bool, 1),

Expand Down Expand Up @@ -343,17 +339,9 @@ func main() {
}()

// TODO(all): Migrate this into prometheus.serve().
for {
select {
case scrapeResult := <-scrapeResults:
if scrapeResult.Err == nil {
ts.AppendSamples(scrapeResult.Samples)
}

case ruleResult := <-ruleResults:
if ruleResult.Err == nil {
ts.AppendSamples(ruleResult.Samples)
}
for block := range unwrittenSamples {
if block.Err == nil {
ts.AppendSamples(block.Samples)
}
}
}
103 changes: 0 additions & 103 deletions model/curation.go
View file

This file was deleted.

Loading

0 comments on commit 30b1cf8

Please sign in to comment.