Skip to content

Commit

Permalink
Merge pull request prometheus#2527 from prometheus/beorn7/storage
Browse files Browse the repository at this point in the history
storage: Evict chunks and calculate persistence pressure...
beorn7 authored Mar 27, 2017
2 parents b5b0e00 + 434ab2a commit e63d079
Showing 6 changed files with 560 additions and 218 deletions.
33 changes: 27 additions & 6 deletions cmd/prometheus/config.go
Original file line number Diff line number Diff line change
@@ -54,6 +54,10 @@ var cfg = struct {

alertmanagerURLs stringset
prometheusURL string

// Deprecated storage flags, kept for backwards compatibility.
deprecatedMemoryChunks uint64
deprecatedMaxChunksToPersist uint64
}{
alertmanagerURLs: stringset{},
}
@@ -145,17 +149,21 @@ func init() {
&cfg.storage.PersistenceStoragePath, "storage.local.path", "data",
"Base path for metrics storage.",
)
cfg.fs.IntVar(
&cfg.storage.MemoryChunks, "storage.local.memory-chunks", 1024*1024,
"How many chunks to keep in memory. While the size of a chunk is 1kiB, the total memory usage will be significantly higher than this value * 1kiB. Furthermore, for various reasons, more chunks might have to be kept in memory temporarily. Sample ingestion will be throttled if the configured value is exceeded by more than 10%.",
cfg.fs.Uint64Var(
&cfg.storage.TargetHeapSize, "storage.local.target-heap-size", 2*1024*1024*1024,
"The metrics storage attempts to limit its own memory usage such that the total heap size approaches this value. Note that this is not a hard limit. Actual heap size might be temporarily or permanently higher for a variety of reasons. The default value is a relatively safe setting to not use more than 3 GiB physical memory.",
)
cfg.fs.Uint64Var(
&cfg.deprecatedMemoryChunks, "storage.local.memory-chunks", 0,
"Deprecated. If set, -storage.local.target-heap-size will be set to this value times 3072.",
)
cfg.fs.DurationVar(
&cfg.storage.PersistenceRetentionPeriod, "storage.local.retention", 15*24*time.Hour,
"How long to retain samples in the local storage.",
)
cfg.fs.IntVar(
&cfg.storage.MaxChunksToPersist, "storage.local.max-chunks-to-persist", 512*1024,
"How many chunks can be waiting for persistence before sample ingestion will be throttled. Many chunks waiting to be persisted will increase the checkpoint size.",
cfg.fs.Uint64Var(
&cfg.deprecatedMaxChunksToPersist, "storage.local.max-chunks-to-persist", 0,
"Deprecated. This flag has no effect anymore.",
)
cfg.fs.DurationVar(
&cfg.storage.CheckpointInterval, "storage.local.checkpoint-interval", 5*time.Minute,
@@ -276,6 +284,10 @@ func parse(args []string) error {
// don't expose it as a separate flag but set it here.
cfg.storage.HeadChunkTimeout = promql.StalenessDelta

if cfg.storage.TargetHeapSize < 1024*1024 {
return fmt.Errorf("target heap size smaller than %d: %d", 1024*1024, cfg.storage.TargetHeapSize)
}

if err := parsePrometheusURL(); err != nil {
return err
}
@@ -292,6 +304,15 @@ func parse(args []string) error {
}
}

// Deal with deprecated storage flags.
if cfg.deprecatedMaxChunksToPersist > 0 {
log.Warn("Flag -storage.local.max-chunks-to-persist is deprecated. It has no effect.")
}
if cfg.deprecatedMemoryChunks > 0 {
cfg.storage.TargetHeapSize = cfg.deprecatedMemoryChunks * 3072
log.Warnf("Flag -storage.local.memory-chunks is deprecated. Its value %d is used to override -storage.local.target-heap-size to %d.", cfg.deprecatedMemoryChunks, cfg.storage.TargetHeapSize)
}

return nil
}

20 changes: 5 additions & 15 deletions storage/local/chunk/instrumentation.go
Original file line number Diff line number Diff line change
@@ -83,18 +83,8 @@ func init() {
prometheus.MustRegister(NumMemDescs)
}

var (
// NumMemChunks is the total number of chunks in memory. This is a
// global counter, also used internally, so not implemented as
// metrics. Collected in MemorySeriesStorage.Collect.
// TODO(beorn7): As it is used internally, it is actually very bad style
// to have it as a global variable.
NumMemChunks int64

// NumMemChunksDesc is the metric descriptor for the above.
NumMemChunksDesc = prometheus.NewDesc(
prometheus.BuildFQName(namespace, subsystem, "memory_chunks"),
"The current number of chunks in memory, excluding cloned chunks (i.e. chunks without a descriptor).",
nil, nil,
)
)
// NumMemChunks is the total number of chunks in memory. This is a global
// counter, also used internally, so not implemented as metrics. Collected in
// MemorySeriesStorage.
// TODO(beorn7): Having this as an exported global variable is really bad.
var NumMemChunks int64
2 changes: 1 addition & 1 deletion storage/local/persistence_test.go
Original file line number Diff line number Diff line change
@@ -176,7 +176,7 @@ func testPersistLoadDropChunks(t *testing.T, encoding chunk.Encoding) {
// Try to drop one chunk, which must be prevented by the shrink
// ratio. Since we do not pass in any chunks to persist, the offset
// should be the number of chunks in the file.
for fp, _ := range fpToChunks {
for fp := range fpToChunks {
firstTime, offset, numDropped, allDropped, err := p.dropAndPersistChunks(fp, 1, nil)
if err != nil {
t.Fatal(err)
Loading
Oops, something went wrong.

0 comments on commit e63d079

Please sign in to comment.