Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add path metric #90

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Add path metric #90

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
2b35f74
add path metric
yurifrlcw Mar 8, 2023
17702a7
feat: query formating
yurifrlcw Mar 14, 2023
1839809
feat: log level
yurifrlcw Apr 11, 2023
6517c65
feat: log on format error
yurifrlcw Apr 11, 2023
2a74d89
feat: log query
yurifrlcw Apr 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 52 additions & 4 deletions cloudflare.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package main

import (
"context"
"encoding/json"
"time"

"github.com/cloudflare/cloudflare-go"
Expand Down Expand Up @@ -156,6 +157,16 @@ type zoneResp struct {
} `json:"dimensions"`
} `json:"httpRequestsAdaptiveGroups"`

HTTPRequestsClientRequestPath []struct {
Count uint64 `json:"count"`
Dimensions struct {
OriginResponseStatus uint16 `json:"originResponseStatus"`
ClientRequestHTTPHost string `json:"clientRequestHTTPHost"`
ClientRequestPath string `json:"clientRequestPath"`
ClientRequestHTTPMethodName string `json:"clientRequestHTTPMethodName"`
} `json:"dimensions"`
} `json:"httpRequestsClientRequestPath"`

HTTPRequestsEdgeCountryHost []struct {
Count uint64 `json:"count"`
Dimensions struct {
Expand Down Expand Up @@ -269,8 +280,8 @@ func fetchZoneTotals(zoneIDs []string) (*cloudflareResponse, error) {
now = now.Truncate(s)
now1mAgo := now.Add(-60 * time.Second)

request := graphql.NewRequest(`
query ($zoneIDs: [String!], $mintime: Time!, $maxtime: Time!, $limit: Int!) {
query := `
query ($zoneIDs: [String!], $mintime: Time!, $maxtime: Time!, $limit: Int!, $httpRequestsClientRequestPathFilter: ZoneHttpRequestsAdaptiveGroupsFilter_InputObject!) {
viewer {
zones(filter: { zoneTag_in: $zoneIDs }) {
zoneTag
Expand Down Expand Up @@ -344,6 +355,19 @@ query ($zoneIDs: [String!], $mintime: Time!, $maxtime: Time!, $limit: Int!) {
clientRequestHTTPHost
}
}
httpRequestsClientRequestPath: httpRequestsAdaptiveGroups(
limit: $limit,
filter: $httpRequestsClientRequestPathFilter,
orderBy: [count_DESC]
) {
count
dimensions {
originResponseStatus
clientRequestPath
clientRequestHTTPHost
clientRequestHTTPMethodName
}
}
httpRequestsEdgeCountryHost: httpRequestsAdaptiveGroups(limit: $limit, filter: { datetime_geq: $mintime, datetime_lt: $maxtime }) {
count
dimensions {
Expand All @@ -363,14 +387,38 @@ query ($zoneIDs: [String!], $mintime: Time!, $maxtime: Time!, $limit: Int!) {
}
}
}
}
`)
}`
request := graphql.NewRequest(query)
if len(cfgCfAPIToken) > 0 {
request.Header.Set("Authorization", "Bearer "+cfgCfAPIToken)
} else {
request.Header.Set("X-AUTH-EMAIL", cfgCfAPIEmail)
request.Header.Set("X-AUTH-KEY", cfgCfAPIKey)
}

var httpRequestsClientRequestPathFilter map[string]interface{}

err := json.Unmarshal([]byte(cfgClientRequestPathFilters), &httpRequestsClientRequestPathFilter)
if err != nil {
log.Error(err)
return nil, err
}

httpRequestsClientRequestPathFilter["AND"] = append(
httpRequestsClientRequestPathFilter["AND"].([]interface{}),
map[string]time.Time{
"datetime_geq": now1mAgo,
"datetime_lt": now,
},
)

if log.DebugLevel == log.GetLevel() {
log.Debug(httpRequestsClientRequestPathFilter)
f, _ := json.Marshal(httpRequestsClientRequestPathFilter)
log.Debug(string(f))
}

request.Var("httpRequestsClientRequestPathFilter", httpRequestsClientRequestPathFilter)
request.Var("limit", 9999)
request.Var("maxtime", now)
request.Var("mintime", now1mAgo)
Expand Down
37 changes: 26 additions & 11 deletions main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,19 @@ import (
)

var (
cfgListen = ":8080"
cfgCfAPIKey = ""
cfgCfAPIEmail = ""
cfgCfAPIToken = ""
cfgMetricsPath = "/metrics"
cfgZones = ""
cfgExcludeZones = ""
cfgScrapeDelay = 300
cfgFreeTier = false
cfgBatchSize = 10
cfgMetricsDenylist = ""
cfgListen = ":8080"
cfgCfAPIKey = ""
cfgCfAPIEmail = ""
cfgCfAPIToken = ""
cfgMetricsPath = "/metrics"
cfgZones = ""
cfgExcludeZones = ""
cfgScrapeDelay = 300
cfgFreeTier = false
cfgBatchSize = 10
cfgMetricsDenylist = ""
cfgClientRequestPathFilters = ""
cfgLogLevel = "info"
)

func getTargetZones() []string {
Expand Down Expand Up @@ -139,8 +141,11 @@ func main() {
flag.StringVar(&cfgExcludeZones, "cf_exclude_zones", cfgExcludeZones, "cloudflare zones to exclude, comma delimited list")
flag.IntVar(&cfgScrapeDelay, "scrape_delay", cfgScrapeDelay, "scrape delay in seconds, defaults to 300")
flag.IntVar(&cfgBatchSize, "cf_batch_size", cfgBatchSize, "cloudflare zones batch size (1-10), defaults to 10")
flag.StringVar(&cfgClientRequestPathFilters, "cf_path_filters", cfgClientRequestPathFilters, "add filters to path query")
flag.BoolVar(&cfgFreeTier, "free_tier", cfgFreeTier, "scrape only metrics included in free plan")
flag.StringVar(&cfgMetricsDenylist, "metrics_denylist", cfgMetricsDenylist, "metrics to not expose, comma delimited list")
flag.StringVar(&cfgLogLevel, "log_level", cfgLogLevel, "log level, defaults to info")

flag.Parse()
if !(len(cfgCfAPIToken) > 0 || (len(cfgCfAPIEmail) > 0 && len(cfgCfAPIKey) > 0)) {
log.Fatal("Please provide CF_API_KEY+CF_API_EMAIL or CF_API_TOKEN")
Expand All @@ -153,6 +158,16 @@ func main() {
log.SetFormatter(customFormatter)
customFormatter.FullTimestamp = true

log_level, err := log.ParseLevel(cfgLogLevel)
if err != nil {
log_level = log.InfoLevel
}

log.SetLevel(log_level)
log.WithFields(log.Fields{
"log_level": log_level,
}).Info()

metricsDenylist := []string{}
if len(cfgMetricsDenylist) > 0 {
metricsDenylist = strings.Split(cfgMetricsDenylist, ",")
Expand Down
22 changes: 22 additions & 0 deletions prometheus.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ const (
zoneRequestHTTPStatusMetricName MetricName = "cloudflare_zone_requests_status"
zoneRequestBrowserMapMetricName MetricName = "cloudflare_zone_requests_browser_map_page_views_count"
zoneRequestOriginStatusCountryHostMetricName MetricName = "cloudflare_zone_requests_origin_status_country_host"
zoneRequestOriginStatusPathHostName MetricName = "cloudflare_zone_requests_origin_status_path_host"
zoneRequestStatusCountryHostMetricName MetricName = "cloudflare_zone_requests_status_country_host"
zoneBandwidthTotalMetricName MetricName = "cloudflare_zone_bandwidth_total"
zoneBandwidthCachedMetricName MetricName = "cloudflare_zone_bandwidth_cached"
Expand Down Expand Up @@ -110,6 +111,12 @@ var (
}, []string{"zone", "status", "country", "host"},
)

zoneRequestOriginStatusPathHost = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: zoneRequestOriginStatusPathHostName.String(),
Help: "Count of requests for zone per edge HTTP status per path per host",
}, []string{"zone", "status", "path", "method", "host"},
)

zoneRequestStatusCountryHost = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: zoneRequestStatusCountryHostMetricName.String(),
Help: "Count of requests for zone per edge HTTP status per country per host",
Expand Down Expand Up @@ -256,6 +263,7 @@ func buildAllMetricsSet() MetricsSet {
allMetricsSet.Add(zoneRequestBrowserMapMetricName)
allMetricsSet.Add(zoneRequestOriginStatusCountryHostMetricName)
allMetricsSet.Add(zoneRequestStatusCountryHostMetricName)
allMetricsSet.Add(zoneRequestOriginStatusPathHostName)
allMetricsSet.Add(zoneBandwidthTotalMetricName)
allMetricsSet.Add(zoneBandwidthCachedMetricName)
allMetricsSet.Add(zoneBandwidthSSLEncryptedMetricName)
Expand Down Expand Up @@ -320,6 +328,9 @@ func mustRegisterMetrics(deniedMetrics MetricsSet) {
if !deniedMetrics.Has(zoneRequestStatusCountryHostMetricName) {
prometheus.MustRegister(zoneRequestStatusCountryHost)
}
if !deniedMetrics.Has(zoneRequestOriginStatusPathHostName) {
prometheus.MustRegister(zoneRequestOriginStatusPathHost)
}
if !deniedMetrics.Has(zoneBandwidthTotalMetricName) {
prometheus.MustRegister(zoneBandwidthTotal)
}
Expand Down Expand Up @@ -565,6 +576,17 @@ func addHTTPAdaptiveGroups(z *zoneResp, name string) {
}).Add(float64(g.Count))
}

for _, g := range z.HTTPRequestsClientRequestPath {
zoneRequestOriginStatusPathHost.With(
prometheus.Labels{
"zone": name,
"status": strconv.Itoa(int(g.Dimensions.OriginResponseStatus)),
"path": g.Dimensions.ClientRequestPath,
"method": g.Dimensions.ClientRequestHTTPMethodName,
"host": g.Dimensions.ClientRequestHTTPHost,
}).Add(float64(g.Count))
}

for _, g := range z.HTTPRequestsEdgeCountryHost {
zoneRequestStatusCountryHost.With(
prometheus.Labels{
Expand Down