Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Basic initial instrumentation of the apiserver #4272

Merged
merged 2 commits into from
Feb 10, 2015
Merged

Basic initial instrumentation of the apiserver #4272

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2463cff
Basic initial instrumentation of the apiserver. This links in the
a-robinson Feb 10, 2015
d39262d
Consolidate the prometheus Counters into a shared CounterVec and remove
a-robinson Feb 10, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion pkg/apiserver/api_installer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,6 @@ func (a *APIInstaller) newWebService() *restful.WebService {
}

func (a *APIInstaller) registerResourceHandlers(path string, storage RESTStorage, ws *restful.WebService, watchHandler http.Handler, redirectHandler http.Handler, proxyHandler http.Handler) error {

// Handler for standard REST verbs (GET, PUT, POST and DELETE).
restVerbHandler := restfulStripPrefix(a.prefix, a.restHandler)
object := storage.New()
Expand Down
31 changes: 25 additions & 6 deletions pkg/apiserver/apiserver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,23 @@ import (

"github.com/emicklei/go-restful"
"github.com/golang/glog"
"github.com/prometheus/client_golang/prometheus"
)

var (
apiserverLatencies = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Name: "apiserver_request_latencies",
Help: "Response latency summary in microseconds for each request handler, verb, and HTTP response code.",
},
[]string{"handler", "verb", "code"},
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What do you envision being the handlers? Today I see redirect and rest. I'm not sure what information you're interested in gathering, would breaking down by resource be more useful?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There's also watch, proxy, and validate handlers, which I was going to add in a followup PR (although validate isn't too important). I can add them to this one if it would make things easier.

But yeah, I think including resource would be reasonable as well. I'll add it in. I'm just worried about growing the cardinality of time series too far, since a Summary metric actually gets converted into 5 exported metrics (0.5, 0.9, and 0.99 quantiles, plus a sum and a count).

What about breaking down the latency summary just by handler and verb, and then having a separate CounterVec metric with handler, verb, resource, and code?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That SGTM. Don't worry about adding the other handlers in this PR.

)
)

func init() {
prometheus.MustRegister(apiserverLatencies)
}

// mux is an object that can register http handlers.
type Mux interface {
Handle(pattern string, handler http.Handler)
Expand Down Expand Up @@ -126,8 +141,9 @@ func InstallValidator(mux Mux, servers func() map[string]Server) {
// TODO: document all handlers
// InstallSupport registers the APIServer support functions
func InstallSupport(mux Mux, ws *restful.WebService) {
// TODO: convert healthz to restful and remove container arg
// TODO: convert healthz and metrics to restful and remove container arg
healthz.InstallHandler(mux)
mux.Handle("/metrics", prometheus.Handler())

// Set up a service to return the git code version.
ws.Path("/version")
Expand Down Expand Up @@ -196,25 +212,28 @@ func writeJSON(statusCode int, codec runtime.Codec, object runtime.Object, w htt
w.Write(formatted.Bytes())
}

// errorJSON renders an error to the response.
func errorJSON(err error, codec runtime.Codec, w http.ResponseWriter) {
// errorJSON renders an error to the response. Returns the HTTP status code of the error.
func errorJSON(err error, codec runtime.Codec, w http.ResponseWriter) int {
status := errToAPIStatus(err)
writeJSON(status.Code, codec, status, w)
return status.Code
}

// errorJSONFatal renders an error to the response, and if codec fails will render plaintext
func errorJSONFatal(err error, codec runtime.Codec, w http.ResponseWriter) {
// errorJSONFatal renders an error to the response, and if codec fails will render plaintext.
// Returns the HTTP status code of the error.
func errorJSONFatal(err error, codec runtime.Codec, w http.ResponseWriter) int {
util.HandleError(fmt.Errorf("apiserver was unable to write a JSON response: %v", err))
status := errToAPIStatus(err)
output, err := codec.Encode(status)
if err != nil {
w.WriteHeader(status.Code)
fmt.Fprintf(w, "%s: %s", status.Reason, status.Message)
return
return status.Code
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status.Code)
w.Write(output)
return status.Code
}

// writeRawJSON writes a non-API object in JSON.
Expand Down
35 changes: 34 additions & 1 deletion pkg/apiserver/redirect.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,23 +18,50 @@ package apiserver

import (
"net/http"
"strconv"
"time"

"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
"github.com/GoogleCloudPlatform/kubernetes/pkg/api/errors"
"github.com/GoogleCloudPlatform/kubernetes/pkg/httplog"
"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"

"github.com/prometheus/client_golang/prometheus"
)

var (
redirectCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "apiserver_redirect_count",
Help: "Counter of redirect requests broken out by apiserver resource and HTTP response code.",
},
[]string{"resource", "code"},
)
)

func init() {
prometheus.MustRegister(redirectCounter)
}

type RedirectHandler struct {
storage map[string]RESTStorage
codec runtime.Codec
apiRequestInfoResolver *APIRequestInfoResolver
}

func (r *RedirectHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
apiResource := ""
var httpCode int
reqStart := time.Now()
defer func() {
redirectCounter.WithLabelValues(apiResource, strconv.Itoa(httpCode)).Inc()
apiserverLatencies.WithLabelValues("redirect", "get", strconv.Itoa(httpCode)).Observe(float64((time.Since(reqStart)) / time.Microsecond))
}()

requestInfo, err := r.apiRequestInfoResolver.GetAPIRequestInfo(req)
if err != nil {
notFound(w, req)
httpCode = http.StatusNotFound
return
}
resource, parts := requestInfo.Resource, requestInfo.Parts
Expand All @@ -43,30 +70,36 @@ func (r *RedirectHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
// redirection requires /resource/resourceName path parts
if len(parts) != 2 || req.Method != "GET" {
notFound(w, req)
httpCode = http.StatusNotFound
return
}
id := parts[1]
storage, ok := r.storage[resource]
if !ok {
httplog.LogOf(req, w).Addf("'%v' has no storage object", resource)
notFound(w, req)
apiResource = "invalidResource"
httpCode = http.StatusNotFound
return
}
apiResource = resource

redirector, ok := storage.(Redirector)
if !ok {
httplog.LogOf(req, w).Addf("'%v' is not a redirector", resource)
errorJSON(errors.NewMethodNotSupported(resource, "redirect"), r.codec, w)
httpCode = errorJSON(errors.NewMethodNotSupported(resource, "redirect"), r.codec, w)
return
}

location, err := redirector.ResourceLocation(ctx, id)
if err != nil {
status := errToAPIStatus(err)
writeJSON(status.Code, r.codec, status, w)
httpCode = status.Code
return
}

w.Header().Set("Location", location)
w.WriteHeader(http.StatusTemporaryRedirect)
httpCode = http.StatusTemporaryRedirect
}
Loading