server: configurable retries

nadiamoe · Jun 9, 2022 · 7e5d3cc · 7e5d3cc
1 parent ca52c8b
commit 7e5d3cc
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 1 deletion.
diff --git a/pool/pool.go b/pool/pool.go
@@ -22,6 +22,7 @@ type Pool struct {
 }
 
 type Config struct {
+	Retries       int
 	Workers       int
 	Stats         *stats.Stats
 	PeekSizeBytes int64
@@ -71,7 +72,7 @@ func (p *Pool) work() {
 func (p *Pool) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
 	retries := 0
 	for {
-		if retries > 3 {
+		if retries > p.Config.Retries {
 			log.Errorf("Max retries for %s exhausted", r.URL.Path)
 			rw.WriteHeader(http.StatusInternalServerError)
 			return

diff --git a/server/server.go b/server/server.go
@@ -18,6 +18,10 @@ type Config struct {
 	// expected connections to refractor, otherwise requests will be serialized.
 	Workers int `yaml:"workers"`
 
+	// Retries controls how many times a request is re-enqueued after a retryable error occurs.
+	// Errors are considered retryable if they occur before writing anything to the client.
+	Retries int `yaml:"retries"`
+
 	// GoodThroughputMiBs is an absolute value, in MiB/s, of what is considered good throughput. Workers that perform
 	// above this throughput will never get rotated out of the worker pool, even if they are in the last 2 positions.
 	GoodThroughputMiBs float64 `yaml:"goodThroughputMiBs"`
@@ -35,6 +39,7 @@ type Config struct {
 const (
 	defaultPeekSizeMiBs = 1.0
 	defaultPeekTimeout  = 4 * time.Second
+	defaultRetries      = 3
 )
 
 type Server struct {
@@ -80,6 +85,11 @@ func New(configFile io.Reader) (*Server, error) {
 		config.PeekTimeout = defaultPeekTimeout
 	}
 
+	if config.Retries == 0 {
+		log.Infof("Defaulting Retries to %s", defaultRetries)
+		config.Retries = defaultRetries
+	}
+
 	s := &Server{
 		pool: pool.New(pool.Config{
 			PeekTimeout:   config.PeekTimeout,