Make a configurable option for the download timeout

Sometimes a remote server gets too slow. It locks download mutex and blocks the pacoloco event loop. To avoid it add a download timeout config option. Previously it has been hardcoded to 20 seconds but it caused issue anatol#19. This commit adds a configurable option `download_timeout` with default value set to 0 (no any timeout). Closes anatol#21
zhulik · Sep 15, 2020 · 5c5e8b6 · 5c5e8b6
1 parent 553fdbf
commit 5c5e8b6
Show file tree

Hide file tree

Showing 6 changed files with 19 additions and 6 deletions.
diff --git a/README.md b/README.md
@@ -31,6 +31,7 @@ The server configuration is located at `/etc/pacoloco.yaml`. Here is an example
 port: 9129
 cache_dir: /var/cache/pacoloco
 purge_files_after: 360000 # 360000 seconds or 100 hours
+download_timeout: 200 # 200 seconds
 repos:
   archlinux:
     urls:
@@ -41,11 +42,11 @@ repos:
   sublime:
     url: https://download.sublimetext.com/arch/stable/x86_64
 ```
-
-`cache_dir` is the cache directory, this location needs to read/writable by the server process.
-`purge_files_after` specifies inactivity duration (in seconds) after which the file should be removed from the cache. This functionality uses unix "AccessTime" field to find out inactive files.
-`port` is the server port.
-`repos` is a list of repositories to mirror. Each repo needs `name` and url of its Arch mirrors. Note that url can be specified either with `url` or `urls` properties, one and only one can be used for each repo configuration.
+* `cache_dir` is the cache directory, this location needs to read/writable by the server process.
+* `purge_files_after` specifies inactivity duration (in seconds) after which the file should be removed from the cache. This functionality uses unix "AccessTime" field to find out inactive files.
+* `port` is the server port.
+* `download_timeout` is a timeout (in seconds) for internet->cache downloads. If a remote server gets slow and file download takes longer than this will be terminated. Default value is `0` that means no timeout.
+* `repos` is a list of repositories to mirror. Each repo needs `name` and url of its Arch mirrors. Note that url can be specified either with `url` or `urls` properties, one and only one can be used for each repo configuration.
 
 With the example configured above `http://YOURSERVER:9129/repo/archlinux` looks exactly like an Arch pacman mirror.
 For example a request to `http://YOURSERVER:9129/repo/archlinux/core/os/x86_64/openssh-8.2p1-3-x86_64.pkg.tar.zst` will be served with file content from `http://mirror.lty.me/archlinux/core/os/x86_64/openssh-8.2p1-3-x86_64.pkg.tar.zst`

diff --git a/config.go b/config.go
@@ -22,6 +22,7 @@ type Config struct {
 	Port            int             `yaml:"port"`
 	Repos           map[string]Repo `yaml:"repos,omitempty"`
 	PurgeFilesAfter int             `yaml:"purge_files_after"`
+	DownloadTimeout int             `yaml:"download_timeout"`
 }
 
 var config *Config

diff --git a/config.test.yaml b/config.test.yaml
@@ -1,6 +1,7 @@
 port: 9129
 cache_dir: /tmp
 purge_files_after: 2592000 # 3600 * 24 * 30days
+download_timeout: 200 # 200 seconds
 repos:
   archlinux:
     urls:

diff --git a/integration_test.go b/integration_test.go
@@ -40,6 +40,7 @@ func TestPacolocoIntegration(t *testing.T) {
 		CacheDir:        testPacolocoDir,
 		Port:            -1,
 		PurgeFilesAfter: -1,
+		DownloadTimeout: 999,
 		Repos:           make(map[string]Repo),
 	}
 

diff --git a/pacoloco.go b/pacoloco.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"flag"
 	"fmt"
 	"io"
@@ -153,7 +154,14 @@ func handleRequest(w http.ResponseWriter, req *http.Request) error {
 // file and sends to `clientWriter` at the same time.
 // The function returns whether the function sent the data to client and error if one occurred
 func downloadFile(url string, filePath string, ifModifiedSince time.Time, clientWriter http.ResponseWriter) (err error, served bool) {
-	req, err := http.NewRequest("GET", url, nil)
+	var req *http.Request
+	if config.DownloadTimeout > 0 {
+		ctx, ctxCancel := context.WithTimeout(context.Background(), time.Duration(config.DownloadTimeout)*time.Second)
+		defer ctxCancel()
+		req, err = http.NewRequestWithContext(ctx, "GET", url, nil)
+	} else {
+		req, err = http.NewRequest("GET", url, nil)
+	}
 	if err != nil {
 		return
 	}

diff --git a/pacoloco.yaml.sample b/pacoloco.yaml.sample
@@ -1,6 +1,7 @@
 # cache_dir: /var/cache/pacoloco
 # port: 9129
 # purge_files_after: 2592000 # purge file after 30 days
+# download_timeout: 200 # stuck downloads will timeout after 200 sec
 repos:
   archlinux:
     urls: