lnwallet: add configurable cache for web fee estimator

Add fee.min-update-timeout and fee.max-update-timeout config options to allow configuration of the web fee estimator cache.
ProofOfKeags · May 4, 2024 · 3837c3f · 3837c3f
1 parent fa616ee
commit 3837c3f
Show file tree

Hide file tree

Showing 15 changed files with 201 additions and 54 deletions.
diff --git a/chainreg/chainregistry.go b/chainreg/chainregistry.go
@@ -77,10 +77,14 @@ type Config struct {
 	// ActiveNetParams details the current chain we are on.
 	ActiveNetParams BitcoinNetParams
 
-	// FeeURL defines the URL for fee estimation we will use. This field is
-	// optional.
+	// Deprecated: Use Fee.URL. FeeURL defines the URL for fee estimation
+	// we will use. This field is optional.
 	FeeURL string
 
+	// Fee defines settings for the web fee estimator. This field is
+	// optional.
+	Fee *lncfg.Fee
+
 	// Dialer is a function closure that will be used to establish outbound
 	// TCP connections to Bitcoin peers in the event of a pruned block being
 	// requested.
@@ -243,6 +247,16 @@ func NewPartialChainControl(cfg *Config) (*PartialChainControl, func(), error) {
 			"cache: %v", err)
 	}
 
+	// Map the deprecated feeurl flag to fee.url.
+	if cfg.FeeURL != "" {
+		if cfg.Fee.URL != "" {
+			return nil, nil, errors.New("fee.url and " +
+				"feeurl are mutually exclusive")
+		}
+
+		cfg.Fee.URL = cfg.FeeURL
+	}
+
 	// If spv mode is active, then we'll be using a distinct set of
 	// chainControl interfaces that interface directly with the p2p network
 	// of the selected chain.
@@ -682,27 +696,34 @@ func NewPartialChainControl(cfg *Config) (*PartialChainControl, func(), error) {
 	// If the fee URL isn't set, and the user is running mainnet, then
 	// we'll return an error to instruct them to set a proper fee
 	// estimator.
-	case cfg.FeeURL == "" && cfg.Bitcoin.MainNet &&
+	case cfg.Fee.URL == "" && cfg.Bitcoin.MainNet &&
 		cfg.Bitcoin.Node == "neutrino":
 
-		return nil, nil, fmt.Errorf("--feeurl parameter required " +
+		return nil, nil, fmt.Errorf("--fee.url parameter required " +
 			"when running neutrino on mainnet")
 
 	// Override default fee estimator if an external service is specified.
-	case cfg.FeeURL != "":
+	case cfg.Fee.URL != "":
 		// Do not cache fees on regtest to make it easier to execute
 		// manual or automated test cases.
 		cacheFees := !cfg.Bitcoin.RegTest
 
-		log.Infof("Using external fee estimator %v: cached=%v",
-			cfg.FeeURL, cacheFees)
+		log.Infof("Using external fee estimator %v: cached=%v: "+
+			"min update timeout=%v, max update timeout=%v",
+			cfg.Fee.URL, cacheFees, cfg.Fee.MinUpdateTimeout,
+			cfg.Fee.MaxUpdateTimeout)
 
-		cc.FeeEstimator = chainfee.NewWebAPIEstimator(
+		cc.FeeEstimator, err = chainfee.NewWebAPIEstimator(
 			chainfee.SparseConfFeeSource{
-				URL: cfg.FeeURL,
+				URL: cfg.Fee.URL,
 			},
 			!cacheFees,
+			cfg.Fee.MinUpdateTimeout,
+			cfg.Fee.MaxUpdateTimeout,
 		)
+		if err != nil {
+			return nil, nil, err
+		}
 	}
 
 	ccCleanup := func() {

diff --git a/config.go b/config.go
@@ -350,7 +350,7 @@ type Config struct {
 	MaxPendingChannels int    `long:"maxpendingchannels" description:"The maximum number of incoming pending channels permitted per peer."`
 	BackupFilePath     string `long:"backupfilepath" description:"The target location of the channel backup file"`
 
-	FeeURL string `long:"feeurl" description:"Optional URL for external fee estimation. If no URL is specified, the method for fee estimation will depend on the chosen backend and network. Must be set for neutrino on mainnet."`
+	FeeURL string `long:"feeurl" description:"DEPRECATED: Use 'fee.url' option. Optional URL for external fee estimation. If no URL is specified, the method for fee estimation will depend on the chosen backend and network. Must be set for neutrino on mainnet." hidden:"true"`
 
 	Bitcoin      *lncfg.Chain    `group:"Bitcoin" namespace:"bitcoin"`
 	BtcdMode     *lncfg.Btcd     `group:"btcd" namespace:"btcd"`
@@ -442,6 +442,8 @@ type Config struct {
 
 	DustThreshold uint64 `long:"dust-threshold" description:"Sets the dust sum threshold in satoshis for a channel after which dust HTLC's will be failed."`
 
+	Fee *lncfg.Fee `group:"fee" namespace:"fee"`
+
 	Invoices *lncfg.Invoices `group:"invoices" namespace:"invoices"`
 
 	Routing *lncfg.Routing `group:"routing" namespace:"routing"`
@@ -582,6 +584,12 @@ func DefaultConfig() Config {
 		MinBackoff:         defaultMinBackoff,
 		MaxBackoff:         defaultMaxBackoff,
 		ConnectionTimeout:  tor.DefaultConnTimeout,
+
+		Fee: &lncfg.Fee{
+			MinUpdateTimeout: lncfg.DefaultMinUpdateTimeout,
+			MaxUpdateTimeout: lncfg.DefaultMaxUpdateTimeout,
+		},
+
 		SubRPCServers: &subRPCServerConfigs{
 			SignRPC:   &signrpc.Config{},
 			RouterRPC: routerrpc.DefaultConfig(),

diff --git a/config_builder.go b/config_builder.go
@@ -550,6 +550,11 @@ func (d *DefaultWalletImpl) BuildWalletConfig(ctx context.Context,
 		NeutrinoCS:                  neutrinoCS,
 		ActiveNetParams:             d.cfg.ActiveNetParams,
 		FeeURL:                      d.cfg.FeeURL,
+		Fee: &lncfg.Fee{
+			URL:              d.cfg.Fee.URL,
+			MinUpdateTimeout: d.cfg.Fee.MinUpdateTimeout,
+			MaxUpdateTimeout: d.cfg.Fee.MaxUpdateTimeout,
+		},
 		Dialer: func(addr string) (net.Conn, error) {
 			return d.cfg.net.Dial(
 				"tcp", addr, d.cfg.ConnectionTimeout,

diff --git a/docs/release-notes/release-notes-0.18.0.md b/docs/release-notes/release-notes-0.18.0.md
@@ -209,6 +209,17 @@
   its bitcoin peers' `feefilter` values into
   account](https://github.com/lightningnetwork/lnd/pull/8418).
 
+* Web fee estimator settings have been moved into a new `fee` config group.
+  A new `fee.url` option has been added within this group that replaces the old
+  `feeurl` option, which is now deprecated. Additionally, [two new config values,
+  fee.min-update-timeout and fee.max-update-timeout](https://github.com/lightningnetwork/lnd/pull/8484)
+  are added to allow users to specify the minimum and maximum time between fee
+  updates from the web fee estimator. The default values are 5 minutes and 20
+  minutes respectively. These values are used to prevent the fee estimator from
+  being queried too frequently. This replaces previously hardcoded values that
+  were set to the same values as the new defaults. The previously deprecated
+  `neutrino.feeurl` option has been removed.
+
 * [Preparatory work](https://github.com/lightningnetwork/lnd/pull/8159) for 
   forwarding of blinded routes was added, along with [support](https://github.com/lightningnetwork/lnd/pull/8160)
   for forwarding blinded payments and [error handling](https://github.com/lightningnetwork/lnd/pull/8485).

diff --git a/htlcswitch/link.go b/htlcswitch/link.go
@@ -202,15 +202,15 @@ type ChannelLinkConfig struct {
 	// receiving node is persistent.
 	UnsafeReplay bool
 
-	// MinFeeUpdateTimeout represents the minimum interval in which a link
+	// MinUpdateTimeout represents the minimum interval in which a link
 	// will propose to update its commitment fee rate. A random timeout will
-	// be selected between this and MaxFeeUpdateTimeout.
-	MinFeeUpdateTimeout time.Duration
+	// be selected between this and MaxUpdateTimeout.
+	MinUpdateTimeout time.Duration
 
-	// MaxFeeUpdateTimeout represents the maximum interval in which a link
+	// MaxUpdateTimeout represents the maximum interval in which a link
 	// will propose to update its commitment fee rate. A random timeout will
-	// be selected between this and MinFeeUpdateTimeout.
-	MaxFeeUpdateTimeout time.Duration
+	// be selected between this and MinUpdateTimeout.
+	MaxUpdateTimeout time.Duration
 
 	// OutgoingCltvRejectDelta defines the number of blocks before expiry of
 	// an htlc where we don't offer an htlc anymore. This should be at least
@@ -1558,8 +1558,8 @@ func getResolutionFailure(resolution *invoices.HtlcFailResolution,
 // within the link's configuration that will be used to determine when the link
 // should propose an update to its commitment fee rate.
 func (l *channelLink) randomFeeUpdateTimeout() time.Duration {
-	lower := int64(l.cfg.MinFeeUpdateTimeout)
-	upper := int64(l.cfg.MaxFeeUpdateTimeout)
+	lower := int64(l.cfg.MinUpdateTimeout)
+	upper := int64(l.cfg.MaxUpdateTimeout)
 	return time.Duration(prand.Int63n(upper-lower) + lower)
 }
 

diff --git a/htlcswitch/link_test.go b/htlcswitch/link_test.go
@@ -2220,11 +2220,11 @@ func newSingleLinkTestHarness(t *testing.T, chanAmt,
 		BatchTicker:          bticker,
 		FwdPkgGCTicker:       ticker.NewForce(15 * time.Second),
 		PendingCommitTicker:  ticker.New(time.Minute),
-		// Make the BatchSize and Min/MaxFeeUpdateTimeout large enough
+		// Make the BatchSize and Min/MaxUpdateTimeout large enough
 		// to not trigger commit updates automatically during tests.
 		BatchSize:               10000,
-		MinFeeUpdateTimeout:     30 * time.Minute,
-		MaxFeeUpdateTimeout:     40 * time.Minute,
+		MinUpdateTimeout:        30 * time.Minute,
+		MaxUpdateTimeout:        40 * time.Minute,
 		MaxOutgoingCltvExpiry:   DefaultMaxOutgoingCltvExpiry,
 		MaxFeeAllocation:        DefaultMaxLinkFeeAllocation,
 		NotifyActiveLink:        func(wire.OutPoint) {},
@@ -4881,11 +4881,11 @@ func (h *persistentLinkHarness) restartLink(
 		BatchTicker:          bticker,
 		FwdPkgGCTicker:       ticker.New(5 * time.Second),
 		PendingCommitTicker:  ticker.New(time.Minute),
-		// Make the BatchSize and Min/MaxFeeUpdateTimeout large enough
+		// Make the BatchSize and Min/MaxUpdateTimeout large enough
 		// to not trigger commit updates automatically during tests.
-		BatchSize:           10000,
-		MinFeeUpdateTimeout: 30 * time.Minute,
-		MaxFeeUpdateTimeout: 40 * time.Minute,
+		BatchSize:        10000,
+		MinUpdateTimeout: 30 * time.Minute,
+		MaxUpdateTimeout: 40 * time.Minute,
 		// Set any hodl flags requested for the new link.
 		HodlMask:                hodl.MaskFromFlags(hodlFlags...),
 		MaxOutgoingCltvExpiry:   DefaultMaxOutgoingCltvExpiry,

diff --git a/htlcswitch/test_utils.go b/htlcswitch/test_utils.go
@@ -1155,8 +1155,8 @@ func (h *hopNetwork) createChannelLink(server, peer *mockServer,
 			BatchTicker:             ticker.NewForce(testBatchTimeout),
 			FwdPkgGCTicker:          ticker.NewForce(fwdPkgTimeout),
 			PendingCommitTicker:     ticker.New(2 * time.Minute),
-			MinFeeUpdateTimeout:     minFeeUpdateTimeout,
-			MaxFeeUpdateTimeout:     maxFeeUpdateTimeout,
+			MinUpdateTimeout:        minFeeUpdateTimeout,
+			MaxUpdateTimeout:        maxFeeUpdateTimeout,
 			OnChannelFailure:        func(lnwire.ChannelID, lnwire.ShortChannelID, LinkFailureError) {},
 			OutgoingCltvRejectDelta: 3,
 			MaxOutgoingCltvExpiry:   DefaultMaxOutgoingCltvExpiry,

diff --git a/lncfg/fee.go b/lncfg/fee.go
@@ -0,0 +1,20 @@
+package lncfg
+
+import "time"
+
+// DefaultMinUpdateTimeout represents the minimum interval in which a
+// WebAPIEstimator will request fresh fees from its API.
+const DefaultMinUpdateTimeout = 5 * time.Minute
+
+// DefaultMaxUpdateTimeout represents the maximum interval in which a
+// WebAPIEstimator will request fresh fees from its API.
+const DefaultMaxUpdateTimeout = 20 * time.Minute
+
+// Fee holds the configuration options for fee estimation.
+//
+//nolint:lll
+type Fee struct {
+	URL              string        `long:"url" description:"Optional URL for external fee estimation. If no URL is specified, the method for fee estimation will depend on the chosen backend and network. Must be set for neutrino on mainnet."`
+	MinUpdateTimeout time.Duration `long:"min-update-timeout" description:"The minimum interval in which fees will be updated from the specified fee URL."`
+	MaxUpdateTimeout time.Duration `long:"max-update-timeout" description:"The maximum interval in which fees will be updated from the specified fee URL."`
+}
diff --git a/lncfg/neutrino.go b/lncfg/neutrino.go
@@ -12,7 +12,6 @@ type Neutrino struct {
 	MaxPeers           int           `long:"maxpeers" description:"Max number of inbound and outbound peers"`
 	BanDuration        time.Duration `long:"banduration" description:"How long to ban misbehaving peers.  Valid time units are {s, m, h}.  Minimum 1 second"`
 	BanThreshold       uint32        `long:"banthreshold" description:"Maximum allowed ban score before disconnecting and banning misbehaving peers."`
-	FeeURL             string        `long:"feeurl" description:"DEPRECATED: Use top level 'feeurl' option. Optional URL for fee estimation. If a URL is not specified, static fees will be used for estimation." hidden:"true"`
 	AssertFilterHeader string        `long:"assertfilterheader" description:"Optional filter header in height:hash format to assert the state of neutrino's filter header chain on startup. If the assertion does not hold, then the filter header chain will be re-synced from the genesis block."`
 	UserAgentName      string        `long:"useragentname" description:"Used to help identify ourselves to other bitcoin peers"`
 	UserAgentVersion   string        `long:"useragentversion" description:"Used to help identify ourselves to other bitcoin peers"`

diff --git a/lntest/fee_service.go b/lntest/fee_service.go
@@ -17,7 +17,7 @@ import (
 
 // WebFeeService defines an interface that's used to provide fee estimation
 // service used in the integration tests. It must provide an URL so that a lnd
-// node can be started with the flag `--feeurl` and uses the customized fee
+// node can be started with the flag `--fee.url` and uses the customized fee
 // estimator.
 type WebFeeService interface {
 	// Start starts the service.

diff --git a/lntest/node/config.go b/lntest/node/config.go
@@ -279,7 +279,7 @@ func (cfg *BaseNodeConfig) GenArgs() []string {
 	}
 
 	if cfg.FeeURL != "" {
-		args = append(args, "--feeurl="+cfg.FeeURL)
+		args = append(args, "--fee.url="+cfg.FeeURL)
 	}
 
 	// Put extra args in the end so the args can be overwritten.

diff --git a/lnwallet/chainfee/estimator.go b/lnwallet/chainfee/estimator.go
@@ -28,14 +28,6 @@ const (
 	// less than this will result in an error.
 	minBlockTarget uint32 = 1
 
-	// minFeeUpdateTimeout represents the minimum interval in which a
-	// WebAPIEstimator will request fresh fees from its API.
-	minFeeUpdateTimeout = 5 * time.Minute
-
-	// maxFeeUpdateTimeout represents the maximum interval in which a
-	// WebAPIEstimator will request fresh fees from its API.
-	maxFeeUpdateTimeout = 20 * time.Minute
-
 	// WebAPIConnectionTimeout specifies the timeout value for connecting
 	// to the api source.
 	WebAPIConnectionTimeout = 5 * time.Second
@@ -739,19 +731,43 @@ type WebAPIEstimator struct {
 	// estimates.
 	noCache bool
 
+	// minFeeUpdateTimeout represents the minimum interval in which the
+	// web estimator will request fresh fees from its API.
+	minFeeUpdateTimeout time.Duration
+
+	// minFeeUpdateTimeout represents the maximum interval in which the
+	// web estimator will request fresh fees from its API.
+	maxFeeUpdateTimeout time.Duration
+
 	quit chan struct{}
 	wg   sync.WaitGroup
 }
 
 // NewWebAPIEstimator creates a new WebAPIEstimator from a given URL and a
 // fallback default fee. The fees are updated whenever a new block is mined.
-func NewWebAPIEstimator(api WebAPIFeeSource, noCache bool) *WebAPIEstimator {
-	return &WebAPIEstimator{
-		apiSource:        api,
-		feeByBlockTarget: make(map[uint32]uint32),
-		noCache:          noCache,
-		quit:             make(chan struct{}),
+func NewWebAPIEstimator(api WebAPIFeeSource, noCache bool,
+	minFeeUpdateTimeout time.Duration,
+	maxFeeUpdateTimeout time.Duration) (*WebAPIEstimator, error) {
+
+	if minFeeUpdateTimeout == 0 || maxFeeUpdateTimeout == 0 {
+		return nil, fmt.Errorf("minFeeUpdateTimeout and " +
+			"maxFeeUpdateTimeout must be greater than 0")
 	}
+
+	if minFeeUpdateTimeout >= maxFeeUpdateTimeout {
+		return nil, fmt.Errorf("minFeeUpdateTimeout target of %v "+
+			"cannot be greater than maxFeeUpdateTimeout of %v",
+			minFeeUpdateTimeout, maxFeeUpdateTimeout)
+	}
+
+	return &WebAPIEstimator{
+		apiSource:           api,
+		feeByBlockTarget:    make(map[uint32]uint32),
+		noCache:             noCache,
+		quit:                make(chan struct{}),
+		minFeeUpdateTimeout: minFeeUpdateTimeout,
+		maxFeeUpdateTimeout: maxFeeUpdateTimeout,
+	}, nil
 }
 
 // EstimateFeePerKW takes in a target for the number of blocks until an initial
@@ -809,7 +825,12 @@ func (w *WebAPIEstimator) Start() error {
 	w.started.Do(func() {
 		log.Infof("Starting web API fee estimator")
 
-		w.updateFeeTicker = time.NewTicker(w.randomFeeUpdateTimeout())
+		feeUpdateTimeout := w.randomFeeUpdateTimeout()
+
+		log.Infof("Web API fee estimator using update timeout of %v",
+			feeUpdateTimeout)
+
+		w.updateFeeTicker = time.NewTicker(feeUpdateTimeout)
 		w.updateFeeEstimates()
 
 		w.wg.Add(1)
@@ -852,9 +873,11 @@ func (w *WebAPIEstimator) RelayFeePerKW() SatPerKWeight {
 // and maxFeeUpdateTimeout that will be used to determine how often the Estimator
 // should retrieve fresh fees from its API.
 func (w *WebAPIEstimator) randomFeeUpdateTimeout() time.Duration {
-	lower := int64(minFeeUpdateTimeout)
-	upper := int64(maxFeeUpdateTimeout)
-	return time.Duration(prand.Int63n(upper-lower) + lower)
+	lower := int64(w.minFeeUpdateTimeout)
+	upper := int64(w.maxFeeUpdateTimeout)
+	return time.Duration(
+		prand.Int63n(upper-lower) + lower, //nolint:gosec
+	).Round(time.Second)
 }
 
 // getCachedFee takes a conf target and returns the cached fee rate. When the