watchtower: Add retry logic for fetching blocks

docs/release-notes/release-notes-0.19.0.md

@@ -71,6 +71,10 @@
 
 ## Code Health
 
+* [Add retry logic](https://github.com/lightningnetwork/lnd/pull/8381) for
+  watchtower block fetching with a max number of attempts and exponential
+  back-off.
+
 * [Moved](https://github.com/lightningnetwork/lnd/pull/9138) profile related
   config settings to its own dedicated group. The old ones still work but will
   be removed in a future release.
@@ -124,9 +128,10 @@
 
 # Contributors (Alphabetical Order)
 
+* Animesh Bilthare
 * Boris Nagaev
 * CharlieZKSmith
 * Elle Mouton
 * Pins
 * Viktor Tigerström
-* Ziggie
+* Ziggie

watchtower/lookout/lookout.go

@@ -1,14 +1,22 @@
 package lookout
 
 import (
+	"errors"
+	"fmt"
 	"sync"
 	"sync/atomic"
+	"time"
 
+	"github.com/btcsuite/btcd/chaincfg/chainhash"
 	"github.com/btcsuite/btcd/wire"
 	"github.com/lightningnetwork/lnd/chainntnfs"
 	"github.com/lightningnetwork/lnd/watchtower/blob"
 )
 
+// ErrLookoutExiting is an error that is returned when the lookout server is
+// in the process of shutting down.
+var ErrLookoutExiting = errors.New("lookout server is shutting down")
+
 // Config houses the Lookout's required resources to properly fulfill it's duty,
 // including block fetching, querying accepted state updates, and construction
 // and publication of justice transactions.
@@ -29,6 +37,18 @@ type Config struct {
 	// Punisher handles the responsibility of crafting and broadcasting
 	// justice transaction for any breached transactions.
 	Punisher Punisher
+
+	// MinBackoff is the minimum amount of time to back-off before
+	// re-attempting to fetch a block.
+	MinBackoff time.Duration
+
+	// MaxBackoff is the maximum amount of time to back-off before
+	// re-attempting to fetch a block.
+	MaxBackoff time.Duration
+
+	// MaxNumRetries is the maximum number of times that we should
+	// re-attempt fetching a block before moving on.
+	MaxNumRetries int
 }
 
 // Lookout will check any incoming blocks against the transactions found in the
@@ -102,6 +122,49 @@ func (l *Lookout) Stop() error {
 	return nil
 }
 
+// fetchBlockWithRetries attempts to fetch a block from the blockchain using
+// its hash. If it fails to fetch the block, it will back-off and retry up to
+// MaxNumRetries times.
+func (l *Lookout) fetchBlockWithRetries(hash *chainhash.Hash) (*wire.MsgBlock,
+	error) {
+
+	backoff := l.cfg.MinBackoff
+
+	updateBackoff := func() {
+		backoff *= 2
+		if backoff > l.cfg.MaxBackoff {
+			backoff = l.cfg.MaxBackoff
+		}
+	}
+
+	var attempt int
+	for {
+		attempt++
+
+		block, err := l.cfg.BlockFetcher.GetBlock(hash)
+		if err == nil {
+			return block, nil
+		}
+
+		if attempt > l.cfg.MaxNumRetries {
+			return nil, fmt.Errorf("failed to fetch block %s "+
+				"after %d attempts: %v", hash, attempt, err)
+		}
+
+		log.Errorf("Failed to fetch block %s (attempt %d): %v. "+
+			"Retrying in %v seconds", hash, attempt, err,
+			backoff.Seconds())
+
+		select {
+		case <-time.After(backoff):
+		case <-l.quit:
+			return nil, ErrLookoutExiting
+		}
+
+		updateBackoff()
+	}
+}
+
 // watchBlocks serially pulls incoming epochs from the epoch source and searches
 // our accepted state updates for any breached transactions. If any are found,
 // we will attempt to decrypt the state updates' encrypted blobs and exact
@@ -118,11 +181,10 @@ func (l *Lookout) watchBlocks(epochs *chainntnfs.BlockEpochEvent) {
 			log.Debugf("Fetching block for (height=%d, hash=%s)",
 				epoch.Height, epoch.Hash)
 
-			// Fetch the full block from the backend corresponding
-			// to the newly arriving epoch.
-			block, err := l.cfg.BlockFetcher.GetBlock(epoch.Hash)
+			// Fetch the full block corresponding to the newly
+			// arriving epoch from the backend.
+			block, err := l.fetchBlockWithRetries(epoch.Hash)
 			if err != nil {
-				// TODO(conner): add retry logic?
 				log.Errorf("Unable to fetch block for "+
 					"(height=%x, hash=%s): %v",
 					epoch.Height, epoch.Hash, err)

watchtower/lookout/lookout_test.go

@@ -90,6 +90,9 @@ func TestLookoutBreachMatching(t *testing.T) {
 		DB:             db,
 		EpochRegistrar: backend,
 		Punisher:       punisher,
+		MinBackoff:     time.Second,
+		MaxBackoff:     time.Minute,
+		MaxNumRetries:  1,
 	})
 	if err := watcher.Start(); err != nil {
 		t.Fatalf("unable to start watcher: %v", err)

watchtower/standalone.go

@@ -3,6 +3,7 @@ package watchtower
 import (
 	"net"
 	"sync/atomic"
+	"time"
 
 	"github.com/btcsuite/btcd/btcec/v2"
 	"github.com/lightningnetwork/lnd/brontide"
@@ -65,6 +66,9 @@ func New(cfg *Config) (*Standalone, error) {
 		DB:             cfg.DB,
 		EpochRegistrar: cfg.EpochRegistrar,
 		Punisher:       punisher,
+		MinBackoff:     time.Second,
+		MaxBackoff:     time.Minute,
+		MaxNumRetries:  5,
 	})
 
 	// Create a brontide listener on each of the provided listening