Merge pull request #299 from xssnick/dev-v1-11

Dict String() & LoadAll from proof, RLDP Unexpected transfers limits,…

Author: xssnick

README.md

@@ -4,7 +4,7 @@
 
 [![Based on TON][ton-svg]][ton]
 [![Telegram Channel][tgc-svg]][tg-channel]
-![Coverage](https://img.shields.io/badge/Coverage-72.6%25-brightgreen)
+![Coverage](https://img.shields.io/badge/Coverage-72.8%25-brightgreen)
 
 Golang library for interacting with TON blockchain.
 

adnl/adnl.go

@@ -9,7 +9,6 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
-	"log"
 	"reflect"
 	"strings"
 	"sync"
@@ -85,7 +84,7 @@ type ADNL struct {
 	mx sync.RWMutex
 }
 
-var Logger = log.Println
+var Logger = func(v ...any) {}
 
 func (g *Gateway) initADNL() *ADNL {
 	tm := int32(time.Now().Unix())

adnl/gateway.go

@@ -68,7 +68,9 @@ type udpPacket struct {
 type Gateway struct {
 	conn net.PacketConn
 
-	dhtIP      net.IP
+	dhtIP   net.IP
+	dhtPort uint16
+
 	addrList   address.List
 	key        ed25519.PrivateKey
 	processors map[string]*srvProcessor
@@ -169,6 +171,10 @@ func (g *Gateway) SetExternalIP(ip net.IP) {
 	g.dhtIP = ip
 }
 
+func (g *Gateway) SetExternalPort(port uint16) {
+	g.dhtPort = port
+}
+
 func (g *Gateway) StartServer(listenAddr string, listenThreads ...int) (err error) {
 	threads := 1
 	if len(listenThreads) > 0 && listenThreads[0] > 0 {
@@ -186,9 +192,12 @@ func (g *Gateway) StartServer(listenAddr string, listenThreads ...int) (err erro
 	}
 
 	if ip = ip.To4(); !ip.Equal(net.IPv4zero) {
-		port, err := strconv.ParseUint(adr[1], 10, 16)
-		if err != nil {
-			return fmt.Errorf("invalid listen port")
+		port := uint64(g.dhtPort)
+		if port == 0 {
+			port, err = strconv.ParseUint(adr[1], 10, 16)
+			if err != nil {
+				return fmt.Errorf("invalid listen port")
+			}
 		}
 
 		tm := int32(time.Now().Unix())

adnl/rldp/client.go

@@ -47,18 +47,23 @@ type activeTransfer struct {
 }
 
 type activeRequest struct {
-	id       string
 	deadline int64
 	result   chan<- AsyncQueryResult
 }
 
+type expectedTransfer struct {
+	deadline int64
+	maxSize  int64
+}
+
 type RLDP struct {
 	adnl  ADNL
 	useV2 bool
 
 	activateRecoverySender chan bool
 	activeRequests         map[string]*activeRequest
 	activeTransfers        map[string]*activeTransfer
+	expectedTransfers      map[string]*expectedTransfer
 
 	recvStreams map[string]*decoderStream
 
@@ -94,6 +99,7 @@ type decoderStream struct {
 }
 
 var DefaultSymbolSize uint32 = 768
+var MaxUnexpectedTransferSize int64 = 1 << 16 // 64 KB
 
 const _MTU = 1 << 37
 
@@ -103,6 +109,7 @@ func NewClient(a ADNL) *RLDP {
 		activeRequests:         map[string]*activeRequest{},
 		activeTransfers:        map[string]*activeTransfer{},
 		recvStreams:            map[string]*decoderStream{},
+		expectedTransfers:      map[string]*expectedTransfer{},
 		activateRecoverySender: make(chan bool, 1),
 	}
 
@@ -168,12 +175,22 @@ func (r *RLDP) handleMessage(msg *adnl.MessageCustom) error {
 		id := string(m.TransferID)
 		r.mx.RLock()
 		stream := r.recvStreams[id]
+		expected := r.expectedTransfers[id]
 		r.mx.RUnlock()
 
 		if stream == nil {
-			// TODO: limit unexpected transfer size to 1024 bytes
 			if m.TotalSize > _MTU || m.TotalSize <= 0 {
-				return fmt.Errorf("bad rldp packet total size")
+				return fmt.Errorf("bad rldp packet total size %d", m.TotalSize)
+			}
+
+			// unexpected transfers limited to this size, for protection
+			var maxTransferSize = MaxUnexpectedTransferSize
+			if expected != nil {
+				maxTransferSize = expected.maxSize
+			}
+
+			if m.TotalSize > maxTransferSize {
+				return fmt.Errorf("too big transfer size %d, max allowed %d", m.TotalSize, maxTransferSize)
 			}
 
 			stream = &decoderStream{
@@ -189,6 +206,7 @@ func (r *RLDP) handleMessage(msg *adnl.MessageCustom) error {
 			} else {
 				r.recvStreams[id] = stream
 			}
+			delete(r.expectedTransfers, id)
 			r.mx.Unlock()
 		}
 
@@ -405,7 +423,8 @@ func (r *RLDP) recoverySender() {
 	packets := make([]tl.Serializable, 0, 1024)
 	transfersToProcess := make([]*activeTransfer, 0, 128)
 	timedOut := make([]*activeTransfer, 0, 32)
-	timedOutReq := make([]*activeRequest, 0, 32)
+	timedOutReq := make([]string, 0, 32)
+	timedOutExp := make([]string, 0, 32)
 	closerCtx := r.adnl.GetCloserCtx()
 	ticker := time.NewTicker(1 * time.Millisecond)
 	defer ticker.Stop()
@@ -419,6 +438,7 @@ func (r *RLDP) recoverySender() {
 			transfersToProcess = transfersToProcess[:0]
 			timedOut = timedOut[:0]
 			timedOutReq = timedOutReq[:0]
+			timedOutExp = timedOutExp[:0]
 
 			ms := time.Now().UnixNano() / int64(time.Millisecond)
 
@@ -436,13 +456,19 @@ func (r *RLDP) recoverySender() {
 				}
 			}
 
-			for _, req := range r.activeRequests {
+			for id, req := range r.activeRequests {
 				if req.deadline < ms {
-					timedOutReq = append(timedOutReq, req)
+					timedOutReq = append(timedOutReq, id)
 				}
 			}
 
-			if len(r.activeRequests)+len(r.activeTransfers) == 0 {
+			for id, req := range r.expectedTransfers {
+				if req.deadline < ms {
+					timedOutExp = append(timedOutExp, id)
+				}
+			}
+
+			if len(r.activeRequests)+len(r.activeTransfers)+len(r.expectedTransfers) == 0 {
 				// stop ticks to not consume resources
 				ticker.Stop()
 			}
@@ -481,13 +507,16 @@ func (r *RLDP) recoverySender() {
 				}
 			}
 
-			if len(timedOut) > 0 || len(timedOutReq) > 0 {
+			if len(timedOut) > 0 || len(timedOutReq) > 0 || len(timedOutExp) > 0 {
 				r.mx.Lock()
 				for _, transfer := range timedOut {
 					delete(r.activeTransfers, string(transfer.id))
 				}
 				for _, req := range timedOutReq {
-					delete(r.activeRequests, req.id)
+					delete(r.activeRequests, req)
+				}
+				for _, req := range timedOutExp {
+					delete(r.expectedTransfers, req)
 				}
 				r.mx.Unlock()
 			}
@@ -580,17 +609,9 @@ func (r *RLDP) DoQuery(ctx context.Context, maxAnswerSize int64, query, result t
 
 	select {
 	case resp := <-res:
-		r.mx.Lock()
-		delete(r.activeRequests, string(qid))
-		r.mx.Unlock()
-
 		reflect.ValueOf(result).Elem().Set(reflect.ValueOf(resp.Result))
 		return nil
 	case <-ctx.Done():
-		r.mx.Lock()
-		delete(r.activeRequests, string(qid))
-		r.mx.Unlock()
-
 		return fmt.Errorf("response deadline exceeded, err: %w", ctx.Err())
 	}
 }
@@ -627,6 +648,7 @@ func (r *RLDP) DoQueryAsync(ctx context.Context, maxAnswerSize int64, id []byte,
 	if err != nil {
 		return err
 	}
+	reverseId := reverseTransferId(transferId)
 
 	out := timeout.UnixNano() / int64(time.Millisecond)
 
@@ -635,6 +657,10 @@ func (r *RLDP) DoQueryAsync(ctx context.Context, maxAnswerSize int64, id []byte,
 		deadline: out,
 		result:   result,
 	}
+	r.expectedTransfers[string(reverseId)] = &expectedTransfer{
+		deadline: out,
+		maxSize:  maxAnswerSize,
+	}
 	r.mx.Unlock()
 
 	if err = r.sendMessageParts(ctx, transferId, data, (time.Duration(q.Timeout)-time.Duration(time.Now().Unix()))*time.Second); err != nil {
@@ -659,17 +685,14 @@ func (r *RLDP) SendAnswer(ctx context.Context, maxAnswerSize int64, queryId, toT
 		return fmt.Errorf("too big answer for that client, client wants no more than %d bytes", maxAnswerSize)
 	}
 
-	transferId := make([]byte, 32)
+	var transferId []byte
 
 	if toTransferId != nil {
 		// if we have transfer to respond, invert it and use id
-		copy(transferId, toTransferId)
-		for i := range transferId {
-			transferId[i] ^= 0xFF
-		}
+		transferId = reverseTransferId(toTransferId)
 	} else {
-		_, err = rand.Read(transferId)
-		if err != nil {
+		transferId = make([]byte, 32)
+		if _, err = rand.Read(transferId); err != nil {
 			return err
 		}
 	}
@@ -680,3 +703,12 @@ func (r *RLDP) SendAnswer(ctx context.Context, maxAnswerSize int64, queryId, toT
 
 	return nil
 }
+
+func reverseTransferId(id []byte) []byte {
+	rev := make([]byte, 32)
+	copy(rev, id)
+	for i := range rev {
+		rev[i] ^= 0xFF
+	}
+	return rev
+}

adnl/rldp/client_test.go

@@ -233,7 +233,6 @@ func TestRLDP_handleMessage(t *testing.T) {
 				queryId := string(tQuery.ID)
 				tChan := make(chan AsyncQueryResult, 2)
 				cli.activeRequests[queryId] = &activeRequest{
-					id:       queryId,
 					deadline: time.Now().Add(time.Second * 10).Unix(),
 					result:   tChan,
 				}
@@ -540,11 +539,6 @@ func TestRLDP_DoQuery(t *testing.T) {
 		if !reflect.DeepEqual(res, answer.Data) {
 			t.Error("got bad response")
 		}
-		time.Sleep(1 * time.Second)
-		if len(cli.activeRequests) != 0 {
-			t.Error("invalid activeRequests and activeTransfers after response", len(cli.activeRequests))
-		}
-
 	})
 
 	t.Run("negative case (deadline exceeded)", func(t *testing.T) {

example/accept-payments/main.go

@@ -74,9 +74,11 @@ func main() {
 			src := ti.SrcAddr
 
 			if dsc, ok := tx.Description.(tlb.TransactionDescriptionOrdinary); ok && dsc.BouncePhase != nil {
-				// transaction was bounced, and coins was returned to sender
-				// this can happen mostly on custom contracts
-				continue
+				if _, ok = dsc.BouncePhase.Phase.(tlb.BouncePhaseOk); ok {
+					// transaction was bounced, and coins were returned to sender
+					// this can happen mostly on custom contracts
+					continue
+				}
 			}
 
 			if !ti.ExtraCurrencies.IsEmpty() {

example/proof-cell/main.go

@@ -132,4 +132,34 @@ func main() {
 
 	println("\n\nProof of 2 keys in 2 dictionaries on diff depth:")
 	println(proof.Dump())
+
+	////////////////////////////////////////////////////
+	///// VERIFY PROOF
+	////////////////////////////////////////////////////
+
+	println("Checking and parsing proof:")
+	expectedHash := exampleCell.Hash()
+
+	proofBody, err := cell.UnwrapProof(proof, expectedHash)
+	if err != nil {
+		panic(err)
+	}
+
+	println("proof verified, now we can trust its body and parse it")
+
+	var dataProof ExampleStruct
+	// LoadFromCellAsProof loads only not pruned branches and maps them to struct field, pruned fields stays empty
+	if err = tlb.LoadFromCellAsProof(&dataProof, proofBody.BeginParse()); err != nil {
+		panic(err)
+	}
+
+	val, err := dataProof.DictA.LoadValueByIntKey(big.NewInt(778))
+	if err != nil {
+		panic(err)
+	}
+
+	println("Trusted data inside proof mapped to struct:")
+	println("Some struct fields:", dataProof.A, val.String())
+
+	println("Not pruned dict keys:\n", dataProof.DictA.String())
 }

tlb/loader.go

@@ -364,7 +364,7 @@ func loadFromCell(v any, slice *cell.Slice, skipProofBranches, skipMagic bool) e
 				Tag:  reflect.StructTag(fmt.Sprintf("tlb:%q", strings.Join(settings[3:], " "))),
 			}})
 
-			values, err := dict.LoadAll()
+			values, err := dict.LoadAll(skipProofBranches)
 			if err != nil {
 				return fmt.Errorf("failed to load dict values for %v: %w", structField.Name, err)
 			}