Skip to content

Commit 51bea3e

Browse files
sstanculeanusstanculeanu
authored
Merge branch 'rc/v1.7.next1' into update_deps
2 parents eaa70a3 + 9f42664 commit 51bea3e

File tree

17 files changed

+648
-54
lines changed

17 files changed

+648
-54
lines changed

‎client/elasticClient.go

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package client
33
import (
44
"bytes"
55
"context"
6+
"errors"
67
"fmt"
78
"io"
89
"net/http"
@@ -84,6 +85,23 @@ func (ec *elasticClient) CheckAndCreateIndex(indexName string) error {
8485
return ec.createIndex(indexName)
8586
}
8687

88+
// PutMappings will put the provided mappings to a given index
89+
func (ec *elasticClient) PutMappings(indexName string, mappings *bytes.Buffer) error {
90+
res, err := ec.client.Indices.PutMapping(
91+
mappings,
92+
ec.client.Indices.PutMapping.WithIndex(indexName),
93+
)
94+
if err != nil {
95+
return err
96+
}
97+
98+
if res.IsError() {
99+
return errors.New(res.String())
100+
}
101+
102+
return nil
103+
}
104+
87105
// CheckAndCreateAlias creates a new alias if it does not already exist
88106
func (ec *elasticClient) CheckAndCreateAlias(alias string, indexName string) error {
89107
if ec.aliasExists(alias) {

‎data/transaction.go

Lines changed: 49 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -2,57 +2,61 @@ package data
22

33
import (
44
"time"
5+
6+
"github.com/multiversx/mx-chain-core-go/data/transaction"
57
)
68

79
// Transaction is a structure containing all the fields that need
810
// to be saved for a transaction. It has all the default fields
911
// plus some extra information for ease of search and filter
1012
type Transaction struct {
11-
MBHash string `json:"miniBlockHash"`
12-
Nonce uint64 `json:"nonce"`
13-
Round uint64 `json:"round"`
14-
Value string `json:"value"`
15-
ValueNum float64 `json:"valueNum"`
16-
Receiver string `json:"receiver"`
17-
Sender string `json:"sender"`
18-
ReceiverShard uint32 `json:"receiverShard"`
19-
SenderShard uint32 `json:"senderShard"`
20-
GasPrice uint64 `json:"gasPrice"`
21-
GasLimit uint64 `json:"gasLimit"`
22-
GasUsed uint64 `json:"gasUsed"`
23-
Fee string `json:"fee"`
24-
FeeNum float64 `json:"feeNum"`
25-
InitialPaidFee string `json:"initialPaidFee,omitempty"`
26-
Data []byte `json:"data"`
27-
Signature string `json:"signature"`
28-
Timestamp time.Duration `json:"timestamp"`
29-
Status string `json:"status"`
30-
SearchOrder uint32 `json:"searchOrder"`
31-
SenderUserName []byte `json:"senderUserName,omitempty"`
32-
ReceiverUserName []byte `json:"receiverUserName,omitempty"`
33-
HasSCR bool `json:"hasScResults,omitempty"`
34-
IsScCall bool `json:"isScCall,omitempty"`
35-
HasOperations bool `json:"hasOperations,omitempty"`
36-
HasLogs bool `json:"hasLogs,omitempty"`
37-
Tokens []string `json:"tokens,omitempty"`
38-
ESDTValues []string `json:"esdtValues,omitempty"`
39-
ESDTValuesNum []float64 `json:"esdtValuesNum,omitempty"`
40-
Receivers []string `json:"receivers,omitempty"`
41-
ReceiversShardIDs []uint32 `json:"receiversShardIDs,omitempty"`
42-
Type string `json:"type,omitempty"`
43-
Operation string `json:"operation,omitempty"`
44-
Function string `json:"function,omitempty"`
45-
IsRelayed bool `json:"isRelayed,omitempty"`
46-
Version uint32 `json:"version,omitempty"`
47-
GuardianAddress string `json:"guardian,omitempty"`
48-
GuardianSignature string `json:"guardianSignature,omitempty"`
49-
ErrorEvent bool `json:"errorEvent,omitempty"`
50-
CompletedEvent bool `json:"completedEvent,omitempty"`
51-
ExecutionOrder int `json:"-"`
52-
SmartContractResults []*ScResult `json:"-"`
53-
Hash string `json:"-"`
54-
BlockHash string `json:"-"`
55-
HadRefund bool `json:"-"`
13+
MBHash string `json:"miniBlockHash"`
14+
Nonce uint64 `json:"nonce"`
15+
Round uint64 `json:"round"`
16+
Value string `json:"value"`
17+
ValueNum float64 `json:"valueNum"`
18+
Receiver string `json:"receiver"`
19+
Sender string `json:"sender"`
20+
ReceiverShard uint32 `json:"receiverShard"`
21+
SenderShard uint32 `json:"senderShard"`
22+
GasPrice uint64 `json:"gasPrice"`
23+
GasLimit uint64 `json:"gasLimit"`
24+
GasUsed uint64 `json:"gasUsed"`
25+
Fee string `json:"fee"`
26+
FeeNum float64 `json:"feeNum"`
27+
InitialPaidFee string `json:"initialPaidFee,omitempty"`
28+
Data []byte `json:"data"`
29+
Signature string `json:"signature"`
30+
Timestamp time.Duration `json:"timestamp"`
31+
Status string `json:"status"`
32+
SearchOrder uint32 `json:"searchOrder"`
33+
SenderUserName []byte `json:"senderUserName,omitempty"`
34+
ReceiverUserName []byte `json:"receiverUserName,omitempty"`
35+
HasSCR bool `json:"hasScResults,omitempty"`
36+
IsScCall bool `json:"isScCall,omitempty"`
37+
HasOperations bool `json:"hasOperations,omitempty"`
38+
HasLogs bool `json:"hasLogs,omitempty"`
39+
Tokens []string `json:"tokens,omitempty"`
40+
ESDTValues []string `json:"esdtValues,omitempty"`
41+
ESDTValuesNum []float64 `json:"esdtValuesNum,omitempty"`
42+
Receivers []string `json:"receivers,omitempty"`
43+
ReceiversShardIDs []uint32 `json:"receiversShardIDs,omitempty"`
44+
Type string `json:"type,omitempty"`
45+
Operation string `json:"operation,omitempty"`
46+
Function string `json:"function,omitempty"`
47+
IsRelayed bool `json:"isRelayed,omitempty"`
48+
Version uint32 `json:"version,omitempty"`
49+
GuardianAddress string `json:"guardian,omitempty"`
50+
GuardianSignature string `json:"guardianSignature,omitempty"`
51+
ErrorEvent bool `json:"errorEvent,omitempty"`
52+
CompletedEvent bool `json:"completedEvent,omitempty"`
53+
RelayedAddr string `json:"relayed,omitempty"`
54+
InnerTransactions []*transaction.FrontendTransaction `json:"innerTransactions,omitempty"`
55+
ExecutionOrder int `json:"-"`
56+
SmartContractResults []*ScResult `json:"-"`
57+
Hash string `json:"-"`
58+
BlockHash string `json:"-"`
59+
HadRefund bool `json:"-"`
5660
}
5761

5862
// Receipt is a structure containing all the fields that need to be safe for a Receipt

‎integrationtests/relayedTxV3_test.go

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
//go:build integrationtests
2+
3+
package integrationtests
4+
5+
import (
6+
"context"
7+
"encoding/hex"
8+
dataBlock "github.com/multiversx/mx-chain-core-go/data/block"
9+
"github.com/multiversx/mx-chain-core-go/data/outport"
10+
"github.com/multiversx/mx-chain-core-go/data/transaction"
11+
indexerdata "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer"
12+
"github.com/stretchr/testify/require"
13+
"math/big"
14+
"testing"
15+
)
16+
17+
func TestRelayedTxV3(t *testing.T) {
18+
setLogLevelDebug()
19+
20+
esClient, err := createESClient(esURL)
21+
require.Nil(t, err)
22+
23+
esProc, err := CreateElasticProcessor(esClient)
24+
require.Nil(t, err)
25+
26+
txHash := []byte("relayedTxV3")
27+
header := &dataBlock.Header{
28+
Round: 50,
29+
TimeStamp: 5040,
30+
}
31+
32+
body := &dataBlock.Body{
33+
MiniBlocks: dataBlock.MiniBlockSlice{
34+
{
35+
Type: dataBlock.TxBlock,
36+
SenderShardID: 0,
37+
ReceiverShardID: 0,
38+
TxHashes: [][]byte{txHash},
39+
},
40+
},
41+
}
42+
43+
address1 := "erd1k7j6ewjsla4zsgv8v6f6fe3dvrkgv3d0d9jerczw45hzedhyed8sh2u34u"
44+
address2 := "erd14eyayfrvlrhzfrwg5zwleua25mkzgncggn35nvc6xhv5yxwml2es0f3dht"
45+
initialTx := &transaction.Transaction{
46+
Nonce: 1000,
47+
SndAddr: decodeAddress(address1),
48+
RcvAddr: decodeAddress(address2),
49+
GasLimit: 15406000,
50+
GasPrice: 1000000000,
51+
Value: big.NewInt(0),
52+
InnerTransactions: []*transaction.Transaction{
53+
{
54+
Nonce: 10,
55+
SndAddr: decodeAddress(address1),
56+
RcvAddr: decodeAddress(address2),
57+
GasLimit: 15406000,
58+
GasPrice: 1000000000,
59+
Value: big.NewInt(0),
60+
},
61+
{
62+
Nonce: 20,
63+
SndAddr: decodeAddress(address1),
64+
RcvAddr: decodeAddress(address2),
65+
GasLimit: 15406000,
66+
GasPrice: 1000000000,
67+
Value: big.NewInt(1000),
68+
},
69+
},
70+
}
71+
72+
txInfo := &outport.TxInfo{
73+
Transaction: initialTx,
74+
FeeInfo: &outport.FeeInfo{
75+
GasUsed: 10556000,
76+
Fee: big.NewInt(2257820000000000),
77+
InitialPaidFee: big.NewInt(2306320000000000),
78+
},
79+
ExecutionOrder: 0,
80+
}
81+
82+
pool := &outport.TransactionPool{
83+
Transactions: map[string]*outport.TxInfo{
84+
hex.EncodeToString(txHash): txInfo,
85+
},
86+
}
87+
err = esProc.SaveTransactions(createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards))
88+
require.Nil(t, err)
89+
90+
ids := []string{hex.EncodeToString(txHash)}
91+
genericResponse := &GenericResponse{}
92+
err = esClient.DoMultiGet(context.Background(), ids, indexerdata.TransactionsIndex, true, genericResponse)
93+
require.Nil(t, err)
94+
95+
require.JSONEq(t,
96+
readExpectedResult("./testdata/relayedTxV3/relayed-tx-v3.json"),
97+
string(genericResponse.Docs[0].Source),
98+
)
99+
}

‎integrationtests/testdata/relayedTxV3/relayed-tx-v3.json

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
{
2+
"miniBlockHash": "b3222be4506429fa237393c56b732a283eea5ccab465deb66b6348083793b6c9",
3+
"nonce": 1000,
4+
"round": 50,
5+
"value": "0",
6+
"valueNum": 0,
7+
"receiver": "erd14eyayfrvlrhzfrwg5zwleua25mkzgncggn35nvc6xhv5yxwml2es0f3dht",
8+
"sender": "erd1k7j6ewjsla4zsgv8v6f6fe3dvrkgv3d0d9jerczw45hzedhyed8sh2u34u",
9+
"receiverShard": 0,
10+
"senderShard": 0,
11+
"gasPrice": 1000000000,
12+
"gasLimit": 15406000,
13+
"gasUsed": 10556000,
14+
"fee": "2257820000000000",
15+
"feeNum": 0.00225782,
16+
"initialPaidFee": "2306320000000000",
17+
"data": null,
18+
"signature": "",
19+
"timestamp": 5040,
20+
"status": "success",
21+
"searchOrder": 0,
22+
"receivers": [
23+
"erd14eyayfrvlrhzfrwg5zwleua25mkzgncggn35nvc6xhv5yxwml2es0f3dht",
24+
"erd14eyayfrvlrhzfrwg5zwleua25mkzgncggn35nvc6xhv5yxwml2es0f3dht"
25+
],
26+
"receiversShardIDs": [
27+
1,
28+
1
29+
],
30+
"operation": "transfer",
31+
"isRelayed": true,
32+
"innerTransactions": [
33+
{
34+
"nonce": 10,
35+
"value": "0",
36+
"receiver": "erd14eyayfrvlrhzfrwg5zwleua25mkzgncggn35nvc6xhv5yxwml2es0f3dht",
37+
"sender": "erd1k7j6ewjsla4zsgv8v6f6fe3dvrkgv3d0d9jerczw45hzedhyed8sh2u34u",
38+
"gasPrice": 1000000000,
39+
"gasLimit": 15406000,
40+
"chainID": "",
41+
"version": 0
42+
},
43+
{
44+
"nonce": 20,
45+
"value": "1000",
46+
"receiver": "erd14eyayfrvlrhzfrwg5zwleua25mkzgncggn35nvc6xhv5yxwml2es0f3dht",
47+
"sender": "erd1k7j6ewjsla4zsgv8v6f6fe3dvrkgv3d0d9jerczw45hzedhyed8sh2u34u",
48+
"gasPrice": 1000000000,
49+
"gasLimit": 15406000,
50+
"chainID": "",
51+
"version": 0
52+
}
53+
]
54+
}

‎mock/databaseWriterStub.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,11 @@ type DatabaseWriterStub struct {
1414
DoScrollRequestCalled func(index string, body []byte, withSource bool, handlerFunc func(responseBytes []byte) error) error
1515
}
1616

17+
// PutMappings -
18+
func (dwm *DatabaseWriterStub) PutMappings(_ string, _ *bytes.Buffer) error {
19+
return nil
20+
}
21+
1722
// UpdateByQuery -
1823
func (dwm *DatabaseWriterStub) UpdateByQuery(_ context.Context, _ string, _ *bytes.Buffer) error {
1924
return nil

‎process/elasticproc/elasticProcessor.go

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters"
2121
"github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tags"
2222
"github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tokeninfo"
23+
"github.com/multiversx/mx-chain-es-indexer-go/templates"
2324
logger "github.com/multiversx/mx-chain-logger-go"
2425
)
2526

@@ -44,6 +45,7 @@ type ArgElasticProcessor struct {
4445
ImportDB bool
4546
IndexTemplates map[string]*bytes.Buffer
4647
IndexPolicies map[string]*bytes.Buffer
48+
ExtraMappings []templates.ExtraMapping
4749
EnabledIndexes map[string]struct{}
4850
TransactionsProc DBTransactionsHandler
4951
AccountsProc DBAccountHandler
@@ -94,7 +96,7 @@ func NewElasticProcessor(arguments *ArgElasticProcessor) (*elasticProcessor, err
9496
bulkRequestMaxSize: arguments.BulkRequestMaxSize,
9597
}
9698

97-
err = ei.init(arguments.UseKibana, arguments.IndexTemplates, arguments.IndexPolicies)
99+
err = ei.init(arguments.UseKibana, arguments.IndexTemplates, arguments.IndexPolicies, arguments.ExtraMappings)
98100
if err != nil {
99101
return nil, err
100102
}
@@ -105,7 +107,7 @@ func NewElasticProcessor(arguments *ArgElasticProcessor) (*elasticProcessor, err
105107
}
106108

107109
// TODO move all the index create part in a new component
108-
func (ei *elasticProcessor) init(useKibana bool, indexTemplates, _ map[string]*bytes.Buffer) error {
110+
func (ei *elasticProcessor) init(useKibana bool, indexTemplates, _ map[string]*bytes.Buffer, extraMappings []templates.ExtraMapping) error {
109111
err := ei.createOpenDistroTemplates(indexTemplates)
110112
if err != nil {
111113
return err
@@ -135,6 +137,17 @@ func (ei *elasticProcessor) init(useKibana bool, indexTemplates, _ map[string]*b
135137
return err
136138
}
137139

140+
return ei.addExtraMappings(extraMappings)
141+
}
142+
143+
func (ei *elasticProcessor) addExtraMappings(extraMappings []templates.ExtraMapping) error {
144+
for _, mappingsTuple := range extraMappings {
145+
err := ei.elasticClient.PutMappings(mappingsTuple.Index, mappingsTuple.Mappings)
146+
if err != nil {
147+
return err
148+
}
149+
}
150+
138151
return nil
139152
}
140153

‎process/elasticproc/factory/elasticProcessorFactory.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,10 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (dataindexer.E
4040
if err != nil {
4141
return nil, err
4242
}
43+
extraMappings, err := templatesAndPoliciesReader.GetExtraMappings()
44+
if err != nil {
45+
return nil, err
46+
}
4347

4448
enabledIndexesMap := make(map[string]struct{})
4549
for _, index := range arguments.EnabledIndexes {
@@ -119,6 +123,7 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (dataindexer.E
119123
UseKibana: arguments.UseKibana,
120124
IndexTemplates: indexTemplates,
121125
IndexPolicies: indexPolicies,
126+
ExtraMappings: extraMappings,
122127
OperationsProc: operationsProc,
123128
ImportDB: arguments.ImportDB,
124129
Version: arguments.Version,

‎process/elasticproc/interface.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ type DatabaseClientHandler interface {
2121
DoCountRequest(ctx context.Context, index string, body []byte) (uint64, error)
2222
UpdateByQuery(ctx context.Context, index string, buff *bytes.Buffer) error
2323

24+
PutMappings(indexName string, mappings *bytes.Buffer) error
2425
CheckAndCreateIndex(index string) error
2526
CheckAndCreateAlias(alias string, index string) error
2627
CheckAndCreateTemplate(templateName string, template *bytes.Buffer) error

‎process/elasticproc/templatesAndPolicies/interface.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,13 @@
11
package templatesAndPolicies
22

3-
import "bytes"
3+
import (
4+
"bytes"
5+
6+
"github.com/multiversx/mx-chain-es-indexer-go/templates"
7+
)
48

59
// TemplatesAndPoliciesHandler defines the actions that a templates and policies handler should do
610
type TemplatesAndPoliciesHandler interface {
711
GetElasticTemplatesAndPolicies() (map[string]*bytes.Buffer, map[string]*bytes.Buffer, error)
12+
GetExtraMappings() ([]templates.ExtraMapping, error)
813
}

0 commit comments

Comments
 (0)