Skip to content

Commit 00240a9

Browse files
joeavanzatojoeavanzato
committed
- Making headers dynamic based on selected arguments so we don't have to constantly populate all of them - only the options we are actually using.
1 parent b84e60c commit 00240a9

File tree

12 files changed

+100
-61
lines changed

12 files changed

+100
-61
lines changed

‎helpers/helpers.go

Lines changed: 29 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,28 @@ func DoesFileExist(filename string) bool {
189189
return true
190190
}
191191

192+
func GetHeaders(tempArgs map[string]any, headers []string) []string {
193+
if !tempArgs["passthrough"].(bool) {
194+
headers = append(headers, vars.GeoFields...)
195+
if tempArgs["use_ti"].(bool) {
196+
headers = append(headers, vars.ThreatFields...)
197+
}
198+
if tempArgs["use_dns"].(bool) {
199+
headers = append(headers, vars.DNSFields...)
200+
}
201+
if tempArgs["use_whois"].(bool) {
202+
if tempArgs["use_dns"].(bool) {
203+
headers = append(headers, vars.WhoisDomainFields...)
204+
}
205+
headers = append(headers, vars.WhoisIPFields...)
206+
}
207+
if tempArgs["use_idb"].(bool) {
208+
headers = append(headers, vars.IDBFields...)
209+
}
210+
}
211+
return headers
212+
}
213+
192214
func ListenOnWriteChannel(c chan []string, w *csv.Writer, logger zerolog.Logger, outputF *os.File, bufferSize int, wait *lbtypes.WaitGroupCount) {
193215
// TODO - Consider having pool of routines appending records to slice [][]string and a single reader drawing from this to avoid any bottle-necks
194216
// TODO - Consider sending writer in a goroutine with wait group, refilling buffer, etc.
@@ -376,8 +398,6 @@ func enrichRecord(logger zerolog.Logger, record []string, asnDB maxminddb.Reader
376398
} else {
377399
record = append(record, "none", "0", "none")
378400
}
379-
} else {
380-
record = append(record, "NA", "NA", "0")
381401
}
382402

383403
domain := ""
@@ -431,14 +451,14 @@ func enrichRecord(logger zerolog.Logger, record []string, asnDB maxminddb.Reader
431451
logger.Error().Msg(setdnserr.Error())
432452
}
433453
}*/
434-
} else {
435-
record = append(record, "")
436454
}
437455
// For TLD
438-
if domain == "." || domain == "" {
439-
record = append(record, "none")
440-
} else {
441-
record = append(record, domain)
456+
if tempArgs["use_dns"].(bool) {
457+
if domain == "." || domain == "" {
458+
record = append(record, "none")
459+
} else {
460+
record = append(record, domain)
461+
}
442462
}
443463

444464
// Removing for now as we can get TLD from live DNS if we are using that
@@ -460,22 +480,18 @@ func enrichRecord(logger zerolog.Logger, record []string, asnDB maxminddb.Reader
460480
// Handling Domain WhoIS lookups if we are using DNS and have a parsed domain with tld for the IP in question
461481
if tempArgs["use_whois"].(bool) && domain != "" && domain != "." {
462482
record = append(record, DoDomainWhoisenrichment(domain)...)
463-
} else {
483+
} else if tempArgs["use_whois"].(bool) && tempArgs["use_dns"].(bool) {
464484
// no whois used OR domain is invalid
465485
record = append(record, "NA", "NA", "NA", "NA")
466486
}
467487

468488
// Handling IP Whois lookups
469489
if tempArgs["use_whois"].(bool) {
470490
record = append(record, DoIPWhoisEnrichment(ipString)...)
471-
} else {
472-
record = append(record, "NA", "NA", "NA", "NA", "NA", "NA", "NA", "NA")
473491
}
474492

475493
if tempArgs["use_idb"].(bool) {
476494
record = append(record, DoIDBEnrichment(ipString)...)
477-
} else {
478-
record = append(record, "NA", "NA", "NA", "NA", "NA")
479495
}
480496

481497
return record

‎main.go

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package main
22

33
import (
44
"database/sql"
5+
"encoding/json"
56
"errors"
67
"flag"
78
"fmt"
@@ -56,6 +57,7 @@ func parseArgs(logger zerolog.Logger) (map[string]any, error) {
5657
updategeo := flag.Bool("updategeo", false, "Update local MaxMind databases, even if they are detected.")
5758
passthrough := flag.Bool("passthrough", false, "Skip all enrichment steps - only perform log conversion to CSV")
5859
includedc := flag.Bool("includedc", false, "Include datacenter list for Threat Intelligence enrichment - will add approximately ~129 million IP addresses to the DB (~7 GB on disk)")
60+
idb := flag.Bool("idb", false, "Perform a live enrichment using the Shodan InternetDB")
5961
ip := flag.String("ip", "", "Provide an IP address for ad-hoc enrichment via stdout")
6062
flag.Parse()
6163

@@ -99,6 +101,7 @@ func parseArgs(logger zerolog.Logger) (map[string]any, error) {
99101
"includedc": *includedc,
100102
"ip": *ip,
101103
"whois": *whois,
104+
"idb": *idb,
102105
}
103106

104107
if (*intelfile != "" && (*inteltype == "" || *intelname == "")) || ((*intelfile == "" || *intelname == "") && *inteltype != "") || ((*intelfile == "" || *inteltype == "") && *intelname != "") {
@@ -209,6 +212,10 @@ func enrichLogs(arguments map[string]any, logFiles []string, logger zerolog.Logg
209212

210213
tempArgs["passthrough"] = arguments["passthrough"].(bool)
211214
tempArgs["use_whois"] = arguments["whois"].(bool)
215+
tempArgs["use_idb"] = arguments["idb"].(bool)
216+
tempArgs["use_dns"] = arguments["dns"].(bool)
217+
tempArgs["passthrough"] = arguments["passthrough"].(bool)
218+
tempArgs["use_ti"] = arguments["useti"].(bool)
212219
//startDate, endDate := getDateBounds(tempArgs)
213220

214221
// TODO - Make this OS independent
@@ -454,9 +461,31 @@ func processFile(arguments map[string]any, inputFile string, outputFile string,
454461
}
455462
}
456463

464+
func test() {
465+
ipaddress := "8.8.8.8"
466+
resp, err := helpers.IDB_Http_Client.Get(fmt.Sprintf("https://internetdb.shodan.io/%s", ipaddress))
467+
if err != nil {
468+
fmt.Println("Error %s", err)
469+
}
470+
defer resp.Body.Close()
471+
if err != nil {
472+
fmt.Println("Error %s", err)
473+
}
474+
dec := json.NewDecoder(resp.Body)
475+
dec.DisallowUnknownFields()
476+
var p lbtypes.ShodanIDBResponse
477+
err = dec.Decode(&p)
478+
if err != nil {
479+
fmt.Println("Error %s", err)
480+
vars.IDBfastcache.Set([]byte(ipaddress), []byte("error"))
481+
}
482+
fmt.Println(p)
483+
}
484+
457485
func main() {
458486
// TODO - Refactor all path handling to use path.Join or similar for OS-transparency
459-
487+
//test()
488+
//return
460489
logger := helpers.SetupLogger()
461490
arguments, err := parseArgs(logger)
462491
if err != nil {
@@ -594,4 +623,8 @@ func main() {
594623
if saveWhoisCacheErr != nil {
595624
logger.Error().Msg(saveWhoisCacheErr.Error())
596625
}
626+
saveIDBCacheErr := vars.IDBfastcache.SaveToFile(vars.IDBCacheFile)
627+
if saveIDBCacheErr != nil {
628+
logger.Error().Msg(saveIDBCacheErr.Error())
629+
}
597630
}

‎parsers/parse_cef.go

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ import (
44
"encoding/csv"
55
"github.com/joeavanzato/logboost/helpers"
66
"github.com/joeavanzato/logboost/lbtypes"
7-
"github.com/joeavanzato/logboost/vars"
87
"github.com/oschwald/maxminddb-golang"
98
"github.com/rs/zerolog"
109
"io"
@@ -164,12 +163,7 @@ func ParseCEF(logger zerolog.Logger, inputFile string, outputFile string, fullPa
164163
}
165164
// TODO - Sort
166165
//sort.Sort(sort.StringSlice(headers))
167-
if !tempArgs["passthrough"].(bool) {
168-
headers = append(headers, vars.GeoFields...)
169-
if tempArgs["use_idb"].(bool) {
170-
headers = append(headers, vars.IDBFields...)
171-
}
172-
}
166+
headers = helpers.GetHeaders(tempArgs, headers)
173167
outputF, err := helpers.CreateOutput(outputFile)
174168
if err != nil {
175169
return err

‎parsers/parse_clf.go

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ import (
44
"encoding/csv"
55
"github.com/joeavanzato/logboost/helpers"
66
"github.com/joeavanzato/logboost/lbtypes"
7-
"github.com/joeavanzato/logboost/vars"
87
"github.com/oschwald/maxminddb-golang"
98
"github.com/rs/zerolog"
109
"io"
@@ -66,12 +65,8 @@ func ParseCLF(logger zerolog.Logger, inputFile string, outputFile string, asnDB
6665
headers = append(headers, combinedHeaders...)
6766
}
6867

69-
if !tempArgs["passthrough"].(bool) {
70-
headers = append(headers, vars.GeoFields...)
71-
if tempArgs["use_idb"].(bool) {
72-
headers = append(headers, vars.IDBFields...)
73-
}
74-
}
68+
headers = helpers.GetHeaders(tempArgs, headers)
69+
7570
outputF, err := helpers.CreateOutput(outputFile)
7671
if err != nil {
7772
return err

‎parsers/parse_csv.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,18 @@ func ProcessCSV(logger zerolog.Logger, asnDB maxminddb.Reader, cityDB maxminddb.
9292
logger.Error().Msgf("Error Processing File: %v", err.Error())
9393
return
9494
}
95+
if tempArgs["use_ti"].(bool) {
96+
headers = append(headers, vars.ThreatFields...)
97+
}
98+
if tempArgs["use_dns"].(bool) {
99+
headers = append(headers, vars.DNSFields...)
100+
}
101+
if tempArgs["use_whois"].(bool) {
102+
if tempArgs["use_dns"].(bool) {
103+
headers = append(headers, vars.WhoisDomainFields...)
104+
}
105+
headers = append(headers, vars.WhoisIPFields...)
106+
}
95107
if tempArgs["use_idb"].(bool) {
96108
headers = append(headers, vars.IDBFields...)
97109
}

‎parsers/parse_iis_w3c.go

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ import (
44
"encoding/csv"
55
"github.com/joeavanzato/logboost/helpers"
66
"github.com/joeavanzato/logboost/lbtypes"
7-
"github.com/joeavanzato/logboost/vars"
87
"github.com/oschwald/maxminddb-golang"
98
"github.com/rs/zerolog"
109
"io"
@@ -85,12 +84,7 @@ func ParseIISStyle(logger zerolog.Logger, asnDB maxminddb.Reader, cityDB maxmind
8584
dateindex = helpers.FindTargetIndexInSlice(headers, arguments["datecol"].(string))
8685
}
8786

88-
if !tempArgs["passthrough"].(bool) {
89-
headers = append(headers, vars.GeoFields...)
90-
if tempArgs["use_idb"].(bool) {
91-
headers = append(headers, vars.IDBFields...)
92-
}
93-
}
87+
headers = helpers.GetHeaders(tempArgs, headers)
9488
err = writer.Write(headers)
9589
if err != nil {
9690
logger.Error().Msg(err.Error())

‎parsers/parse_json.go

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -92,12 +92,7 @@ func ParseJSON(logger zerolog.Logger, asnDB maxminddb.Reader, cityDB maxminddb.R
9292

9393
headers = append(headers, jsonkeys...)
9494
//sort.Sort(sort.StringSlice(headers))
95-
if !arguments["passthrough"].(bool) {
96-
headers = append(headers, vars.GeoFields...)
97-
if tempArgs["use_idb"].(bool) {
98-
headers = append(headers, vars.IDBFields...)
99-
}
100-
}
95+
headers = helpers.GetHeaders(tempArgs, headers)
10196
err = writer.Write(headers)
10297
if err != nil {
10398
logger.Error().Msg(err.Error())

‎parsers/parse_json_multi.go

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -180,12 +180,7 @@ func ParseMultiLineJSON(logger zerolog.Logger, asnDB maxminddb.Reader, cityDB ma
180180
// Sort JSONKeys alphabetically
181181
//sort.Sort(sort.StringSlice(headers))
182182

183-
if !arguments["passthrough"].(bool) {
184-
headers = append(headers, vars.GeoFields...)
185-
if tempArgs["use_idb"].(bool) {
186-
headers = append(headers, vars.IDBFields...)
187-
}
188-
}
183+
headers = helpers.GetHeaders(tempArgs, headers)
189184
err = writer.Write(headers)
190185
if err != nil {
191186
logger.Error().Msg(err.Error())

‎parsers/parse_kv.go

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -95,12 +95,7 @@ func ParseKV(logger zerolog.Logger, inputFile string, outputFile string, asnDB m
9595
headers = append(headers, kvheaders...)
9696
// TODO - Sort
9797
//sort.Sort(sort.StringSlice(headers))
98-
if !arguments["passthrough"].(bool) {
99-
headers = append(headers, vars.GeoFields...)
100-
if tempArgs["use_idb"].(bool) {
101-
headers = append(headers, vars.IDBFields...)
102-
}
103-
}
98+
headers = helpers.GetHeaders(tempArgs, headers)
10499
err = writer.Write(headers)
105100
if err != nil {
106101
logger.Error().Msg(err.Error())

‎parsers/parse_raw.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,18 @@ func ParseRaw(logger zerolog.Logger, asnDB maxminddb.Reader, cityDB maxminddb.Re
2828
headers := make([]string, 0)
2929
headers = append(headers, "line")
3030
headers = append(headers, vars.GeoFields...)
31+
if tempArgs["use_ti"].(bool) {
32+
headers = append(headers, vars.ThreatFields...)
33+
}
34+
if tempArgs["use_dns"].(bool) {
35+
headers = append(headers, vars.DNSFields...)
36+
}
37+
if tempArgs["use_whois"].(bool) {
38+
if tempArgs["use_dns"].(bool) {
39+
headers = append(headers, vars.WhoisDomainFields...)
40+
}
41+
headers = append(headers, vars.WhoisIPFields...)
42+
}
3143
if tempArgs["use_idb"].(bool) {
3244
headers = append(headers, vars.IDBFields...)
3345
}

0 commit comments

Comments
 (0)