Skip to content

Commit b84e60c

Browse files
joeavanzatojoeavanzato
committed
- Adding initial functionality to do live Shodan IDB queries
1 parent ad25089 commit b84e60c

File tree

13 files changed

+189
-75
lines changed

13 files changed

+189
-75
lines changed

‎helpers/helpers.go

Lines changed: 144 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"compress/gzip"
77
"database/sql"
88
"encoding/csv"
9+
"encoding/json"
910
"fmt"
1011
"github.com/joeavanzato/logboost/lbtypes"
1112
tldparser "github.com/joeavanzato/logboost/tldparserr"
@@ -456,97 +457,163 @@ func enrichRecord(logger zerolog.Logger, record []string, asnDB maxminddb.Reader
456457
record = append(record, "NA")
457458
}*/
458459

459-
// Handling Domain WhoIS lookups if we are using DNS and have a parsed domain with tld
460+
// Handling Domain WhoIS lookups if we are using DNS and have a parsed domain with tld for the IP in question
460461
if tempArgs["use_whois"].(bool) && domain != "" && domain != "." {
461-
// "lb_DomainWhois_CreatedDate", "lb_DomainWhois_UpdatedDate", "lb_DomainWhois_Country", "lb_DomainWhois_Organization"
462-
//u, _ := tld.Parse(dnsRecords[i])
463-
//domain := fmt.Sprintf("%s.%s", u.Domain, u.TLD)
464-
// Check for cached data using this domain
465-
_value := make([]byte, 0)
466-
cacheValue, _existsInCache := vars.Whoisfastcache.HasGet(_value, []byte(domain))
467-
if _existsInCache {
468-
// If it exists, check if it was stored with an error and it not, parse it out and append the values
469-
if string(cacheValue) == "error" {
470-
record = append(record, "err", "err", "err", "err")
471-
} else {
472-
whoisData := strings.Split(string(cacheValue), "|")
473-
record = append(record, whoisData...)
474-
}
475-
} else {
476-
// Domain Does not exist in cache yet
477-
result2, whoiserr := Whois(domain)
478-
if whoiserr != nil {
479-
vars.Whoisfastcache.Set([]byte(domain), []byte("error"))
480-
record = append(record, "err", "err", "err", "err")
481-
} else {
482-
// Set cache
483-
parsedresult, parseerr := whoisparser.Parse(result2)
484-
if parseerr == nil {
485-
whoIsData := make([]string, 0)
486-
if parsedresult.Domain != nil {
487-
whoIsData = append(whoIsData, parsedresult.Domain.CreatedDate, parsedresult.Domain.UpdatedDate)
488-
} else {
489-
whoIsData = append(whoIsData, "NA", "NA")
490-
}
491-
if parsedresult.Registrant != nil {
492-
whoIsData = append(whoIsData, parsedresult.Registrant.Country, parsedresult.Registrant.Organization)
493-
} else {
494-
whoIsData = append(whoIsData, "NA", "NA")
495-
}
496-
record = append(record, whoIsData...)
497-
vars.Whoisfastcache.Set([]byte(domain), []byte(strings.Join(whoIsData, "|")))
498-
} else {
499-
vars.Whoisfastcache.Set([]byte(domain), []byte("error"))
500-
record = append(record, "err", "err", "err", "err")
501-
}
502-
}
503-
}
462+
record = append(record, DoDomainWhoisenrichment(domain)...)
504463
} else {
505464
// no whois used OR domain is invalid
506465
record = append(record, "NA", "NA", "NA", "NA")
507466
}
508467

509468
// Handling IP Whois lookups
510-
// This is pretty slow - probably will adopt just specific source code from the current used library to streamline this
511469
if tempArgs["use_whois"].(bool) {
512-
value := make([]byte, 0)
513-
cacheValue, existsInCache := vars.Whoisfastcache.HasGet(value, []byte(ipString))
514-
if existsInCache {
515-
if string(cacheValue) == "error" {
516-
record = append(record, "NA", "NA", "NA", "NA", "NA", "NA", "NA", "NA")
517-
} else {
518-
parsedResult, parseerr := ParseIPWhoisLookup(string(cacheValue))
519-
if parseerr == nil {
520-
// "lb_Whois_CIDR", "lb_Whois_NetName", "lb_Whois_NetType", "lb_Whois_Organization", "lb_Whois_Created", "lb_Whois_Updated", "lb_Whois_Organization", "lb_Whois_Country", "lb_Whois_Parent"
521-
record = append(record, parsedResult.CIDR, parsedResult.NetName, parsedResult.NetType, parsedResult.Customer, parsedResult.RegistrationDate, parsedResult.RegistrationUpdated, parsedResult.Country, parsedResult.Parent)
522-
//recordsJoined := strings.Join(dnsRecords, "|")
470+
record = append(record, DoIPWhoisEnrichment(ipString)...)
471+
} else {
472+
record = append(record, "NA", "NA", "NA", "NA", "NA", "NA", "NA", "NA")
473+
}
474+
475+
if tempArgs["use_idb"].(bool) {
476+
record = append(record, DoIDBEnrichment(ipString)...)
477+
} else {
478+
record = append(record, "NA", "NA", "NA", "NA", "NA")
479+
}
480+
481+
return record
482+
}
483+
484+
// IntSlicetoStringSlice converts a slice of ints to a slice of the same length but of type string
485+
func IntSlicetoStringSlice(s []int) []string {
486+
os := make([]string, len(s))
487+
for k, v := range s {
488+
os[k] = strconv.Itoa(v)
489+
}
490+
return os
491+
}
492+
493+
// DoIDBEnrichment returns a slice representing enrichments from Shodan's InternetDB project using the provided IP Address as the enrichment target.
494+
func DoIDBEnrichment(ipaddress string) []string {
495+
_value := make([]byte, 0)
496+
cacheValue, _existsInCache := vars.IDBfastcache.HasGet(_value, []byte(ipaddress))
497+
errorString := []string{"err", "err", "err", "err", "err"}
498+
if _existsInCache {
499+
if string(cacheValue) == "error" {
500+
return errorString
501+
} else {
502+
idbData := strings.Split(string(cacheValue), "&&")
503+
return idbData
504+
}
505+
}
506+
resp, err := IDB_Http_Client.Get(fmt.Sprintf("https://internetdb.shodan.io/%s", ipaddress))
507+
if err != nil {
508+
vars.IDBfastcache.Set([]byte(ipaddress), []byte("error"))
509+
return errorString
510+
}
511+
defer resp.Body.Close()
512+
if err != nil {
513+
vars.IDBfastcache.Set([]byte(ipaddress), []byte("error"))
514+
return errorString
515+
}
516+
dec := json.NewDecoder(resp.Body)
517+
dec.DisallowUnknownFields()
518+
var p lbtypes.ShodanIDBResponse
519+
err = dec.Decode(&p)
520+
if err != nil {
521+
vars.IDBfastcache.Set([]byte(ipaddress), []byte("error"))
522+
return errorString
523+
}
524+
idbdata := make([]string, 0)
525+
idbdata = append(idbdata, strings.Join(p.Cpes, "|"), strings.Join(p.Hostnames, "|"), strings.Join(IntSlicetoStringSlice(p.Ports), "|"), strings.Join(p.Tags, "|"), strings.Join(p.Vulns, "|"))
526+
vars.IDBfastcache.Set([]byte(ipaddress), []byte(strings.Join(idbdata, "&&")))
527+
return idbdata
528+
}
529+
530+
// DoDomainWhoisenrichment returns a slice representing enrichments from a Domain-based WhoIS lookup.
531+
func DoDomainWhoisenrichment(domain string) []string {
532+
// "lb_DomainWhois_CreatedDate", "lb_DomainWhois_UpdatedDate", "lb_DomainWhois_Country", "lb_DomainWhois_Organization"
533+
//u, _ := tld.Parse(dnsRecords[i])
534+
//domain := fmt.Sprintf("%s.%s", u.Domain, u.TLD)
535+
// Check for cached data using this domain
536+
_value := make([]byte, 0)
537+
cacheValue, _existsInCache := vars.Whoisfastcache.HasGet(_value, []byte(domain))
538+
if _existsInCache {
539+
// If it exists, check if it was stored with an error and it not, parse it out and append the values
540+
if string(cacheValue) == "error" {
541+
return []string{"err", "err", "err", "err"}
542+
} else {
543+
whoisData := strings.Split(string(cacheValue), "|")
544+
return whoisData
545+
}
546+
} else {
547+
// Domain Does not exist in cache yet
548+
result2, whoiserr := Whois(domain)
549+
if whoiserr != nil {
550+
vars.Whoisfastcache.Set([]byte(domain), []byte("error"))
551+
return []string{"err", "err", "err", "err"}
552+
} else {
553+
// Set cache
554+
parsedresult, parseerr := whoisparser.Parse(result2)
555+
if parseerr == nil {
556+
whoIsData := make([]string, 0)
557+
if parsedresult.Domain != nil {
558+
whoIsData = append(whoIsData, parsedresult.Domain.CreatedDate, parsedresult.Domain.UpdatedDate)
523559
} else {
524-
record = append(record, "NA", "NA", "NA", "NA", "NA", "NA", "NA", "NA")
560+
whoIsData = append(whoIsData, "NA", "NA")
525561
}
562+
if parsedresult.Registrant != nil {
563+
whoIsData = append(whoIsData, parsedresult.Registrant.Country, parsedresult.Registrant.Organization)
564+
} else {
565+
whoIsData = append(whoIsData, "NA", "NA")
566+
}
567+
vars.Whoisfastcache.Set([]byte(domain), []byte(strings.Join(whoIsData, "|")))
568+
return whoIsData
569+
} else {
570+
vars.Whoisfastcache.Set([]byte(domain), []byte("error"))
571+
return []string{"err", "err", "err", "err"}
526572
}
573+
}
574+
}
575+
return []string{"NA", "NA", "NA", "NA"}
576+
}
577+
578+
// DoIPWhoisEnrichment returns a slice representing enrichments from an IP-based WhoIS lookup.
579+
func DoIPWhoisEnrichment(ipaddress string) []string {
580+
// This is pretty slow - probably will adopt just specific source code from the current used library to streamline this
581+
value := make([]byte, 0)
582+
cacheValue, existsInCache := vars.Whoisfastcache.HasGet(value, []byte(ipaddress))
583+
if existsInCache {
584+
if string(cacheValue) == "error" {
585+
return []string{"NA", "NA", "NA", "NA", "NA", "NA", "NA", "NA"}
527586
} else {
528-
//result2, whoiserr := whois.Whois(ipString)
529-
// IP Does not exist in cache yet
530-
result2, whoiserr := Whois(ipString)
531-
if result2 == "" {
532-
vars.Whoisfastcache.Set([]byte(ipString), []byte("error"))
587+
parsedResult, parseerr := ParseIPWhoisLookup(string(cacheValue))
588+
if parseerr == nil {
589+
// "lb_Whois_CIDR", "lb_Whois_NetName", "lb_Whois_NetType", "lb_Whois_Organization", "lb_Whois_Created", "lb_Whois_Updated", "lb_Whois_Organization", "lb_Whois_Country", "lb_Whois_Parent"
590+
return []string{parsedResult.CIDR, parsedResult.NetName, parsedResult.NetType, parsedResult.Customer, parsedResult.RegistrationDate, parsedResult.RegistrationUpdated, parsedResult.Country, parsedResult.Parent}
591+
//recordsJoined := strings.Join(dnsRecords, "|")
533592
} else {
534-
vars.Whoisfastcache.Set([]byte(ipString), []byte(result2))
593+
return []string{"NA", "NA", "NA", "NA", "NA", "NA", "NA", "NA"}
535594
}
536-
if whoiserr == nil {
537-
parsedResult, parseerr := ParseIPWhoisLookup(result2)
538-
if parseerr == nil {
539-
// "lb_Whois_CIDR", "lb_Whois_NetName", "lb_Whois_NetType", "lb_Whois_Organization", "lb_Whois_Created", "lb_Whois_Updated", "lb_Whois_Organization", "lb_Whois_Country", "lb_Whois_Parent"
540-
record = append(record, parsedResult.CIDR, parsedResult.NetName, parsedResult.NetType, parsedResult.Customer, parsedResult.RegistrationDate, parsedResult.RegistrationUpdated, parsedResult.Country, parsedResult.Parent)
541-
//recordsJoined := strings.Join(dnsRecords, "|")
542-
} else {
543-
record = append(record, "NA", "NA", "NA", "NA", "NA", "NA", "NA", "NA")
544-
}
595+
}
596+
} else {
597+
//result2, whoiserr := whois.Whois(ipString)
598+
// IP Does not exist in cache yet
599+
result2, whoiserr := Whois(ipaddress)
600+
if result2 == "" {
601+
vars.Whoisfastcache.Set([]byte(ipaddress), []byte("error"))
602+
} else {
603+
vars.Whoisfastcache.Set([]byte(ipaddress), []byte(result2))
604+
}
605+
if whoiserr == nil {
606+
parsedResult, parseerr := ParseIPWhoisLookup(result2)
607+
if parseerr == nil {
608+
// "lb_Whois_CIDR", "lb_Whois_NetName", "lb_Whois_NetType", "lb_Whois_Organization", "lb_Whois_Created", "lb_Whois_Updated", "lb_Whois_Organization", "lb_Whois_Country", "lb_Whois_Parent"
609+
return []string{parsedResult.CIDR, parsedResult.NetName, parsedResult.NetType, parsedResult.Customer, parsedResult.RegistrationDate, parsedResult.RegistrationUpdated, parsedResult.Country, parsedResult.Parent}
610+
//recordsJoined := strings.Join(dnsRecords, "|")
611+
} else {
612+
return []string{"NA", "NA", "NA", "NA", "NA", "NA", "NA", "NA"}
545613
}
546614
}
547615
}
548-
549-
return record
616+
return []string{"NA", "NA", "NA", "NA", "NA", "NA", "NA", "NA"}
550617
}
551618

552619
type IPWhoisResult struct {
@@ -639,7 +706,9 @@ func ParseIPWhoisLookup(data string) (IPWhoisResult, error) {
639706
} else if strings.HasPrefix(v, "RegDate:") {
640707
result.RegistrationDate = value
641708
} else if strings.HasPrefix(v, "Updated:") {
642-
result.AddressUpdated = value
709+
if value != "" {
710+
result.AddressUpdated = value
711+
}
643712
} else if strings.HasPrefix(v, "OrgNOCHandle:") {
644713
} else if strings.HasPrefix(v, "OrgNOCName:") {
645714
result.OrgNOCName = value

‎helpers/network.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ var resolver = &net.Resolver{
2424
},
2525
}
2626

27+
var IDB_Http_Client = http.Client{Timeout: time.Duration(2) * time.Second}
28+
2729
func SetupPrivateNetworks() error {
2830
for _, cidr := range []string{
2931
"127.0.0.0/8", // IPv4 loopback

‎lbtypes/typesMethods.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,19 @@ type DBRefs struct {
4949
ASN *maxminddb.Reader
5050
}
5151

52+
// IDB Data Struct
53+
type ShodanIDBResponse struct {
54+
Cpes []string `json:"cpes"`
55+
Hostnames []string `json:"hostnames"`
56+
IP string `json:"ip"`
57+
Ports []int `json:"ports"`
58+
Tags []string `json:"tags"`
59+
Vulns []string `json:"vulns"`
60+
}
61+
type ShodanIDBResponseFail struct {
62+
Detail string `json:"detail"`
63+
}
64+
5265
// TODO - Put lock and map in single struct for organization - then refactor CheckIP and AddIP to just take the original cachemap struct
5366
var IPCacheMap = make(map[string]IPCache)
5467
var IPCacheMapLock = sync.RWMutex{}

‎parsers/parse_cef.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,9 @@ func ParseCEF(logger zerolog.Logger, inputFile string, outputFile string, fullPa
166166
//sort.Sort(sort.StringSlice(headers))
167167
if !tempArgs["passthrough"].(bool) {
168168
headers = append(headers, vars.GeoFields...)
169+
if tempArgs["use_idb"].(bool) {
170+
headers = append(headers, vars.IDBFields...)
171+
}
169172
}
170173
outputF, err := helpers.CreateOutput(outputFile)
171174
if err != nil {

‎parsers/parse_clf.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,9 @@ func ParseCLF(logger zerolog.Logger, inputFile string, outputFile string, asnDB
6868

6969
if !tempArgs["passthrough"].(bool) {
7070
headers = append(headers, vars.GeoFields...)
71+
if tempArgs["use_idb"].(bool) {
72+
headers = append(headers, vars.IDBFields...)
73+
}
7174
}
7275
outputF, err := helpers.CreateOutput(outputFile)
7376
if err != nil {

‎parsers/parse_csv.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,9 @@ func ProcessCSV(logger zerolog.Logger, asnDB maxminddb.Reader, cityDB maxminddb.
9292
logger.Error().Msgf("Error Processing File: %v", err.Error())
9393
return
9494
}
95+
if tempArgs["use_idb"].(bool) {
96+
headers = append(headers, vars.IDBFields...)
97+
}
9598

9699
newParse, newWrite, NewInputF, NewOutputF, err := helpers.GetNewPW(logger, inputFile, outputFile)
97100
defer NewInputF.Close()

‎parsers/parse_iis_w3c.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,9 @@ func ParseIISStyle(logger zerolog.Logger, asnDB maxminddb.Reader, cityDB maxmind
8787

8888
if !tempArgs["passthrough"].(bool) {
8989
headers = append(headers, vars.GeoFields...)
90+
if tempArgs["use_idb"].(bool) {
91+
headers = append(headers, vars.IDBFields...)
92+
}
9093
}
9194
err = writer.Write(headers)
9295
if err != nil {

‎parsers/parse_json.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,9 @@ func ParseJSON(logger zerolog.Logger, asnDB maxminddb.Reader, cityDB maxminddb.R
9494
//sort.Sort(sort.StringSlice(headers))
9595
if !arguments["passthrough"].(bool) {
9696
headers = append(headers, vars.GeoFields...)
97+
if tempArgs["use_idb"].(bool) {
98+
headers = append(headers, vars.IDBFields...)
99+
}
97100
}
98101
err = writer.Write(headers)
99102
if err != nil {

‎parsers/parse_json_multi.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,9 @@ func ParseMultiLineJSON(logger zerolog.Logger, asnDB maxminddb.Reader, cityDB ma
182182

183183
if !arguments["passthrough"].(bool) {
184184
headers = append(headers, vars.GeoFields...)
185+
if tempArgs["use_idb"].(bool) {
186+
headers = append(headers, vars.IDBFields...)
187+
}
185188
}
186189
err = writer.Write(headers)
187190
if err != nil {

‎parsers/parse_kv.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,9 @@ func ParseKV(logger zerolog.Logger, inputFile string, outputFile string, asnDB m
9797
//sort.Sort(sort.StringSlice(headers))
9898
if !arguments["passthrough"].(bool) {
9999
headers = append(headers, vars.GeoFields...)
100+
if tempArgs["use_idb"].(bool) {
101+
headers = append(headers, vars.IDBFields...)
102+
}
100103
}
101104
err = writer.Write(headers)
102105
if err != nil {

0 commit comments

Comments
 (0)